Compare commits

...

19 Commits

Author SHA1 Message Date
nym21 4daabcee2c release: v0.3.0-beta.10 2026-05-17 22:20:30 +02:00
nym21 a6021b26cc docs: update generated docs 2026-05-17 22:19:58 +02:00
nym21 1a706da13c deps: bumped 2026-05-17 22:14:05 +02:00
nym21 20c4a113c9 oracle: v2 2026-05-17 22:13:03 +02:00
nym21 e5819769e8 website: snap 2026-05-17 22:12:44 +02:00
nym21 421e5286ce global: snap 2026-05-15 23:53:55 +02:00
nym21 68db22b9e8 mempool: polish/cleanup 2026-05-14 23:29:10 +02:00
nym21 90aca2e048 mmpl: new, mempool + rpc: fixes 2026-05-14 13:59:15 +02:00
nym21 528c134f26 mempool: fixes 2026-05-13 18:36:02 +02:00
nym21 5cc3fbfa6e crates: snapshot 2026-05-12 22:33:09 +02:00
nym21 8fc2e71492 website: snap 2026-05-12 22:32:53 +02:00
nym21 445c60a6f1 mempool: fixes 2026-05-10 19:40:02 +02:00
nym21 dd6eca138b mempool: fixes 2026-05-10 16:23:06 +02:00
nym21 774580ee11 mempool: fixes 2026-05-10 14:04:08 +02:00
nym21 fe5f30bca6 mempool: snap 2026-05-10 00:24:02 +02:00
nym21 c52a076bfc mempool: fix 2026-05-09 13:36:06 +02:00
nym21 e62b0ac2a5 global: next block template (+ diff) 2026-05-09 12:56:11 +02:00
nym21 3f2b5d3084 mempool: cleanup 2026-05-08 12:26:01 +02:00
nym21 aab16f8832 clients: bump versions 2026-05-08 12:14:34 +02:00
753 changed files with 11695 additions and 6314 deletions
Generated
+109 -49
View File
@@ -295,7 +295,7 @@ checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "blk"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_error",
@@ -308,7 +308,7 @@ dependencies = [
[[package]]
name = "brk"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_bencher",
"brk_bindgen",
@@ -333,7 +333,7 @@ dependencies = [
[[package]]
name = "brk_alloc"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"libmimalloc-sys",
"mimalloc",
@@ -341,7 +341,7 @@ dependencies = [
[[package]]
name = "brk_bencher"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_error",
"brk_logger",
@@ -351,14 +351,14 @@ dependencies = [
[[package]]
name = "brk_bencher_visualizer"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"plotters",
]
[[package]]
name = "brk_bindgen"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_cohort",
"brk_query",
@@ -371,7 +371,7 @@ dependencies = [
[[package]]
name = "brk_cli"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"anyhow",
"brk_alloc",
@@ -396,7 +396,7 @@ dependencies = [
[[package]]
name = "brk_client"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_cohort",
"brk_types",
@@ -407,7 +407,7 @@ dependencies = [
[[package]]
name = "brk_cohort"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_error",
"brk_traversable",
@@ -419,7 +419,7 @@ dependencies = [
[[package]]
name = "brk_computer"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_alloc",
@@ -448,7 +448,7 @@ dependencies = [
[[package]]
name = "brk_error"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"fjall",
@@ -464,7 +464,7 @@ dependencies = [
[[package]]
name = "brk_fetcher"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_error",
"brk_logger",
@@ -476,7 +476,7 @@ dependencies = [
[[package]]
name = "brk_indexer"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_alloc",
@@ -502,7 +502,7 @@ dependencies = [
[[package]]
name = "brk_iterator"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_error",
@@ -513,7 +513,7 @@ dependencies = [
[[package]]
name = "brk_logger"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"jiff",
"owo-colors",
@@ -524,14 +524,14 @@ dependencies = [
[[package]]
name = "brk_mempool"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_error",
"brk_logger",
"brk_oracle",
"brk_rpc",
"brk_types",
"derive_more",
"parking_lot",
"rustc-hash",
"smallvec",
@@ -540,7 +540,7 @@ dependencies = [
[[package]]
name = "brk_oracle"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_indexer",
"brk_types",
@@ -550,13 +550,14 @@ dependencies = [
[[package]]
name = "brk_query"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_computer",
"brk_error",
"brk_indexer",
"brk_mempool",
"brk_oracle",
"brk_reader",
"brk_rpc",
"brk_traversable",
@@ -574,7 +575,7 @@ dependencies = [
[[package]]
name = "brk_reader"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_error",
@@ -590,7 +591,7 @@ dependencies = [
[[package]]
name = "brk_rpc"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_error",
@@ -607,7 +608,7 @@ dependencies = [
[[package]]
name = "brk_server"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"aide",
"axum",
@@ -639,7 +640,7 @@ dependencies = [
[[package]]
name = "brk_store"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_error",
"brk_types",
@@ -650,7 +651,7 @@ dependencies = [
[[package]]
name = "brk_traversable"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"brk_traversable_derive",
"brk_types",
@@ -663,7 +664,7 @@ dependencies = [
[[package]]
name = "brk_traversable_derive"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"proc-macro2",
"quote",
@@ -672,7 +673,7 @@ dependencies = [
[[package]]
name = "brk_types"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"bitcoin",
"brk_error",
@@ -695,7 +696,7 @@ dependencies = [
[[package]]
name = "brk_website"
version = "0.3.0-beta.9"
version = "0.3.0-beta.10"
dependencies = [
"axum",
"brk_logger",
@@ -767,9 +768,9 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9"
[[package]]
name = "cc"
version = "1.2.61"
version = "1.2.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -967,9 +968,9 @@ dependencies = [
[[package]]
name = "corepc-types"
version = "0.12.0"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1583872320eb2ac629c36753023fd072f1ca1b3b74b20cc62bab055b54278789"
checksum = "b96c7869aa8234d10a41cbe3a1697bcb3a2482c48d9eb3541b3a4014a81afdad"
dependencies = [
"bitcoin",
"serde",
@@ -1499,9 +1500,9 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "hashbrown"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "heck"
@@ -1509,6 +1510,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "hex-conservative"
version = "0.2.2"
@@ -1794,7 +1801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown 0.17.0",
"hashbrown 0.17.1",
"serde",
"serde_core",
]
@@ -1808,6 +1815,23 @@ dependencies = [
"compare",
]
[[package]]
name = "is-terminal"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
]
[[package]]
name = "is_ci"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
[[package]]
name = "itertools"
version = "0.13.0"
@@ -2017,9 +2041,9 @@ dependencies = [
[[package]]
name = "lz4_flex"
version = "0.13.0"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a"
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
[[package]]
name = "matchit"
@@ -2084,6 +2108,19 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "mmpl"
version = "0.3.0-beta.10"
dependencies = [
"brk_error",
"brk_mempool",
"brk_rpc",
"brk_types",
"rustc-hash",
"serde",
"serde_json",
]
[[package]]
name = "nom"
version = "7.1.3"
@@ -2153,6 +2190,10 @@ name = "owo-colors"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
dependencies = [
"supports-color 2.1.0",
"supports-color 3.0.2",
]
[[package]]
name = "parking_lot"
@@ -2198,9 +2239,9 @@ dependencies = [
[[package]]
name = "pco"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89d71ab3c07ed898defa4915bdc2a963131d811a1eab0eeacfac65c94cdeae8"
checksum = "553ccdc7f6999785559af4998c79712a5ab820e26b68bad9146609c19587ec82"
dependencies = [
"better_io",
"dtype_dispatch",
@@ -2336,9 +2377,9 @@ dependencies = [
[[package]]
name = "quick_cache"
version = "0.6.21"
version = "0.6.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3"
checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
@@ -2757,9 +2798,9 @@ dependencies = [
[[package]]
name = "serde_qs"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2316d01592c3382277c5062105510e35e0a6bfb2851e30028485f7af8cf1240"
checksum = "67d525c8ff68aa99e5818302259bdd02d86d0303710616f39c0f44846ff6d332"
dependencies = [
"axum",
"itoa",
@@ -2885,6 +2926,25 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "supports-color"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
dependencies = [
"is-terminal",
"is_ci",
]
[[package]]
name = "supports-color"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6"
dependencies = [
"is_ci",
]
[[package]]
name = "syn"
version = "2.0.117"
@@ -2998,9 +3058,9 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.52.2"
version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"libc",
"mio",
@@ -3300,7 +3360,7 @@ dependencies = [
"itoa",
"libc",
"log",
"lz4_flex 0.13.0",
"lz4_flex 0.13.1",
"parking_lot",
"pco",
"rawdb",
@@ -3642,9 +3702,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0"
checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
[[package]]
name = "wio"
@@ -3817,9 +3877,9 @@ dependencies = [
[[package]]
name = "zerofrom"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
dependencies = [
"zerofrom-derive",
]
+27 -27
View File
@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.3.0-beta.9"
package.version = "0.3.0-beta.10"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md"
@@ -39,40 +39,40 @@ debug = true
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.9", features = ["serde"] }
brk_alloc = { version = "0.3.0-beta.9", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-beta.9", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-beta.9", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.0-beta.9", path = "crates/brk_cli" }
brk_client = { version = "0.3.0-beta.9", path = "crates/brk_client" }
brk_cohort = { version = "0.3.0-beta.9", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.0-beta.9", path = "crates/brk_computer" }
brk_error = { version = "0.3.0-beta.9", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.0-beta.9", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.0-beta.9", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.0-beta.9", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.0-beta.9", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.0-beta.9", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.0-beta.9", path = "crates/brk_oracle" }
brk_query = { version = "0.3.0-beta.9", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.0-beta.9", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.0-beta.9", path = "crates/brk_rpc" }
brk_server = { version = "0.3.0-beta.9", path = "crates/brk_server" }
brk_store = { version = "0.3.0-beta.9", path = "crates/brk_store" }
brk_traversable = { version = "0.3.0-beta.9", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.0-beta.9", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.0-beta.9", path = "crates/brk_types" }
brk_website = { version = "0.3.0-beta.9", path = "crates/brk_website" }
brk_alloc = { version = "0.3.0-beta.10", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-beta.10", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-beta.10", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.0-beta.10", path = "crates/brk_cli" }
brk_client = { version = "0.3.0-beta.10", path = "crates/brk_client" }
brk_cohort = { version = "0.3.0-beta.10", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.0-beta.10", path = "crates/brk_computer" }
brk_error = { version = "0.3.0-beta.10", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.0-beta.10", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.0-beta.10", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.0-beta.10", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.0-beta.10", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.0-beta.10", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.0-beta.10", path = "crates/brk_oracle" }
brk_query = { version = "0.3.0-beta.10", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.0-beta.10", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.0-beta.10", path = "crates/brk_rpc" }
brk_server = { version = "0.3.0-beta.10", path = "crates/brk_server" }
brk_store = { version = "0.3.0-beta.10", path = "crates/brk_store" }
brk_traversable = { version = "0.3.0-beta.10", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.0-beta.10", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.0-beta.10", path = "crates/brk_types" }
brk_website = { version = "0.3.0-beta.10", path = "crates/brk_website" }
byteview = "0.10.1"
color-eyre = "0.6.5"
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-types = { version = "0.12.0", features = ["std"], default-features = false }
corepc-types = { version = "0.13.0", features = ["std"], default-features = false }
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
fjall = "=3.0.4"
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.24", features = ["perf-inline", "tz-system"], default-features = false }
owo-colors = "4.3.0"
parking_lot = "0.12.5"
pco = "1.0.1"
pco = "1.0.2"
rayon = "1.12.0"
rustc-hash = "2.1.2"
schemars = { version = "1.2.1", features = ["indexmap2"] }
@@ -81,7 +81,7 @@ serde_bytes = "0.11.19"
serde_derive = "1.0.228"
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.1"
tokio = { version = "1.52.2", features = ["rt-multi-thread"] }
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.10", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] }
+1 -1
View File
@@ -13,7 +13,7 @@ brk_error = { workspace = true }
brk_reader = { workspace = true }
brk_rpc = { workspace = true }
brk_types = { workspace = true }
owo-colors = { workspace = true }
owo-colors = { workspace = true, features = ["supports-colors"] }
serde_json = { workspace = true }
[[bin]]
+10 -1
View File
@@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::{collections::HashSet, path::PathBuf};
use brk_error::{Error, Result};
use brk_rpc::{Auth, Client};
@@ -66,6 +66,9 @@ impl Args {
}
continue;
}
if a.starts_with('-') {
return Err(Error::Parse(format!("unknown flag {a}")));
}
positional.push(a);
}
@@ -74,6 +77,12 @@ impl Args {
.next()
.ok_or_else(|| Error::Parse("missing selector".into()))?;
let paths: Vec<Path> = iter.map(|f| Path::parse(&f)).collect::<Result<_>>()?;
let mut seen = HashSet::with_capacity(paths.len());
for p in &paths {
if !seen.insert(p.raw.as_str()) {
return Err(Error::Parse(format!("duplicate field '{}'", p.raw)));
}
}
Ok(Self {
selector,
paths,
+235 -171
View File
@@ -6,29 +6,126 @@ use bitcoin::{
};
use brk_error::{Error, Result};
use brk_types::ReadBlock;
use serde_json::{Value, json};
use serde_json::{Map, Value, json};
use crate::path::{Path, Step};
// `hex` is intentionally absent: matches `bitcoin-cli getblock <hash> 2`
// and keeps NDJSON dumps tractable. Still reachable explicitly via `blk N hex`.
const BLOCK_FIELDS: &[&str] = &[
"height",
"hash",
"version",
"version_hex",
"merkle",
"time",
"nonce",
"bits",
"difficulty",
"prev",
"txs",
"n_inputs",
"n_outputs",
"witness_txs",
"size",
"strippedsize",
"weight",
"subsidy",
"coinbase",
"coinbase_hex",
"header_hex",
"tx",
];
const TX_FIELDS: &[&str] = &[
"txid",
"wtxid",
"version",
"locktime",
"size",
"base_size",
"vsize",
"weight",
"inputs",
"outputs",
"is_coinbase",
"has_witness",
"is_rbf",
"total_out",
"hex",
"vin",
"vout",
];
const VIN_FIELDS: &[&str] = &[
"prev_txid",
"prev_vout",
"sequence",
"script_sig",
"script_sig_asm",
"witness",
"has_witness",
"is_rbf",
"coinbase",
];
const VOUT_FIELDS: &[&str] = &[
"value",
"script_pubkey",
"script_pubkey_asm",
"type",
"address",
];
pub struct Ctx<'a> {
block: &'a ReadBlock,
network: Network,
size_weight: OnceCell<(usize, usize)>,
}
impl<'a> Ctx<'a> {
pub fn new(block: &'a ReadBlock) -> Self {
pub fn new(block: &'a ReadBlock, network: Network) -> Self {
Self {
block,
network,
size_weight: OnceCell::new(),
}
}
pub fn resolve(&self, path: &Path) -> Result<Value> {
let (step, rest) = pop(&path.steps)?;
self.block_field(&step.name, step.index, rest)
}
pub fn resolve_str(&self, path: &Path) -> Result<String> {
Ok(match self.resolve(path)? {
Value::String(s) => s,
other => other.to_string(),
})
}
pub fn full(&self) -> Value {
let mut obj = Map::with_capacity(BLOCK_FIELDS.len());
for &name in BLOCK_FIELDS {
obj.insert(
name.into(),
self.block_field(name, None, &[]).expect("known block field"),
);
}
Value::Object(obj)
}
fn size_and_weight(&self) -> (usize, usize) {
*self
.size_weight
.get_or_init(|| self.block.total_size_and_weight())
}
fn block_field(&self, name: &str, index: Option<usize>, rest: &[Step]) -> Result<Value> {
let b = self.block;
let raw: &Block = b;
let scalar = |v| scalar_leaf(v, step, rest);
match step.name.as_str() {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"height" => scalar(json!(*b.height())),
"hash" => scalar(json!(b.hash().to_string())),
"time" => scalar(json!(b.header.time)),
@@ -37,7 +134,7 @@ impl<'a> Ctx<'a> {
"{:08x}",
b.header.version.to_consensus() as u32
))),
"bits" => scalar(json!(b.header.bits.to_consensus())),
"bits" => scalar(json!(format!("{:08x}", b.header.bits.to_consensus()))),
"nonce" => scalar(json!(b.header.nonce)),
"prev" => scalar(json!(b.header.prev_blockhash.to_string())),
"merkle" => scalar(json!(b.header.merkle_root.to_string())),
@@ -62,102 +159,154 @@ impl<'a> Ctx<'a> {
"header_hex" => scalar(json!(serialize_hex(&b.header))),
"hex" => scalar(json!(serialize_hex(raw))),
"coinbase" => scalar(json!(b.coinbase_tag().as_str())),
"tx" => pick(&b.txdata, step, rest, |i, tx| resolve_tx(tx, i == 0, rest)),
"coinbase_hex" => {
debug_assert!(
!b.txdata.is_empty() && !b.txdata[0].input.is_empty(),
"consensus-valid block has a coinbase tx with at least one input"
);
scalar(json!(b.txdata[0].input[0].script_sig.to_hex_string()))
}
"tx" => pick(&b.txdata, name, index, |i, tx| {
self.resolve_tx(tx, i == 0, rest)
}),
other => Err(unknown("block", other)),
}
}
pub fn resolve_str(&self, path: &Path) -> Result<String> {
Ok(match self.resolve(path)? {
Value::String(s) => s,
other => other.to_string(),
})
fn resolve_tx(&self, tx: &Transaction, is_coinbase: bool, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
let mut obj = Map::with_capacity(TX_FIELDS.len());
for &name in TX_FIELDS {
obj.insert(
name.into(),
self.tx_field(tx, is_coinbase, name, None, &[])
.expect("known tx field"),
);
}
return Ok(Value::Object(obj));
}
let (step, rest) = pop(steps)?;
self.tx_field(tx, is_coinbase, &step.name, step.index, rest)
}
pub fn full(&self) -> Value {
let b = self.block;
let (size, weight) = self.size_and_weight();
let tx: Vec<Value> = b
.txdata
.iter()
.enumerate()
.map(|(i, tx)| tx_to_value(tx, i == 0))
.collect();
json!({
"height": *b.height(),
"hash": b.hash().to_string(),
"version": b.header.version.to_consensus(),
"version_hex": format!("{:08x}", b.header.version.to_consensus() as u32),
"merkle": b.header.merkle_root.to_string(),
"time": b.header.time,
"nonce": b.header.nonce,
"bits": b.header.bits.to_consensus(),
"difficulty": b.header.difficulty_float(),
"prev": b.header.prev_blockhash.to_string(),
"txs": b.txdata.len(),
"n_inputs": b.txdata.iter().map(|t| t.input.len()).sum::<usize>(),
"n_outputs": b.txdata.iter().map(|t| t.output.len()).sum::<usize>(),
"witness_txs": b.txdata.iter().filter(|t| tx_has_witness(t)).count(),
"size": size,
"strippedsize": (weight - size) / 3,
"weight": weight,
"subsidy": subsidy_sats(*b.height()),
"coinbase": b.coinbase_tag().as_str(),
"header_hex": serialize_hex(&b.header),
"tx": tx,
})
fn tx_field(
&self,
tx: &Transaction,
is_coinbase: bool,
name: &str,
index: Option<usize>,
rest: &[Step],
) -> Result<Value> {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"txid" => scalar(json!(tx.compute_txid().to_string())),
"wtxid" => scalar(json!(tx.compute_wtxid().to_string())),
"version" => scalar(json!(tx.version.0)),
"locktime" => scalar(json!(tx.lock_time.to_consensus_u32())),
"size" => scalar(json!(tx.total_size())),
"base_size" => scalar(json!(tx.base_size())),
"vsize" => scalar(json!(tx.vsize())),
"weight" => scalar(json!(tx.weight().to_wu())),
"inputs" => scalar(json!(tx.input.len())),
"outputs" => scalar(json!(tx.output.len())),
"is_coinbase" => scalar(json!(is_coinbase)),
"has_witness" => scalar(json!(tx_has_witness(tx))),
"is_rbf" => scalar(json!(tx_is_rbf(tx))),
"total_out" => scalar(json!(tx_total_out(tx))),
"hex" => scalar(json!(serialize_hex(tx))),
"vin" => pick(&tx.input, name, index, |j, vin| {
resolve_vin(vin, is_coinbase && j == 0, rest)
}),
"vout" => pick(&tx.output, name, index, |_, vout| {
self.resolve_vout(vout, rest)
}),
other => Err(unknown("tx", other)),
}
}
fn size_and_weight(&self) -> (usize, usize) {
*self
.size_weight
.get_or_init(|| self.block.total_size_and_weight())
fn resolve_vout(&self, vout: &TxOut, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
let mut obj = Map::with_capacity(VOUT_FIELDS.len());
for &name in VOUT_FIELDS {
obj.insert(
name.into(),
self.vout_field(vout, name, None, &[])
.expect("known vout field"),
);
}
return Ok(Value::Object(obj));
}
let (step, rest) = pop(steps)?;
self.vout_field(vout, &step.name, step.index, rest)
}
}
fn resolve_tx(tx: &Transaction, is_coinbase: bool, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
return Ok(tx_to_value(tx, is_coinbase));
fn vout_field(
&self,
vout: &TxOut,
name: &str,
index: Option<usize>,
rest: &[Step],
) -> Result<Value> {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"value" => scalar(json!(vout.value.to_sat())),
"script_pubkey" => scalar(json!(vout.script_pubkey.to_hex_string())),
"script_pubkey_asm" => scalar(json!(vout.script_pubkey.to_asm_string())),
"type" => scalar(json!(script_type(&vout.script_pubkey))),
"address" => scalar(self.address_value(&vout.script_pubkey)),
other => Err(unknown("vout", other)),
}
}
let (step, rest) = pop(steps)?;
let scalar = |v| scalar_leaf(v, step, rest);
match step.name.as_str() {
"txid" => scalar(json!(tx.compute_txid().to_string())),
"wtxid" => scalar(json!(tx.compute_wtxid().to_string())),
"version" => scalar(json!(tx.version.0)),
"locktime" => scalar(json!(tx.lock_time.to_consensus_u32())),
"size" => scalar(json!(tx.total_size())),
"base_size" => scalar(json!(tx.base_size())),
"vsize" => scalar(json!(tx.vsize())),
"weight" => scalar(json!(tx.weight().to_wu())),
"inputs" => scalar(json!(tx.input.len())),
"outputs" => scalar(json!(tx.output.len())),
"is_coinbase" => scalar(json!(is_coinbase)),
"has_witness" => scalar(json!(tx_has_witness(tx))),
"is_rbf" => scalar(json!(tx_is_rbf(tx))),
"total_out" => scalar(json!(tx_total_out(tx))),
"hex" => scalar(json!(serialize_hex(tx))),
"vin" => pick(&tx.input, step, rest, |j, vin| {
resolve_vin(vin, is_coinbase && j == 0, rest)
}),
"vout" => pick(&tx.output, step, rest, |_, vout| resolve_vout(vout, rest)),
other => Err(unknown("tx", other)),
fn address_value(&self, s: &ScriptBuf) -> Value {
Address::from_script(s, self.network)
.map(|a| Value::String(a.to_string()))
.unwrap_or(Value::Null)
}
}
fn resolve_vin(vin: &TxIn, is_coinbase: bool, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
return Ok(vin_to_value(vin, is_coinbase));
let mut obj = Map::with_capacity(VIN_FIELDS.len());
for &name in VIN_FIELDS {
obj.insert(
name.into(),
vin_field(vin, is_coinbase, name, None, &[]).expect("known vin field"),
);
}
return Ok(Value::Object(obj));
}
let (step, rest) = pop(steps)?;
let scalar = |v| scalar_leaf(v, step, rest);
match step.name.as_str() {
vin_field(vin, is_coinbase, &step.name, step.index, rest)
}
fn vin_field(
vin: &TxIn,
is_coinbase: bool,
name: &str,
index: Option<usize>,
rest: &[Step],
) -> Result<Value> {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"prev_txid" => scalar(json!(vin.previous_output.txid.to_string())),
"prev_vout" => scalar(json!(vin.previous_output.vout)),
"sequence" => scalar(json!(vin.sequence.0)),
"script_sig" => scalar(json!(vin.script_sig.to_hex_string())),
"script_sig_asm" => scalar(json!(vin.script_sig.to_asm_string())),
"witness" => scalar(witness_to_value(vin)),
"witness" => {
if !rest.is_empty() {
return Err(Error::Parse(
"'witness' element has no fields to drill into".into(),
));
}
let items: Vec<String> = vin
.witness
.iter()
.map(|w| w.to_lower_hex_string())
.collect();
pick(&items, name, index, |_, hex| Ok(Value::String(hex.clone())))
}
"has_witness" => scalar(json!(!vin.witness.is_empty())),
"is_rbf" => scalar(json!(vin.sequence.is_rbf())),
"coinbase" => scalar(json!(is_coinbase)),
@@ -165,33 +314,17 @@ fn resolve_vin(vin: &TxIn, is_coinbase: bool, steps: &[Step]) -> Result<Value> {
}
}
fn resolve_vout(vout: &TxOut, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
return Ok(vout_to_value(vout));
}
let (step, rest) = pop(steps)?;
let scalar = |v| scalar_leaf(v, step, rest);
match step.name.as_str() {
"value" => scalar(json!(vout.value.to_sat())),
"script_pubkey" => scalar(json!(vout.script_pubkey.to_hex_string())),
"script_pubkey_asm" => scalar(json!(vout.script_pubkey.to_asm_string())),
"type" => scalar(json!(script_type(&vout.script_pubkey))),
"address" => scalar(address_value(&vout.script_pubkey)),
other => Err(unknown("vout", other)),
}
}
fn pick<T>(
items: &[T],
step: &Step,
_rest: &[Step],
name: &str,
index: Option<usize>,
mut resolve: impl FnMut(usize, &T) -> Result<Value>,
) -> Result<Value> {
match step.index {
match index {
Some(i) => {
let item = items
.get(i)
.ok_or_else(|| out_of_range(&step.name, i, items.len()))?;
.ok_or_else(|| out_of_range(name, i, items.len()))?;
resolve(i, item)
}
None => Ok(Value::Array(
@@ -210,14 +343,13 @@ fn pop(steps: &[Step]) -> Result<(&Step, &[Step])> {
.ok_or_else(|| Error::Parse("empty path segment".into()))
}
fn scalar_leaf(v: Value, step: &Step, rest: &[Step]) -> Result<Value> {
if step.index.is_some() {
return Err(Error::Parse(format!("'{}' is not an array", step.name)));
fn scalar_leaf(v: Value, name: &str, index: Option<usize>, rest: &[Step]) -> Result<Value> {
if index.is_some() {
return Err(Error::Parse(format!("'{name}' is not an array")));
}
if !rest.is_empty() {
return Err(Error::Parse(format!(
"'{}' is a scalar; nothing to drill into",
step.name
"'{name}' has no fields to drill into"
)));
}
Ok(v)
@@ -233,59 +365,6 @@ fn unknown(level: &str, name: &str) -> Error {
))
}
fn tx_to_value(tx: &Transaction, is_coinbase: bool) -> Value {
let vin: Vec<Value> = tx
.input
.iter()
.enumerate()
.map(|(j, v)| vin_to_value(v, is_coinbase && j == 0))
.collect();
let vout: Vec<Value> = tx.output.iter().map(vout_to_value).collect();
json!({
"txid": tx.compute_txid().to_string(),
"wtxid": tx.compute_wtxid().to_string(),
"version": tx.version.0,
"locktime": tx.lock_time.to_consensus_u32(),
"size": tx.total_size(),
"base_size": tx.base_size(),
"vsize": tx.vsize(),
"weight": tx.weight().to_wu(),
"inputs": tx.input.len(),
"outputs": tx.output.len(),
"is_coinbase": is_coinbase,
"has_witness": tx_has_witness(tx),
"is_rbf": tx_is_rbf(tx),
"total_out": tx_total_out(tx),
"hex": serialize_hex(tx),
"vin": vin,
"vout": vout,
})
}
fn vin_to_value(vin: &TxIn, is_coinbase: bool) -> Value {
json!({
"prev_txid": vin.previous_output.txid.to_string(),
"prev_vout": vin.previous_output.vout,
"sequence": vin.sequence.0,
"script_sig": vin.script_sig.to_hex_string(),
"script_sig_asm": vin.script_sig.to_asm_string(),
"witness": witness_to_value(vin),
"has_witness": !vin.witness.is_empty(),
"is_rbf": vin.sequence.is_rbf(),
"coinbase": is_coinbase,
})
}
fn vout_to_value(vout: &TxOut) -> Value {
json!({
"value": vout.value.to_sat(),
"script_pubkey": vout.script_pubkey.to_hex_string(),
"script_pubkey_asm": vout.script_pubkey.to_asm_string(),
"type": script_type(&vout.script_pubkey),
"address": address_value(&vout.script_pubkey),
})
}
fn tx_has_witness(tx: &Transaction) -> bool {
tx.input.iter().any(|i| !i.witness.is_empty())
}
@@ -307,15 +386,6 @@ fn subsidy_sats(height: u32) -> u64 {
}
}
fn witness_to_value(vin: &TxIn) -> Value {
Value::Array(
vin.witness
.iter()
.map(|w| Value::String(w.to_lower_hex_string()))
.collect(),
)
}
fn script_type(s: &ScriptBuf) -> &'static str {
if s.is_p2pkh() {
"p2pkh"
@@ -335,9 +405,3 @@ fn script_type(s: &ScriptBuf) -> &'static str {
"unknown"
}
}
fn address_value(s: &ScriptBuf) -> Value {
Address::from_script(s, Network::Bitcoin)
.map(|a| Value::String(a.to_string()))
.unwrap_or(Value::Null)
}
+4 -2
View File
@@ -15,16 +15,18 @@ impl Formatter {
pub fn format(&self, ctx: &Ctx) -> Result<String> {
match self.mode {
Mode::Bare => self.bare(ctx),
Mode::Bare => self.bare(ctx, false),
Mode::Tsv => self.tsv(ctx),
Mode::Json => Ok(serde_json::to_string(&self.object(ctx)?)?),
Mode::Pretty if self.fields.len() == 1 => self.bare(ctx, true),
Mode::Pretty => Ok(serde_json::to_string_pretty(&self.object(ctx)?)?),
}
}
fn bare(&self, ctx: &Ctx) -> Result<String> {
fn bare(&self, ctx: &Ctx, pretty: bool) -> Result<String> {
Ok(match ctx.resolve(&self.fields[0])? {
Value::String(s) => s,
other if pretty => serde_json::to_string_pretty(&other)?,
other => other.to_string(),
})
}
+6 -4
View File
@@ -37,17 +37,19 @@ fn run() -> Result<()> {
let client = args.rpc()?;
let (start, end) = Selector::parse(&args.selector, &client)?;
let network = client.get_network()?;
let mode = Mode::pick(args.pretty, args.compact, args.paths.len());
let mode = Mode::pick(args.pretty, args.compact, args.paths.len())?;
let reader = Reader::new(args.blocks_dir(), &client);
let formatter = Formatter::new(mode, args.paths);
let parser_threads = std::thread::available_parallelism()
let parser_threads = (std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
/ 2;
/ 2)
.max(1);
for block in reader.range_with(start, end, parser_threads)?.iter() {
let block = block?;
let line = formatter.format(&Ctx::new(&block))?;
let line = formatter.format(&Ctx::new(&block, network))?;
if !line.is_empty() {
println!("{line}");
}
+15 -3
View File
@@ -1,3 +1,5 @@
use brk_error::{Error, Result};
#[derive(Clone, Copy)]
pub enum Mode {
Bare,
@@ -7,8 +9,18 @@ pub enum Mode {
}
impl Mode {
pub fn pick(pretty: bool, compact: bool, n_fields: usize) -> Self {
if pretty {
pub fn pick(pretty: bool, compact: bool, n_fields: usize) -> Result<Self> {
if pretty && compact {
return Err(Error::Parse(
"--pretty and --compact are mutually exclusive".into(),
));
}
if compact && n_fields == 0 {
return Err(Error::Parse(
"--compact requires at least one field".into(),
));
}
Ok(if pretty {
Self::Pretty
} else if n_fields == 0 {
Self::Json
@@ -18,6 +30,6 @@ impl Mode {
Self::Tsv
} else {
Self::Json
}
})
}
}
+11 -9
View File
@@ -6,13 +6,15 @@ pub struct Selector;
impl Selector {
pub fn parse(s: &str, client: &Client) -> Result<(Height, Height)> {
let (start, end) = match s.split_once("..") {
Some((a, b)) => (Self::endpoint(a, client)?, Self::endpoint(b, client)?),
None => {
let h = Self::endpoint(s, client)?;
(h, h)
}
let (a, b) = s.split_once("..").unwrap_or((s, s));
let needs_tip = |p: &str| p == "tip" || p.starts_with("tip-");
let tip = if needs_tip(a) || needs_tip(b) {
Some(client.get_last_height()?)
} else {
None
};
let start = Self::endpoint(a, tip)?;
let end = Self::endpoint(b, tip)?;
if end < start {
return Err(Error::Parse(format!(
"range end {end} before start {start}"
@@ -21,15 +23,15 @@ impl Selector {
Ok((start, end))
}
fn endpoint(s: &str, client: &Client) -> Result<Height> {
fn endpoint(s: &str, tip: Option<Height>) -> Result<Height> {
if s == "tip" {
return client.get_last_height();
return Ok(tip.expect("tip pre-resolved when input contains 'tip'"));
}
if let Some(rest) = s.strip_prefix("tip-") {
let n: u32 = rest
.parse()
.map_err(|_| Error::Parse(format!("bad tip offset: {s}")))?;
let tip = client.get_last_height()?;
let tip = tip.expect("tip pre-resolved when input contains 'tip'");
return tip
.checked_sub(n)
.ok_or_else(|| Error::Parse(format!("tip-{n} underflows genesis")));
+27 -18
View File
@@ -1,4 +1,4 @@
use owo_colors::OwoColorize;
use owo_colors::{OwoColorize, Stream};
const SEL_W: usize = 5; // longest selector token: "tip-N"
const LABEL_W: usize = 28; // longest label across OUTPUT/OPTIONS/EXAMPLES (= example cmd "blk 800000 tx.0.vout.0.value")
@@ -7,18 +7,18 @@ const PH_W: usize = LABEL_W - FLAG_W - 1; // placeholder column width so flag+ph
const GAP: usize = 4;
pub fn print() {
println!("{} - inspect a Bitcoin Core block", "blk".bold());
println!("{} - inspect a Bitcoin Core block", bold("blk"));
println!();
section("USAGE");
println!(
" blk {} [{} ...] [OPTIONS]",
"<selector>".bright_black(),
"<field>".bright_black()
dim("<selector>"),
dim("<field>")
);
println!(
" {}",
"no fields = full block as JSON (analog of `bitcoin-cli getblock <hash> 2`)".bright_black()
dim("no fields = full block as JSON (analog of `bitcoin-cli getblock <hash> 2`)")
);
println!();
@@ -32,15 +32,15 @@ pub fn print() {
section("FIELDS");
println!(
" {}",
"dotted paths drill into nested data; omit an index for arrays".bright_black()
dim("dotted paths drill into nested data, omit an index for arrays")
);
println!();
group("block");
fields(&[
"height, hash, time, version, version_hex, bits, nonce,",
"prev, merkle, difficulty, txs, n_inputs, n_outputs,",
"witness_txs, size, strippedsize, weight, subsidy, coinbase,",
"header_hex, hex",
"witness_txs, size, strippedsize, weight, subsidy,",
"coinbase, coinbase_hex, header_hex, hex",
]);
println!();
group_note("tx.i", "omit i for all txs");
@@ -61,14 +61,14 @@ pub fn print() {
println!();
println!(
" {}",
"Naked tx / tx.i / vin / vout returns the whole sub-object as JSON.".bright_black()
dim("Naked tx / tx.i / vin / vout returns the whole sub-object as JSON.")
);
println!();
section("OUTPUT");
out("no fields", "full block JSON object, one per line (NDJSON)");
out("1 field", "bare value, one per line");
out("2+ fields", "compact JSON object, one per line (NDJSON)");
out("2+ fields", "JSON object, one per line (NDJSON)");
out("-p, --pretty", "pretty JSON object instead");
out(
"-c, --compact",
@@ -115,18 +115,18 @@ pub fn print() {
}
fn section(name: &str) {
println!("{}", format!("{name}:").bold());
println!("{}", bold(&format!("{name}:")));
}
fn group(name: &str) {
println!(" {}", format!("{name}:").bold());
println!(" {}", bold(&format!("{name}:")));
}
fn group_note(name: &str, note: &str) {
println!(
" {} {}",
format!("{name}:").bold(),
format!("({note})").bright_black()
bold(&format!("{name}:")),
dim(&format!("({note})"))
);
}
@@ -143,7 +143,7 @@ fn pad(s: &str, width: usize) -> String {
fn sel(token: &str, desc: &str) {
println!(
" {}{}{}{desc}",
token.bright_black(),
dim(token),
pad(token, SEL_W),
" ".repeat(GAP),
);
@@ -161,12 +161,12 @@ fn opt(flag: &str, ph: &str, desc: &str, default: Option<&str>) {
let head = format!(
" {flag}{} {}{}{}",
pad(flag, FLAG_W),
ph.bright_black(),
dim(ph),
pad(ph, PH_W),
" ".repeat(GAP),
);
match default {
Some(d) => println!("{head}{desc} {}", d.bright_black()),
Some(d) => println!("{head}{desc} {}", dim(d)),
None => println!("{head}{desc}"),
}
}
@@ -176,6 +176,15 @@ fn ex(cmd: &str, note: &str) {
" {cmd}{}{}{}",
pad(cmd, LABEL_W),
" ".repeat(GAP),
format!("# {note}").bright_black()
dim(&format!("# {note}"))
);
}
fn bold(s: &str) -> String {
s.if_supports_color(Stream::Stdout, |t| t.bold()).to_string()
}
fn dim(s: &str) -> String {
s.if_supports_color(Stream::Stdout, |t| t.bright_black())
.to_string()
}
+26 -8
View File
@@ -8953,7 +8953,7 @@ pub struct BrkClient {
impl BrkClient {
/// Client version.
pub const VERSION: &'static str = "v0.3.0-beta.8";
pub const VERSION: &'static str = "v0.3.0-beta.9";
/// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Self {
@@ -9009,7 +9009,7 @@ impl BrkClient {
/// Health check
///
/// Returns the health status of the API server, including uptime information.
/// Liveness probe. Returns server identity, uptime, and indexed/computed heights from local state only (no bitcoind round-trip). For real chain-tip catch-up, see `/api/server/sync`.
///
/// Endpoint: `GET /health`
pub fn get_health(&self) -> Result<Health> {
@@ -9294,7 +9294,7 @@ impl BrkClient {
/// Address transactions
///
/// Get transaction history for an address, sorted with newest first. Returns up to 50 entries: mempool transactions first, then confirmed transactions filling the remainder. To paginate further confirmed transactions, use `/address/{address}/txs/chain/{last_seen_txid}`.
/// Get transaction history for an address, newest first. Returns up to 50 mempool transactions plus a confirmed page sized to fill the response to 50 total (chain floor of 25, so 25-50 confirmed depending on mempool weight). To paginate further confirmed history, use `/address/{address}/txs/chain/{last_seen_txid}`.
///
/// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-address-transactions)*
///
@@ -9734,7 +9734,7 @@ impl BrkClient {
/// Projected mempool blocks
///
/// Get projected blocks from the mempool for fee estimation.
/// Projected blocks for fee estimation. Block 0 reflects Bitcoin Core's actual next-block selection; blocks 1+ are a fee-tier approximation.
///
/// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-mempool-blocks-fees)*
///
@@ -9745,7 +9745,7 @@ impl BrkClient {
/// Recommended fees
///
/// Get recommended fee rates for different confirmation targets.
/// Recommended fee rates by confirmation target.
///
/// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-recommended-fees)*
///
@@ -9756,7 +9756,7 @@ impl BrkClient {
/// Precise recommended fees
///
/// Get recommended fee rates with up to 3 decimal places.
/// Recommended fee rates with sub-integer precision.
///
/// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-recommended-fees-precise)*
///
@@ -9778,10 +9778,10 @@ impl BrkClient {
/// Mempool content hash
///
/// Returns an opaque `u64` that changes whenever the projected next block changes. Same value as the mempool ETag. Useful as a freshness/liveness signal: if it stays constant for tens of seconds on a live network, the mempool sync loop has stalled.
/// Returns an opaque hash that changes whenever the projected next block changes. Same value as the mempool ETag. Useful as a freshness/liveness signal: if it stays constant for tens of seconds on a live network, the mempool sync loop has stalled.
///
/// Endpoint: `GET /api/mempool/hash`
pub fn get_mempool_hash(&self) -> Result<i64> {
pub fn get_mempool_hash(&self) -> Result<NextBlockHash> {
self.base.get_json(&format!("/api/mempool/hash"))
}
@@ -9829,6 +9829,24 @@ impl BrkClient {
self.base.get_json(&format!("/api/v1/fullrbf/replacements"))
}
/// Projected next block template
///
/// Bitcoin Core's `getblocktemplate` selection: full transaction bodies in GBT order with aggregate stats. The returned `hash` is an opaque content token; pass it as `<hash>` on `/api/v1/mempool/block-template/diff/{hash}` to fetch deltas instead of refetching the whole template.
///
/// Endpoint: `GET /api/v1/mempool/block-template`
pub fn get_block_template(&self) -> Result<BlockTemplate> {
self.base.get_json(&format!("/api/v1/mempool/block-template"))
}
/// Block template diff since hash
///
/// Delta of the projected next block since `<hash>`. `order` is the full new template in order: each entry is either a number (index into the prior template the client cached at `<hash>`) or a transaction object (new body to insert at this position). Walk `order` once to rebuild; `removed` is a convenience list of txids that left so clients can evict cached bodies. After applying, use the response `hash` as `<hash>` on the next call to keep iterating. Returns `404` when `<hash>` has aged out of server history; clients should fall back to `/api/v1/mempool/block-template`.
///
/// Endpoint: `GET /api/v1/mempool/block-template/diff/{hash}`
pub fn get_block_template_diff(&self, hash: NextBlockHash) -> Result<BlockTemplateDiff> {
self.base.get_json(&format!("/api/v1/mempool/block-template/diff/{hash}"))
}
/// Live BTC/USD price
///
/// Returns the current BTC/USD price in dollars, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool.
+1 -1
View File
@@ -41,7 +41,7 @@ use super::{
metrics::AvgAmountMetrics,
};
const VERSION: Version = Version::new(24);
const VERSION: Version = Version::new(24 + brk_oracle::VERSION);
#[derive(Traversable)]
pub struct AddrMetricsVecs<M: StorageMode = Rw> {
+51 -91
View File
@@ -1,8 +1,8 @@
use std::ops::Range;
use brk_error::{Error, Result};
use brk_error::Result;
use brk_indexer::{Indexer, Lengths};
use brk_oracle::{Config, NUM_BINS, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
use brk_oracle::{Config, Histogram, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
use tracing::info;
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, StorageMode, VecIndex, WritableVec};
@@ -61,8 +61,8 @@ impl Vecs {
fn compute_prices(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let source_version =
indexer.vecs.outputs.value.version() + indexer.vecs.outputs.output_type.version();
let source_version = indexer.vecs.outputs.value.version()
+ indexer.vecs.outputs.output_type.version();
self.spot
.cents
.height
@@ -112,7 +112,7 @@ impl Vecs {
let seed_bin = cents_to_bin(prev_cents.inner() as f64);
let warmup = config.window_size.min(committed - START_HEIGHT);
let mut oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
Self::feed_blocks(o, indexer, (committed - warmup)..committed);
Self::feed_blocks(o, indexer, (committed - warmup)..committed, None);
});
let num_new = total_heights - committed;
@@ -121,7 +121,8 @@ impl Vecs {
committed, total_heights
);
let ref_bins = Self::feed_blocks(&mut oracle, indexer, committed..total_heights);
let ref_bins =
Self::feed_blocks(&mut oracle, indexer, committed..total_heights, None);
for (i, ref_bin) in ref_bins.into_iter().enumerate() {
self.spot
@@ -150,32 +151,18 @@ impl Vecs {
}
/// Feed a range of blocks from the indexer into an Oracle (skipping coinbase),
/// returning per-block ref_bin values. Uncapped: derives boundaries from
/// raw indexer vec lengths. Use during compute, when the indexer is
/// quiescent and `safe_lengths` is still pinned at the pre-pass value.
fn feed_blocks<M: StorageMode>(
/// returning per-block ref_bin values.
///
/// A transaction carrying an `OP_RETURN` output is protocol machinery, not a
/// dollar-denominated payment, so all of its outputs are dropped from the
/// histogram. This needs per-transaction grouping of a block's outputs.
///
/// Pass `cap = None` from compute paths, when the indexer is quiescent and
/// raw vec lengths are authoritative. Pass `cap = Some(&safe_lengths)` from
/// reader paths so concurrent writer pushes past the cap are invisible.
pub fn feed_blocks<IM: StorageMode>(
oracle: &mut Oracle,
indexer: &Indexer<M>,
range: Range<usize>,
) -> Vec<f64> {
Self::feed_blocks_inner(oracle, indexer, range, None)
}
/// Capped variant: derives boundaries from `cap` instead of raw vec
/// lengths, so concurrent writer pushes past `cap` are invisible.
/// Reader paths (live_oracle) use this with the current `safe_lengths`.
fn feed_blocks_capped<M: StorageMode>(
oracle: &mut Oracle,
indexer: &Indexer<M>,
range: Range<usize>,
cap: &Lengths,
) -> Vec<f64> {
Self::feed_blocks_inner(oracle, indexer, range, Some(cap))
}
fn feed_blocks_inner<M: StorageMode>(
oracle: &mut Oracle,
indexer: &Indexer<M>,
indexer: &Indexer<IM>,
range: Range<usize>,
cap: Option<&Lengths>,
) -> Vec<f64> {
@@ -208,36 +195,38 @@ impl Vecs {
let mut ref_bins = Vec::with_capacity(range.len());
// Cursor avoids per-block PcoVec page decompression for
// the tx-indexed first_txout_index lookup. The accessed
// tx_index values (first_tx_index + 1) are strictly increasing
// across blocks, so the cursor only advances forward.
// Cursor avoids per-block PcoVec page decompression for the
// tx-indexed first_txout_index lookup. Accessed tx_index values
// are strictly increasing across blocks, so it only advances forward.
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
// Reusable buffers avoid per-block allocation
// Reusable buffers: avoid per-block allocation. `tx_starts` holds the
// first txout index of each non-coinbase tx in the current block.
let mut values: Vec<Sats> = Vec::new();
let mut output_types: Vec<OutputType> = Vec::new();
let mut tx_starts: Vec<usize> = Vec::new();
for (idx, _h) in range.enumerate() {
let first_tx_index = first_tx_indexes[idx];
for idx in 0..range.len() {
let next_first_tx_index = first_tx_indexes
.get(idx + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
.unwrap_or(TxIndex::from(total_txs))
.to_usize();
let block_first_tx = first_tx_indexes[idx].to_usize() + 1;
let tx_count = next_first_tx_index - block_first_tx;
let next_out_first = out_firsts
let out_end = out_firsts
.get(idx + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
let out_start = if first_tx_index.to_usize() + 1 < next_first_tx_index.to_usize() {
let target = first_tx_index.to_usize() + 1;
txout_cursor.advance(target - txout_cursor.position());
txout_cursor.next().unwrap().to_usize()
} else {
next_out_first
};
let out_end = next_out_first;
txout_cursor.advance(block_first_tx - txout_cursor.position());
tx_starts.clear();
for _ in 0..tx_count {
tx_starts.push(txout_cursor.next().unwrap().to_usize());
}
let out_start = tx_starts.first().copied().unwrap_or(out_end);
indexer
.vecs
@@ -250,10 +239,20 @@ impl Vecs {
&mut output_types,
);
let mut hist = [0u32; NUM_BINS];
for i in 0..values.len() {
if let Some(bin) = oracle.output_to_bin(values[i], output_types[i]) {
hist[bin] += 1;
let mut hist = Histogram::zeros();
for tx in 0..tx_count {
let lo = tx_starts[tx] - out_start;
let hi = tx_starts
.get(tx + 1)
.map(|s| s - out_start)
.unwrap_or(out_end - out_start);
if output_types[lo..hi].contains(&OutputType::OpReturn) {
continue;
}
for i in lo..hi {
if let Some(bin) = oracle.output_to_bin(values[i], output_types[i]) {
hist.increment(bin);
}
}
}
@@ -263,42 +262,3 @@ impl Vecs {
ref_bins
}
}
impl<M: StorageMode> Vecs<M> {
/// Returns an Oracle seeded from the last committed price, with the last
/// window_size blocks already processed. Ready for additional blocks (e.g. mempool).
pub fn live_oracle<IM: StorageMode>(&self, indexer: &Indexer<IM>) -> Result<Oracle> {
let config = Config::default();
let safe_lengths = indexer.safe_lengths();
let height = safe_lengths.height.to_usize();
let last_idx = self
.spot
.cents
.height
.len()
.checked_sub(1)
.ok_or(Error::NotFound(
"oracle prices not yet computed".to_string(),
))?;
let last_cents = self
.spot
.cents
.height
.collect_one_at(last_idx)
.ok_or(Error::NotFound(
"oracle prices not yet computed".to_string(),
))?;
let seed_bin = cents_to_bin(last_cents.inner() as f64);
let window_size = config.window_size;
let oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
Vecs::feed_blocks_capped(
o,
indexer,
height.saturating_sub(window_size)..height,
&safe_lengths,
);
});
Ok(oracle)
}
}
+4 -1
View File
@@ -4,6 +4,7 @@ pub(crate) mod ohlcs;
use std::path::Path;
use brk_oracle::VERSION as ORACLE_VERSION;
use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{Database, ReadOnlyClone, Rw, StorageMode};
@@ -49,7 +50,9 @@ impl Vecs {
version: Version,
indexes: &indexes::Vecs,
) -> brk_error::Result<Self> {
let version = version + Version::new(11);
// `ORACLE_VERSION` folds in the on-chain oracle algorithm version so
// every price-derived module invalidates when computed prices change.
let version = version + Version::new(11 + ORACLE_VERSION);
let price_cents = CachedPerBlock::forced_import(db, "price_cents", version, indexes)?;
+14 -14
View File
@@ -3,7 +3,6 @@
use std::{
fs,
path::{Path, PathBuf},
sync::Arc,
thread,
time::{Duration, Instant},
};
@@ -13,7 +12,6 @@ use brk_reader::{Reader, XORBytes};
use brk_rpc::Client;
use brk_types::{BlockHash, Height};
use fjall::PersistMode;
use parking_lot::RwLock;
use tracing::{debug, error, info};
use vecdb::{
Exit, RawDBError, ReadOnlyClone, ReadableVec, Ro, Rw, StorageMode, WritableVec, unlikely,
@@ -39,13 +37,25 @@ pub struct Indexer<M: StorageMode = Rw> {
path: PathBuf,
pub vecs: Vecs<M>,
pub stores: Stores,
tip_blockhash: Arc<RwLock<BlockHash>>,
safe_lengths: SafeLengths,
}
impl<M: StorageMode> Indexer<M> {
/// Tip block hash at the pipeline-safe ceiling.
///
/// Reads the on-disk blockhash vec at `safe_lengths.height - 1` so
/// the answer always agrees with `safe_lengths`. The indexer's loop
/// pushes new hashes per block before `safe_lengths` advances (that
/// only happens after the compute pass via
/// [`Indexer::advance_safe_lengths`]); reading from a live cache
/// here would mint a tip ahead of every safe-bound endpoint and
/// cause cache etags to invalidate before the data they cover is
/// actually queryable.
pub fn tip_blockhash(&self) -> BlockHash {
*self.tip_blockhash.read()
match self.safe_lengths().height.decremented() {
Some(h) => self.vecs.blocks.blockhash.collect_one(h).unwrap_or_default(),
None => BlockHash::default(),
}
}
/// Pipeline-safe `Lengths` snapshot shared with `Query`. Writers
@@ -83,8 +93,6 @@ impl Indexer {
let stores = Stores::forced_import(&indexed_path, VERSION)?;
info!("Imported stores in {:?}", i.elapsed());
let tip_blockhash = vecs.blocks.blockhash.collect_last().unwrap_or_default();
let safe_lengths = SafeLengths::new();
if let Some(lengths) = Lengths::from_local(&vecs, &stores) {
safe_lengths.advance(lengths);
@@ -94,7 +102,6 @@ impl Indexer {
path: indexed_path.clone(),
vecs,
stores,
tip_blockhash: Arc::new(RwLock::new(tip_blockhash)),
safe_lengths,
})
};
@@ -122,7 +129,6 @@ impl Indexer {
fn full_reset(&mut self) -> Result<()> {
info!("Full reset...");
self.safe_lengths.reset();
*self.tip_blockhash.write() = BlockHash::default();
self.vecs.reset()?;
let stores_path = self.path.join("stores");
fs::remove_dir_all(&stores_path).ok();
@@ -188,9 +194,6 @@ impl Indexer {
debug!("Rollback stores done.");
self.vecs.rollback_if_needed(&starting_lengths)?;
debug!("Rollback vecs done.");
if let Some(hash) = prev_hash.as_ref() {
*self.tip_blockhash.write() = *hash;
}
drop(lock);
let mut lengths = starting_lengths;
@@ -312,8 +315,6 @@ impl Indexer {
export(stores, vecs, height)?;
readers = Readers::new(vecs);
}
*self.tip_blockhash.write() = block.block_hash().into();
}
drop(readers);
@@ -388,7 +389,6 @@ impl ReadOnlyClone for Indexer {
path: self.path.clone(),
vecs: self.vecs.read_only_clone(),
stores: self.stores.clone(),
tip_blockhash: self.tip_blockhash.clone(),
safe_lengths: self.safe_lengths.clone(),
}
}
+1 -1
View File
@@ -118,7 +118,7 @@ impl Blocks {
BlockRange::After { hash } => {
let start = if let Some(hash) = hash.as_ref() {
let block_info = client.get_block_header_info(hash)?;
(block_info.height + 1).into()
Height::from((block_info.height + 1) as u64)
} else {
Height::ZERO
};
+19
View File
@@ -23,6 +23,7 @@ const MAX_LOG_AGE_DAYS: u64 = 7;
/// `*.txt` file older than 7 days is pruned on startup.
pub fn init(dir: Option<&Path>) -> io::Result<()> {
tracing_log::LogTracer::init().ok();
install_panic_hook();
#[cfg(debug_assertions)]
const DEFAULT_LEVEL: &str = "debug";
@@ -65,6 +66,24 @@ pub fn init(dir: Option<&Path>) -> io::Result<()> {
Ok(())
}
fn install_panic_hook() {
std::panic::set_hook(Box::new(|info| {
let location = info
.location()
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
.unwrap_or_else(|| "unknown".to_string());
let payload = info.payload();
let msg = payload
.downcast_ref::<&str>()
.copied()
.map(str::to_owned)
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "Box<dyn Any>".to_owned());
let backtrace = std::backtrace::Backtrace::capture();
tracing::error!(location, backtrace = %backtrace, "panic: {msg}");
}));
}
/// Register a hook that gets called for every log message.
pub fn register_hook<F>(hook: F) -> Result<(), &'static str>
where
+1 -1
View File
@@ -11,9 +11,9 @@ exclude = ["examples/"]
[dependencies]
bitcoin = { workspace = true }
brk_error = { workspace = true }
brk_oracle = { workspace = true }
brk_rpc = { workspace = true }
brk_types = { workspace = true }
derive_more = { workspace = true }
tracing = { workspace = true }
parking_lot = { workspace = true }
rustc-hash = { workspace = true }
+2 -3
View File
@@ -26,13 +26,13 @@ fn main() -> Result<()> {
let info_count = mempool.info().count;
let stats = mempool.stats();
let snapshot = mempool.snapshot();
let blocks_tx_total: usize = snapshot.blocks.iter().map(|b| b.len()).sum();
let blocks_tx_total: usize = snapshot.blocks.iter().map(Vec::len).sum();
println!(
"info.count={} txs={} unresolved={} addrs={} outpoints={} \
graveyard.tombstones={} graveyard.order={} \
snap.txs.len={} snap.blocks={} snap.blocks_txs={} \
rebuilds={} skip.clean={}",
rebuilds={}",
info_count,
stats.txs,
stats.unresolved,
@@ -44,7 +44,6 @@ fn main() -> Result<()> {
snapshot.blocks.len(),
blocks_tx_total,
stats.rebuilds,
stats.skip_cleans,
);
}
}
+45
View File
@@ -0,0 +1,45 @@
//! Address-keyed reads.
use std::cmp::Reverse;
use brk_types::{AddrBytes, AddrMempoolStats, Timestamp, Transaction, TxidPrefix};
use crate::Mempool;
impl Mempool {
/// Hash of the address's mempool state, `None` if the address has
/// no live mempool activity. Used as an `ETag` for address-keyed
/// mempool responses. Route handlers may fall back to a 0 sentinel.
pub fn addr_state_hash(&self, addr: &AddrBytes) -> Option<u64> {
self.read().addrs.stats_hash(addr)
}
/// Per-address mempool stats. `None` if the address has no live mempool activity.
pub fn addr_stats(&self, addr: &AddrBytes) -> Option<AddrMempoolStats> {
self.read().addrs.get(addr).map(|e| e.stats.clone())
}
/// Live mempool txs touching `addr`, newest first by `first_seen`,
/// capped at `limit`. Returns owned `Transaction`s.
#[must_use]
pub fn addr_txs(&self, addr: &AddrBytes, limit: usize) -> Vec<Transaction> {
let state = self.read();
let Some(entry) = state.addrs.get(addr) else {
return vec![];
};
let mut ordered: Vec<(Timestamp, &Transaction)> = entry
.txids
.iter()
.filter_map(|txid| {
let record = state.txs.record_by_prefix(&TxidPrefix::from(txid))?;
Some((record.entry.first_seen, &record.tx))
})
.collect();
ordered.sort_unstable_by_key(|b| Reverse(b.0));
ordered
.into_iter()
.take(limit)
.map(|(_, tx)| tx.clone())
.collect()
}
}
@@ -0,0 +1,199 @@
//! Projected next block: full template and incremental diff.
use brk_types::{
BlockTemplate, BlockTemplateDiff, BlockTemplateDiffEntry, MempoolBlock, NextBlockHash,
Transaction, Txid,
};
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::warn;
use crate::{Mempool, state::State};
impl Mempool {
pub fn next_block_hash(&self) -> NextBlockHash {
self.snapshot().next_block_hash
}
/// Full projected next block: Core's `getblocktemplate` selection
/// (block 0) with aggregate stats and full tx bodies in GBT order.
#[must_use]
pub fn block_template(&self) -> BlockTemplate {
let snap = self.snapshot();
BlockTemplate {
hash: snap.next_block_hash,
stats: snap
.block_stats
.first()
.map(MempoolBlock::from)
.unwrap_or_default(),
transactions: self.collect_txs(snap.block0_txids()),
}
}
/// Delta of the projected next block since `since`. `None` when
/// `since` has aged out of the rebuilder's history (server should
/// 404 -> client falls back to `block_template`).
///
/// `order` walks the new template in template order. Each entry is
/// either a `Retained` index into the prior template (which the
/// client cached when it obtained `since`) or a `New` inline body.
/// `removed` is the convenience list of txids that left.
#[must_use]
pub fn block_template_diff(&self, since: NextBlockHash) -> Option<BlockTemplateDiff> {
let past = self.rebuilder().historical_block0(since)?;
let prior_index: FxHashMap<Txid, u32> = past
.iter()
.enumerate()
.map(|(idx, txid)| (*txid, idx as u32))
.collect();
let snap = self.snapshot();
let state = self.read();
let mut order = Vec::with_capacity(snap.blocks.first().map_or(0, Vec::len));
let mut current: FxHashSet<Txid> = FxHashSet::default();
for txid in snap.block0_txids() {
current.insert(txid);
match prior_index.get(&txid) {
Some(&idx) => order.push(BlockTemplateDiffEntry::Retained(idx)),
None => match Self::lookup_body(&state, &txid) {
Some(tx) => order.push(BlockTemplateDiffEntry::New(tx)),
None => warn!(?txid, "block_template_diff: snapshot tx body missing"),
},
}
}
drop(state);
let removed = past.into_iter().filter(|t| !current.contains(t)).collect();
Some(BlockTemplateDiff {
hash: snap.next_block_hash,
since,
order,
removed,
})
}
fn collect_txs(&self, txids: impl IntoIterator<Item = Txid>) -> Vec<Transaction> {
let state = self.read();
txids
.into_iter()
.filter_map(|txid| {
let body = Self::lookup_body(&state, &txid);
if body.is_none() {
warn!(?txid, "block_template: snapshot tx body missing");
}
body
})
.collect()
}
/// Body for a txid in a published snapshot. Graveyard fallback
/// covers the eviction race: an Applier may have buried the tx
/// after the snapshot was built. Burial retention (1h) >> snapshot
/// cycle (~1s), so the invariant holds in practice. A `None` here
/// is a soft anomaly the caller logs and drops.
fn lookup_body(state: &State, txid: &Txid) -> Option<Transaction> {
state
.txs
.get(txid)
.or_else(|| state.graveyard.get(txid).map(|t| &t.tx))
.cloned()
}
}
#[cfg(test)]
mod tests {
use brk_types::{BlockTemplateDiffEntry, FeeRate};
use super::*;
use crate::{
state::TxEntry,
test_support::{fake_entry_info, fake_tx, p2wpkh_script},
};
fn insert_tx(mempool: &Mempool, seed: u8, fee: u64, vsize: u64) -> Txid {
let tx = fake_tx(seed, &[None], &[(p2wpkh_script(seed + 1), 1_234)]);
let txid = tx.txid;
let info = fake_entry_info(txid, fee, vsize);
let entry = TxEntry::new(&info, vsize, false);
let mut state = mempool.test_state_lock().write();
state.txs.insert(tx, entry);
txid
}
#[test]
fn block_template_hash_matches_next_block_hash() {
let mempool = Mempool::for_test();
let txid = insert_tx(&mempool, 0xA0, 1_234, 100);
mempool.test_tick(&[txid], FeeRate::new(1.0));
let template = mempool.block_template();
assert_eq!(template.hash, mempool.next_block_hash());
assert_eq!(template.transactions.len(), 1);
assert_eq!(template.transactions[0].txid, txid);
}
#[test]
fn block_template_diff_round_trip_reconstructs_t1_from_t0() {
// T0: pool has two txs, both in gbt -> block 0.
let mempool = Mempool::for_test();
let txid_a = insert_tx(&mempool, 0xA1, 1_111, 100);
let txid_b = insert_tx(&mempool, 0xA2, 2_222, 100);
mempool.test_tick(&[txid_a, txid_b], FeeRate::new(1.0));
let t0 = mempool.block_template();
// T1: add a third tx, advance gbt. block_template_diff(t0.hash) must
// be reconstructible into the new block 0 ordering by combining the
// retained prior-indexed bodies from T0 with the New bodies inline.
let txid_c = insert_tx(&mempool, 0xA3, 3_333, 100);
mempool.test_tick(&[txid_a, txid_b, txid_c], FeeRate::new(1.0));
let t1 = mempool.block_template();
let diff = mempool
.block_template_diff(t0.hash)
.expect("t0 is still in history");
assert_eq!(diff.since, t0.hash);
assert_eq!(diff.hash, t1.hash);
let mut reconstructed = Vec::with_capacity(diff.order.len());
for entry in &diff.order {
match entry {
BlockTemplateDiffEntry::Retained(idx) => {
reconstructed.push(t0.transactions[*idx as usize].clone());
}
BlockTemplateDiffEntry::New(tx) => reconstructed.push(tx.clone()),
}
}
let expected: Vec<_> = t1.transactions.iter().map(|tx| tx.txid).collect();
let got: Vec<_> = reconstructed.iter().map(|tx| tx.txid).collect();
assert_eq!(got, expected, "diff round-trips back into T1 ordering");
assert!(diff.removed.is_empty());
}
#[test]
fn block_template_diff_removed_lists_evicted_txs() {
let mempool = Mempool::for_test();
let txid_a = insert_tx(&mempool, 0xA4, 1_111, 100);
let txid_b = insert_tx(&mempool, 0xA5, 2_222, 100);
mempool.test_tick(&[txid_a, txid_b], FeeRate::new(1.0));
let t0 = mempool.block_template();
// T1: txid_a no longer in gbt.
mempool.test_tick(&[txid_b], FeeRate::new(1.0));
let diff = mempool.block_template_diff(t0.hash).unwrap();
assert_eq!(diff.removed, vec![txid_a]);
}
#[test]
fn block_template_diff_unknown_since_returns_none() {
let mempool = Mempool::for_test();
mempool.test_tick(&[], FeeRate::new(1.0));
let bogus = NextBlockHash::new(0xDEAD_BEEF);
assert!(mempool.block_template_diff(bogus).is_none());
}
#[test]
fn block_template_empty_pool_has_no_transactions() {
let mempool = Mempool::for_test();
mempool.test_tick(&[], FeeRate::new(2.0));
let template = mempool.block_template();
assert!(template.transactions.is_empty());
}
}
+38
View File
@@ -0,0 +1,38 @@
//! Fee reads: tier recommendations, projected-block stats, per-tx rates.
use brk_types::{FeeRate, RecommendedFees, TxidPrefix, Txid};
use crate::{Mempool, snapshot::BlockStats};
impl Mempool {
#[must_use]
pub fn fees(&self) -> RecommendedFees {
self.snapshot().fees.clone()
}
#[must_use]
pub fn block_stats(&self) -> Vec<BlockStats> {
self.snapshot().block_stats.clone()
}
/// Effective fee rate for a live tx: snapshot's linearized chunk
/// rate. Falls back to `fee/vsize` for txs added since the latest
/// snapshot was built (apply -> same-cycle tick gap).
pub fn live_effective_fee_rate(&self, prefix: &TxidPrefix) -> Option<FeeRate> {
if let Some(rate) = self.snapshot().chunk_rate_for(prefix) {
return Some(rate);
}
self.read()
.txs
.entry_by_prefix(prefix)
.map(|e| e.fee_rate())
}
/// Linearized chunk rate captured at burial - same value
/// `live_effective_fee_rate` returned while the tx was alive, so an
/// evicted RBF predecessor reports the package-effective rate it
/// had in the mempool, not a misleading isolated `fee/vsize`.
pub fn graveyard_fee_rate(&self, txid: &Txid) -> Option<FeeRate> {
self.read().graveyard.get(txid).map(|tomb| tomb.chunk_rate)
}
}
+23
View File
@@ -0,0 +1,23 @@
//! Mempool info + price-blending output histogram.
use brk_oracle::Histogram;
use brk_types::MempoolInfo;
use crate::Mempool;
impl Mempool {
#[must_use]
pub fn info(&self) -> MempoolInfo {
self.read().info.clone()
}
/// Snapshot of pre-bucketed oracle bins across all live mempool tx
/// outputs. The total is maintained incrementally by `TxStore` on
/// every insert/remove, so this hot path is `O(NUM_BINS)` regardless
/// of pool size. Used by `live_price` to blend the mempool into the
/// committed oracle without re-parsing scripts per request.
#[must_use]
pub fn live_histogram(&self) -> Histogram {
self.read().txs.live_histogram()
}
}
+11
View File
@@ -0,0 +1,11 @@
//! Read-side accessors on [`crate::Mempool`]. Each submodule groups a
//! cohesive method set. Types flow back through `pub use`.
mod addr;
mod block_template;
mod fees;
mod histogram;
mod rbf;
mod tx;
pub use rbf::{RbfForTx, RbfNode};
+214
View File
@@ -0,0 +1,214 @@
//! RBF tree extraction. Returns owned trees so the caller can enrich
//! with indexer data (`mined`, effective fee rate) after the lock
//! drops: enriching under the lock re-enters `Mempool` and would
//! recursively acquire the same read lock.
use brk_types::{Sats, Timestamp, Transaction, Txid, TxidPrefix, VSize};
use rustc_hash::FxHashSet;
use crate::{
Mempool,
state::TxEntry,
stores::{TxGraveyard, TxStore},
};
#[derive(Debug, Clone)]
pub struct RbfNode {
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
pub value: Sats,
pub first_seen: Timestamp,
/// BIP-125 signaling: at least one input has sequence < 0xffffffff-1.
pub rbf: bool,
/// `true` iff any predecessor in this subtree was non-signaling.
pub full_rbf: bool,
pub replaces: Vec<RbfNode>,
}
#[derive(Debug, Clone, Default)]
pub struct RbfForTx {
/// Tree rooted at the terminal replacer. `None` if `txid` is unknown.
pub root: Option<RbfNode>,
/// Direct predecessors of the requested tx (txids only).
pub replaces: Vec<Txid>,
}
impl Mempool {
/// Walk forward through `Replaced { by }` to the terminal replacer
/// and return its full predecessor tree, plus the requested tx's
/// direct predecessors. Single read-lock window.
#[must_use]
pub fn rbf_for_tx(&self, txid: &Txid) -> RbfForTx {
let state = self.read();
let root_txid = state.graveyard.replacement_root_of(*txid);
let replaces: Vec<Txid> = state
.graveyard
.predecessors_of(txid)
.map(|(p, _)| *p)
.collect();
let root = Self::build_rbf_node(&root_txid, &state.txs, &state.graveyard);
RbfForTx { root, replaces }
}
/// Recent terminal-replacer trees, most-recent first, deduplicated
/// by root, capped at `limit`. `full_rbf_only` drops trees with no
/// non-signaling predecessor.
#[must_use]
pub fn recent_rbf_trees(&self, full_rbf_only: bool, limit: usize) -> Vec<RbfNode> {
let state = self.read();
let mut seen: FxHashSet<Txid> = FxHashSet::default();
state
.graveyard
.replaced_iter_recent_first()
.filter_map(|(_, by)| {
let root = state.graveyard.replacement_root_of(*by);
seen.insert(root).then_some(root)
})
.filter_map(|root| Self::build_rbf_node(&root, &state.txs, &state.graveyard))
.filter(|n| !full_rbf_only || n.full_rbf)
.take(limit)
.collect()
}
fn build_rbf_node(txid: &Txid, txs: &TxStore, graveyard: &TxGraveyard) -> Option<RbfNode> {
let (tx, entry) = Self::resolve_rbf_node(txid, txs, graveyard)?;
let replaces: Vec<RbfNode> = graveyard
.predecessors_of(txid)
.filter_map(|(pred, _)| Self::build_rbf_node(pred, txs, graveyard))
.collect();
let full_rbf = replaces.iter().any(|c| !c.rbf || c.full_rbf);
let value: Sats = tx.output.iter().map(|o| o.value).sum();
Some(RbfNode {
txid: *txid,
fee: entry.fee,
vsize: entry.vsize,
value,
first_seen: entry.first_seen,
rbf: entry.rbf,
full_rbf,
replaces,
})
}
fn resolve_rbf_node<'a>(
txid: &Txid,
txs: &'a TxStore,
graveyard: &'a TxGraveyard,
) -> Option<(&'a Transaction, &'a TxEntry)> {
txs.record_by_prefix(&TxidPrefix::from(txid))
.map(|r| (&r.tx, &r.entry))
.or_else(|| graveyard.get(txid).map(|t| (&t.tx, &t.entry)))
}
}
#[cfg(test)]
mod tests {
use brk_types::FeeRate;
use super::*;
use crate::{
Mempool, TxRemoval,
state::TxEntry,
test_support::{fake_entry_info, fake_tx, fake_txid, p2wpkh_script},
};
/// Place a live tx (the replacer) and bury one or more predecessors
/// pointing at it. `bury_chain` carries `(seed, predecessor_of_next)`
/// pairs in oldest-first order. Each links forward to the next entry
/// or to `live_seed` when last.
fn build_rbf_world(live_seed: u8, predecessors: &[u8]) -> (Mempool, Txid, Vec<Txid>) {
let mempool = Mempool::for_test();
let live_tx = fake_tx(live_seed, &[None], &[(p2wpkh_script(live_seed + 1), 1_234)]);
let live_txid = live_tx.txid;
let live_entry = TxEntry::new(&fake_entry_info(live_txid, 5_000, 100), 100, true);
let mut pred_txids = Vec::with_capacity(predecessors.len());
let mut state = mempool.test_state_lock().write();
for (i, seed) in predecessors.iter().enumerate() {
let tx = fake_tx(*seed, &[None], &[(p2wpkh_script(seed + 1), 1_234)]);
let txid = tx.txid;
// Each predecessor signals BIP-125 (rbf=true) so full_rbf stays clear.
let entry = TxEntry::new(&fake_entry_info(txid, 1_000, 100), 100, true);
let by = predecessors
.get(i + 1)
.map(|next_seed| fake_txid(*next_seed))
.unwrap_or(live_txid);
let rate = FeeRate::from((entry.fee, entry.vsize));
state.graveyard.bury(tx, entry, rate, TxRemoval::Replaced { by });
pred_txids.push(txid);
}
state.txs.insert(live_tx, live_entry);
drop(state);
(mempool, live_txid, pred_txids)
}
#[test]
fn rbf_for_tx_single_replacement_returns_root_and_replaces() {
// pred -> live. rbf_for_tx(pred) walks forward to live and lists
// pred under its `replaces` tree.
let (mempool, live, preds) = build_rbf_world(0xC0, &[0xC1]);
let pred = preds[0];
let rbf = mempool.rbf_for_tx(&pred);
let root = rbf.root.expect("terminal replacer reachable");
assert_eq!(root.txid, live);
let replaced_txids: Vec<Txid> = root.replaces.iter().map(|n| n.txid).collect();
assert_eq!(replaced_txids, vec![pred]);
// Convenience list: direct predecessors of the requested tx.
assert!(rbf.replaces.is_empty(), "pred has no predecessors of its own");
}
#[test]
fn rbf_for_tx_chain_walks_to_terminal_root() {
// A -> B -> C(live). rbf_for_tx(A) walks A -> B -> C, root is C.
// root.replaces is B, B.replaces is A.
let (mempool, live, preds) = build_rbf_world(0xC2, &[0xC3, 0xC4]);
let a = preds[0];
let b = preds[1];
let rbf = mempool.rbf_for_tx(&a);
let root = rbf.root.expect("terminal replacer reachable");
assert_eq!(root.txid, live);
assert_eq!(root.replaces.len(), 1);
assert_eq!(root.replaces[0].txid, b);
assert_eq!(root.replaces[0].replaces.len(), 1);
assert_eq!(root.replaces[0].replaces[0].txid, a);
}
#[test]
fn rbf_for_tx_unknown_tx_returns_none_root() {
let mempool = Mempool::for_test();
let bogus = Txid::COINBASE;
let rbf = mempool.rbf_for_tx(&bogus);
assert!(rbf.root.is_none());
assert!(rbf.replaces.is_empty());
}
#[test]
fn recent_rbf_trees_dedup_by_root_and_respect_limit() {
// Chain 0xC6 -> 0xC7 -> live plus a sibling 0xC8 also replaced by
// live. All paths roll up to the same root, so the recent listing
// dedups them down to a single tree.
let (mempool, live, _preds) = build_rbf_world(0xC5, &[0xC6, 0xC7]);
{
let mut state = mempool.test_state_lock().write();
let extra = fake_tx(0xC8, &[None], &[(p2wpkh_script(0xC9), 1_234)]);
let extra_txid = extra.txid;
let entry = TxEntry::new(&fake_entry_info(extra_txid, 999, 100), 100, true);
let rate = FeeRate::from((entry.fee, entry.vsize));
state.graveyard.bury(extra, entry, rate, TxRemoval::Replaced { by: live });
}
let trees = mempool.recent_rbf_trees(false, 10);
assert_eq!(trees.len(), 1, "all paths roll up to one root");
assert_eq!(trees[0].txid, live);
let capped = mempool.recent_rbf_trees(false, 0);
assert!(capped.is_empty(), "limit honored");
}
}
+68
View File
@@ -0,0 +1,68 @@
//! Tx-keyed reads.
use brk_types::{
MempoolRecentTx, OutpointPrefix, Transaction, Txid, TxidPrefix, Vin, Vout,
};
use crate::Mempool;
impl Mempool {
pub fn contains_txid(&self, txid: &Txid) -> bool {
self.read().txs.contains(txid)
}
/// Apply `f` to the live tx body if present.
pub fn with_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> {
self.read().txs.get(txid).map(f)
}
/// Apply `f` to a `Vanished` tombstone's tx body if present.
/// `Replaced` tombstones return `None` because the tx will not confirm.
pub fn with_vanished_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> {
self.read().graveyard.get_vanished(txid).map(|t| f(&t.tx))
}
/// Mempool tx spending `(txid, vout)`, or `None`. The spender's
/// input list is walked to rule out `TxidPrefix` collisions.
pub fn lookup_spender(&self, txid: &Txid, vout: Vout) -> Option<(Txid, Vin)> {
let key = OutpointPrefix::new(TxidPrefix::from(txid), vout);
let state = self.read();
let spender_prefix = state.outpoint_spends.get(&key)?;
let spender = state.txs.record_by_prefix(&spender_prefix)?;
let vin_pos = spender
.tx
.input
.iter()
.position(|inp| inp.txid == *txid && inp.vout == vout)?;
Some((spender.entry.txid, Vin::from(vin_pos)))
}
/// Snapshot of all live mempool txids.
///
/// Allocates `32 * len(mempool)` bytes under the read guard. Sized for
/// diagnostics. Route layers serving large pools should paginate at
/// their boundary rather than calling this per request.
#[must_use]
pub fn txids(&self) -> Vec<Txid> {
self.read().txs.txids().copied().collect()
}
/// Snapshot of recent live txs.
#[must_use]
pub fn recent_txs(&self) -> Vec<MempoolRecentTx> {
self.read().txs.recent().to_vec()
}
/// `first_seen` Unix-second timestamps for `txids`, in input order.
/// Returns 0 for unknown txids. `Vanished` tombstones fall back to
/// the buried entry's `first_seen` to avoid flicker between drop
/// and indexer catch-up.
#[must_use]
pub fn transaction_times(&self, txids: &[Txid]) -> Vec<u64> {
let state = self.read();
txids
.iter()
.map(|txid| state.first_seen(txid).map_or(0, u64::from))
.collect()
}
}
-162
View File
@@ -1,162 +0,0 @@
//! Cluster mempool linearization (Core 31's "Single Fee Linearization").
//!
//! Given a topologically ordered cluster (parents before children) with
//! per-tx `(fee, vsize)` and parent edges as local indices, partition the
//! cluster into chunks ordered by descending feerate, where each chunk is
//! the highest-rate ancestor-closed set of remaining txs.
//!
//! The "lift" merging this implements is what makes CPFP visible at the
//! cluster level: a child whose rate exceeds its parent's rate gets folded
//! into a chunk with the parent, and the chunk's rate is the combined
//! `(parent_fee + child_fee) / (parent_vsize + child_vsize)`. Cascades
//! upward through any further parents until rates are non-increasing.
//!
//! This is the proxy-fallback case; under Core 31+ each tx's `fees.chunk`
//! / `chunkweight` already encodes the chunked rate, so all members of a
//! chunk would share that rate. Computing locally from `(fee, vsize)`
//! gives the same answer either way and works on older Core too.
//!
//! Complexity is `O(n^2)` per linearization (n bounded by cluster cap),
//! matching mempool.space's frontend implementation.
use brk_types::{CpfpClusterChunk, CpfpClusterTxIndex, FeeRate, Sats, VSize};
use rustc_hash::{FxBuildHasher, FxHashSet};
/// One cluster member: its `(fee, vsize)` and parent edges as
/// local indices into the same array.
pub struct ChunkInput<'a> {
pub fee: Sats,
pub vsize: VSize,
pub parents: &'a [CpfpClusterTxIndex],
}
/// Linearize `items` into chunks. `items` must be in topological order
/// (parents before children); `parents` indices must point earlier in
/// the slice. Returns chunks sorted by descending feerate, with each
/// chunk's `txs` listed in the input topological order.
pub fn linearize(items: &[ChunkInput<'_>]) -> Vec<CpfpClusterChunk> {
let n = items.len();
if n == 0 {
return Vec::new();
}
let mut remaining: Vec<bool> = vec![true; n];
let mut chunks: Vec<CpfpClusterChunk> = Vec::new();
let empty: FxHashSet<u32> = FxHashSet::default();
while remaining.iter().any(|&r| r) {
// Pick the top single-anchored ancestor-closed set. On rate
// ties the larger set wins so a uniform-rate chain emits one
// chunk instead of n singletons. The extension loop below
// catches the same case at zero extra cost, but starting big
// shaves iterations.
let mut best: Option<(FeeRate, FxHashSet<u32>, Sats, VSize)> = None;
for i in 0..n {
if !remaining[i] {
continue;
}
let anc = closure(items, &remaining, &empty, i as u32);
let (fee, vsize) = sum_fee_vsize(items, &anc);
let rate = FeeRate::from((fee, vsize));
let replace = match &best {
None => true,
Some((br, ba, _, _)) => rate > *br || (rate == *br && anc.len() > ba.len()),
};
if replace {
best = Some((rate, anc, fee, vsize));
}
}
let (mut chunk_rate, mut anc, mut chunk_fee, mut chunk_vsize) =
best.expect("at least one remaining tx");
// Extend the chunk with any other remaining ancestor-closed
// subset whose union keeps the chunk rate >= current rate.
// SFL chunks are the *maximum* ancestor-closed set at the top
// rate, but a single anchor only sees one connected component
// up to the cluster root: a parent with one long chain plus
// additional same-rate sibling children leaves the siblings
// stranded as same-rate singleton chunks - which can even
// appear "above" the main chunk under integer-vsize rounding,
// breaking the descending-rate invariant.
loop {
let mut best_ext: Option<(FeeRate, FxHashSet<u32>, Sats, VSize)> = None;
for i in 0..n {
if !remaining[i] || anc.contains(&(i as u32)) {
continue;
}
let extra = closure(items, &remaining, &anc, i as u32);
if extra.is_empty() {
continue;
}
let (ef, ev) = sum_fee_vsize(items, &extra);
let new_fee = chunk_fee + ef;
let new_vsize = chunk_vsize + ev;
let new_rate = FeeRate::from((new_fee, new_vsize));
if new_rate < chunk_rate {
continue;
}
let replace = match &best_ext {
None => true,
Some((br, _, _, _)) => new_rate > *br,
};
if replace {
best_ext = Some((new_rate, extra, new_fee, new_vsize));
}
}
match best_ext {
Some((r, e, f, v)) => {
anc.extend(&e);
chunk_fee = f;
chunk_vsize = v;
chunk_rate = r;
}
None => break,
}
}
let mut indices: Vec<u32> = anc.into_iter().collect();
indices.sort_unstable();
for &x in &indices {
remaining[x as usize] = false;
}
let txs: Vec<CpfpClusterTxIndex> =
indices.into_iter().map(CpfpClusterTxIndex::from).collect();
chunks.push(CpfpClusterChunk { txs, feerate: chunk_rate });
}
chunks
}
fn closure(
items: &[ChunkInput<'_>],
remaining: &[bool],
excluded: &FxHashSet<u32>,
start: u32,
) -> FxHashSet<u32> {
let mut set: FxHashSet<u32> = FxHashSet::with_capacity_and_hasher(8, FxBuildHasher);
if !remaining[start as usize] || excluded.contains(&start) {
return set;
}
let mut stack: Vec<u32> = vec![start];
while let Some(x) = stack.pop() {
if !set.insert(x) {
continue;
}
for &p in items[x as usize].parents {
let pu: u32 = u32::from(p);
if remaining[pu as usize] && !excluded.contains(&pu) && !set.contains(&pu) {
stack.push(pu);
}
}
}
set
}
fn sum_fee_vsize(items: &[ChunkInput<'_>], set: &FxHashSet<u32>) -> (Sats, VSize) {
let mut fee = Sats::ZERO;
let mut vsize = VSize::from(0u64);
for &x in set {
fee += items[x as usize].fee;
vsize += items[x as usize].vsize;
}
(fee, vsize)
}
-305
View File
@@ -1,305 +0,0 @@
//! CPFP (Child Pays For Parent) walk over a `Snapshot`'s adjacency.
//!
//! The snapshot stores per-tx parent/child edges in `TxIndex` space and
//! per-tx `(fee, vsize)` we need for chunking.
//!
//! Three independent walks:
//! - `ancestors_idx`: capped DFS up `parents` only.
//! - `descendants_idx`: capped DFS down `children` only.
//! - cluster `members`: capped DFS over `parents children`, i.e. the
//! connected component of the seed in the in-mempool dependency
//! graph. Required to match Core 31's cluster mempool semantics:
//! siblings (sharing a parent) and cousins (sharing a descendant)
//! belong to the same cluster but are missed by ancestor/descendant
//! walks alone.
//!
//! The cluster is then linearized via `brk_types::linearize` (single fee
//! linearization) so chunks reflect Core's CPFP "lift": a child whose
//! rate exceeds its parent's gets folded into a chunk with the parent
//! at the combined feerate. The seed's chunk feerate is what
//! `effective_fee_per_vsize` reports.
use std::collections::VecDeque;
use brk_types::{
CpfpCluster, CpfpClusterChunk, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, CpfpInfo, FeeRate,
SigOps, TxidPrefix, VSize,
};
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use crate::{
Mempool,
chunking::{ChunkInput, linearize},
steps::{SnapTx, TxIndex},
};
/// Cap matches Bitcoin Core's default mempool ancestor/descendant
/// chain limits and mempool.space's truncation.
const MAX: usize = 25;
/// Cluster cap matches Bitcoin Core 31's `MAX_CLUSTER_COUNT_LIMIT`
/// (max txs in a single cluster-mempool cluster). Sized large enough
/// to hold the whole connected component for any policy-conformant
/// cluster, then truncated.
const MAX_CLUSTER: usize = 64;
impl Mempool {
/// CPFP info for a live mempool tx. Returns `None` only when the
/// tx isn't in the mempool, so callers can fall through to the
/// confirmed path.
pub fn cpfp_info(&self, prefix: &TxidPrefix) -> Option<CpfpInfo> {
let snapshot = self.snapshot();
let seed_idx = snapshot.idx_of(prefix)?;
let seed = snapshot.tx(seed_idx)?;
let sigops = self
.read()
.txs
.get(&seed.txid)
.map(|tx| tx.total_sigop_cost)
.unwrap_or(SigOps::ZERO);
Some(build_cpfp_info(&snapshot.txs, seed_idx, seed, sigops))
}
}
fn build_cpfp_info(
txs: &[SnapTx],
seed_idx: TxIndex,
seed: &SnapTx,
sigops: SigOps,
) -> CpfpInfo {
let ancestors = collect_entries(txs, seed_idx, |t| &t.parents);
let descendants = collect_entries(txs, seed_idx, |t| &t.children);
let best_descendant = descendants
.iter()
.max_by_key(|e| FeeRate::from((e.fee, e.weight)))
.cloned();
let (cluster, effective_fee_per_vsize) = build_cluster(txs, seed_idx, seed);
let vsize = VSize::from(seed.weight);
CpfpInfo {
ancestors,
best_descendant,
descendants,
effective_fee_per_vsize,
sigops,
fee: seed.fee,
vsize,
adjusted_vsize: sigops.adjust_vsize(vsize),
cluster,
}
}
/// Walk the graph from `seed` along `next` and lift the visited indices
/// into wire-shape `CpfpEntry`s in one go.
fn collect_entries(
txs: &[SnapTx],
seed: TxIndex,
next: impl Fn(&SnapTx) -> &[TxIndex],
) -> Vec<CpfpEntry> {
walk(txs, seed, next)
.iter()
.filter_map(|&i| txs.get(i.as_usize()).map(CpfpEntry::from))
.collect()
}
/// Capped DFS from `seed` (exclusive), following the neighbors yielded
/// by `next`. Used for both the ancestor and descendant walks.
fn walk(txs: &[SnapTx], seed: TxIndex, next: impl Fn(&SnapTx) -> &[TxIndex]) -> Vec<TxIndex> {
let Some(seed_node) = txs.get(seed.as_usize()) else {
return Vec::new();
};
let mut visited: FxHashSet<TxIndex> =
FxHashSet::with_capacity_and_hasher(MAX + 1, FxBuildHasher);
visited.insert(seed);
let mut out: Vec<TxIndex> = Vec::with_capacity(MAX);
let mut stack: Vec<TxIndex> = next(seed_node).to_vec();
while let Some(idx) = stack.pop() {
if out.len() >= MAX {
break;
}
if !visited.insert(idx) {
continue;
}
out.push(idx);
if let Some(t) = txs.get(idx.as_usize()) {
stack.extend(next(t).iter().copied());
}
}
out
}
/// Wire-shape `CpfpCluster` plus the seed's chunk feerate. Members are
/// the connected component of the seed in the dependency graph, then
/// topologically sorted (parents before children) so wire indices and
/// chunk-internal ordering are valid for client-side reconstruction.
/// Returns `(None, seed_per_tx_rate)` for singletons (matches
/// mempool.space, which omits `cluster` when no relations exist).
fn build_cluster(
txs: &[SnapTx],
seed_idx: TxIndex,
seed: &SnapTx,
) -> (Option<CpfpCluster>, FeeRate) {
let seed_per_tx_rate = FeeRate::from((seed.fee, seed.vsize));
let component = walk_cluster(txs, seed_idx);
if component.len() <= 1 {
return (None, seed_per_tx_rate);
}
let members = topo_sort(txs, &component);
let local_of = build_local_index(&members);
let (cluster_txs, vsizes) = collect_cluster_members(txs, &members, &local_of);
let chunks = linearize_cluster(&cluster_txs, &vsizes);
let (chunk_index, seed_chunk_rate) =
locate_seed_chunk(local_of[&seed_idx], &chunks, seed_per_tx_rate);
(
Some(CpfpCluster {
txs: cluster_txs,
chunks,
chunk_index,
}),
seed_chunk_rate,
)
}
/// `members[i]`'s wire index, keyed by snapshot `TxIndex`. Built once
/// so per-tx parent edges can be remapped without a linear scan.
fn build_local_index(members: &[TxIndex]) -> FxHashMap<TxIndex, CpfpClusterTxIndex> {
members
.iter()
.enumerate()
.map(|(i, &idx)| (idx, CpfpClusterTxIndex::from(i as u32)))
.collect()
}
/// Materialize wire-shape `CpfpClusterTx`s for every member with parent
/// edges remapped to local indices, plus the parallel `vsize` column the
/// linearizer needs (not carried on `CpfpClusterTx`, which only stores
/// weight).
fn collect_cluster_members(
txs: &[SnapTx],
members: &[TxIndex],
local_of: &FxHashMap<TxIndex, CpfpClusterTxIndex>,
) -> (Vec<CpfpClusterTx>, Vec<VSize>) {
let mut cluster_txs: Vec<CpfpClusterTx> = Vec::with_capacity(members.len());
let mut vsizes: Vec<VSize> = Vec::with_capacity(members.len());
for &idx in members {
let Some(t) = txs.get(idx.as_usize()) else {
continue;
};
let parents: Vec<CpfpClusterTxIndex> = t
.parents
.iter()
.filter_map(|p| local_of.get(p).copied())
.collect();
cluster_txs.push(CpfpClusterTx {
txid: t.txid,
weight: t.weight,
fee: t.fee,
parents,
});
vsizes.push(t.vsize);
}
(cluster_txs, vsizes)
}
/// Single-fee-linearize the cluster, borrowing parents from the
/// already-built `cluster_txs` so no re-allocation is needed.
fn linearize_cluster(cluster_txs: &[CpfpClusterTx], vsizes: &[VSize]) -> Vec<CpfpClusterChunk> {
let inputs: Vec<ChunkInput<'_>> = cluster_txs
.iter()
.zip(vsizes)
.map(|(c, &vsize)| ChunkInput {
fee: c.fee,
vsize,
parents: &c.parents,
})
.collect();
linearize(&inputs)
}
/// Find the chunk containing the seed and return its index plus rate.
/// Falls back to `(0, seed_per_tx_rate)` when the seed isn't in any
/// chunk - shouldn't happen but keeps the wire shape valid.
fn locate_seed_chunk(
seed_local: CpfpClusterTxIndex,
chunks: &[CpfpClusterChunk],
seed_per_tx_rate: FeeRate,
) -> (u32, FeeRate) {
chunks
.iter()
.enumerate()
.find(|(_, ch)| ch.txs.contains(&seed_local))
.map(|(i, ch)| (i as u32, ch.feerate))
.unwrap_or((0, seed_per_tx_rate))
}
/// Capped DFS over the undirected dependency graph (`parents
/// children`) starting from `seed`. Returns the connected component
/// truncated to `MAX_CLUSTER`, with `seed` at index 0.
fn walk_cluster(txs: &[SnapTx], seed: TxIndex) -> Vec<TxIndex> {
if txs.get(seed.as_usize()).is_none() {
return Vec::new();
}
let mut visited: FxHashSet<TxIndex> =
FxHashSet::with_capacity_and_hasher(MAX_CLUSTER, FxBuildHasher);
visited.insert(seed);
let mut out: Vec<TxIndex> = Vec::with_capacity(MAX_CLUSTER);
out.push(seed);
let mut stack: Vec<TxIndex> = vec![seed];
while let Some(idx) = stack.pop() {
let Some(t) = txs.get(idx.as_usize()) else {
continue;
};
for &n in t.parents.iter().chain(t.children.iter()) {
if out.len() >= MAX_CLUSTER {
return out;
}
if visited.insert(n) {
out.push(n);
stack.push(n);
}
}
}
out
}
/// Kahn's topological sort over the connected component, restricted to
/// in-cluster parent edges. Returns members in an order where every tx
/// follows all its in-cluster parents.
fn topo_sort(txs: &[SnapTx], component: &[TxIndex]) -> Vec<TxIndex> {
let n = component.len();
let pos: FxHashMap<TxIndex, usize> = component
.iter()
.enumerate()
.map(|(i, &x)| (x, i))
.collect();
let mut indeg: Vec<u32> = vec![0; n];
let mut children: Vec<Vec<usize>> = vec![Vec::new(); n];
for (i, &idx) in component.iter().enumerate() {
let Some(t) = txs.get(idx.as_usize()) else {
continue;
};
indeg[i] = t.parents.iter().filter(|p| pos.contains_key(p)).count() as u32;
for &c in t.children.iter() {
if let Some(&ci) = pos.get(&c) {
children[i].push(ci);
}
}
}
let mut queue: VecDeque<usize> = (0..n).filter(|&i| indeg[i] == 0).collect();
let mut out: Vec<TxIndex> = Vec::with_capacity(n);
while let Some(i) = queue.pop_front() {
out.push(component[i]);
for &c in &children[i] {
indeg[c] -= 1;
if indeg[c] == 0 {
queue.push_back(c);
}
}
}
out
}
@@ -0,0 +1,7 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AddedKind {
/// First time we've seen this txid.
Fresh,
/// Re-entered the pool after a prior removal still in the graveyard.
Revived,
}
@@ -0,0 +1,104 @@
//! Per-cycle 0↔1+ address transition buffer. Same-cycle cancellation
//! (enter→leave, leave→enter) is encapsulated on the recording methods.
use brk_types::AddrBytes;
use rustc_hash::FxHashSet;
#[derive(Default)]
pub struct AddrTransitions {
enters: FxHashSet<AddrBytes>,
leaves: FxHashSet<AddrBytes>,
}
impl AddrTransitions {
pub fn record_enter(&mut self, bytes: AddrBytes) {
if !self.leaves.remove(&bytes) {
self.enters.insert(bytes);
}
}
pub fn record_leave(&mut self, bytes: AddrBytes) {
if !self.enters.remove(&bytes) {
self.leaves.insert(bytes);
}
}
pub fn into_vecs(self) -> (Vec<AddrBytes>, Vec<AddrBytes>) {
(
self.enters.into_iter().collect(),
self.leaves.into_iter().collect(),
)
}
}
#[cfg(test)]
mod tests {
use bitcoin::{ScriptBuf, hashes::Hash};
use super::*;
fn addr(seed: u8) -> AddrBytes {
let mut bytes = [0u8; 20];
bytes[0] = seed;
let script = ScriptBuf::new_p2wpkh(&bitcoin::WPubkeyHash::from_byte_array(bytes));
AddrBytes::try_from(&script).expect("p2wpkh -> AddrBytes")
}
#[test]
fn enter_then_leave_cancels() {
let mut t = AddrTransitions::default();
t.record_enter(addr(1));
t.record_leave(addr(1));
let (enters, leaves) = t.into_vecs();
assert!(enters.is_empty());
assert!(leaves.is_empty());
}
#[test]
fn leave_then_enter_cancels() {
let mut t = AddrTransitions::default();
t.record_leave(addr(2));
t.record_enter(addr(2));
let (enters, leaves) = t.into_vecs();
assert!(enters.is_empty());
assert!(leaves.is_empty());
}
#[test]
fn enter_leave_enter_collapses_to_single_enter() {
let mut t = AddrTransitions::default();
let a = addr(3);
t.record_enter(a.clone());
t.record_leave(a.clone());
t.record_enter(a.clone());
let (enters, leaves) = t.into_vecs();
assert_eq!(enters, vec![a]);
assert!(leaves.is_empty());
}
#[test]
fn leave_enter_leave_collapses_to_single_leave() {
let mut t = AddrTransitions::default();
let a = addr(4);
t.record_leave(a.clone());
t.record_enter(a.clone());
t.record_leave(a.clone());
let (enters, leaves) = t.into_vecs();
assert!(enters.is_empty());
assert_eq!(leaves, vec![a]);
}
#[test]
fn distinct_addrs_dont_interfere() {
let mut t = AddrTransitions::default();
let a = addr(5);
let b = addr(6);
t.record_enter(a.clone());
t.record_leave(b.clone());
let (mut enters, mut leaves) = t.into_vecs();
enters.sort_by_key(|x| x.as_slice()[0]);
leaves.sort_by_key(|x| x.as_slice()[0]);
assert_eq!(enters, vec![a]);
assert_eq!(leaves, vec![b]);
}
}
+10
View File
@@ -0,0 +1,10 @@
use crate::cycle::{AddrTransitions, TxAdded, TxRemoved};
/// Per-cycle accumulator threaded through the pipeline steps and
/// drained into the public [`crate::Cycle`] at end of cycle.
#[derive(Default)]
pub struct CycleDiff {
pub added: Vec<TxAdded>,
pub removed: Vec<TxRemoved>,
pub addrs: AddrTransitions,
}
+30
View File
@@ -0,0 +1,30 @@
use std::{sync::Arc, time::Duration};
use brk_types::{AddrBytes, BlockHash, Height, MempoolInfo};
use crate::{
Snapshot,
cycle::{TxAdded, TxRemoved},
};
/// One pull cycle's worth of changes. Produced by
/// [`crate::Mempool::tick`] after fetch → prepare → apply → prevouts →
/// rebuild. The snapshot is always present (the rebuilder runs every
/// cycle). Compare `next_block_hash` across cycles if you need to
/// detect whether the projection actually changed.
pub struct Cycle {
pub added: Vec<TxAdded>,
pub removed: Vec<TxRemoved>,
/// Addresses that went from 0 → 1+ live mempool txs this cycle.
/// Same-cycle enter-then-leave is collapsed (no event in either list).
pub addr_enters: Vec<AddrBytes>,
/// Addresses that went from 1+ → 0 live mempool txs this cycle.
pub addr_leaves: Vec<AddrBytes>,
/// Latest confirmed block. Compare to the prior cycle's `tip_hash`
/// to detect a new block.
pub tip_hash: BlockHash,
pub tip_height: Height,
pub info: MempoolInfo,
pub snapshot: Arc<Snapshot>,
pub took: Duration,
}
+15
View File
@@ -0,0 +1,15 @@
//! Per-cycle types. Every type here lives exactly one tick.
mod added_kind;
mod addr_transitions;
mod diff;
mod event;
mod tx_added;
mod tx_removed;
pub use added_kind::AddedKind;
pub use addr_transitions::AddrTransitions;
pub use diff::CycleDiff;
pub use event::Cycle;
pub use tx_added::TxAdded;
pub use tx_removed::TxRemoved;
+13
View File
@@ -0,0 +1,13 @@
use brk_types::{FeeRate, Sats, Timestamp, Txid, VSize};
use crate::cycle::AddedKind;
#[derive(Debug, Clone, Copy)]
pub struct TxAdded {
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
pub fee_rate: FeeRate,
pub first_seen: Timestamp,
pub kind: AddedKind,
}
@@ -0,0 +1,13 @@
use brk_types::{FeeRate, Txid};
use crate::TxRemoval;
#[derive(Debug, Clone, Copy)]
pub struct TxRemoved {
pub txid: Txid,
pub reason: TxRemoval,
/// Package-effective rate at burial. Same value the tx reported
/// while alive - RBF predecessors keep their package rate, not a
/// misleading isolated fee/vsize.
pub chunk_rate: FeeRate,
}
+7 -9
View File
@@ -13,22 +13,20 @@ pub struct MempoolStats {
pub graveyard_tombstones: usize,
pub graveyard_order: usize,
pub rebuilds: u64,
pub skip_cleans: u64,
}
impl From<&Mempool> for MempoolStats {
fn from(mempool: &Mempool) -> Self {
let inner = mempool.read();
let state = mempool.read();
let rebuilder = mempool.rebuilder();
Self {
txs: inner.txs.len(),
unresolved: inner.txs.unresolved().len(),
addrs: inner.addrs.len(),
outpoint_spends: inner.outpoint_spends.len(),
graveyard_tombstones: inner.graveyard.tombstones_len(),
graveyard_order: inner.graveyard.order_len(),
txs: state.txs.len(),
unresolved: state.txs.unresolved().len(),
addrs: state.addrs.len(),
outpoint_spends: state.outpoint_spends.len(),
graveyard_tombstones: state.graveyard.tombstones_len(),
graveyard_order: state.graveyard.order_len(),
rebuilds: rebuilder.rebuild_count(),
skip_cleans: rebuilder.skip_clean_count(),
}
}
}
+194
View File
@@ -0,0 +1,194 @@
//! Cycle loop. `start_with` drives [`Mempool::tick_with`] every
//! [`PERIOD`]. Each cycle is wrapped in `catch_unwind` so a panic
//! doesn't freeze the snapshot. `parking_lot` locks don't poison.
use std::{
any::Any,
panic::{AssertUnwindSafe, catch_unwind},
sync::atomic::Ordering,
thread,
time::{Duration, Instant},
};
use brk_error::Result;
use brk_types::{TxOut, Txid, Vout};
use rustc_hash::FxHashMap;
use tracing::error;
use crate::{
Inner, Mempool,
cycle::{Cycle, CycleDiff},
steps::{Applier, Fetched, Fetcher, Preparer, Prevouts},
};
const PERIOD: Duration = Duration::from_millis(1000);
impl Mempool {
/// Infinite update loop with a 1s interval. Resolves
/// confirmed-parent prevouts via the default `getrawtransaction`
/// resolver. Requires bitcoind started with `txindex=1`. Discards
/// per-cycle [`Cycle`] events - use [`Mempool::tick`] to consume them.
pub fn start(&self) {
self.start_with(Prevouts::rpc_resolver(self.0.client.clone()));
}
/// Variant of `start` that uses a caller-supplied resolver for
/// confirmed-parent prevouts (typically backed by an indexer).
///
/// Sleep is `PERIOD - work_duration`, so a 350ms cycle followed by
/// a 100ms cycle still ticks roughly every `PERIOD`. When work
/// overruns `PERIOD`, the next cycle starts immediately.
///
/// # Panics
///
/// Panics if a driver is already running on this `Mempool` instance.
/// One `Mempool` may host at most one driver. Spawn another instance
/// for additional loops.
pub fn start_with<F>(&self, resolver: F)
where
F: Fn(&[(Txid, Vout)]) -> FxHashMap<(Txid, Vout), TxOut> + Send,
{
if self
.0
.started
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
panic!("Mempool::start_with already running on this instance");
}
loop {
let started = Instant::now();
let outcome = catch_unwind(AssertUnwindSafe(|| {
if let Err(e) = self.tick_with(&resolver) {
error!("update failed: {e}");
}
}));
if let Err(payload) = outcome {
error!(
"mempool update panicked, continuing loop: {}",
Self::panic_msg(&payload)
);
}
if let Some(rest) = PERIOD.checked_sub(started.elapsed()) {
thread::sleep(rest);
}
}
}
/// One sync cycle: fetch, prepare, apply, fill prevouts, rebuild.
/// Returns a [`Cycle`] reporting everything that changed. Uses the
/// default `getrawtransaction` resolver for confirmed-parent
/// prevouts (requires `txindex=1`).
///
/// # Errors
///
/// Propagates any failure from the initial RPC fetch (network drop,
/// auth, bitcoind error). Steps after `Fetcher::fetch` are infallible
/// today. The resolver itself swallows its own errors and retries
/// next cycle.
pub fn tick(&self) -> Result<Cycle> {
self.tick_with(Prevouts::rpc_resolver(self.0.client.clone()))
}
/// Variant of [`Mempool::tick`] with a caller-supplied resolver for
/// confirmed-parent prevouts. The resolver MUST resolve confirmed
/// prevouts only. Mempool-to-mempool chains are wired internally
/// and the resolver is never called for them.
///
/// # Errors
///
/// Same as [`Mempool::tick`]: only the RPC fetch is fallible.
pub fn tick_with<F>(&self, resolver: F) -> Result<Cycle>
where
F: Fn(&[(Txid, Vout)]) -> FxHashMap<(Txid, Vout), TxOut>,
{
let started = Instant::now();
let Inner {
client,
state,
rebuilder,
..
} = &*self.0;
let Fetched {
state: rpc,
new_entries,
new_txs,
block_template_txids,
} = Fetcher::fetch(client, state)?;
let pulled = Preparer::prepare(&rpc.live_txids, new_entries, new_txs, state);
let mut diff = CycleDiff::default();
let prev_snapshot = rebuilder.snapshot();
Applier::apply(state, &prev_snapshot, pulled, &mut diff);
drop(prev_snapshot);
Prevouts::fill(state, &mut diff, resolver);
rebuilder.tick(state, &block_template_txids, rpc.min_fee);
let CycleDiff {
added,
removed,
addrs,
} = diff;
let (addr_enters, addr_leaves) = addrs.into_vecs();
Ok(Cycle {
added,
removed,
addr_enters,
addr_leaves,
tip_hash: rpc.tip_hash,
tip_height: rpc.tip_height,
info: self.info(),
snapshot: rebuilder.snapshot(),
took: started.elapsed(),
})
}
fn panic_msg(payload: &(dyn Any + Send)) -> &str {
payload
.downcast_ref::<&'static str>()
.copied()
.or_else(|| payload.downcast_ref::<String>().map(String::as_str))
.unwrap_or("<non-string panic payload>")
}
}
#[cfg(test)]
mod tests {
use std::panic::catch_unwind;
use rustc_hash::FxHashMap;
use super::*;
#[test]
#[should_panic(expected = "Mempool::start_with already running on this instance")]
fn double_start_panics_with_documented_message() {
let mempool = Mempool::for_test();
// Simulate a prior `start_with` having grabbed the latch. We
// can't actually call it first because the real call enters an
// infinite loop. Flipping the atomic is what the runtime check
// observes anyway.
mempool.0.started.store(true, Ordering::Release);
mempool.start_with(|_: &[(Txid, Vout)]| FxHashMap::default());
}
#[test]
fn panic_msg_extracts_static_str_payload() {
let payload = catch_unwind(|| panic!("boom static")).unwrap_err();
assert_eq!(Mempool::panic_msg(payload.as_ref()), "boom static");
}
#[test]
fn panic_msg_extracts_string_payload() {
let payload = catch_unwind(|| panic!("boom owned {}", 42)).unwrap_err();
assert_eq!(Mempool::panic_msg(payload.as_ref()), "boom owned 42");
}
#[test]
fn panic_msg_falls_back_for_non_string_payload() {
// Payload that isn't &str or String: the helper labels it
// explicitly instead of dropping it on the floor.
let payload = catch_unwind(|| std::panic::panic_any(42u32)).unwrap_err();
assert_eq!(Mempool::panic_msg(payload.as_ref()), "<non-string panic payload>");
}
}
+98 -259
View File
@@ -2,85 +2,106 @@
//!
//! One pull cycle, five steps:
//!
//! 1. [`steps::fetcher::Fetcher`] - one mixed batched RPC for
//! `getrawmempool verbose` + `getblocktemplate` + `getmempoolinfo`,
//! then a second batch for `getrawtransaction` on new entries. The
//! GBT is validated to be a subset of the verbose listing; on
//! mismatch the cycle is skipped.
//! 2. [`steps::preparer::Preparer`] - decode and classify into
//! ```text
//! Fetcher -> Preparer -> Applier -> Prevouts -> Rebuilder
//! RPC decode & write to fill build
//! classify State missing Snapshot
//! prevouts
//! ```
//!
//! 1. [`steps::Fetcher`] - one mixed batched RPC for
//! `getblocktemplate` + `getrawmempool false` + `getmempoolinfo`,
//! then a single mixed `getmempoolentry`+`getrawtransaction` batch
//! on new txids only. GBT-only txs are synthesized inline from the
//! GBT payload so block 0 matches Core's selection exactly without
//! a follow-up entry fetch that could race the listing.
//! 2. [`steps::Preparer`] - decode and classify into
//! `TxsPulled { added, removed }`. Pure CPU.
//! 3. [`steps::applier::Applier`] - apply the diff to
//! [`state::State`] under a single write lock.
//! 3. [`steps::Applier`] - apply the diff to [`state::State`] under a
//! single write lock.
//! 4. [`steps::Prevouts::fill`] - fills `prevout: None` inputs in one
//! pass, using same-cycle in-mempool parents directly and the
//! caller-supplied resolver (default: `getrawtransaction`) for
//! confirmed parents.
//! 5. [`steps::rebuilder::Rebuilder`] - throttled rebuild of the
//! projected-blocks `Snapshot` from the same-cycle GBT and min fee.
//! 5. [`snapshot::Rebuilder`] - rebuilds the projected-blocks
//! [`Snapshot`] from the same-cycle GBT and min fee.
//!
//! # Locking domains
//!
//! Two independent locks. No path holds both simultaneously.
//!
//! - `State` (`RwLock<State>`): the live mempool. Cycle steps 3 and 4
//! take the write guard. Every read-side accessor takes a read guard.
//! - `Rebuilder.{snapshot, history}` (two `RwLock`s, written in that
//! order each cycle): the published projection. Readers grab one or
//! the other. The cycle drops its `State` guard before touching them.
//!
//! # Usage
//!
//! Drive the loop on a worker thread and read from any clone:
//!
//! ```no_run
//! use brk_mempool::Mempool;
//! # fn make_client() -> brk_rpc::Client { unimplemented!() }
//! let client = make_client();
//! let mempool = Mempool::new(&client);
//! let reader = mempool.clone();
//! std::thread::spawn(move || mempool.start());
//! // `reader.snapshot()`, `reader.block_template()`, etc. on this thread.
//! # let _ = reader;
//! ```
//!
//! A `Mempool` hosts at most one driver. Calling `start` / `start_with`
//! a second time on the same instance panics. Spawn a separate
//! `Mempool::new` if you need more loops.
use std::{
any::Any,
cmp::Reverse,
panic::{AssertUnwindSafe, catch_unwind},
sync::Arc,
thread,
time::Duration,
};
use std::sync::{Arc, atomic::AtomicBool};
use brk_error::Result;
use brk_rpc::Client;
use brk_types::{
AddrBytes, AddrMempoolStats, FeeRate, MempoolInfo, MempoolRecentTx, OutpointPrefix, OutputType,
Sats, Timestamp, Transaction, TxOut, Txid, TxidPrefix, Vin, Vout,
};
use parking_lot::{RwLock, RwLockReadGuard};
use tracing::error;
pub mod chunking;
mod cpfp;
mod api;
mod cycle;
mod diagnostics;
mod rbf;
mod driver;
mod snapshot;
mod state;
pub(crate) mod steps;
pub(crate) mod stores;
mod steps;
mod stores;
pub use chunking::{ChunkInput, linearize};
#[cfg(test)]
mod test_support;
pub use api::{RbfForTx, RbfNode};
pub use cycle::{AddedKind, Cycle, TxAdded, TxRemoved};
pub use diagnostics::MempoolStats;
pub use rbf::{RbfForTx, RbfNode};
use steps::{Applier, Fetched, Fetcher, Preparer, Prevouts, Rebuilder};
pub use steps::{BlockStats, RecommendedFees, Snapshot, TxEntry, TxRemoval};
pub use stores::{TxGraveyard, TxStore, TxTombstone};
pub use snapshot::Snapshot;
pub use steps::TxRemoval;
/// Confirmed-parent prevout resolver passed to [`Mempool::update_with`] /
/// [`Mempool::start_with`]. Receives `(parent_txid, vout)`, returns the
/// `TxOut` if the parent is reachable, `None` otherwise.
pub type PrevoutResolver = Box<dyn Fn(&Txid, Vout) -> Option<TxOut> + Send + Sync>;
pub(crate) use state::State;
use snapshot::Rebuilder;
use state::State;
/// Cheaply cloneable: clones share one live mempool via `Arc`.
#[derive(Clone)]
pub struct Mempool(Arc<Shared>);
pub struct Mempool(Arc<Inner>);
struct Shared {
struct Inner {
client: Client,
state: RwLock<State>,
rebuilder: Rebuilder,
started: AtomicBool,
}
impl Mempool {
pub fn new(client: &Client) -> Self {
Self(Arc::new(Shared {
Self(Arc::new(Inner {
client: client.clone(),
state: RwLock::new(State::default()),
rebuilder: Rebuilder::default(),
started: AtomicBool::new(false),
}))
}
pub fn info(&self) -> MempoolInfo {
self.read().info.clone()
}
pub fn snapshot(&self) -> Arc<Snapshot> {
self.0.rebuilder.snapshot()
}
@@ -90,223 +111,41 @@ impl Mempool {
MempoolStats::from(self)
}
pub(crate) fn rebuilder(&self) -> &Rebuilder {
fn rebuilder(&self) -> &Rebuilder {
&self.0.rebuilder
}
pub fn fees(&self) -> RecommendedFees {
self.snapshot().fees.clone()
}
pub fn block_stats(&self) -> Vec<BlockStats> {
self.snapshot().block_stats.clone()
}
pub fn next_block_hash(&self) -> u64 {
self.snapshot().next_block_hash
}
pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 {
self.read().addrs.stats_hash(addr)
}
/// Mempool tx spending `(txid, vout)`, or `None`. The spender's
/// input list is walked to rule out `TxidPrefix` collisions.
pub fn lookup_spender(&self, txid: &Txid, vout: Vout) -> Option<(Txid, Vin)> {
let key = OutpointPrefix::new(TxidPrefix::from(txid), vout);
let state = self.read();
let spender_prefix = state.outpoint_spends.get(&key)?;
let spender = state.txs.record_by_prefix(&spender_prefix)?;
let vin_pos = spender
.tx
.input
.iter()
.position(|inp| inp.txid == *txid && inp.vout == vout)?;
Some((spender.entry.txid, Vin::from(vin_pos)))
}
pub(crate) fn read(&self) -> RwLockReadGuard<'_, State> {
fn read(&self) -> RwLockReadGuard<'_, State> {
self.0.state.read()
}
pub fn contains_txid(&self, txid: &Txid) -> bool {
self.read().txs.contains(txid)
}
/// Apply `f` to the live tx body if present.
pub fn with_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> {
self.read().txs.get(txid).map(f)
}
/// Apply `f` to a `Vanished` tombstone's tx body if present.
/// `Replaced` tombstones return `None` because the tx will not confirm.
pub fn with_vanished_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> {
let state = self.read();
let tomb = state.graveyard.get(txid)?;
matches!(tomb.reason(), TxRemoval::Vanished).then(|| f(&tomb.tx))
}
/// Snapshot of all live mempool txids.
pub fn txids(&self) -> Vec<Txid> {
self.read().txs.txids().copied().collect()
}
/// Snapshot of recent live txs.
pub fn recent_txs(&self) -> Vec<MempoolRecentTx> {
self.read().txs.recent().to_vec()
}
/// Per-address mempool stats. `None` if the address has no live mempool activity.
pub fn addr_stats(&self, addr: &AddrBytes) -> Option<AddrMempoolStats> {
self.read().addrs.get(addr).map(|e| e.stats.clone())
}
/// Live mempool txs touching `addr`, newest first by `first_seen`,
/// capped at `limit`. Returns owned `Transaction`s.
pub fn addr_txs(&self, addr: &AddrBytes, limit: usize) -> Vec<Transaction> {
let state = self.read();
let Some(entry) = state.addrs.get(addr) else {
return vec![];
};
let mut ordered: Vec<(Timestamp, &Transaction)> = entry
.txids
.iter()
.filter_map(|txid| {
let record = state.txs.record_by_prefix(&TxidPrefix::from(txid))?;
Some((record.entry.first_seen, &record.tx))
})
.collect();
ordered.sort_unstable_by_key(|b| Reverse(b.0));
ordered
.into_iter()
.take(limit)
.map(|(_, tx)| tx.clone())
.collect()
}
/// Apply `f` to an iterator over `(value, output_type)` for every output
/// of every live mempool tx. The lock is held for the duration of the call.
pub fn process_live_outputs<R>(
&self,
f: impl FnOnce(&mut dyn Iterator<Item = (Sats, OutputType)>) -> R,
) -> R {
let inner = self.read();
let mut iter = inner
.txs
.values()
.flat_map(|tx| &tx.output)
.map(|txout| (txout.value, txout.type_()));
f(&mut iter)
}
/// Effective fee rate for a live tx: snapshot's chunk rate when
/// the tx is in the latest snapshot, falling back to the entry's
/// `fee/vsize` if not yet ingested.
pub fn live_effective_fee_rate(&self, prefix: &TxidPrefix) -> Option<FeeRate> {
if let Some(rate) = self.snapshot().chunk_rate_for(prefix) {
return Some(rate);
}
self.read()
.txs
.entry_by_prefix(prefix)
.map(|e| e.fee_rate())
}
/// Effective fee rate (Core's chunk rate) snapshotted into the
/// tomb's entry at burial - same value `live_effective_fee_rate`
/// returns while the tx is alive, so an evicted RBF predecessor
/// reports the package-effective rate it had in the mempool, not a
/// misleading isolated `fee/vsize`.
pub fn graveyard_fee_rate(&self, txid: &Txid) -> Option<FeeRate> {
self.read()
.graveyard
.get(txid)
.map(|tomb| tomb.entry.chunk_rate)
}
/// `first_seen` Unix-second timestamps for `txids`, in input order.
/// Returns 0 for unknown txids. `Vanished` tombstones fall back to
/// the buried entry's `first_seen` to avoid flicker between drop
/// and indexer catch-up.
pub fn transaction_times(&self, txids: &[Txid]) -> Vec<u64> {
let state = self.read();
txids
.iter()
.map(|txid| state.first_seen(txid).map_or(0, u64::from))
.collect()
}
/// Infinite update loop with a 1 second interval. Resolves
/// confirmed-parent prevouts via the default `getrawtransaction`
/// resolver; requires bitcoind started with `txindex=1`.
pub fn start(&self) {
self.start_with(Prevouts::rpc_resolver(self.0.client.clone()));
}
/// Variant of `start` that uses a caller-supplied resolver for
/// confirmed-parent prevouts (typically backed by an indexer).
/// Each cycle is wrapped in `catch_unwind` so a panic doesn't
/// freeze the snapshot; `parking_lot` locks don't poison.
pub fn start_with<F>(&self, resolver: F)
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
loop {
let outcome = catch_unwind(AssertUnwindSafe(|| {
if let Err(e) = self.update_with(&resolver) {
error!("update failed: {e}");
}
}));
if let Err(payload) = outcome {
error!("mempool update panicked, continuing loop: {}", panic_msg(&payload));
}
thread::sleep(Duration::from_secs(1));
}
}
/// One sync cycle with the default RPC resolver. Equivalent to
/// `update_with(rpc_resolver)`. Standalone consumers (Core +
/// `txindex=1`) get a one-line driver loop.
pub fn update(&self) -> Result<()> {
self.update_with(Prevouts::rpc_resolver(self.0.client.clone()))
}
/// One sync cycle: fetch, prepare, apply, fill prevouts, maybe
/// rebuild. The resolver MUST resolve confirmed prevouts only;
/// mempool-to-mempool chains are wired internally and the
/// resolver is never called for them.
pub fn update_with<F>(&self, resolver: F) -> Result<()>
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
let Shared {
client,
state,
rebuilder,
} = &*self.0;
let Some(Fetched {
entries_info,
new_raws,
gbt,
min_fee,
}) = Fetcher::fetch(client, state)?
else {
return Ok(());
};
let pulled = Preparer::prepare(entries_info, new_raws, state);
let changed = Applier::apply(state, pulled);
Prevouts::fill(state, resolver);
rebuilder.tick(state, changed, &gbt, min_fee);
Ok(())
}
}
fn panic_msg(payload: &(dyn Any + Send)) -> &str {
payload
.downcast_ref::<&'static str>()
.copied()
.or_else(|| payload.downcast_ref::<String>().map(String::as_str))
.unwrap_or("<non-string panic payload>")
#[cfg(test)]
mod test_helpers {
use brk_rpc::Auth;
use brk_types::{FeeRate, Txid};
use super::*;
impl Mempool {
/// Test-only constructor that wires a Client at the default URL without
/// touching the network. `simple_http` only parses the URL on init.
pub(crate) fn for_test() -> Self {
let client = Client::new(Client::default_url(), Auth::None).unwrap();
Self(Arc::new(Inner {
client,
state: RwLock::new(State::default()),
rebuilder: Rebuilder::default(),
started: AtomicBool::new(false),
}))
}
pub(crate) fn test_state_lock(&self) -> &RwLock<State> {
&self.0.state
}
pub(crate) fn test_tick(&self, gbt_txids: &[Txid], min_fee: FeeRate) {
self.0.rebuilder.tick(&self.0.state, gbt_txids, min_fee);
}
}
}
-109
View File
@@ -1,109 +0,0 @@
//! RBF tree extraction. Returns owned trees so the caller can enrich
//! with indexer data (`mined`, effective fee rate) after the lock
//! drops: enriching under the lock re-enters `Mempool` and would
//! recursively acquire the same read lock.
use brk_types::{Sats, Timestamp, Transaction, Txid, TxidPrefix, VSize};
use rustc_hash::FxHashSet;
use crate::{Mempool, TxEntry, TxRemoval, TxStore, stores::TxGraveyard};
#[derive(Debug, Clone)]
pub struct RbfNode {
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
pub value: Sats,
pub first_seen: Timestamp,
/// BIP-125 signaling: at least one input has sequence < 0xffffffff-1.
pub rbf: bool,
/// `true` iff any predecessor in this subtree was non-signaling.
pub full_rbf: bool,
pub replaces: Vec<RbfNode>,
}
#[derive(Debug, Clone, Default)]
pub struct RbfForTx {
/// Tree rooted at the terminal replacer. `None` if `txid` is unknown.
pub root: Option<RbfNode>,
/// Direct predecessors of the requested tx (txids only).
pub replaces: Vec<Txid>,
}
impl Mempool {
/// Walk forward through `Replaced { by }` to the terminal replacer
/// and return its full predecessor tree, plus the requested tx's
/// direct predecessors. Single read-lock window.
pub fn rbf_for_tx(&self, txid: &Txid) -> RbfForTx {
let inner = self.read();
let root_txid = walk_to_replacement_root(&inner.graveyard, *txid);
let replaces: Vec<Txid> = inner
.graveyard
.predecessors_of(txid)
.map(|(p, _)| *p)
.collect();
let root = build_node(&root_txid, &inner.txs, &inner.graveyard);
RbfForTx { root, replaces }
}
/// Recent terminal-replacer trees, most-recent first, deduplicated
/// by root, capped at `limit`. `full_rbf_only` drops trees with no
/// non-signaling predecessor.
pub fn recent_rbf_trees(&self, full_rbf_only: bool, limit: usize) -> Vec<RbfNode> {
let inner = self.read();
let mut seen: FxHashSet<Txid> = FxHashSet::default();
inner
.graveyard
.replaced_iter_recent_first()
.filter_map(|(_, by)| {
let root = walk_to_replacement_root(&inner.graveyard, *by);
seen.insert(root).then_some(root)
})
.filter_map(|root| build_node(&root, &inner.txs, &inner.graveyard))
.filter(|n| !full_rbf_only || n.full_rbf)
.take(limit)
.collect()
}
}
fn walk_to_replacement_root(graveyard: &TxGraveyard, mut root: Txid) -> Txid {
while let Some(TxRemoval::Replaced { by }) = graveyard.get(&root).map(|t| t.reason()) {
root = *by;
}
root
}
fn build_node(txid: &Txid, txs: &TxStore, graveyard: &TxGraveyard) -> Option<RbfNode> {
let (tx, entry) = resolve_node(txid, txs, graveyard)?;
let replaces: Vec<RbfNode> = graveyard
.predecessors_of(txid)
.filter_map(|(pred, _)| build_node(pred, txs, graveyard))
.collect();
let full_rbf = replaces.iter().any(|c| !c.rbf || c.full_rbf);
let value: Sats = tx.output.iter().map(|o| o.value).sum();
Some(RbfNode {
txid: *txid,
fee: entry.fee,
vsize: entry.vsize,
value,
first_seen: entry.first_seen,
rbf: entry.rbf,
full_rbf,
replaces,
})
}
fn resolve_node<'a>(
txid: &Txid,
txs: &'a TxStore,
graveyard: &'a TxGraveyard,
) -> Option<(&'a Transaction, &'a TxEntry)> {
txs.record_by_prefix(&TxidPrefix::from(txid))
.map(|r| (&r.tx, &r.entry))
.or_else(|| graveyard.get(txid).map(|t| (&t.tx, &t.entry)))
}
@@ -1,4 +1,4 @@
use brk_types::{FeeRate, Sats, VSize, get_weighted_percentile};
use brk_types::{FeeRate, MempoolBlock, Sats, VSize, get_weighted_percentile};
use super::{SnapTx, TxIndex};
@@ -32,14 +32,18 @@ pub struct BlockStats {
}
impl BlockStats {
/// Block 0 (Core's actual selection): exact 0/10/25/50/75/90/100.
pub fn compute_core(block: &[TxIndex], txs: &[SnapTx]) -> Self {
Self::compute(block, txs, CORE_PERCENTILES)
}
/// Blocks 1..N (projected): clipped 5/95 bounds to hide outliers.
pub fn compute_projected(block: &[TxIndex], txs: &[SnapTx]) -> Self {
Self::compute(block, txs, PROJECTED_PERCENTILES)
/// Stats for every projected block in `blocks`, in order. `blocks[0]`
/// uses Core's exact 0..100 percentiles. The rest use the clipped
/// 5..95 range to hide CPFP / stale-GBT outliers.
pub fn for_blocks(blocks: &[Vec<TxIndex>], txs: &[SnapTx]) -> Vec<Self> {
blocks
.iter()
.enumerate()
.map(|(i, block)| {
let pct = if i == 0 { CORE_PERCENTILES } else { PROJECTED_PERCENTILES };
Self::compute(block, txs, pct)
})
.collect()
}
/// Vsize-weighted percentile distribution over `chunk_rate` -
@@ -78,8 +82,10 @@ impl BlockStats {
fee_range,
}
}
}
pub fn median_fee_rate(&self) -> FeeRate {
self.fee_range[3]
impl From<&BlockStats> for MempoolBlock {
fn from(s: &BlockStats) -> Self {
Self::new(s.tx_count, s.total_size, s.total_vsize, s.total_fee, s.fee_range)
}
}
+233
View File
@@ -0,0 +1,233 @@
//! Build per-tx adjacency from the live `TxStore`, then run Single Fee
//! Linearization over every multi-tx cluster.
use std::mem;
use brk_types::TxidPrefix;
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use crate::{state::TxEntry, stores::TxStore};
use super::{Cluster, SnapTx, Snapshot, TxIndex};
pub type PrefixIndex = FxHashMap<TxidPrefix, TxIndex>;
impl Snapshot {
pub fn build_txs(txs: &TxStore) -> (Vec<SnapTx>, PrefixIndex) {
let n = txs.len();
let mut prefix_to_idx: PrefixIndex =
FxHashMap::with_capacity_and_hasher(n, FxBuildHasher);
for (i, (prefix, _)) in txs.records().enumerate() {
prefix_to_idx.insert(*prefix, TxIndex::from(i));
}
let mut snap_txs: Vec<SnapTx> = txs
.records()
.map(|(_, record)| Self::live_tx(&record.entry, &prefix_to_idx))
.collect();
Self::mirror_children(&mut snap_txs);
Self::refresh_chunk_rates(&mut snap_txs);
(snap_txs, prefix_to_idx)
}
fn live_tx(e: &TxEntry, prefix_to_idx: &PrefixIndex) -> SnapTx {
let parents: SmallVec<[TxIndex; 2]> = e
.depends
.iter()
.filter_map(|p| prefix_to_idx.get(p).copied())
.collect();
SnapTx {
txid: e.txid,
fee: e.fee,
vsize: e.vsize,
weight: e.weight,
size: e.size,
chunk_rate: e.fee_rate(),
parents,
children: SmallVec::new(),
}
}
fn mirror_children(txs: &mut [SnapTx]) {
for i in 0..txs.len() {
let child = TxIndex::from(i);
let parents = mem::take(&mut txs[i].parents);
for &p in &parents {
if let Some(t) = txs.get_mut(p.as_usize()) {
t.children.push(child);
}
}
txs[i].parents = parents;
}
}
/// Walk every multi-tx connected component once and overwrite each
/// member's `chunk_rate` with the linearized chunk's feerate.
/// Visited bitmap ensures each cluster is linearized exactly once.
fn refresh_chunk_rates(snap_txs: &mut [SnapTx]) {
let n = snap_txs.len();
let mut visited = vec![false; n];
for seed in 0..n {
if visited[seed] {
continue;
}
let t = &snap_txs[seed];
if t.parents.is_empty() && t.children.is_empty() {
visited[seed] = true;
continue;
}
let component = Cluster::walk(snap_txs, TxIndex::from(seed));
for &m in &component {
visited[m.as_usize()] = true;
}
if component.len() <= 1 {
continue;
}
let (members, chunks) = Cluster::linearize(snap_txs, &component);
for chunk in &chunks {
for &local in &chunk.txs {
let m = members[u32::from(local) as usize];
snap_txs[m.as_usize()].chunk_rate = chunk.feerate;
}
}
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicU32, Ordering};
use bitcoin::hashes::Hash;
use brk_types::{FeeRate, Sats, Txid, VSize, Weight};
use super::*;
/// Build a `SnapTx` for tests. `txid` is auto-assigned from a
/// process-wide counter so each tx is distinguishable in
/// debug output. The cluster code itself keys off `TxIndex`,
/// not `txid`.
fn snap_tx(fee: Sats, vsize: VSize) -> SnapTx {
static COUNTER: AtomicU32 = AtomicU32::new(0);
let mut bytes = [0u8; 32];
bytes[..4].copy_from_slice(&COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes());
SnapTx {
txid: Txid::from(bitcoin::Txid::from_byte_array(bytes)),
fee,
vsize,
weight: Weight::from(vsize),
size: u64::from(vsize),
chunk_rate: FeeRate::from((fee, vsize)),
parents: SmallVec::new(),
children: SmallVec::new(),
}
}
fn link(txs: &mut [SnapTx], parent: usize, child: usize) {
txs[child].parents.push(TxIndex::from(parent));
txs[parent].children.push(TxIndex::from(child));
}
fn sats(n: u64) -> Sats {
Sats::from(n)
}
fn vsize(n: u64) -> VSize {
VSize::from(n)
}
#[test]
fn singleton_keeps_fee_per_vsize() {
let mut txs = vec![snap_tx(sats(1000), vsize(100))];
let seed = txs[0].chunk_rate;
Snapshot::refresh_chunk_rates(&mut txs);
assert_eq!(txs[0].chunk_rate, seed);
}
#[test]
fn two_tx_cpfp_lift() {
let mut txs = vec![
snap_tx(sats(100), vsize(100)),
snap_tx(sats(1900), vsize(100)),
];
link(&mut txs, 0, 1);
let parent_seed = txs[0].chunk_rate;
Snapshot::refresh_chunk_rates(&mut txs);
assert!(txs[0].chunk_rate > parent_seed);
assert_eq!(txs[0].chunk_rate, txs[1].chunk_rate);
assert_eq!(txs[0].chunk_rate, FeeRate::from((sats(2000), vsize(200))));
}
#[test]
fn three_tx_chain_chunks_correctly() {
let mut txs = vec![
snap_tx(sats(100), vsize(100)),
snap_tx(sats(100), vsize(100)),
snap_tx(sats(5800), vsize(100)),
];
link(&mut txs, 0, 1);
link(&mut txs, 1, 2);
Snapshot::refresh_chunk_rates(&mut txs);
let combined = FeeRate::from((sats(6000), vsize(300)));
assert_eq!(txs[0].chunk_rate, combined);
assert_eq!(txs[1].chunk_rate, combined);
assert_eq!(txs[2].chunk_rate, combined);
}
#[test]
fn disjoint_clusters_linearized_independently() {
let mut txs = vec![
snap_tx(sats(100), vsize(100)),
snap_tx(sats(1900), vsize(100)),
snap_tx(sats(500), vsize(100)),
snap_tx(sats(4500), vsize(100)),
];
link(&mut txs, 0, 1);
link(&mut txs, 2, 3);
Snapshot::refresh_chunk_rates(&mut txs);
assert_eq!(txs[0].chunk_rate, txs[1].chunk_rate);
assert_eq!(txs[2].chunk_rate, txs[3].chunk_rate);
assert_ne!(txs[0].chunk_rate, txs[2].chunk_rate);
}
#[test]
fn cluster_cap_does_not_panic() {
let n = 100;
let mut txs: Vec<SnapTx> = (0..n).map(|_| snap_tx(sats(1000), vsize(100))).collect();
for i in 1..n {
link(&mut txs, i - 1, i);
}
Snapshot::refresh_chunk_rates(&mut txs);
}
#[test]
fn refresh_chunk_rates_is_order_independent_within_clusters() {
let mut a = vec![
snap_tx(sats(1_000), vsize(100)),
snap_tx(sats(100), vsize(100)),
snap_tx(sats(5_000), vsize(100)),
snap_tx(sats(200), vsize(100)),
];
link(&mut a, 0, 1);
link(&mut a, 2, 3);
Snapshot::refresh_chunk_rates(&mut a);
// Same pool, members of each cluster reordered.
let mut b = vec![
snap_tx(sats(100), vsize(100)),
snap_tx(sats(1_000), vsize(100)),
snap_tx(sats(200), vsize(100)),
snap_tx(sats(5_000), vsize(100)),
];
link(&mut b, 1, 0);
link(&mut b, 3, 2);
Snapshot::refresh_chunk_rates(&mut b);
let mut rates_a: Vec<f64> = a.iter().map(|t| f64::from(t.chunk_rate)).collect();
let mut rates_b: Vec<f64> = b.iter().map(|t| f64::from(t.chunk_rate)).collect();
rates_a.sort_by(|x, y| x.partial_cmp(y).unwrap());
rates_b.sort_by(|x, y| x.partial_cmp(y).unwrap());
assert_eq!(rates_a, rates_b);
}
}
+133
View File
@@ -0,0 +1,133 @@
//! Cluster primitives over `SnapTx` adjacency: connected-component
//! discovery, topo-sort, and the glue to Single Fee Linearization
//! ([`brk_types::linearize`], shared with brk_query's confirmed-cpfp).
use std::collections::VecDeque;
use brk_types::{ChunkInput, CpfpClusterChunk, CpfpClusterTxIndex, linearize};
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use smallvec::SmallVec;
use super::{SnapTx, TxIndex};
/// Matches Bitcoin Core 31's `MAX_CLUSTER_COUNT_LIMIT`.
pub const MAX_CLUSTER: usize = 64;
pub struct Cluster;
impl Cluster {
/// Capped DFS over the undirected dependency graph (`parents
/// children`) starting from `seed`. Returns the connected component
/// truncated to `MAX_CLUSTER`, with `seed` at index 0.
pub fn walk(txs: &[SnapTx], seed: TxIndex) -> Vec<TxIndex> {
if txs.get(seed.as_usize()).is_none() {
return Vec::new();
}
let mut visited: FxHashSet<TxIndex> =
FxHashSet::with_capacity_and_hasher(MAX_CLUSTER, FxBuildHasher);
visited.insert(seed);
let mut out: Vec<TxIndex> = Vec::with_capacity(MAX_CLUSTER);
out.push(seed);
let mut stack: Vec<TxIndex> = vec![seed];
while let Some(idx) = stack.pop() {
let Some(t) = txs.get(idx.as_usize()) else {
continue;
};
for &n in t.parents.iter().chain(t.children.iter()) {
if out.len() >= MAX_CLUSTER {
return out;
}
if visited.insert(n) {
out.push(n);
stack.push(n);
}
}
}
out
}
/// Linearize the connected component into chunks. Topo-sorts members,
/// remaps parent edges to cluster-local indices, and runs SFL. Returns
/// `(members, chunks)` where `members` is the topo-ordered `TxIndex`
/// list and `chunks[*].txs` are local indices into `members`. Callers
/// must filter singletons before calling - the singleton's `chunk_rate`
/// is `fee/vsize`, set elsewhere.
pub fn linearize(
txs: &[SnapTx],
component: &[TxIndex],
) -> (Vec<TxIndex>, Vec<CpfpClusterChunk>) {
let members = Self::topo_sort(txs, component);
let local_of = Self::local_index(&members);
let parents_local: Vec<SmallVec<[CpfpClusterTxIndex; 2]>> = members
.iter()
.map(|idx| {
txs[idx.as_usize()]
.parents
.iter()
.filter_map(|p| local_of.get(p).copied())
.collect()
})
.collect();
let inputs: Vec<ChunkInput<'_>> = members
.iter()
.zip(&parents_local)
.map(|(idx, ps)| {
let t = &txs[idx.as_usize()];
ChunkInput {
fee: t.fee,
vsize: t.vsize,
parents: ps.as_slice(),
}
})
.collect();
let chunks = linearize(&inputs);
(members, chunks)
}
/// `members[i]`'s wire index, keyed by snapshot `TxIndex`. Built once
/// so per-tx parent edges can be remapped without a linear scan.
pub fn local_index(members: &[TxIndex]) -> FxHashMap<TxIndex, CpfpClusterTxIndex> {
members
.iter()
.enumerate()
.map(|(i, &idx)| (idx, CpfpClusterTxIndex::from(i as u32)))
.collect()
}
/// Kahn's topological sort over the connected component, restricted to
/// in-cluster parent edges. Returns members in an order where every tx
/// follows all its in-cluster parents.
fn topo_sort(txs: &[SnapTx], component: &[TxIndex]) -> Vec<TxIndex> {
let n = component.len();
let pos: FxHashMap<TxIndex, usize> = component
.iter()
.enumerate()
.map(|(i, &x)| (x, i))
.collect();
let mut indeg: Vec<u32> = vec![0; n];
let mut children: Vec<Vec<usize>> = vec![Vec::new(); n];
for (i, &idx) in component.iter().enumerate() {
let Some(t) = txs.get(idx.as_usize()) else {
continue;
};
indeg[i] = t.parents.iter().filter(|p| pos.contains_key(p)).count() as u32;
for &c in t.children.iter() {
if let Some(&ci) = pos.get(&c) {
children[i].push(ci);
}
}
}
let mut queue: VecDeque<usize> = (0..n).filter(|&i| indeg[i] == 0).collect();
let mut out: Vec<TxIndex> = Vec::with_capacity(n);
while let Some(i) = queue.pop_front() {
out.push(component[i]);
for &c in &children[i] {
indeg[c] -= 1;
if indeg[c] == 0 {
queue.push_back(c);
}
}
}
out
}
}
+248
View File
@@ -0,0 +1,248 @@
//! CPFP (Child Pays For Parent) walk over a `Snapshot`'s adjacency.
//!
//! Three independent walks:
//! - `ancestors`: capped DFS up `parents` only.
//! - `descendants`: capped DFS down `children` only.
//! - cluster: connected component over `parents children`,
//! linearized for wire shape and seed chunk feerate.
use brk_types::{
CPFP_CHAIN_LIMIT, CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, CpfpInfo, FeeRate,
SigOps, TxidPrefix, VSize, find_seed_chunk,
};
use rustc_hash::{FxBuildHasher, FxHashSet};
use crate::Mempool;
use super::{Cluster, SnapTx, Snapshot, TxIndex};
impl Mempool {
/// CPFP info for a live mempool tx. Returns `None` when the tx
/// isn't in the live pool, so callers can fall through to the
/// confirmed path. The snapshot can lag `state.txs` by up to one
/// cycle: if the seed is in the snapshot but no longer in live
/// state we return `None` rather than a half-stale report.
pub fn cpfp_info(&self, prefix: &TxidPrefix) -> Option<CpfpInfo> {
let snapshot = self.snapshot();
let seed_idx = snapshot.idx_of(prefix)?;
let seed = snapshot.tx(seed_idx)?;
let sigops = self.read().txs.get(&seed.txid)?.total_sigop_cost;
Some(snapshot.cpfp_info_at(seed_idx, seed, sigops))
}
}
impl Snapshot {
fn cpfp_info_at(&self, seed_idx: TxIndex, seed: &SnapTx, sigops: SigOps) -> CpfpInfo {
let ancestors = Self::collect_cpfp_entries(&self.txs, seed_idx, |t| &t.parents);
let descendants = Self::collect_cpfp_entries(&self.txs, seed_idx, |t| &t.children);
let best_descendant = descendants
.iter()
.max_by_key(|e| FeeRate::from((e.fee, e.weight)))
.cloned();
let (cluster, effective_fee_per_vsize) = Self::build_cpfp_cluster(&self.txs, seed_idx, seed);
let vsize = VSize::from(seed.weight);
CpfpInfo {
ancestors,
best_descendant,
descendants,
effective_fee_per_vsize,
sigops,
fee: seed.fee,
vsize,
adjusted_vsize: sigops.adjust_vsize(vsize),
cluster,
}
}
/// Capped DFS from `seed` (exclusive) along `next`, lifted directly
/// to wire-shape `CpfpEntry`s. Used for both ancestor and descendant
/// walks.
fn collect_cpfp_entries(
txs: &[SnapTx],
seed: TxIndex,
next: impl Fn(&SnapTx) -> &[TxIndex],
) -> Vec<CpfpEntry> {
let Some(seed_node) = txs.get(seed.as_usize()) else {
return Vec::new();
};
let mut visited: FxHashSet<TxIndex> =
FxHashSet::with_capacity_and_hasher(CPFP_CHAIN_LIMIT + 1, FxBuildHasher);
visited.insert(seed);
let mut out: Vec<CpfpEntry> = Vec::with_capacity(CPFP_CHAIN_LIMIT);
let mut stack: Vec<TxIndex> = next(seed_node).to_vec();
while let Some(idx) = stack.pop() {
if out.len() >= CPFP_CHAIN_LIMIT {
break;
}
if !visited.insert(idx) {
continue;
}
if let Some(t) = txs.get(idx.as_usize()) {
out.push(CpfpEntry::from(t));
stack.extend(next(t).iter().copied());
}
}
out
}
/// Wire-shape `CpfpCluster` plus the seed's chunk feerate. Members
/// are the connected component of the seed in the dependency graph,
/// topologically ordered (parents before children) so wire indices
/// and chunk-internal ordering are valid for client-side
/// reconstruction. Returns `(None, seed_per_tx_rate)` for singletons
/// (matches mempool.space, which omits `cluster` when no relations
/// exist).
fn build_cpfp_cluster(
txs: &[SnapTx],
seed_idx: TxIndex,
seed: &SnapTx,
) -> (Option<CpfpCluster>, FeeRate) {
let seed_per_tx_rate = FeeRate::from((seed.fee, seed.vsize));
let component = Cluster::walk(txs, seed_idx);
if component.len() <= 1 {
return (None, seed_per_tx_rate);
}
let (members, chunks) = Cluster::linearize(txs, &component);
let cluster_txs = Self::wire_cluster_members(txs, &members);
let seed_local = CpfpClusterTxIndex::from(
members
.iter()
.position(|&i| i == seed_idx)
.map_or(0, |p| p as u32),
);
let (chunk_index, seed_chunk_rate) = find_seed_chunk(&chunks, seed_local, seed_per_tx_rate);
(
Some(CpfpCluster {
txs: cluster_txs,
chunks,
chunk_index,
}),
seed_chunk_rate,
)
}
/// Materialize wire-shape `CpfpClusterTx`s for every topo-ordered
/// member with parent edges remapped to local indices.
fn wire_cluster_members(txs: &[SnapTx], members: &[TxIndex]) -> Vec<CpfpClusterTx> {
let local_of = Cluster::local_index(members);
members
.iter()
.map(|&idx| {
let t = &txs[idx.as_usize()];
CpfpClusterTx {
txid: t.txid,
weight: t.weight,
fee: t.fee,
parents: t
.parents
.iter()
.filter_map(|p| local_of.get(p).copied())
.collect(),
}
})
.collect()
}
}
#[cfg(test)]
mod tests {
use brk_types::{FeeRate, Txid};
use super::*;
use crate::{
state::TxEntry,
test_support::{fake_entry_info, fake_tx, p2wpkh_script},
};
/// Insert a tx, optionally declaring parent dependencies for the
/// snapshot builder's adjacency wire-up.
fn insert_with_depends(
mempool: &Mempool,
seed: u8,
fee: u64,
vsize: u64,
parents: &[Txid],
) -> Txid {
let tx = fake_tx(seed, &[None], &[(p2wpkh_script(seed + 1), 1_234)]);
let txid = tx.txid;
let mut info = fake_entry_info(txid, fee, vsize);
info.depends = parents.to_vec();
let entry = TxEntry::new(&info, vsize, false);
let mut state = mempool.test_state_lock().write();
state.txs.insert(tx, entry);
txid
}
#[test]
fn singleton_cpfp_info_has_no_cluster() {
let mempool = Mempool::for_test();
let txid = insert_with_depends(&mempool, 0xB0, 10_000, 100, &[]);
mempool.test_tick(&[txid], FeeRate::new(1.0));
let info = mempool
.cpfp_info(&TxidPrefix::from(&txid))
.expect("tx is in mempool");
assert!(info.cluster.is_none(), "singletons emit no cluster");
assert!(info.ancestors.is_empty());
assert!(info.descendants.is_empty());
// Effective rate equals isolated rate when there's no package lift.
let isolated = FeeRate::from((info.fee, info.vsize));
assert_eq!(info.effective_fee_per_vsize, isolated);
}
#[test]
fn two_tx_cpfp_cluster_has_both_members_and_lifted_rate() {
let mempool = Mempool::for_test();
let parent = insert_with_depends(&mempool, 0xB1, 100, 100, &[]);
let child = insert_with_depends(&mempool, 0xB2, 1_900, 100, &[parent]);
mempool.test_tick(&[parent, child], FeeRate::new(1.0));
let parent_info = mempool.cpfp_info(&TxidPrefix::from(&parent)).unwrap();
let cluster = parent_info.cluster.expect("two-tx cluster present");
assert_eq!(cluster.txs.len(), 2);
// Topological order: parent first.
assert_eq!(cluster.txs[0].txid, parent);
assert_eq!(cluster.txs[1].txid, child);
// Child reports the parent as its only local parent.
assert_eq!(cluster.txs[1].parents.len(), 1);
// CPFP lift: parent's effective rate exceeds its isolated rate.
let parent_isolated = FeeRate::from((parent_info.fee, parent_info.vsize));
assert!(parent_info.effective_fee_per_vsize > parent_isolated);
// Same package -> child's reported chunk rate matches parent's.
let child_info = mempool.cpfp_info(&TxidPrefix::from(&child)).unwrap();
assert_eq!(parent_info.effective_fee_per_vsize, child_info.effective_fee_per_vsize);
}
#[test]
fn cpfp_ancestor_and_descendant_walks_are_directional() {
// chain: A -> B -> C
let mempool = Mempool::for_test();
let a = insert_with_depends(&mempool, 0xB3, 100, 100, &[]);
let b = insert_with_depends(&mempool, 0xB4, 100, 100, &[a]);
let c = insert_with_depends(&mempool, 0xB5, 5_800, 100, &[b]);
mempool.test_tick(&[a, b, c], FeeRate::new(1.0));
// B sees A as an ancestor and C as a descendant.
let info_b = mempool.cpfp_info(&TxidPrefix::from(&b)).unwrap();
let ancestor_ids: Vec<_> = info_b.ancestors.iter().map(|e| e.txid).collect();
let descendant_ids: Vec<_> = info_b.descendants.iter().map(|e| e.txid).collect();
assert_eq!(ancestor_ids, vec![a]);
assert_eq!(descendant_ids, vec![c]);
// best_descendant picks the highest-rate descendant.
assert_eq!(info_b.best_descendant.as_ref().map(|e| e.txid), Some(c));
}
#[test]
fn cpfp_info_returns_none_for_unknown_txid() {
let mempool = Mempool::for_test();
mempool.test_tick(&[], FeeRate::new(1.0));
let bogus = TxidPrefix::from(&Txid::COINBASE);
assert!(mempool.cpfp_info(&bogus).is_none());
}
}
@@ -1,6 +1,6 @@
use brk_types::{FeeRate, RecommendedFees};
use super::stats::BlockStats;
use super::block_stats::BlockStats;
/// Output rounding granularity in sat/vB. mempool.space's
/// `/api/v1/fees/recommended` uses `1.0`, their `/precise`
@@ -74,7 +74,7 @@ impl Fees {
previous_fee: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
let median = block.median_fee_rate();
let median = block.fee_range[3];
let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev));
let vsize = u64::from(block.total_vsize);
if vsize <= EMPTY_BLOCK_VSIZE || median < min_fee {
@@ -87,3 +87,79 @@ impl Fees {
use_fee.ceil_to(MIN_INCREMENT).max(min_fee)
}
}
#[cfg(test)]
mod tests {
use brk_types::{Sats, VSize};
use super::*;
fn block(vsize: u64, median_fee: f64) -> BlockStats {
let median = FeeRate::new(median_fee);
BlockStats {
tx_count: 1,
total_size: vsize,
total_vsize: VSize::from(vsize),
total_fee: Sats::from((vsize as f64 * median_fee) as u64),
fee_range: [
median, median, median, median, median, median, median,
],
}
}
#[test]
fn empty_stats_collapses_every_tier_to_min_fee() {
let min = FeeRate::new(2.0);
let fees = Fees::compute(&[], min);
let priority_fastest = FeeRate::new(2.5); // min + PRIORITY_FACTOR
let priority_half_hour = FeeRate::new(2.25); // min + PRIORITY_FACTOR/2
assert_eq!(fees.minimum_fee, min);
assert_eq!(fees.economy_fee, min);
assert_eq!(fees.hour_fee, min);
assert_eq!(fees.fastest_fee, priority_fastest);
assert_eq!(fees.half_hour_fee, priority_half_hour);
}
#[test]
fn min_fee_floor_lifts_below_one_sat_rates() {
// `mempoolminfee` below MIN_INCREMENT: result is clamped up.
let min = FeeRate::new(0.0);
let fees = Fees::compute(&[], min);
assert!(f64::from(fees.minimum_fee) >= 0.001);
// `fastest_fee` always at least MIN_FASTEST_FEE.
assert!(f64::from(fees.fastest_fee) >= 1.0);
}
#[test]
fn small_partial_final_block_collapses_to_min_fee() {
// vsize <= EMPTY_BLOCK_VSIZE: returns min_fee unconditionally.
let stats = vec![block(400_000, 12.5)];
let min = FeeRate::new(1.0);
let fees = Fees::compute(&stats, min);
assert_eq!(fees.hour_fee, min);
assert_eq!(fees.economy_fee, min);
}
#[test]
fn full_block_carries_signal_into_top_tier() {
let stats = vec![block(1_000_000, 25.0), block(1_000_000, 10.0)];
let min = FeeRate::new(1.0);
let fees = Fees::compute(&stats, min);
// fastest gets PRIORITY_FACTOR (0.5) added.
assert_eq!(fees.fastest_fee, FeeRate::new(25.5));
// hour comes from block[2], which doesn't exist -> collapses to min.
assert_eq!(fees.hour_fee, min);
}
#[test]
fn partial_final_block_tapers_linearly() {
// vsize in (EMPTY, FULL]: rate = use_fee * (vsize - EMPTY)/EMPTY.
// 725_000 vsize -> multiplier = 225_000 / 500_000 = 0.45.
let stats = vec![block(725_000, 10.0)];
let min = FeeRate::new(1.0);
let fees = Fees::compute(&stats, min);
// economy/hour come from the same (only) block, both tapered.
// 10.0 * 0.45 = 4.5, fastest = 4.5 + 0.5 = 5.0.
assert_eq!(fees.fastest_fee, FeeRate::new(5.0));
}
}
+215
View File
@@ -0,0 +1,215 @@
mod block_stats;
mod builder;
mod cluster;
mod cpfp;
mod fees;
mod partition;
mod rebuilder;
mod snap_tx;
mod tx_index;
pub use block_stats::BlockStats;
pub use cluster::Cluster;
pub use rebuilder::Rebuilder;
pub use snap_tx::SnapTx;
pub use tx_index::TxIndex;
use builder::PrefixIndex;
use fees::Fees;
use partition::Partitioner;
use std::hash::{Hash, Hasher};
use brk_types::{FeeRate, NextBlockHash, RecommendedFees, Txid, TxidPrefix};
use rustc_hash::FxHasher;
#[derive(Default)]
pub struct Snapshot {
/// Dense per-tx data indexed by `TxIndex`. Each entry carries the
/// linearized chunk rate plus parent/child adjacency.
pub txs: Vec<SnapTx>,
/// Projected blocks. `blocks[0]` is Core's `getblocktemplate`
/// (Bitcoin Core's actual selection). The rest are greedy-packed
/// by descending chunk rate, with a final overflow block.
pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees,
/// Content hash of the projected next block. Same value as the
/// mempool `ETag`.
pub next_block_hash: NextBlockHash,
prefix_to_idx: PrefixIndex,
}
impl Snapshot {
/// `min_fee` is bitcoind's live `mempoolminfee`, the floor for
/// every recommended-fee tier.
fn build(
txs: Vec<SnapTx>,
blocks: Vec<Vec<TxIndex>>,
prefix_to_idx: PrefixIndex,
min_fee: FeeRate,
) -> Self {
let block_stats = BlockStats::for_blocks(&blocks, &txs);
let fees = Fees::compute(&block_stats, min_fee);
let next_block_hash = Self::hash_next_block(&blocks, &txs);
Self {
txs,
blocks,
block_stats,
fees,
next_block_hash,
prefix_to_idx,
}
}
/// Content tag over block 0 in template order. Hashes txids, not
/// `TxIndex` slots, because slot assignment is per-cycle.
fn hash_next_block(blocks: &[Vec<TxIndex>], txs: &[SnapTx]) -> NextBlockHash {
let Some(block) = blocks.first() else {
return NextBlockHash::ZERO;
};
let mut hasher = FxHasher::default();
for idx in block {
txs[idx.as_usize()].txid.hash(&mut hasher);
}
NextBlockHash::new(hasher.finish())
}
pub fn tx(&self, idx: TxIndex) -> Option<&SnapTx> {
self.txs.get(idx.as_usize())
}
pub fn idx_of(&self, prefix: &TxidPrefix) -> Option<TxIndex> {
self.prefix_to_idx.get(prefix).copied()
}
/// Txids of `blocks[0]` (Core's `getblocktemplate` selection),
/// in template order. Empty for a default snapshot.
pub fn block0_txids(&self) -> impl Iterator<Item = Txid> + '_ {
self.blocks
.first()
.into_iter()
.flatten()
.map(|idx| self.txs[idx.as_usize()].txid)
}
/// Linearized chunk rate for a live tx by prefix. Recomputed each
/// snapshot, package-aware (CPFP lifts apply), equals `fee/vsize`
/// for singletons.
pub fn chunk_rate_for(&self, prefix: &TxidPrefix) -> Option<FeeRate> {
let idx = self.idx_of(prefix)?;
Some(self.txs[idx.as_usize()].chunk_rate)
}
/// Test-only: stitch a snapshot from `(prefix, chunk_rate)` pairs
/// without running the full builder.
#[cfg(test)]
pub(crate) fn for_test_with_chunk_rates(entries: &[(TxidPrefix, FeeRate, Txid)]) -> Self {
use brk_types::{Sats, VSize, Weight};
use smallvec::SmallVec;
let mut prefix_to_idx = PrefixIndex::default();
let mut txs = Vec::with_capacity(entries.len());
for (i, (prefix, rate, txid)) in entries.iter().enumerate() {
prefix_to_idx.insert(*prefix, TxIndex::from(i));
txs.push(SnapTx {
txid: *txid,
fee: Sats::ZERO,
vsize: VSize::from(0u64),
weight: Weight::from(0u64),
size: 0,
chunk_rate: *rate,
parents: SmallVec::new(),
children: SmallVec::new(),
});
}
Self {
txs,
blocks: vec![],
block_stats: vec![],
fees: RecommendedFees::default(),
next_block_hash: NextBlockHash::ZERO,
prefix_to_idx,
}
}
}
#[cfg(test)]
mod tests {
use bitcoin::hashes::Hash;
use brk_types::{Sats, VSize, Weight};
use smallvec::SmallVec;
use super::*;
fn snap_tx(seed: u8) -> SnapTx {
let mut bytes = [0u8; 32];
bytes[0] = seed;
SnapTx {
txid: Txid::from(bitcoin::Txid::from_byte_array(bytes)),
fee: Sats::from(1_234u64),
vsize: VSize::from(100u64),
weight: Weight::from(400u64),
size: 100,
chunk_rate: FeeRate::from((Sats::from(1_234u64), VSize::from(100u64))),
parents: SmallVec::new(),
children: SmallVec::new(),
}
}
#[test]
fn next_block_hash_is_deterministic_across_runs() {
let txs = vec![snap_tx(1), snap_tx(2), snap_tx(3)];
let blocks = vec![vec![
TxIndex::from(0usize),
TxIndex::from(1usize),
TxIndex::from(2usize),
]];
let h1 = Snapshot::hash_next_block(&blocks, &txs);
let h2 = Snapshot::hash_next_block(&blocks, &txs);
assert_eq!(h1, h2);
}
#[test]
fn next_block_hash_changes_with_block0_membership() {
let txs = vec![snap_tx(1), snap_tx(2), snap_tx(3)];
let two_member = vec![vec![TxIndex::from(0usize), TxIndex::from(1usize)]];
let three_member = vec![vec![
TxIndex::from(0usize),
TxIndex::from(1usize),
TxIndex::from(2usize),
]];
assert_ne!(
Snapshot::hash_next_block(&two_member, &txs),
Snapshot::hash_next_block(&three_member, &txs),
);
}
#[test]
fn next_block_hash_changes_with_block0_order() {
// hash_next_block hashes txids in template order: reordering
// block 0 must produce a different hash.
let txs = vec![snap_tx(1), snap_tx(2), snap_tx(3)];
let forward = vec![vec![
TxIndex::from(0usize),
TxIndex::from(1usize),
TxIndex::from(2usize),
]];
let reversed = vec![vec![
TxIndex::from(2usize),
TxIndex::from(1usize),
TxIndex::from(0usize),
]];
assert_ne!(
Snapshot::hash_next_block(&forward, &txs),
Snapshot::hash_next_block(&reversed, &txs),
);
}
#[test]
fn empty_blocks_hash_is_zero() {
let txs = vec![snap_tx(1)];
let blocks: Vec<Vec<TxIndex>> = vec![];
assert_eq!(Snapshot::hash_next_block(&blocks, &txs), NextBlockHash::ZERO);
}
}
@@ -0,0 +1,142 @@
//! Pack live txs into projected blocks 1..N by descending `chunk_rate`.
//! Block 0 is filled by the caller from `getblocktemplate`. Final block
//! is a catch-all (no vsize cap).
use brk_types::{FeeRate, VSize};
use rustc_hash::FxHashSet;
use super::{SnapTx, TxIndex};
pub struct Partitioner;
impl Partitioner {
pub fn partition(
txs: &[SnapTx],
excluded: &FxHashSet<TxIndex>,
num_remaining_blocks: usize,
) -> Vec<Vec<TxIndex>> {
if num_remaining_blocks == 0 {
return Vec::new();
}
let sorted = Self::sorted_candidates(txs, excluded);
let mut blocks: Vec<Vec<TxIndex>> = (0..num_remaining_blocks).map(|_| Vec::new()).collect();
let mut block_vsize = VSize::default();
let mut current = 0;
let last = num_remaining_blocks - 1;
for (idx, vsize, _) in sorted {
let fits = vsize <= VSize::MAX_BLOCK.saturating_sub(block_vsize);
if !fits && current < last && !blocks[current].is_empty() {
current += 1;
block_vsize = VSize::default();
}
blocks[current].push(idx);
block_vsize += vsize;
}
blocks
}
fn sorted_candidates(
txs: &[SnapTx],
excluded: &FxHashSet<TxIndex>,
) -> Vec<(TxIndex, VSize, FeeRate)> {
let mut cands: Vec<(TxIndex, VSize, FeeRate)> = txs
.iter()
.enumerate()
.filter_map(|(i, t)| {
let idx = TxIndex::from(i);
(!excluded.contains(&idx)).then_some((idx, t.vsize, t.chunk_rate))
})
.collect();
cands.sort_by(|(a_idx, _, a_rate), (b_idx, _, b_rate)| {
b_rate
.cmp(a_rate)
.then_with(|| txs[a_idx.as_usize()].txid.cmp(&txs[b_idx.as_usize()].txid))
});
cands
}
}
#[cfg(test)]
mod tests {
use bitcoin::hashes::Hash;
use brk_types::{Sats, Txid, Weight};
use smallvec::SmallVec;
use super::*;
fn snap_tx(seed: u8, fee: u64, vsize: u64) -> SnapTx {
let mut bytes = [0u8; 32];
bytes[0] = seed;
SnapTx {
txid: Txid::from(bitcoin::Txid::from_byte_array(bytes)),
fee: Sats::from(fee),
vsize: VSize::from(vsize),
weight: Weight::from(vsize * 4),
size: vsize,
chunk_rate: FeeRate::from((Sats::from(fee), VSize::from(vsize))),
parents: SmallVec::new(),
children: SmallVec::new(),
}
}
#[test]
fn zero_blocks_returns_empty() {
let txs = vec![snap_tx(1, 100, 100)];
let blocks = Partitioner::partition(&txs, &FxHashSet::default(), 0);
assert!(blocks.is_empty());
}
#[test]
fn higher_chunk_rate_packs_first() {
let txs = vec![snap_tx(1, 100, 100), snap_tx(2, 1_000, 100)];
let blocks = Partitioner::partition(&txs, &FxHashSet::default(), 3);
assert_eq!(blocks[0][0], TxIndex::from(1usize));
assert_eq!(blocks[0][1], TxIndex::from(0usize));
}
#[test]
fn excluded_txs_are_skipped() {
let txs = vec![snap_tx(1, 100, 100), snap_tx(2, 1_000, 100)];
let mut excluded = FxHashSet::default();
excluded.insert(TxIndex::from(1usize));
let blocks = Partitioner::partition(&txs, &excluded, 3);
let flat: Vec<TxIndex> = blocks.into_iter().flatten().collect();
assert_eq!(flat, vec![TxIndex::from(0usize)]);
}
#[test]
fn vsize_cap_respected_except_for_last_block() {
let big = u64::from(VSize::MAX_BLOCK) - 100;
// Three "fills the rest of a block" sized txs, one block window.
// Final block has no cap, so all three end up in it when the
// request is one block deep.
let txs = vec![
snap_tx(1, 1_000, big),
snap_tx(2, 900, big),
snap_tx(3, 800, big),
];
let one_block = Partitioner::partition(&txs, &FxHashSet::default(), 1);
assert_eq!(one_block.len(), 1);
assert_eq!(one_block[0].len(), 3, "final block ignores vsize cap");
// With three slots, the first two get one tx each, last block
// soaks up the rest.
let three_blocks = Partitioner::partition(&txs, &FxHashSet::default(), 3);
assert_eq!(three_blocks[0].len(), 1);
assert_eq!(three_blocks[1].len(), 1);
assert_eq!(three_blocks[2].len(), 1);
}
#[test]
fn txid_breaks_ties_within_same_rate() {
// Identical rate, distinct txids: order must follow ascending txid.
let txs = vec![
snap_tx(0x20, 100, 100),
snap_tx(0x10, 100, 100),
snap_tx(0x30, 100, 100),
];
let blocks = Partitioner::partition(&txs, &FxHashSet::default(), 1);
let txids: Vec<u8> = blocks[0].iter().map(|i| txs[i.as_usize()].txid[0]).collect();
assert_eq!(txids, vec![0x10, 0x20, 0x30]);
}
}
@@ -0,0 +1,109 @@
//! # Locking
//!
//! Two locks live on `Rebuilder`: `history` and `snapshot`. Writes always
//! land on `history` first, then `snapshot`, so any `next_block_hash` a
//! reader sees in the published snapshot is already recorded in
//! `historical_block0`. No read path ever holds both, and no path holds
//! a `State` guard together with either Rebuilder lock - the cycle reads
//! `State` once to build the snapshot, then drops it before touching
//! these locks.
use std::{
collections::VecDeque,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
};
use brk_types::{FeeRate, NextBlockHash, Txid, TxidPrefix};
use parking_lot::RwLock;
use rustc_hash::FxHashSet;
use crate::State;
use super::{Partitioner, Snapshot, TxIndex};
const NUM_BLOCKS: usize = 8;
const HISTORY: usize = 10;
#[derive(Default)]
pub struct Rebuilder {
snapshot: RwLock<Arc<Snapshot>>,
/// Past block-0 txid lists keyed by `next_block_hash`, oldest first.
/// Ordered so `block_template_diff` can emit `Retained(prior_index)`
/// entries that line up with the client's cached prior template.
history: RwLock<VecDeque<(NextBlockHash, Vec<Txid>)>>,
rebuild_count: AtomicU64,
}
impl Rebuilder {
/// Rebuild every cycle. `min_fee` participates in the result, so a
/// "skip if no add/remove" gate would freeze served fees when Core's
/// `mempoolminfee` drifts on a quiet pool.
///
/// History is updated before the snapshot Arc is swapped so a reader
/// can never observe a `next_block_hash` that hasn't been recorded
/// yet. `block_template_diff(current_hash)` returning 404 in the
/// publish gap would force unnecessary client refetches.
pub fn tick(&self, lock: &RwLock<State>, gbt_txids: &[Txid], min_fee: FeeRate) {
let snap = Self::build_snapshot(lock, gbt_txids, min_fee);
let block0: Vec<Txid> = snap.block0_txids().collect();
let next_hash = snap.next_block_hash;
let mut hist = self.history.write();
hist.retain(|(h, _)| *h != next_hash);
hist.push_back((next_hash, block0));
while hist.len() > HISTORY {
hist.pop_front();
}
drop(hist);
*self.snapshot.write() = Arc::new(snap);
self.rebuild_count.fetch_add(1, Ordering::Relaxed);
}
/// Past block-0 ordered txid list for `hash`, or `None` if it has
/// aged out (or was never seen). Used by `block_template_diff` to
/// decide 200 vs 404 and to resolve `Retained(prior_index)` entries.
pub fn historical_block0(&self, hash: NextBlockHash) -> Option<Vec<Txid>> {
self.history
.read()
.iter()
.find(|(h, _)| *h == hash)
.map(|(_, block0)| block0.clone())
}
pub fn rebuild_count(&self) -> u64 {
self.rebuild_count.load(Ordering::Relaxed)
}
fn build_snapshot(
lock: &RwLock<State>,
gbt_txids: &[Txid],
min_fee: FeeRate,
) -> Snapshot {
let (txs, prefix_to_idx) = {
let state = lock.read();
Snapshot::build_txs(&state.txs)
};
let block0: Vec<TxIndex> = gbt_txids
.iter()
.filter_map(|txid| prefix_to_idx.get(&TxidPrefix::from(txid)).copied())
.collect();
let excluded: FxHashSet<TxIndex> = block0.iter().copied().collect();
let rest = Partitioner::partition(&txs, &excluded, NUM_BLOCKS.saturating_sub(1));
let mut blocks = Vec::with_capacity(NUM_BLOCKS);
blocks.push(block0);
blocks.extend(rest);
Snapshot::build(txs, blocks, prefix_to_idx, min_fee)
}
pub fn snapshot(&self) -> Arc<Snapshot> {
self.snapshot.read().clone()
}
}
@@ -3,10 +3,11 @@ use smallvec::SmallVec;
use super::TxIndex;
/// Frozen per-tx view used by the snapshot. Holds the chunk rate
/// (Core's `fees.chunk` / `chunkweight` when available, else proxy)
/// plus resolved parent/child adjacency in `TxIndex` space, so
/// CPFP queries are a pure walk over `Snapshot.txs`.
/// Frozen per-tx view used by the snapshot. `chunk_rate` is the
/// linearized chunk feerate (local Single Fee Linearization, run fresh
/// every snapshot). Singletons report `fee/vsize`. Parent/child
/// adjacency in `TxIndex` space, so CPFP queries are a pure walk over
/// `Snapshot.txs`.
#[derive(Clone, Debug)]
pub struct SnapTx {
pub txid: Txid,
@@ -17,7 +18,7 @@ pub struct SnapTx {
pub size: u64,
pub chunk_rate: FeeRate,
/// Direct parents in the live pool (resolved against entry slots
/// at build time; cross-pool / confirmed parents are dropped).
/// at build time. Cross-pool / confirmed parents are dropped).
pub parents: SmallVec<[TxIndex; 2]>,
pub children: SmallVec<[TxIndex; 4]>,
}
-35
View File
@@ -1,35 +0,0 @@
//! Single-locked container for the live mempool.
//!
//! All cycle steps and read-side accessors take a guard on this one
//! lock. The substructures are plain owned types — they used to each
//! own a RwLock, but the canonical lock-order discipline disappears
//! when there's nothing to order.
use brk_types::{MempoolInfo, Timestamp, Txid};
use crate::{
TxRemoval,
stores::{AddrTracker, OutpointSpends, TxGraveyard, TxStore},
};
#[derive(Default)]
pub struct State {
pub info: MempoolInfo,
pub txs: TxStore,
pub addrs: AddrTracker,
pub outpoint_spends: OutpointSpends,
pub graveyard: TxGraveyard,
}
impl State {
/// `first_seen` for a tx that's live or in a `Vanished` tombstone.
/// Smooths the flicker between drop and indexer catch-up; `Replaced`
/// tombstones are excluded since the tx will not confirm.
pub fn first_seen(&self, txid: &Txid) -> Option<Timestamp> {
if let Some(e) = self.txs.entry(txid) {
return Some(e.first_seen);
}
let tomb = self.graveyard.get(txid)?;
matches!(tomb.reason(), TxRemoval::Vanished).then_some(tomb.entry.first_seen)
}
}
+38
View File
@@ -0,0 +1,38 @@
//! Single-locked container for the live mempool. All cycle steps and
//! read-side accessors take a guard on this one lock.
//!
//! # Concurrency
//!
//! `State` is held under one `RwLock` at the crate root. The cycle
//! takes the write guard for `Applier` and `Prevouts`, then drops it
//! before the [`crate::snapshot::Rebuilder`] runs. No code path holds
//! a `State` guard at the same time as a `Rebuilder` lock, so the two
//! domains are independent and lock-ordering between them is moot.
mod tx_entry;
pub use tx_entry::TxEntry;
use brk_types::{MempoolInfo, Timestamp, Txid};
use crate::stores::{AddrTracker, OutpointSpends, TxGraveyard, TxStore};
#[derive(Default)]
pub struct State {
pub info: MempoolInfo,
pub txs: TxStore,
pub addrs: AddrTracker,
pub outpoint_spends: OutpointSpends,
pub graveyard: TxGraveyard,
}
impl State {
/// Smooths the flicker between drop and indexer catch-up. `Replaced`
/// tombstones are excluded since the tx will not confirm.
pub fn first_seen(&self, txid: &Txid) -> Option<Timestamp> {
if let Some(e) = self.txs.entry(txid) {
return Some(e.first_seen);
}
self.graveyard.get_vanished(txid).map(|t| t.entry.first_seen)
}
}
@@ -1,10 +1,8 @@
use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, Txid, TxidPrefix, VSize, Weight};
use smallvec::SmallVec;
/// A mempool transaction entry. Carries the per-tx facts needed for
/// projection, plus the snapshot-time `chunk_rate` (Core's cluster-mempool
/// chunk fee rate, or the proxy fallback) used as the effective rate
/// for partitioning, fee tiers, and CPFP.
/// Per-tx facts needed for projection. Chunk rates live on the snapshot
/// (linearized fresh each cycle), not here.
#[derive(Debug, Clone)]
pub struct TxEntry {
pub txid: Txid,
@@ -17,15 +15,10 @@ pub struct TxEntry {
pub first_seen: Timestamp,
/// BIP-125 explicit signaling: any input has sequence < 0xfffffffe.
pub rbf: bool,
/// Effective per-vbyte rate Core would mine this tx at. From
/// `MempoolEntryInfo::chunk_rate()`: Core 31+ uses `fees.chunk /
/// (chunkweight/4)`, older Core falls back to
/// `max(ancestor_rate, descendant_pkg_rate)`.
pub chunk_rate: FeeRate,
}
impl TxEntry {
pub(super) fn new(info: &MempoolEntryInfo, size: u64, rbf: bool) -> Self {
pub fn new(info: &MempoolEntryInfo, size: u64, rbf: bool) -> Self {
Self {
txid: info.txid,
fee: info.fee,
@@ -35,7 +28,6 @@ impl TxEntry {
depends: info.depends.iter().map(TxidPrefix::from).collect(),
first_seen: info.first_seen,
rbf,
chunk_rate: info.chunk_rate(),
}
}
+228 -22
View File
@@ -2,49 +2,91 @@ use brk_types::{Transaction, TxidPrefix};
use parking_lot::RwLock;
use crate::{
State, TxEntry, TxRemoval,
Snapshot, TxRemoval,
cycle::{AddrTransitions, CycleDiff, TxAdded, TxRemoved},
state::{State, TxEntry},
steps::preparer::{TxAddition, TxsPulled},
};
/// Applies a prepared diff to in-memory mempool state under one write
/// guard. Body proceeds: bury removed → publish added → evict.
/// guard. Body proceeds: bury removed → publish added → evict. Events
/// are pushed into the caller-supplied [`CycleDiff`] accumulator.
pub struct Applier;
impl Applier {
/// Returns true iff anything changed.
pub fn apply(lock: &RwLock<State>, pulled: TxsPulled) -> bool {
/// `prev_snapshot` supplies the previous cycle's snapshot. Burial
/// reads each tomb's `chunk_rate` from it (always-fresh,
/// package-aware via local linearization). The fallback to
/// `entry.fee_rate()` is unreachable in steady state - every burial
/// target was alive at the previous tick, so the snapshot has it.
pub fn apply(
lock: &RwLock<State>,
prev_snapshot: &Snapshot,
pulled: TxsPulled,
diff: &mut CycleDiff,
) {
let TxsPulled { added, removed } = pulled;
let has_changes = !added.is_empty() || !removed.is_empty();
let mut state = lock.write();
Self::bury_removals(&mut state, removed);
Self::publish_additions(&mut state, added);
Self::bury_removals(&mut state, prev_snapshot, &mut diff.addrs, &mut diff.removed, removed);
Self::publish_additions(&mut state, &mut diff.addrs, &mut diff.added, added);
state.graveyard.evict_old();
has_changes
}
fn bury_removals(state: &mut State, removed: Vec<(TxidPrefix, TxRemoval)>) {
fn bury_removals(
state: &mut State,
snapshot: &Snapshot,
transitions: &mut AddrTransitions,
events: &mut Vec<TxRemoved>,
removed: Vec<(TxidPrefix, TxRemoval)>,
) {
events.reserve(removed.len());
for (prefix, reason) in removed {
Self::bury_one(state, &prefix, reason);
if let Some(ev) = Self::bury_one(state, snapshot, transitions, &prefix, reason) {
events.push(ev);
}
}
}
fn bury_one(state: &mut State, prefix: &TxidPrefix, reason: TxRemoval) {
let Some(record) = state.txs.remove_by_prefix(prefix) else {
return;
};
fn bury_one(
state: &mut State,
prev_snapshot: &Snapshot,
transitions: &mut AddrTransitions,
prefix: &TxidPrefix,
reason: TxRemoval,
) -> Option<TxRemoved> {
let record = state.txs.remove_by_prefix(prefix)?;
let chunk_rate = prev_snapshot
.chunk_rate_for(prefix)
.unwrap_or_else(|| record.entry.fee_rate());
let txid = record.entry.txid;
state.info.remove(&record.tx, record.entry.fee);
state.addrs.remove_tx(&record.tx, &txid);
state.addrs.remove_tx(transitions, &record.tx);
state.outpoint_spends.remove_spends(&record.tx, *prefix);
state.graveyard.bury(txid, record.tx, record.entry, reason);
state
.graveyard
.bury(record.tx, record.entry, chunk_rate, reason);
Some(TxRemoved { txid, reason, chunk_rate })
}
fn publish_additions(state: &mut State, added: Vec<TxAddition>) {
fn publish_additions(
state: &mut State,
transitions: &mut AddrTransitions,
events: &mut Vec<TxAdded>,
added: Vec<TxAddition>,
) {
events.reserve(added.len());
for addition in added {
let kind = addition.kind();
if let Some((tx, entry)) = Self::resolve_addition(state, addition) {
Self::publish_one(state, tx, entry);
events.push(TxAdded {
txid: entry.txid,
fee: entry.fee,
vsize: entry.vsize,
fee_rate: entry.fee_rate(),
first_seen: entry.first_seen,
kind,
});
Self::publish_one(state, transitions, tx, entry);
}
}
}
@@ -59,11 +101,175 @@ impl Applier {
}
}
fn publish_one(state: &mut State, tx: Transaction, entry: TxEntry) {
fn publish_one(
state: &mut State,
transitions: &mut AddrTransitions,
tx: Transaction,
entry: TxEntry,
) {
let prefix = entry.txid_prefix();
state.info.add(&tx, entry.fee);
state.addrs.add_tx(&tx, &entry.txid);
state.addrs.add_tx(transitions, &tx);
state.outpoint_spends.insert_spends(&tx, prefix);
state.txs.insert(tx, entry);
}
}
#[cfg(test)]
mod tests {
use brk_types::{FeeRate, Sats, TxOut, Txid, VSize};
use super::*;
use crate::{
AddedKind,
cycle::CycleDiff,
steps::preparer::{TxAddition, TxsPulled},
test_support::{fake_entry_info, fake_tx, p2wpkh_script},
};
fn fresh_addition(seed: u8, fee: u64, vsize: u64) -> (TxAddition, Txid) {
let prev = Some(TxOut::from((p2wpkh_script(seed), Sats::from(2_500u64))));
let tx = fake_tx(seed, &[prev], &[(p2wpkh_script(seed + 1), 1_234)]);
let txid = tx.txid;
let info = fake_entry_info(txid, fee, vsize);
let entry = TxEntry::new(&info, vsize, false);
(TxAddition::Fresh { tx, entry }, txid)
}
fn fresh_pulled(addition: TxAddition) -> TxsPulled {
TxsPulled {
added: vec![addition],
removed: vec![],
}
}
#[test]
fn publish_one_inserts_into_all_stores() {
let lock = RwLock::new(State::default());
let snapshot = Snapshot::default();
let mut diff = CycleDiff::default();
let (addition, txid) = fresh_addition(0xC0, 200, 100);
Applier::apply(&lock, &snapshot, fresh_pulled(addition), &mut diff);
let state = lock.read();
assert!(state.txs.contains(&txid));
assert_eq!(diff.added.len(), 1);
assert_eq!(diff.added[0].txid, txid);
}
#[test]
fn revived_path_exhumes_body_from_graveyard() {
let lock = RwLock::new(State::default());
let snapshot = Snapshot::default();
let (addition, txid) = fresh_addition(0xC1, 300, 100);
let TxAddition::Fresh { tx, entry } = addition else {
unreachable!();
};
// Pre-load the graveyard with this tx, then submit a Revived
// addition that re-publishes it without a raw body.
let rate = FeeRate::from((entry.fee, entry.vsize));
lock.write()
.graveyard
.bury(tx, entry.clone(), rate, TxRemoval::Vanished);
let mut diff = CycleDiff::default();
Applier::apply(
&lock,
&snapshot,
fresh_pulled(TxAddition::Revived { entry }),
&mut diff,
);
let state = lock.read();
assert!(state.txs.contains(&txid), "revived tx republished");
assert!(state.graveyard.get(&txid).is_none(), "tomb consumed");
assert_eq!(diff.added.len(), 1);
assert!(matches!(diff.added[0].kind, AddedKind::Revived));
}
#[test]
fn revived_with_empty_graveyard_is_dropped() {
let lock = RwLock::new(State::default());
let snapshot = Snapshot::default();
let info = fake_entry_info(Txid::COINBASE, 100, 100);
let entry = TxEntry::new(&info, 100, false);
let mut diff = CycleDiff::default();
Applier::apply(
&lock,
&snapshot,
fresh_pulled(TxAddition::Revived { entry }),
&mut diff,
);
let state = lock.read();
assert!(!state.txs.contains(&Txid::COINBASE));
assert!(diff.added.is_empty(), "no body, no event");
}
#[test]
fn bury_preserves_chunk_rate_from_snapshot() {
let lock = RwLock::new(State::default());
let (addition, txid) = fresh_addition(0xC2, 100, 100);
// Publish first to plant the tx, with a fee-rate that differs
// from the snapshot's stub rate so we can tell them apart.
Applier::apply(
&lock,
&Snapshot::default(),
fresh_pulled(addition),
&mut CycleDiff::default(),
);
let isolated_rate = FeeRate::from((Sats::from(100u64), VSize::from(100u64)));
let cpfp_rate = FeeRate::from((Sats::from(500u64), VSize::from(100u64)));
let prefix = TxidPrefix::from(&txid);
let snapshot = Snapshot::for_test_with_chunk_rates(&[(prefix, cpfp_rate, txid)]);
let mut diff = CycleDiff::default();
Applier::apply(
&lock,
&snapshot,
TxsPulled {
added: vec![],
removed: vec![(prefix, TxRemoval::Vanished)],
},
&mut diff,
);
assert_eq!(diff.removed.len(), 1);
assert_eq!(diff.removed[0].chunk_rate, cpfp_rate);
assert_ne!(diff.removed[0].chunk_rate, isolated_rate);
let state = lock.read();
assert_eq!(state.graveyard.get(&txid).unwrap().chunk_rate, cpfp_rate);
}
#[test]
fn bury_falls_back_to_isolated_rate_when_snapshot_misses() {
let lock = RwLock::new(State::default());
let (addition, txid) = fresh_addition(0xC3, 700, 100);
Applier::apply(
&lock,
&Snapshot::default(),
fresh_pulled(addition),
&mut CycleDiff::default(),
);
let isolated_rate = FeeRate::from((Sats::from(700u64), VSize::from(100u64)));
let prefix = TxidPrefix::from(&txid);
let mut diff = CycleDiff::default();
Applier::apply(
&lock,
&Snapshot::default(),
TxsPulled {
added: vec![],
removed: vec![(prefix, TxRemoval::Vanished)],
},
&mut diff,
);
assert_eq!(diff.removed[0].chunk_rate, isolated_rate);
}
}
@@ -1,10 +1,20 @@
use brk_rpc::{BlockTemplateTx, RawTx};
use brk_types::{FeeRate, MempoolEntryInfo, Txid};
use brk_rpc::MempoolState;
use brk_types::{MempoolEntryInfo, Txid};
use rustc_hash::FxHashMap;
pub struct Fetched {
pub entries_info: Vec<MempoolEntryInfo>,
pub new_raws: FxHashMap<Txid, RawTx>,
pub gbt: Vec<BlockTemplateTx>,
pub min_fee: FeeRate,
/// Passthrough fields from the batched RPC fetch: live txid set,
/// fee floor, chain tip. `live_txids` is the union of
/// `getrawmempool` and `getblocktemplate` (see [`super::Fetcher::fetch`]),
/// so downstream sees a single coherent "live" view.
pub state: MempoolState,
/// `MempoolEntryInfo` for newly-observed txids only (existing ones
/// keep their first-sight entry on the live store).
pub new_entries: Vec<MempoolEntryInfo>,
pub new_txs: FxHashMap<Txid, bitcoin::Transaction>,
/// Block 0 ordering from `getblocktemplate`. Bodies and stats have
/// already been folded into `new_entries`/`new_txs` (or were already
/// in the pool). The Rebuilder only needs the txid sequence to
/// project Core's exact selection.
pub block_template_txids: Vec<Txid>,
}
+97 -45
View File
@@ -3,64 +3,116 @@ mod fetched;
pub use fetched::Fetched;
use brk_error::Result;
use brk_rpc::{Client, MempoolState};
use brk_types::{MempoolEntryInfo, Txid};
use brk_rpc::Client;
use brk_types::{MempoolEntryInfo, Timestamp, Txid, VSize};
use parking_lot::RwLock;
use rustc_hash::FxHashSet;
use tracing::warn;
use crate::{
State,
stores::{TxGraveyard, TxStore},
};
use crate::State;
/// Cap before the batch RPC so we never hand bitcoind an unbounded batch.
/// GBT-synthesized entries are not subject to this cap: they're bounded
/// by the block weight limit Core enforces on its own template.
const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000;
/// Two batched round-trips per cycle regardless of mempool size:
/// `getrawmempool verbose` + `getblocktemplate` + `getmempoolinfo` in
/// one mixed batch, then `getrawtransaction` for new txs.
/// Two batched round-trips per cycle, scaling with churn rather than
/// mempool size: `getblocktemplate` + `getrawmempool false` +
/// `getmempoolinfo` in one mixed batch, then `getmempoolentry` +
/// `getrawtransaction` for *new* non-GBT txids in a second mixed batch.
///
/// `getblocktemplate` is validated to be a subset of the verbose
/// listing inside the RPC layer; mismatches return `Ok(None)` so the
/// cycle is skipped without polluting downstream state.
/// GBT entries already carry the full tx body and stats, so any GBT tx
/// not yet in the local pool is materialized inline from the GBT
/// payload instead of being refetched. That removes the GBT/listing
/// race that used to skip cycles when a tx vanished from the mempool
/// between the GBT and `getrawmempool` calls: block 0 always reflects
/// Core's exact selection because we never ask for that data twice.
///
/// Confirmed prevouts are resolved post-apply by the caller-supplied
/// resolver passed to `Mempool::update_with`, so the in-crate path no
/// resolver passed to `Mempool::tick_with`, so the in-crate path no
/// longer issues a third batch for parents.
pub struct Fetcher;
impl Fetcher {
pub fn fetch(client: &Client, lock: &RwLock<State>) -> Result<Option<Fetched>> {
let Some(MempoolState {
entries,
gbt,
min_fee,
}) = client.fetch_mempool_state()?
else {
return Ok(None);
};
let new_txids = {
let state = lock.read();
Self::new_txids(&entries, &state.txs, &state.graveyard)
};
let new_raws = client.get_raw_transactions(&new_txids)?;
Ok(Some(Fetched {
entries_info: entries,
new_raws,
gbt,
min_fee,
}))
}
pub fn fetch(client: &Client, lock: &RwLock<State>) -> Result<Fetched> {
let (mut state, block_template) = client.fetch_mempool_state()?;
fn new_txids(
entries_info: &[MempoolEntryInfo],
known: &TxStore,
graveyard: &TxGraveyard,
) -> Vec<Txid> {
entries_info
.iter()
.filter(|info| !known.contains(&info.txid) && !graveyard.contains(&info.txid))
.take(MAX_TX_FETCHES_PER_CYCLE)
.map(|info| info.txid)
.collect()
// One read snapshot decides both the RPC fetch list and the
// GBT-synthesis set, so they agree on what's "already known".
let (new_txids, gbt_synth_set) = {
let mempool = lock.read();
let mut gbt_txids: FxHashSet<Txid> =
FxHashSet::with_capacity_and_hasher(block_template.len(), Default::default());
let mut gbt_synth_set: FxHashSet<Txid> = FxHashSet::default();
for g in &block_template {
gbt_txids.insert(g.txid);
if !mempool.txs.contains(&g.txid) {
gbt_synth_set.insert(g.txid);
}
}
let new_txids: Vec<Txid> = state
.live_txids
.iter()
.filter(|t| !mempool.txs.contains(t) && !gbt_txids.contains(t))
.take(MAX_TX_FETCHES_PER_CYCLE)
.copied()
.collect();
if new_txids.len() == MAX_TX_FETCHES_PER_CYCLE {
warn!(
cap = MAX_TX_FETCHES_PER_CYCLE,
"Fetcher: new-tx batch hit the per-cycle cap; remainder defers to the next cycle"
);
}
(new_txids, gbt_synth_set)
};
let (mut new_entries, mut new_txs) = client.fetch_new_pool_data(&new_txids)?;
new_entries.reserve(gbt_synth_set.len());
new_txs.reserve(gbt_synth_set.len());
// Consume `block_template` by value: GBT-only txs move their
// body and depends into the synthesis path (no clones), and
// the GBT ordering is captured as a `Vec<Txid>` for the
// Rebuilder, which is the only downstream consumer and only
// reads txids.
//
// GBT carries no per-tx arrival timestamp. `now` is correct to
// within ~1 cycle for a tx that just entered Core's mempool
// (the only kind that triggers synthesis: not in our pool yet
// means it just appeared this cycle).
let now = Timestamp::now();
let block_template_txids: Vec<Txid> = block_template
.into_iter()
.map(|g| {
let txid = g.txid;
if gbt_synth_set.contains(&txid) {
new_entries.push(MempoolEntryInfo {
txid,
vsize: VSize::from(g.weight),
weight: g.weight,
fee: g.fee,
first_seen: now,
depends: g.depends,
});
new_txs.insert(txid, g.tx);
}
txid
})
.collect();
// Promote `live_txids` to the union of `getrawmempool` and GBT:
// the two RPC views can disagree by a cycle, so a tx visible to
// GBT but missing from `getrawmempool` (or vice versa) is still
// alive. Without the union, GBT-only txs would oscillate enter ↔
// leave every cycle as `Preparer::classify_removals` buried what
// GBT had just resurrected.
state.live_txids.extend(block_template_txids.iter().copied());
Ok(Fetched {
state,
new_entries,
new_txs,
block_template_txids,
})
}
}
+3 -5
View File
@@ -1,13 +1,11 @@
//! The five pipeline steps. See the crate-level docs for the cycle.
//! Cycle stages in pipeline order.
mod applier;
mod fetcher;
pub(crate) mod preparer;
mod preparer;
mod prevouts;
pub(crate) mod rebuilder;
pub use applier::Applier;
pub use fetcher::{Fetched, Fetcher};
pub use preparer::{Preparer, TxEntry, TxRemoval};
pub use preparer::{Preparer, TxRemoval};
pub use prevouts::Prevouts;
pub use rebuilder::{BlockStats, Rebuilder, RecommendedFees, SnapTx, Snapshot, TxIndex};
+288 -25
View File
@@ -1,18 +1,18 @@
//! Turn `Fetched` raws into a typed diff for the Applier. Pure CPU,
//! holds a read guard on `State` for the cycle. New txs are
//! classified into three buckets:
//! holds a read guard on `State` for the cycle. New entries are
//! classified into two buckets:
//!
//! - **live** - already in `known`, skipped.
//! - **revivable** - in the graveyard, resurrected from the tombstone.
//! - **fresh** - decoded from `new_raws`, prevouts resolved against
//! the live mempool only. Confirmed-parent prevouts land as
//! `prevout: None` and are filled post-apply by the resolver passed
//! to `Mempool::update_with`.
//! to `Mempool::tick_with`.
//!
//! Removals are inferred by cross-referencing inputs.
//! Existing entries are not re-classified - they keep their first-sight
//! state on the live store. Removals are inferred by cross-referencing
//! inputs against the full `live_txids` set from the cycle's pull.
use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Txid, TxidPrefix};
use brk_types::{MempoolEntryInfo, Transaction, Txid, TxidPrefix, Vout};
use parking_lot::RwLock;
use rustc_hash::{FxHashMap, FxHashSet};
@@ -22,52 +22,50 @@ use crate::{
};
mod tx_addition;
mod tx_entry;
mod tx_removal;
mod txs_pulled;
pub use tx_addition::TxAddition;
pub use tx_entry::TxEntry;
pub use tx_removal::TxRemoval;
pub use txs_pulled::TxsPulled;
type SpentBy = FxHashMap<(Txid, Vout), Txid>;
pub struct Preparer;
impl Preparer {
pub fn prepare(
entries_info: Vec<MempoolEntryInfo>,
new_raws: FxHashMap<Txid, RawTx>,
live_txids: &[Txid],
new_entries: Vec<MempoolEntryInfo>,
new_txs: FxHashMap<Txid, bitcoin::Transaction>,
lock: &RwLock<State>,
) -> TxsPulled {
let state = lock.read();
let live: FxHashSet<TxidPrefix> = entries_info
.iter()
.map(|info| TxidPrefix::from(&info.txid))
.collect();
let added = Self::classify_additions(entries_info, new_raws, &state.txs, &state.graveyard);
let removed = TxRemoval::classify(&live, &added, &state.txs);
let live: FxHashSet<TxidPrefix> = live_txids.iter().map(TxidPrefix::from).collect();
let added = Self::classify_additions(new_entries, new_txs, &state.txs, &state.graveyard);
let removed = Self::classify_removals(&live, &added, &state.txs);
TxsPulled { added, removed }
}
fn classify_additions(
entries_info: Vec<MempoolEntryInfo>,
mut new_raws: FxHashMap<Txid, RawTx>,
new_entries: Vec<MempoolEntryInfo>,
mut new_txs: FxHashMap<Txid, bitcoin::Transaction>,
known: &TxStore,
graveyard: &TxGraveyard,
) -> Vec<TxAddition> {
entries_info
new_entries
.iter()
.filter_map(|info| Self::classify(info, known, graveyard, &mut new_raws))
.filter_map(|info| Self::classify_addition(info, known, graveyard, &mut new_txs))
.collect()
}
fn classify(
fn classify_addition(
info: &MempoolEntryInfo,
known: &TxStore,
graveyard: &TxGraveyard,
new_raws: &mut FxHashMap<Txid, RawTx>,
new_txs: &mut FxHashMap<Txid, bitcoin::Transaction>,
) -> Option<TxAddition> {
if known.contains(&info.txid) {
return None;
@@ -75,7 +73,272 @@ impl Preparer {
if let Some(tomb) = graveyard.get(&info.txid) {
return Some(TxAddition::revived(info, tomb));
}
let raw = new_raws.remove(&info.txid)?;
Some(TxAddition::fresh(info, raw, known))
let tx = new_txs.remove(&info.txid)?;
Some(TxAddition::fresh(info, tx, known))
}
/// One `(prefix, reason)` per known tx that's gone from the live set,
/// in `known` iteration order.
///
/// Cost is `O(R * avg_inputs)` where R is the removed-tx count and
/// `avg_inputs` is small for non-pathological txs. Worst case is a
/// `mempoolminfee` jump dropping ~10k txs in one cycle - still well
/// under the cycle budget.
fn classify_removals(
live: &FxHashSet<TxidPrefix>,
added: &[TxAddition],
known: &TxStore,
) -> Vec<(TxidPrefix, TxRemoval)> {
let spent_by = Self::build_spent_by(added);
known
.records()
.filter_map(|(prefix, record)| {
if live.contains(prefix) {
return None;
}
Some((*prefix, Self::removal_reason(&record.tx, &spent_by)))
})
.collect()
}
fn removal_reason(tx: &Transaction, spent_by: &SpentBy) -> TxRemoval {
tx.input
.iter()
.find_map(|i| spent_by.get(&(i.txid, i.vout)).copied())
.map_or(TxRemoval::Vanished, |by| TxRemoval::Replaced { by })
}
/// Only `Fresh` additions carry tx input data. Revived txs were
/// already in-pool, so they can't be new spenders of anything.
fn build_spent_by(added: &[TxAddition]) -> SpentBy {
let mut spent_by: SpentBy = FxHashMap::default();
for addition in added {
if let TxAddition::Fresh { tx, .. } = addition {
for txin in &tx.input {
spent_by.insert((txin.txid, txin.vout), tx.txid);
}
}
}
spent_by
}
}
#[cfg(test)]
mod tests {
use bitcoin::hashes::Hash;
use brk_types::{FeeRate, Sats, VSize};
use super::*;
use crate::{
AddedKind, TxRemoval,
state::TxEntry,
test_support::{fake_bitcoin_tx, fake_entry_info, fake_tx, fake_txid, p2wpkh_script},
};
fn empty_state() -> RwLock<State> {
RwLock::new(State::default())
}
fn seed_known(state: &RwLock<State>, txid: Txid) {
let tx = fake_tx(0xA0, &[None], &[(p2wpkh_script(50), 5_000)]);
let mut altered = tx;
altered.txid = txid;
for input in altered.input.iter_mut() {
input.prevout = Some(brk_types::TxOut::from((
p2wpkh_script(51),
Sats::from(1_000u64),
)));
}
let info = fake_entry_info(txid, 1_000, 100);
let entry = TxEntry::new(&info, 100, false);
state.write().txs.insert(altered, entry);
}
fn seed_graveyard(state: &RwLock<State>, txid: Txid) {
let tx = fake_tx(0xB0, &[None], &[(p2wpkh_script(60), 5_000)]);
let mut altered = tx;
altered.txid = txid;
let info = fake_entry_info(txid, 500, 100);
let entry = TxEntry::new(&info, 100, false);
let rate = FeeRate::from((Sats::from(500u64), VSize::from(100u64)));
state
.write()
.graveyard
.bury(altered, entry, rate, TxRemoval::Vanished);
}
#[test]
fn classify_addition_skips_already_known() {
let state = empty_state();
let known_txid = fake_txid(0x10);
seed_known(&state, known_txid);
let info = fake_entry_info(known_txid, 100, 100);
let mut new_txs: FxHashMap<Txid, bitcoin::Transaction> = FxHashMap::default();
new_txs.insert(known_txid, fake_bitcoin_tx(0x11, &[(p2wpkh_script(7), 1_234)]));
let pulled = Preparer::prepare(&[known_txid], vec![info], new_txs, &state);
assert!(pulled.added.is_empty(), "known tx must be filtered out");
assert!(pulled.removed.is_empty(), "still live, nothing removed");
}
#[test]
fn classify_addition_emits_revived_for_graveyard_hit() {
let state = empty_state();
let txid = fake_txid(0x20);
seed_graveyard(&state, txid);
let info = fake_entry_info(txid, 100, 100);
let pulled = Preparer::prepare(&[txid], vec![info], FxHashMap::default(), &state);
assert_eq!(pulled.added.len(), 1);
assert!(matches!(pulled.added[0].kind(), AddedKind::Revived));
}
#[test]
fn classify_addition_emits_fresh_with_raw_payload() {
let state = empty_state();
let txid = fake_txid(0x30);
// Make the bitcoin tx hash to `txid`: we instead key `new_txs`
// by the synthetic txid, since classify_addition keys lookup by
// info.txid, not by tx.compute_txid().
let info = fake_entry_info(txid, 200, 120);
let raw = fake_bitcoin_tx(0x31, &[(p2wpkh_script(8), 2_345)]);
let mut new_txs: FxHashMap<Txid, bitcoin::Transaction> = FxHashMap::default();
new_txs.insert(txid, raw);
let pulled = Preparer::prepare(&[txid], vec![info], new_txs, &state);
assert_eq!(pulled.added.len(), 1);
assert!(matches!(pulled.added[0].kind(), AddedKind::Fresh));
}
#[test]
fn classify_addition_drops_entry_with_no_raw_and_no_graveyard() {
let state = empty_state();
let txid = fake_txid(0x40);
let info = fake_entry_info(txid, 100, 100);
let pulled = Preparer::prepare(&[txid], vec![info], FxHashMap::default(), &state);
assert!(pulled.added.is_empty(), "no payload, no tomb -> filtered");
}
#[test]
fn classify_removal_marks_replaced_when_outpoint_is_spent_by_new_tx() {
let state = empty_state();
// Loser: spends (parent, vout=0). We arrange the new fresh tx
// to spend the same outpoint.
let parent_txid = fake_txid(0x50);
let loser_txid = fake_txid(0x51);
let replacer_txid = fake_txid(0x52);
{
let prev = Some(brk_types::TxOut::from((
p2wpkh_script(80),
Sats::from(10_000u64),
)));
let mut tx = fake_tx(0x51, &[prev], &[(p2wpkh_script(81), 5_000)]);
tx.txid = loser_txid;
tx.input[0].txid = parent_txid;
tx.input[0].vout = Vout::ZERO;
let info = fake_entry_info(loser_txid, 100, 100);
let entry = TxEntry::new(&info, 100, false);
state.write().txs.insert(tx, entry);
}
let info = fake_entry_info(replacer_txid, 200, 120);
let mut new_txs: FxHashMap<Txid, bitcoin::Transaction> = FxHashMap::default();
let mut raw = fake_bitcoin_tx(0x52, &[(p2wpkh_script(82), 4_321)]);
raw.input[0].previous_output = bitcoin::OutPoint {
txid: bitcoin::Txid::from_byte_array({
let mut b = [0u8; 32];
b[0] = 0x50;
b
}),
vout: 0,
};
new_txs.insert(replacer_txid, raw);
let pulled = Preparer::prepare(&[replacer_txid], vec![info], new_txs, &state);
assert_eq!(pulled.removed.len(), 1);
let (_, reason) = pulled.removed[0];
match reason {
TxRemoval::Replaced { by } => assert_eq!(by, replacer_txid),
TxRemoval::Vanished => panic!("expected Replaced, got Vanished"),
}
}
#[test]
fn classify_removal_marks_vanished_when_no_new_tx_spends_outpoint() {
let state = empty_state();
let gone_txid = fake_txid(0x60);
{
let prev = Some(brk_types::TxOut::from((
p2wpkh_script(90),
Sats::from(10_000u64),
)));
let mut tx = fake_tx(0x60, &[prev], &[(p2wpkh_script(91), 6_000)]);
tx.txid = gone_txid;
tx.input[0].txid = fake_txid(0xAA);
let info = fake_entry_info(gone_txid, 100, 100);
let entry = TxEntry::new(&info, 100, false);
state.write().txs.insert(tx, entry);
}
// No live txids in this cycle, no replacers staged.
let pulled = Preparer::prepare(&[], vec![], FxHashMap::default(), &state);
assert_eq!(pulled.removed.len(), 1);
assert!(matches!(pulled.removed[0].1, TxRemoval::Vanished));
}
#[test]
fn fresh_resolves_prevout_from_same_cycle_mempool_parent() {
// Same-cycle ordering: parent inserted first, then child whose
// input points at parent.vout=0. We exercise the path by
// putting the parent into the live store via a Fresh add, then
// a second Preparer call where the child's `info` references
// the parent's outpoint.
let state = empty_state();
let parent_txid = fake_txid(0x70);
let child_txid = fake_txid(0x71);
// Stage parent directly in the live store so resolve_prevout
// sees it when the child is decoded.
{
let mut parent = fake_tx(0x70, &[], &[(p2wpkh_script(100), 7_777)]);
parent.txid = parent_txid;
parent.input.clear();
let info = fake_entry_info(parent_txid, 100, 80);
let entry = TxEntry::new(&info, 80, false);
state.write().txs.insert(parent, entry);
}
let info = fake_entry_info(child_txid, 200, 120);
let mut raw = fake_bitcoin_tx(0x70, &[(p2wpkh_script(101), 6_000)]);
raw.input[0].previous_output = bitcoin::OutPoint {
txid: bitcoin::Txid::from_byte_array({
let mut b = [0u8; 32];
b[0] = 0x70;
b
}),
vout: 0,
};
let mut new_txs: FxHashMap<Txid, bitcoin::Transaction> = FxHashMap::default();
new_txs.insert(child_txid, raw);
let pulled = Preparer::prepare(
&[parent_txid, child_txid],
vec![info],
new_txs,
&state,
);
let TxAddition::Fresh { tx, .. } = &pulled.added[0] else {
panic!("expected Fresh classification");
};
let prevout = tx.input[0]
.prevout
.as_ref()
.expect("parent in same-cycle pool must resolve");
assert_eq!(prevout.value, Sats::from(7_777u64));
// No removal: parent + child both in live set.
assert!(pulled.removed.is_empty());
}
}
@@ -4,19 +4,18 @@
//! prevouts against the live mempool (same-cycle parents), build a
//! full `Transaction` + `Entry`. Confirmed parents land as
//! `prevout: None` and are filled post-apply by the resolver passed
//! to `Mempool::update_with`.
//! to `Mempool::tick_with`.
//! - **Revived** - tx in the graveyard. Rebuild the `Entry` only
//! (preserving `rbf`, `size`). The Applier exhumes the cached tx
//! body. No raw decoding.
use std::mem;
use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, SigOps, Transaction, TxIn, TxOut, TxStatus, Txid, Vout};
use crate::{TxTombstone, stores::TxStore};
use super::TxEntry;
use crate::{
cycle::AddedKind,
state::TxEntry,
stores::{TxStore, TxTombstone},
};
pub enum TxAddition {
Fresh { tx: Transaction, entry: TxEntry },
@@ -24,52 +23,68 @@ pub enum TxAddition {
}
impl TxAddition {
pub fn kind(&self) -> AddedKind {
match self {
Self::Fresh { .. } => AddedKind::Fresh,
Self::Revived { .. } => AddedKind::Revived,
}
}
/// Resolves prevouts against the live mempool only. Confirmed
/// parents land with `prevout: None` and are filled by the
/// resolver supplied to `Mempool::update_with` in the same cycle.
pub(super) fn fresh(info: &MempoolEntryInfo, raw: RawTx, mempool_txs: &TxStore) -> Self {
let total_size = raw.hex.len() / 2;
let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf());
let tx = Self::build_tx(info, raw, total_size, mempool_txs);
/// resolver supplied to `Mempool::tick_with` in the same cycle.
pub(super) fn fresh(
info: &MempoolEntryInfo,
tx: bitcoin::Transaction,
mempool_txs: &TxStore,
) -> Self {
let total_size = tx.total_size();
let rbf = tx.input.iter().any(|i| i.sequence.is_rbf());
let built = Self::build_tx(info, tx, total_size, mempool_txs);
let entry = TxEntry::new(info, total_size as u64, rbf);
Self::Fresh { tx, entry }
Self::Fresh { tx: built, entry }
}
fn build_tx(
info: &MempoolEntryInfo,
mut raw: RawTx,
tx: bitcoin::Transaction,
total_size: usize,
mempool_txs: &TxStore,
) -> Transaction {
let input = mem::take(&mut raw.tx.input)
let input = tx
.input
.into_iter()
.map(|txin| Self::build_txin(txin, mempool_txs))
.collect();
let mut tx = Transaction {
let mut built = Transaction {
index: None,
txid: info.txid,
version: raw.tx.version.into(),
version: tx.version.into(),
total_sigop_cost: SigOps::ZERO,
weight: info.weight,
lock_time: raw.tx.lock_time.into(),
lock_time: tx.lock_time.into(),
total_size,
fee: info.fee,
input,
output: raw.tx.output.into_iter().map(TxOut::from).collect(),
output: tx.output.into_iter().map(TxOut::from).collect(),
status: TxStatus::UNCONFIRMED,
};
tx.total_sigop_cost = tx.total_sigop_cost();
tx
built.refresh_sigops();
built
}
/// Preserves the tomb's original `first_seen`: bitcoind resets the
/// timestamp on re-acceptance (and GBT synthesis carries "now"), but
/// the consumer wants the first-ever sighting, not the latest one.
pub(super) fn revived(info: &MempoolEntryInfo, tomb: &TxTombstone) -> Self {
let entry = TxEntry::new(info, tomb.entry.size, tomb.entry.rbf);
let mut entry = TxEntry::new(info, tomb.entry.size, tomb.entry.rbf);
entry.first_seen = tomb.entry.first_seen;
Self::Revived { entry }
}
fn build_txin(txin: bitcoin::TxIn, mempool_txs: &TxStore) -> TxIn {
let prev_txid: Txid = txin.previous_output.txid.into();
let prev_vout = usize::from(Vout::from(txin.previous_output.vout));
let prev_vout = Vout::from(txin.previous_output.vout);
let prevout = Self::resolve_prevout(&prev_txid, prev_vout, mempool_txs);
TxIn {
@@ -78,7 +93,7 @@ impl TxAddition {
is_coinbase: false,
prevout,
txid: prev_txid,
vout: txin.previous_output.vout.into(),
vout: prev_vout,
script_sig: txin.script_sig,
script_sig_asm: (),
witness: txin.witness.into(),
@@ -88,10 +103,10 @@ impl TxAddition {
}
}
fn resolve_prevout(prev_txid: &Txid, prev_vout: usize, mempool_txs: &TxStore) -> Option<TxOut> {
fn resolve_prevout(prev_txid: &Txid, prev_vout: Vout, mempool_txs: &TxStore) -> Option<TxOut> {
let prev = mempool_txs.get(prev_txid)?;
prev.output
.get(prev_vout)
.get(usize::from(prev_vout))
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value)))
}
}
@@ -1,64 +1,18 @@
//! Why a tx left the mempool between two pull cycles, plus the
//! classifier that diffs the live prefix set against `known` to
//! produce one [`TxRemoval`] per loser.
//! Why a tx left the mempool between two pull cycles. The diff that
//! produces one [`TxRemoval`] per loser lives on [`super::Preparer`].
use brk_types::{Transaction, Txid, TxidPrefix, Vout};
use rustc_hash::{FxHashMap, FxHashSet};
use super::TxAddition;
use crate::stores::TxStore;
use brk_types::Txid;
/// `Replaced` = at least one freshly added tx this cycle spends one of
/// its inputs (BIP-125 replacement inferred from conflicting outpoints).
/// `by` is the immediate successor. The chain extends if `by` is itself
/// later replaced. Walk it forward via `TxGraveyard::replacement_root_of`.
///
/// `Vanished` = any other reason we can't distinguish from the data at
/// hand (mined, expired, evicted, or replaced by a tx we didn't fetch
/// due to the per-cycle fetch cap).
#[derive(Debug)]
#[derive(Debug, Clone, Copy)]
pub enum TxRemoval {
Replaced { by: Txid },
Vanished,
}
type SpentBy = FxHashMap<(Txid, Vout), Txid>;
impl TxRemoval {
/// Returns `(prefix, reason)` pairs in iteration order of `known`.
pub(super) fn classify(
live: &FxHashSet<TxidPrefix>,
added: &[TxAddition],
known: &TxStore,
) -> Vec<(TxidPrefix, Self)> {
let spent_by = Self::build_spent_by(added);
known
.records()
.filter_map(|(prefix, record)| {
if live.contains(prefix) {
return None;
}
Some((*prefix, Self::find_removal(&record.tx, &spent_by)))
})
.collect()
}
fn find_removal(tx: &Transaction, spent_by: &SpentBy) -> Self {
tx.input
.iter()
.find_map(|i| spent_by.get(&(i.txid, i.vout)).cloned())
.map_or(Self::Vanished, |by| Self::Replaced { by })
}
/// Only `Fresh` additions carry tx input data. Revived txs were
/// already in-pool, so they can't be new spenders of anything.
fn build_spent_by(added: &[TxAddition]) -> SpentBy {
let mut spent_by: SpentBy = FxHashMap::default();
for addition in added {
if let TxAddition::Fresh { tx, .. } = addition {
for txin in &tx.input {
spent_by.insert((txin.txid, txin.vout), tx.txid);
}
}
}
spent_by
}
}
+63 -42
View File
@@ -11,7 +11,7 @@
//! a fill directly (cheap, lock-local). Otherwise we record the
//! hole for external resolution.
//! 2. Drop the read guard. Call `resolver` on the remaining holes
//! (typically `getrawtransaction` or an indexer lookup); failures
//! (typically `getrawtransaction` or an indexer lookup). Failures
//! are simply skipped and retried next cycle.
//! 3. Take the write guard once and fold both fill batches into the
//! `TxStore` via `apply_fills` -> `add_input`. Idempotent: each
@@ -23,9 +23,10 @@ use std::sync::atomic::{AtomicBool, Ordering};
use brk_rpc::Client;
use brk_types::{TxOut, Txid, TxidPrefix, Vin, Vout};
use parking_lot::RwLock;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::warn;
use crate::{State, stores::TxStore};
use crate::{cycle::CycleDiff, state::State, stores::TxStore};
pub struct Prevouts;
@@ -33,15 +34,15 @@ type Fills = Vec<(Vin, TxOut)>;
type Holes = Vec<(Vin, Txid, Vout)>;
type FillBatch = Vec<(Txid, Fills)>;
type HoleBatch = Vec<(Txid, Holes)>;
type Resolved = FxHashMap<(Txid, Vout), TxOut>;
impl Prevouts {
/// Fill every unfilled prevout the cycle can resolve. Same-cycle
/// in-mempool parents are filled lock-locally; the remainder go
/// through `resolver` outside any lock. Returns true iff anything
/// was written.
pub fn fill<F>(lock: &RwLock<State>, resolver: F) -> bool
/// in-mempool parents are filled lock-locally. The remainder go
/// through `resolver` (one batched call) outside any lock.
pub fn fill<F>(lock: &RwLock<State>, diff: &mut CycleDiff, resolver: F)
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
F: Fn(&[(Txid, Vout)]) -> Resolved,
{
let (in_mempool, holes) = {
let state = lock.read();
@@ -50,37 +51,53 @@ impl Prevouts {
let external = Self::resolve_external(holes, resolver);
if in_mempool.is_empty() && external.is_empty() {
return false;
return;
}
let mut state = lock.write();
Self::write_fills(&mut state, in_mempool);
Self::write_fills(&mut state, external);
true
for (txid, fills) in in_mempool.into_iter().chain(external) {
let prefix = TxidPrefix::from(&txid);
for prevout in state.txs.apply_fills(&prefix, fills) {
state.addrs.add_input(&mut diff.addrs, &txid, &prevout);
}
}
}
/// Default resolver: per-call `getrawtransaction` against the
/// bitcoind RPC client `Mempool` already holds. Requires
/// `txindex=1`. On any failure logs once with a hint, then returns
/// `None`; the next cycle retries automatically.
pub fn rpc_resolver(client: Client) -> impl Fn(&Txid, Vout) -> Option<TxOut> {
/// Default resolver: one batched `getrawtransaction` per cycle,
/// deduped by parent txid. Requires bitcoind with `txindex=1`.
pub fn rpc_resolver(client: Client) -> impl Fn(&[(Txid, Vout)]) -> Resolved {
let warned = AtomicBool::new(false);
move |txid, vout| {
let bt: &bitcoin::Txid = txid.into();
match client.get_raw_transaction(bt, None as Option<&bitcoin::BlockHash>) {
Ok(tx) => tx
.output
.get(usize::from(vout))
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into()))),
move |holes: &[(Txid, Vout)]| {
if holes.is_empty() {
return Resolved::default();
}
let mut seen: FxHashSet<Txid> = FxHashSet::default();
let unique: Vec<Txid> = holes
.iter()
.filter_map(|(t, _)| seen.insert(*t).then_some(*t))
.collect();
let parents = match client.get_raw_transactions(&unique) {
Ok(map) => {
warned.store(false, Ordering::Relaxed);
map
}
Err(_) => {
if !warned.swap(true, Ordering::Relaxed) {
warn!(
"mempool: getrawtransaction missed for {txid}; ensure bitcoind is running with txindex=1"
"mempool: getrawtransaction batch failed; ensure bitcoind is running with txindex=1"
);
}
None
return Resolved::default();
}
}
};
holes
.iter()
.filter_map(|(txid, vout)| {
let o = parents.get(txid)?.output.get(usize::from(*vout))?;
let txout = TxOut::from((o.script_pubkey.clone(), o.value.into()));
Some(((*txid, *vout), txout))
})
.collect()
}
}
@@ -88,9 +105,6 @@ impl Prevouts {
/// same-cycle in-mempool fill (parent is live) or an external hole
/// (parent is confirmed or unknown).
fn gather(txs: &TxStore) -> (FillBatch, HoleBatch) {
if txs.unresolved().is_empty() {
return (Vec::new(), Vec::new());
}
let mut filled: FillBatch = Vec::new();
let mut holes: HoleBatch = Vec::new();
for prefix in txs.unresolved() {
@@ -123,30 +137,37 @@ impl Prevouts {
(filled, holes)
}
/// Flatten holes into one `(prev_txid, vout)` slice, invoke the
/// resolver once, then re-attribute resolved entries to their
/// consumer txs. Mempool double-spend rules guarantee every
/// `(prev_txid, vout)` key is unique across the batch, so no
/// dedup is needed before calling.
fn resolve_external<F>(holes: HoleBatch, resolver: F) -> FillBatch
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
F: Fn(&[(Txid, Vout)]) -> Resolved,
{
let total: usize = holes.iter().map(|(_, h)| h.len()).sum();
let mut flat: Vec<(Txid, Vout)> = Vec::with_capacity(total);
for (_, tx_holes) in &holes {
for (_, prev_txid, vout) in tx_holes {
flat.push((*prev_txid, *vout));
}
}
let mut resolved = resolver(&flat);
if resolved.is_empty() {
return Vec::new();
}
holes
.into_iter()
.filter_map(|(txid, holes)| {
let fills: Fills = holes
.filter_map(|(txid, tx_holes)| {
let fills: Fills = tx_holes
.into_iter()
.filter_map(|(vin, prev_txid, vout)| {
resolver(&prev_txid, vout).map(|o| (vin, o))
resolved.remove(&(prev_txid, vout)).map(|o| (vin, o))
})
.collect();
(!fills.is_empty()).then_some((txid, fills))
})
.collect()
}
fn write_fills(state: &mut State, fills: FillBatch) {
for (txid, tx_fills) in fills {
let prefix = TxidPrefix::from(&txid);
for prevout in state.txs.apply_fills(&prefix, tx_fills) {
state.addrs.add_input(&txid, &prevout);
}
}
}
}
@@ -1,100 +0,0 @@
use std::sync::{
Arc,
atomic::{AtomicBool, AtomicU64, Ordering},
};
use brk_rpc::BlockTemplateTx;
use brk_types::{FeeRate, TxidPrefix};
use parking_lot::RwLock;
use rustc_hash::FxHashSet;
use crate::State;
use partition::Partitioner;
use snapshot::{PrefixIndex, builder};
mod partition;
mod snapshot;
pub use brk_types::RecommendedFees;
pub use snapshot::{BlockStats, SnapTx, Snapshot, TxIndex};
const NUM_BLOCKS: usize = 8;
#[derive(Default)]
pub struct Rebuilder {
snapshot: RwLock<Arc<Snapshot>>,
dirty: AtomicBool,
rebuild_count: AtomicU64,
skip_clean: AtomicU64,
}
impl Rebuilder {
/// Mark dirty if the cycle changed mempool state, then rebuild iff
/// the dirty bit is set. Cycle pacing is the driver loop's job; the
/// rebuild itself is pure CPU on already-fetched data. The dirty
/// bit is cleared only after the snapshot is published, so a panic
/// in `build_snapshot` retries on the next cycle.
pub fn tick(
&self,
lock: &RwLock<State>,
changed: bool,
gbt: &[BlockTemplateTx],
min_fee: FeeRate,
) {
if changed {
self.dirty.store(true, Ordering::Release);
}
if !self.dirty.load(Ordering::Acquire) {
self.skip_clean.fetch_add(1, Ordering::Relaxed);
return;
}
*self.snapshot.write() = Arc::new(Self::build_snapshot(lock, gbt, min_fee));
self.dirty.store(false, Ordering::Release);
self.rebuild_count.fetch_add(1, Ordering::Relaxed);
}
pub fn rebuild_count(&self) -> u64 {
self.rebuild_count.load(Ordering::Relaxed)
}
pub fn skip_clean_count(&self) -> u64 {
self.skip_clean.load(Ordering::Relaxed)
}
fn build_snapshot(
lock: &RwLock<State>,
gbt: &[BlockTemplateTx],
min_fee: FeeRate,
) -> Snapshot {
let (txs, prefix_to_idx) = {
let state = lock.read();
builder::build_txs(&state.txs)
};
let block0 = Self::block_from_gbt(gbt, &prefix_to_idx);
let excluded: FxHashSet<TxIndex> = block0.iter().copied().collect();
let rest = Partitioner::partition(&txs, &excluded, NUM_BLOCKS.saturating_sub(1));
let mut blocks = Vec::with_capacity(NUM_BLOCKS);
blocks.push(block0);
blocks.extend(rest);
Snapshot::build(txs, blocks, prefix_to_idx, min_fee)
}
/// Block 0 from `getblocktemplate`: Core's actual selection. Maps
/// each GBT txid back to its `TxIndex` via the per-build prefix
/// index. Fetcher already validated GBT ⊆ verbose mempool, so any
/// drop here is a same-cycle race and the partitioner picks up the
/// slack so callers always see eight blocks.
fn block_from_gbt(gbt: &[BlockTemplateTx], prefix_to_idx: &PrefixIndex) -> Vec<TxIndex> {
gbt.iter()
.filter_map(|t| prefix_to_idx.get(&TxidPrefix::from(&t.txid)).copied())
.collect()
}
pub fn snapshot(&self) -> Arc<Snapshot> {
self.snapshot.read().clone()
}
}
@@ -1,65 +0,0 @@
//! Pack live txs into projected blocks 1..N, sorted by descending
//! `chunk_rate`. Block 0 is filled by the caller from `getblocktemplate`
//! (Core's actual selection); blocks 1..N feed
//! `/api/v1/fees/mempool-blocks` as a coarse fee-tier gradient.
//!
//! No topological gate: a child can sit before its parent within a
//! tied-rate run, but cluster members share a `chunk_rate` so they
//! land in the same block in the common case, and the only output is
//! a per-block rate distribution where intra-block order is invisible.
//!
//! The final block is a catch-all (no vsize cap) so leftover tail
//! vsize is accounted for instead of silently dropped.
//!
//! Walk sorted candidates once. For each, push into the current
//! block if it fits; otherwise advance to the next block (unless we
//! are already on the last one, which absorbs everything remaining).
use std::cmp::Reverse;
use brk_types::VSize;
use rustc_hash::FxHashSet;
use super::snapshot::{SnapTx, TxIndex};
pub struct Partitioner;
impl Partitioner {
pub fn partition(
txs: &[SnapTx],
excluded: &FxHashSet<TxIndex>,
num_remaining_blocks: usize,
) -> Vec<Vec<TxIndex>> {
if num_remaining_blocks == 0 {
return Vec::new();
}
let sorted = sorted_indices(txs, excluded);
let mut blocks: Vec<Vec<TxIndex>> = (0..num_remaining_blocks).map(|_| Vec::new()).collect();
let mut block_vsize = VSize::default();
let mut current = 0;
let last = num_remaining_blocks - 1;
for (idx, vsize) in sorted {
let fits = vsize <= VSize::MAX_BLOCK.saturating_sub(block_vsize);
if !fits && current < last && !blocks[current].is_empty() {
current += 1;
block_vsize = VSize::default();
}
blocks[current].push(idx);
block_vsize += vsize;
}
blocks
}
}
fn sorted_indices(txs: &[SnapTx], excluded: &FxHashSet<TxIndex>) -> Vec<(TxIndex, VSize)> {
let mut cands: Vec<(TxIndex, VSize, _)> = txs
.iter()
.enumerate()
.filter_map(|(i, t)| {
let idx = TxIndex::from(i);
(!excluded.contains(&idx)).then_some((idx, t.vsize, t.chunk_rate))
})
.collect();
cands.sort_by_key(|(_, _, rate)| Reverse(*rate));
cands.into_iter().map(|(i, v, _)| (i, v)).collect()
}
@@ -1,75 +0,0 @@
//! Build the per-tx adjacency for a snapshot from the live `TxStore`.
//!
//! One pass over the live records to assign compact `TxIndex`es and a
//! `prefix -> TxIndex` map, then per entry resolve `depends` against
//! it to produce parent edges. Children are mirrored from parents in
//! a second pass. Cross-pool parents (confirmed or evicted) are
//! dropped silently - the live pool reflects what miners actually see,
//! and any stale `depends` entry is self-healing.
//!
//! The prefix map is returned alongside the txs so the rebuilder can
//! reuse it for GBT mapping and the final `Snapshot::build` step
//! without reconstructing it.
use brk_types::TxidPrefix;
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use crate::TxEntry;
use crate::stores::TxStore;
use super::{SnapTx, TxIndex};
pub type PrefixIndex = FxHashMap<TxidPrefix, TxIndex>;
pub fn build_txs(txs: &TxStore) -> (Vec<SnapTx>, PrefixIndex) {
let (prefix_to_idx, ordered) = compact_index(txs);
let mut snap_txs: Vec<SnapTx> = ordered.iter().map(|e| live_tx(e, &prefix_to_idx)).collect();
mirror_children(&mut snap_txs);
(snap_txs, prefix_to_idx)
}
fn compact_index(txs: &TxStore) -> (PrefixIndex, Vec<&TxEntry>) {
let mut map: PrefixIndex = FxHashMap::with_capacity_and_hasher(txs.len(), FxBuildHasher);
let mut ordered: Vec<&TxEntry> = Vec::with_capacity(txs.len());
for (i, (prefix, record)) in txs.records().enumerate() {
map.insert(*prefix, TxIndex::from(i));
ordered.push(&record.entry);
}
(map, ordered)
}
fn live_tx(e: &TxEntry, prefix_to_idx: &PrefixIndex) -> SnapTx {
let parents: SmallVec<[TxIndex; 2]> = e
.depends
.iter()
.filter_map(|p| prefix_to_idx.get(p).copied())
.collect();
SnapTx {
txid: e.txid,
fee: e.fee,
vsize: e.vsize,
weight: e.weight,
size: e.size,
chunk_rate: e.chunk_rate,
parents,
children: SmallVec::new(),
}
}
fn mirror_children(txs: &mut [SnapTx]) {
let edges: Vec<(TxIndex, TxIndex)> = txs
.iter()
.enumerate()
.flat_map(|(i, t)| {
let child = TxIndex::from(i);
t.parents.iter().map(move |&p| (p, child))
})
.collect();
for (parent, child) in edges {
if let Some(t) = txs.get_mut(parent.as_usize()) {
t.children.push(child);
}
}
}
@@ -1,96 +0,0 @@
pub mod builder;
mod fees;
mod stats;
mod tx;
mod tx_index;
pub use builder::PrefixIndex;
pub use stats::BlockStats;
pub use tx::SnapTx;
pub use tx_index::TxIndex;
use std::hash::{DefaultHasher, Hash, Hasher};
use brk_types::{FeeRate, RecommendedFees, TxidPrefix};
use fees::Fees;
#[derive(Default)]
pub struct Snapshot {
/// Dense per-tx data indexed by `TxIndex`. Each entry carries the
/// chunk rate (Core's chunk-mempool truth or proxy fallback) plus
/// resolved parent/child adjacency, so CPFP queries don't re-read
/// any external state.
pub txs: Vec<SnapTx>,
/// Projected blocks. `blocks[0]` is Core's `getblocktemplate`
/// (Bitcoin Core's actual selection); the rest are greedy-packed
/// by descending chunk rate, with a final overflow block.
pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees,
/// Content hash of the projected next block. Same value as the
/// mempool ETag.
pub next_block_hash: u64,
/// Per-snapshot `TxidPrefix -> TxIndex` index, so live queries can
/// resolve a prefix to the snapshot's compact index without
/// re-walking `txs`. Built once by `builder::build_txs` and reused
/// by the rebuilder for GBT mapping.
prefix_to_idx: PrefixIndex,
}
impl Snapshot {
/// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor
/// for every recommended-fee tier.
pub fn build(
txs: Vec<SnapTx>,
blocks: Vec<Vec<TxIndex>>,
prefix_to_idx: PrefixIndex,
min_fee: FeeRate,
) -> Self {
let block_stats: Vec<BlockStats> = blocks
.iter()
.enumerate()
.map(|(i, block)| {
if i == 0 {
BlockStats::compute_core(block, &txs)
} else {
BlockStats::compute_projected(block, &txs)
}
})
.collect();
let fees = Fees::compute(&block_stats, min_fee);
let next_block_hash = Self::hash_next_block(&blocks);
Self {
txs,
blocks,
block_stats,
fees,
next_block_hash,
prefix_to_idx,
}
}
fn hash_next_block(blocks: &[Vec<TxIndex>]) -> u64 {
let Some(block) = blocks.first() else {
return 0;
};
let mut hasher = DefaultHasher::new();
block.hash(&mut hasher);
hasher.finish()
}
pub fn tx(&self, idx: TxIndex) -> Option<&SnapTx> {
self.txs.get(idx.as_usize())
}
pub fn idx_of(&self, prefix: &TxidPrefix) -> Option<TxIndex> {
self.prefix_to_idx.get(prefix).copied()
}
/// Effective chunk rate for a live tx by prefix, or `None` if the
/// tx isn't in this snapshot.
pub fn chunk_rate_for(&self, prefix: &TxidPrefix) -> Option<FeeRate> {
let idx = self.idx_of(prefix)?;
Some(self.txs[idx.as_usize()].chunk_rate)
}
}
+180 -31
View File
@@ -1,58 +1,65 @@
use std::{
collections::hash_map::Entry as MapEntry,
hash::{DefaultHasher, Hash, Hasher},
hash::{Hash, Hasher},
};
use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid};
use derive_more::Deref;
use rustc_hash::FxHashMap;
use rustc_hash::{FxHashMap, FxHasher};
use crate::cycle::AddrTransitions;
mod addr_entry;
use addr_entry::AddrEntry;
pub use addr_entry::AddrEntry;
#[derive(Default, Deref)]
#[derive(Default)]
pub struct AddrTracker(FxHashMap<AddrBytes, AddrEntry>);
impl AddrTracker {
pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) {
pub fn get(&self, addr: &AddrBytes) -> Option<&AddrEntry> {
self.0.get(addr)
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn add_tx(&mut self, transitions: &mut AddrTransitions, tx: &Transaction) {
let txid = &tx.txid;
for txin in &tx.input {
if let Some(prevout) = txin.prevout.as_ref() {
self.add_input(txid, prevout);
self.add_input(transitions, txid, prevout);
}
}
for txout in &tx.output {
if let Some(bytes) = txout.addr_bytes() {
self.apply_add(bytes, txid, |stats| stats.receiving(txout));
self.apply_add(transitions, bytes, txid, |stats| stats.receiving(txout));
}
}
}
pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) {
pub fn remove_tx(&mut self, transitions: &mut AddrTransitions, tx: &Transaction) {
let txid = &tx.txid;
for txin in &tx.input {
if let Some(prevout) = txin.prevout.as_ref() {
self.remove_input(txid, prevout);
self.remove_input(transitions, txid, prevout);
}
}
for txout in &tx.output {
if let Some(bytes) = txout.addr_bytes() {
self.apply_remove(bytes, txid, |stats| stats.received(txout));
self.apply_remove(transitions, bytes, txid, |stats| stats.received(txout));
}
}
}
/// Hash of an address's per-mempool stats. Stable while the address
/// is unchanged. Cheaper to recompute than to track invalidation.
/// Returns 0 for unknown addresses (collision with a real hash is
/// astronomically unlikely and only costs one ETag false-hit if it
/// ever happens).
pub fn stats_hash(&self, addr: &AddrBytes) -> u64 {
let Some(entry) = self.0.get(addr) else {
return 0;
};
let mut hasher = DefaultHasher::new();
/// Hash of an address's per-mempool stats, `None` if the address
/// has no live mempool activity. Stable while the address is
/// unchanged. Cheaper to recompute than to track invalidation.
pub fn stats_hash(&self, addr: &AddrBytes) -> Option<u64> {
let entry = self.0.get(addr)?;
let mut hasher = FxHasher::default();
entry.stats.hash(&mut hasher);
hasher.finish()
Some(hasher.finish())
}
/// Fold a single newly-resolved input into the per-address stats.
@@ -60,34 +67,58 @@ impl AddrTracker {
/// previously `None` has been filled, and by `add_tx` for each
/// resolved input. Inputs whose prevout doesn't resolve to an addr
/// are no-ops.
pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) {
pub fn add_input(
&mut self,
transitions: &mut AddrTransitions,
txid: &Txid,
prevout: &TxOut,
) {
let Some(bytes) = prevout.addr_bytes() else {
return;
};
self.apply_add(bytes, txid, |stats| stats.sending(prevout));
self.apply_add(transitions, bytes, txid, |stats| stats.sending(prevout));
}
fn remove_input(&mut self, txid: &Txid, prevout: &TxOut) {
fn remove_input(
&mut self,
transitions: &mut AddrTransitions,
txid: &Txid,
prevout: &TxOut,
) {
let Some(bytes) = prevout.addr_bytes() else {
return;
};
self.apply_remove(bytes, txid, |stats| stats.sent(prevout));
self.apply_remove(transitions, bytes, txid, |stats| stats.sent(prevout));
}
fn apply_add(
&mut self,
transitions: &mut AddrTransitions,
bytes: AddrBytes,
txid: &Txid,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let entry = self.0.entry(bytes).or_default();
entry.txids.insert(*txid);
update_stats(&mut entry.stats);
entry.stats.update_tx_count(entry.txids.len() as u32);
match self.0.entry(bytes) {
MapEntry::Occupied(mut occupied) => {
let entry = occupied.get_mut();
entry.txids.insert(*txid);
update_stats(&mut entry.stats);
entry.stats.update_tx_count(entry.txids.len() as u32);
}
MapEntry::Vacant(vacant) => {
let key = vacant.key().clone();
let entry = vacant.insert(AddrEntry::default());
entry.txids.insert(*txid);
update_stats(&mut entry.stats);
entry.stats.update_tx_count(entry.txids.len() as u32);
transitions.record_enter(key);
}
}
}
fn apply_remove(
&mut self,
transitions: &mut AddrTransitions,
bytes: AddrBytes,
txid: &Txid,
update_stats: impl FnOnce(&mut AddrMempoolStats),
@@ -100,9 +131,127 @@ impl AddrTracker {
update_stats(&mut entry.stats);
let len = entry.txids.len();
if len == 0 {
occupied.remove();
let (bytes, _) = occupied.remove_entry();
transitions.record_leave(bytes);
} else {
entry.stats.update_tx_count(len as u32);
}
}
}
#[cfg(test)]
mod tests {
use brk_types::{Sats, TxOut};
use super::*;
use crate::test_support::{fake_tx, p2wpkh_script};
fn addr_of(script: &bitcoin::ScriptBuf) -> AddrBytes {
AddrBytes::try_from(script).expect("p2wpkh script must yield AddrBytes")
}
#[test]
fn add_tx_records_enter_for_new_addr() {
let mut tracker = AddrTracker::default();
let mut transitions = AddrTransitions::default();
let out_script = p2wpkh_script(1);
let tx = fake_tx(1, &[], &[(out_script.clone(), 5_000)]);
let bytes = addr_of(&out_script);
tracker.add_tx(&mut transitions, &tx);
assert_eq!(tracker.len(), 1);
let entry = tracker.get(&bytes).expect("addr indexed");
assert_eq!(entry.stats.funded_txo_count, 1);
assert_eq!(entry.stats.funded_txo_sum, Sats::from(5_000u64));
assert_eq!(entry.stats.tx_count, 1);
let (enters, leaves) = transitions.into_vecs();
assert_eq!(enters, vec![bytes]);
assert!(leaves.is_empty());
}
#[test]
fn add_then_remove_tx_returns_to_zero_addrs() {
let mut tracker = AddrTracker::default();
let mut transitions = AddrTransitions::default();
let out_script = p2wpkh_script(2);
let prev_script = p2wpkh_script(3);
let tx = fake_tx(
2,
&[Some(TxOut::from((prev_script.clone(), Sats::from(4_000u64))))],
&[(out_script.clone(), 3_500)],
);
let recv = addr_of(&out_script);
let spend = addr_of(&prev_script);
tracker.add_tx(&mut transitions, &tx);
tracker.remove_tx(&mut transitions, &tx);
assert_eq!(tracker.len(), 0);
assert!(tracker.get(&recv).is_none());
assert!(tracker.get(&spend).is_none());
// add+remove in the same cycle: enter/leave cancel out.
let (enters, leaves) = transitions.into_vecs();
assert!(enters.is_empty(), "enter cancelled by same-cycle leave");
assert!(leaves.is_empty(), "leave cancelled by same-cycle enter");
}
#[test]
fn second_tx_touching_addr_does_not_re_enter() {
let mut tracker = AddrTracker::default();
let mut transitions = AddrTransitions::default();
let shared = p2wpkh_script(4);
let tx_a = fake_tx(3, &[], &[(shared.clone(), 2_500)]);
let tx_b = fake_tx(4, &[], &[(shared.clone(), 7_500)]);
tracker.add_tx(&mut transitions, &tx_a);
tracker.add_tx(&mut transitions, &tx_b);
let entry = tracker.get(&addr_of(&shared)).expect("addr indexed");
assert_eq!(entry.stats.funded_txo_count, 2);
assert_eq!(entry.stats.funded_txo_sum, Sats::from(10_000u64));
assert_eq!(entry.stats.tx_count, 2);
// Only one enter, even though two txs landed on the addr.
let (enters, _) = transitions.into_vecs();
assert_eq!(enters.len(), 1);
}
#[test]
fn stats_hash_is_none_for_untracked_addr() {
let tracker = AddrTracker::default();
let bytes = addr_of(&p2wpkh_script(5));
assert!(tracker.stats_hash(&bytes).is_none());
}
#[test]
fn stats_hash_stable_for_repeat_reads() {
let mut tracker = AddrTracker::default();
let mut transitions = AddrTransitions::default();
let script = p2wpkh_script(6);
let tx = fake_tx(5, &[], &[(script.clone(), 3_333)]);
tracker.add_tx(&mut transitions, &tx);
let bytes = addr_of(&script);
let first = tracker.stats_hash(&bytes).expect("addr tracked");
let second = tracker.stats_hash(&bytes).expect("addr tracked");
assert_eq!(first, second);
}
#[test]
fn stats_hash_changes_after_a_mutation() {
let mut tracker = AddrTracker::default();
let mut transitions = AddrTransitions::default();
let script = p2wpkh_script(7);
let bytes = addr_of(&script);
let tx_a = fake_tx(6, &[], &[(script.clone(), 1_111)]);
tracker.add_tx(&mut transitions, &tx_a);
let before = tracker.stats_hash(&bytes).expect("tracked after first add");
let tx_b = fake_tx(7, &[], &[(script, 2_222)]);
tracker.add_tx(&mut transitions, &tx_b);
let after = tracker.stats_hash(&bytes).expect("tracked after second add");
assert_ne!(before, after, "second funding tx must shift the hash");
}
}
+11 -8
View File
@@ -1,13 +1,16 @@
//! Stateful in-memory holders. After Phase 3 they're plain owned
//! types (no internal locks) — `State` aggregates them under a
//! single `RwLock` in `crate::state`.
//! In-memory holders for live mempool state. Plain owned types with
//! no internal locks: `crate::state::State` aggregates them under a
//! single `RwLock` so the cycle steps and read-side accessors share
//! one lock-order discipline.
pub mod addr_tracker;
pub(crate) mod outpoint_spends;
pub mod tx_graveyard;
pub mod tx_store;
mod addr_tracker;
mod outpoint_spends;
mod output_bins;
mod tx_graveyard;
mod tx_store;
pub use addr_tracker::AddrTracker;
pub(crate) use outpoint_spends::OutpointSpends;
pub use outpoint_spends::OutpointSpends;
pub use output_bins::OutputBins;
pub use tx_graveyard::{TxGraveyard, TxTombstone};
pub use tx_store::TxStore;
@@ -1,23 +1,22 @@
use brk_types::{OutpointPrefix, Transaction, TxidPrefix};
use derive_more::Deref;
use rustc_hash::FxHashMap;
/// Mempool index from spent outpoint to spending mempool tx.
///
/// Keys are `OutpointPrefix` (8 bytes txid + 2 bytes vout); prefix
/// Keys are `OutpointPrefix` (8 bytes txid + 2 bytes vout). Prefix
/// collisions are possible, so callers must verify the candidate
/// spender's input list. Values are the spender's `TxidPrefix`,
/// looked up against `TxStore` to recover the full spender record.
#[derive(Default, Deref)]
#[derive(Default)]
pub struct OutpointSpends(FxHashMap<OutpointPrefix, TxidPrefix>);
impl OutpointSpends {
pub fn len(&self) -> usize {
self.0.len()
}
pub fn insert_spends(&mut self, tx: &Transaction, spender: TxidPrefix) {
for input in &tx.input {
if input.is_coinbase {
continue;
}
let key = OutpointPrefix::new(TxidPrefix::from(&input.txid), input.vout);
for key in Self::spent_outpoints(tx) {
self.0.insert(key, spender);
}
}
@@ -25,11 +24,7 @@ impl OutpointSpends {
/// Only removes entries whose stored prefix still matches `spender`,
/// so an outpoint already re-claimed by a later spender is left alone.
pub fn remove_spends(&mut self, tx: &Transaction, spender: TxidPrefix) {
for input in &tx.input {
if input.is_coinbase {
continue;
}
let key = OutpointPrefix::new(TxidPrefix::from(&input.txid), input.vout);
for key in Self::spent_outpoints(tx) {
if self.0.get(&key) == Some(&spender) {
self.0.remove(&key);
}
@@ -40,4 +35,11 @@ impl OutpointSpends {
pub fn get(&self, key: &OutpointPrefix) -> Option<TxidPrefix> {
self.0.get(key).copied()
}
fn spent_outpoints(tx: &Transaction) -> impl Iterator<Item = OutpointPrefix> + '_ {
tx.input
.iter()
.filter(|i| !i.is_coinbase)
.map(|i| OutpointPrefix::new(TxidPrefix::from(&i.txid), i.vout))
}
}
@@ -0,0 +1,23 @@
use brk_oracle::default_eligible_bin;
use brk_types::Transaction;
use smallvec::SmallVec;
/// Pre-bucketed oracle bins for a tx's eligible outputs. Computed once on
/// insert so `Mempool::live_histogram` can bin all live outputs without
/// re-parsing scripts or recomputing eligibility per request.
pub struct OutputBins(SmallVec<[u16; 4]>);
impl OutputBins {
pub fn from_tx(tx: &Transaction) -> Self {
Self(
tx.output
.iter()
.filter_map(|o| default_eligible_bin(o.value, o.type_()))
.collect(),
)
}
pub fn iter(&self) -> impl Iterator<Item = u16> + '_ {
self.0.iter().copied()
}
}
+214 -16
View File
@@ -3,14 +3,14 @@ use std::{
time::{Duration, Instant},
};
use brk_types::{Transaction, Txid};
use brk_types::{FeeRate, Transaction, Txid};
use rustc_hash::FxHashMap;
mod tombstone;
pub use tombstone::TxTombstone;
use crate::{TxEntry, TxRemoval};
use crate::{TxRemoval, state::TxEntry};
const RETENTION: Duration = Duration::from_hours(1);
@@ -23,10 +23,6 @@ pub struct TxGraveyard {
}
impl TxGraveyard {
pub fn contains(&self, txid: &Txid) -> bool {
self.tombstones.contains_key(txid)
}
pub fn tombstones_len(&self) -> usize {
self.tombstones.len()
}
@@ -39,6 +35,27 @@ impl TxGraveyard {
self.tombstones.get(txid)
}
/// Tombstone iff the tx vanished from the pool (mined, expired, or
/// dropped). `Replaced` tombstones return `None` because the tx
/// will not confirm.
pub fn get_vanished(&self, txid: &Txid) -> Option<&TxTombstone> {
let tomb = self.tombstones.get(txid)?;
matches!(tomb.removal, TxRemoval::Vanished).then_some(tomb)
}
/// Walk forward through `Replaced { by }` to the terminal replacer.
/// Returns the first txid in the chain that isn't a `Replaced`
/// tombstone: live, `Vanished`, or unknown (chain broken because an
/// intermediate `by` aged out of the graveyard).
pub fn replacement_root_of(&self, mut txid: Txid) -> Txid {
while let Some(TxRemoval::Replaced { by }) =
self.tombstones.get(&txid).map(|t| &t.removal)
{
txid = *by;
}
txid
}
/// Tombstones marked as `Replaced { by: replacer }`. Used to walk
/// backward through RBF history: given a tx that's still live (or
/// in the graveyard), find every tx it displaced.
@@ -51,28 +68,43 @@ impl TxGraveyard {
})
}
/// Every `Replaced` tombstone, yielded as (predecessor_txid,
/// replacer_txid) in reverse bury order (most recent replacement
/// Every `Replaced` tombstone, yielded as (`predecessor_txid`,
/// `replacer_txid`) in reverse bury order (most recent replacement
/// event first). Caller walks the replacer chain forward to find
/// each tree's terminal replacer.
///
/// `order` may carry stale entries (re-buries, prior exhumes); the
/// `order` may carry stale entries (re-buries, prior exhumes). The
/// `removed_at == t` check skips those.
pub fn replaced_iter_recent_first(&self) -> impl Iterator<Item = (&Txid, &Txid)> {
self.order.iter().rev().filter_map(|(t, txid)| {
let ts = self.tombstones.get(txid)?;
if ts.removed_at() != *t {
if ts.removed_at != *t {
return None;
}
Some((txid, ts.replaced_by()?))
})
}
pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: TxEntry, removal: TxRemoval) {
let now = Instant::now();
self.tombstones
.insert(txid, TxTombstone::new(tx, entry, removal, now));
self.order.push_back((now, txid));
pub fn bury(
&mut self,
tx: Transaction,
entry: TxEntry,
chunk_rate: FeeRate,
removal: TxRemoval,
) {
let txid = entry.txid;
let removed_at = Instant::now();
self.tombstones.insert(
txid,
TxTombstone {
tx,
entry,
chunk_rate,
removal,
removed_at,
},
);
self.order.push_back((removed_at, txid));
}
/// Remove and return the tombstone, e.g. when the tx comes back to life.
@@ -92,10 +124,176 @@ impl TxGraveyard {
}
let (_, txid) = self.order.pop_front().unwrap();
if let Some(ts) = self.tombstones.get(&txid)
&& ts.removed_at() == t
&& ts.removed_at == t
{
self.tombstones.remove(&txid);
}
}
}
/// Test-only: force the oldest `order` entries to look older than
/// `RETENTION`. Splits `Instant::now()` arithmetic out of the test
/// bodies and avoids real-time sleeps.
#[cfg(test)]
fn shift_oldest_back(&mut self, count: usize) {
let bumped = Instant::now() - (RETENTION + Duration::from_secs(1));
for entry in self.order.iter_mut().take(count) {
let txid = entry.1;
entry.0 = bumped;
if let Some(ts) = self.tombstones.get_mut(&txid) {
ts.removed_at = bumped;
}
}
}
}
#[cfg(test)]
mod tests {
use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, VSize, Weight};
use super::*;
use crate::test_support::{fake_tx, fake_txid};
fn tomb_inputs(seed: u8) -> (Transaction, TxEntry, FeeRate) {
let tx = fake_tx(seed, &[], &[]);
let info = MempoolEntryInfo {
txid: tx.txid,
vsize: VSize::from(100u64),
weight: Weight::from(400u64),
fee: Sats::from(100u64),
first_seen: Timestamp::from(0u32),
depends: vec![],
};
let entry = TxEntry::new(&info, 100, false);
let rate = FeeRate::from((Sats::from(100u64), VSize::from(100u64)));
(tx, entry, rate)
}
#[test]
fn bury_then_exhume_roundtrips_the_tombstone() {
let mut g = TxGraveyard::default();
let (tx, entry, rate) = tomb_inputs(1);
let txid = entry.txid;
g.bury(tx, entry, rate, TxRemoval::Vanished);
assert_eq!(g.tombstones_len(), 1);
assert!(g.get(&txid).is_some());
let resurrected = g.exhume(&txid).expect("tombstone present");
assert_eq!(resurrected.entry.txid, txid);
assert!(g.get(&txid).is_none());
assert_eq!(g.tombstones_len(), 0);
// `order` still references the exhumed entry until evict_old
// runs. The timestamp-match check on evict skips stale rows.
assert_eq!(g.order_len(), 1);
}
#[test]
fn get_vanished_filters_out_replaced_tombstones() {
let mut g = TxGraveyard::default();
let (tx_a, entry_a, rate) = tomb_inputs(2);
let (tx_b, entry_b, _) = tomb_inputs(3);
let txid_a = entry_a.txid;
let txid_b = entry_b.txid;
g.bury(tx_a, entry_a, rate, TxRemoval::Replaced { by: txid_b });
g.bury(tx_b, entry_b, rate, TxRemoval::Vanished);
assert!(g.get_vanished(&txid_a).is_none());
assert!(g.get_vanished(&txid_b).is_some());
}
#[test]
fn replacement_root_walks_replaced_chain() {
let mut g = TxGraveyard::default();
let (tx_a, entry_a, rate) = tomb_inputs(4);
let (tx_b, entry_b, _) = tomb_inputs(5);
let (tx_c, entry_c, _) = tomb_inputs(6);
let a = entry_a.txid;
let b = entry_b.txid;
let c = entry_c.txid;
g.bury(tx_a, entry_a, rate, TxRemoval::Replaced { by: b });
g.bury(tx_b, entry_b, rate, TxRemoval::Replaced { by: c });
g.bury(tx_c, entry_c, rate, TxRemoval::Vanished);
assert_eq!(g.replacement_root_of(a), c);
assert_eq!(g.replacement_root_of(c), c);
let unknown = fake_txid(99);
assert_eq!(g.replacement_root_of(unknown), unknown);
}
#[test]
fn predecessors_of_returns_direct_replacers() {
let mut g = TxGraveyard::default();
let (tx_a, entry_a, rate) = tomb_inputs(7);
let (tx_b, entry_b, _) = tomb_inputs(8);
let (tx_c, entry_c, _) = tomb_inputs(9);
let replacer = entry_c.txid;
let a = entry_a.txid;
let b = entry_b.txid;
g.bury(tx_a, entry_a, rate, TxRemoval::Replaced { by: replacer });
g.bury(tx_b, entry_b, rate, TxRemoval::Replaced { by: replacer });
g.bury(tx_c, entry_c, rate, TxRemoval::Vanished);
let mut preds: Vec<Txid> = g.predecessors_of(&replacer).map(|(t, _)| *t).collect();
preds.sort_unstable_by_key(|t| t.as_slice()[0]);
let mut expected = vec![a, b];
expected.sort_unstable_by_key(|t| t.as_slice()[0]);
assert_eq!(preds, expected);
assert_eq!(g.predecessors_of(&fake_txid(123)).count(), 0);
}
#[test]
fn replaced_iter_recent_first_skips_stale_order_entries() {
let mut g = TxGraveyard::default();
let (tx_a, entry_a, rate) = tomb_inputs(10);
let (tx_b, entry_b, _) = tomb_inputs(11);
let replacer = entry_b.txid;
let pred = entry_a.txid;
g.bury(tx_a.clone(), entry_a.clone(), rate, TxRemoval::Replaced { by: replacer });
g.bury(tx_b, entry_b, rate, TxRemoval::Vanished);
// Re-bury the predecessor: its `order` entry is now stale.
g.bury(tx_a, entry_a, rate, TxRemoval::Replaced { by: replacer });
let collected: Vec<(Txid, Txid)> = g
.replaced_iter_recent_first()
.map(|(p, by)| (*p, *by))
.collect();
assert_eq!(collected, vec![(pred, replacer)]);
}
#[test]
fn evict_old_drops_aged_tombstones() {
let mut g = TxGraveyard::default();
let (tx_a, entry_a, rate) = tomb_inputs(12);
let (tx_b, entry_b, _) = tomb_inputs(13);
let txid_a = entry_a.txid;
let txid_b = entry_b.txid;
g.bury(tx_a, entry_a, rate, TxRemoval::Vanished);
g.bury(tx_b, entry_b, rate, TxRemoval::Vanished);
g.shift_oldest_back(1);
g.evict_old();
assert!(g.get(&txid_a).is_none(), "aged tombstone evicted");
assert!(g.get(&txid_b).is_some(), "fresh tombstone retained");
}
#[test]
fn re_bury_mid_retention_resets_age() {
let mut g = TxGraveyard::default();
let (tx, entry, rate) = tomb_inputs(14);
let txid = entry.txid;
g.bury(tx.clone(), entry.clone(), rate, TxRemoval::Vanished);
g.shift_oldest_back(1);
// Re-bury: a stale order entry remains pointing at the old time,
// but `removed_at` on the tombstone is now fresh. evict_old's
// timestamp-match check should drop the stale order entry without
// touching the live tombstone.
g.bury(tx, entry, rate, TxRemoval::Vanished);
g.evict_old();
assert!(g.get(&txid).is_some());
}
}
@@ -1,42 +1,24 @@
use std::time::Instant;
use brk_types::{Transaction, Txid};
use brk_types::{FeeRate, Transaction, Txid};
use crate::{TxEntry, TxRemoval};
use crate::{TxRemoval, state::TxEntry};
/// A buried mempool tx, retained for reappearance detection and
/// post-mine analytics.
/// post-mine analytics. `chunk_rate` is the linearized chunk feerate at
/// burial time - same value `live_effective_fee_rate` reported while
/// the tx was alive, so an evicted RBF predecessor reports the
/// package-effective rate, not a misleading isolated `fee/vsize`.
pub struct TxTombstone {
pub tx: Transaction,
pub entry: TxEntry,
removal: TxRemoval,
removed_at: Instant,
pub chunk_rate: FeeRate,
pub removal: TxRemoval,
pub removed_at: Instant,
}
impl TxTombstone {
pub(crate) fn new(
tx: Transaction,
entry: TxEntry,
removal: TxRemoval,
removed_at: Instant,
) -> Self {
Self {
tx,
entry,
removal,
removed_at,
}
}
pub fn reason(&self) -> &TxRemoval {
&self.removal
}
pub(crate) fn removed_at(&self) -> Instant {
self.removed_at
}
pub(crate) fn replaced_by(&self) -> Option<&Txid> {
pub fn replaced_by(&self) -> Option<&Txid> {
match &self.removal {
TxRemoval::Replaced { by } => Some(by),
TxRemoval::Vanished => None,
+219 -16
View File
@@ -1,15 +1,29 @@
use brk_oracle::Histogram;
use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, TxidPrefix, Vin};
use rustc_hash::{FxHashMap, FxHashSet};
use crate::TxEntry;
use crate::{state::TxEntry, stores::OutputBins};
const RECENT_CAP: usize = 10;
/// Per-tx record: live tx body and its mempool entry, kept under one
/// key so a single map probe returns both.
/// Per-tx record: live tx body, its mempool entry, and the pre-bucketed
/// oracle bins for its outputs. Kept under one key so a single map probe
/// returns everything readers need.
pub struct TxRecord {
pub tx: Transaction,
pub entry: TxEntry,
pub output_bins: OutputBins,
}
impl TxRecord {
pub fn new(tx: Transaction, entry: TxEntry) -> Self {
let output_bins = OutputBins::from_tx(&tx);
Self {
tx,
entry,
output_bins,
}
}
}
/// Live-pool index keyed by `TxidPrefix`. The full `Txid` lives in
@@ -18,11 +32,15 @@ pub struct TxRecord {
/// set of prefixes whose tx still has at least one `prevout: None`,
/// maintained on every `insert` / `remove_by_prefix` / `apply_fills`
/// so the post-update prevout filler can early-exit when empty.
/// `live_histogram` mirrors the union of each record's `OutputBins`,
/// kept in sync on `insert` / `remove_by_prefix` so the oracle-blend
/// read path is a single array clone, not a full pool walk.
#[derive(Default)]
pub struct TxStore {
records: FxHashMap<TxidPrefix, TxRecord>,
recent: Vec<MempoolRecentTx>,
unresolved: FxHashSet<TxidPrefix>,
live_histogram: Histogram,
}
impl TxStore {
@@ -34,10 +52,6 @@ impl TxStore {
self.records.len()
}
pub fn is_empty(&self) -> bool {
self.records.is_empty()
}
pub fn get(&self, txid: &Txid) -> Option<&Transaction> {
self.records.get(&TxidPrefix::from(txid)).map(|r| &r.tx)
}
@@ -56,7 +70,7 @@ impl TxStore {
self.records.get(prefix)
}
/// `(prefix, record)` pairs in HashMap iteration order. Used by
/// `(prefix, record)` pairs in `HashMap` iteration order. Used by
/// the snapshot builder to assign a compact `TxIndex` to each
/// live tx in one pass.
pub fn records(&self) -> impl Iterator<Item = (&TxidPrefix, &TxRecord)> {
@@ -67,10 +81,6 @@ impl TxStore {
self.records.values().map(|r| &r.entry.txid)
}
pub fn values(&self) -> impl Iterator<Item = &Transaction> {
self.records.values().map(|r| &r.tx)
}
pub fn insert(&mut self, tx: Transaction, entry: TxEntry) {
let prefix = entry.txid_prefix();
debug_assert!(
@@ -81,7 +91,11 @@ impl TxStore {
if tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.insert(prefix);
}
self.records.insert(prefix, TxRecord { tx, entry });
let record = TxRecord::new(tx, entry);
for bin in record.output_bins.iter() {
self.live_histogram[bin as usize] += 1;
}
self.records.insert(prefix, record);
}
fn sample_recent(&mut self, txid: &Txid, tx: &Transaction) {
@@ -98,9 +112,18 @@ impl TxStore {
pub fn remove_by_prefix(&mut self, prefix: &TxidPrefix) -> Option<TxRecord> {
let record = self.records.remove(prefix)?;
self.unresolved.remove(prefix);
for bin in record.output_bins.iter() {
self.live_histogram[bin as usize] -= 1;
}
Some(record)
}
/// Snapshot the live oracle-bin histogram. Maintained incrementally
/// on insert/remove, so this is `O(NUM_BINS)`, not `O(live_outputs)`.
pub fn live_histogram(&self) -> Histogram {
self.live_histogram.clone()
}
/// Set of prefixes with at least one unfilled prevout. Used by the
/// prevout filler as a cheap "is there any work?" gate.
pub fn unresolved(&self) -> &FxHashSet<TxidPrefix> {
@@ -110,8 +133,10 @@ impl TxStore {
/// Apply resolved prevouts to a tx in place. `fills` is `(vin, prevout)`.
/// Returns the prevouts actually written (so the caller can fold them
/// into `AddrTracker`). Updates `unresolved` if fully resolved after
/// the fill, and recomputes `total_sigop_cost` (P2SH and witness
/// components depend on prevouts).
/// the fill, and refreshes `total_sigop_cost` (P2SH and witness
/// components depend on prevouts). `entry.vsize` is Core's value from
/// `MempoolEntryInfo` and is not recomputed here - the sigops shift
/// belongs to the `Transaction`, not the entry.
pub fn apply_fills(&mut self, prefix: &TxidPrefix, fills: Vec<(Vin, TxOut)>) -> Vec<TxOut> {
let Some(record) = self.records.get_mut(prefix) else {
return Vec::new();
@@ -120,7 +145,7 @@ impl TxStore {
if applied.is_empty() {
return applied;
}
record.tx.total_sigop_cost = record.tx.total_sigop_cost();
record.tx.refresh_sigops();
if record.tx.input.iter().all(|i| i.prevout.is_some()) {
self.unresolved.remove(prefix);
}
@@ -140,3 +165,181 @@ impl TxStore {
applied
}
}
#[cfg(test)]
mod tests {
use bitcoin::ScriptBuf;
use brk_types::{MempoolEntryInfo, Sats, Timestamp, VSize, Weight};
use super::*;
use crate::test_support::{fake_tx, fake_txid, p2wpkh_script};
fn entry_for(tx: &Transaction, fee: u64, vsize: u64) -> TxEntry {
let info = MempoolEntryInfo {
txid: tx.txid,
vsize: VSize::from(vsize),
weight: Weight::from(VSize::from(vsize)),
fee: Sats::from(fee),
first_seen: Timestamp::from(0u32),
depends: vec![],
};
TxEntry::new(&info, vsize, false)
}
fn tx_without_prevouts(seed: u8) -> Transaction {
fake_tx(seed, &[None, None], &[(p2wpkh_script(1), 1_000)])
}
fn tx_with_prevouts(seed: u8) -> Transaction {
let prev = Some(TxOut::from((p2wpkh_script(2), Sats::from(2_000u64))));
fake_tx(seed, &[prev], &[(p2wpkh_script(3), 500)])
}
#[test]
fn insert_records_unresolved_when_prevouts_missing() {
let mut store = TxStore::default();
let tx = tx_without_prevouts(1);
let entry = entry_for(&tx, 100, 100);
let prefix = entry.txid_prefix();
store.insert(tx, entry);
assert!(store.unresolved().contains(&prefix));
assert_eq!(store.len(), 1);
}
#[test]
fn insert_skips_unresolved_when_all_prevouts_present() {
let mut store = TxStore::default();
let tx = tx_with_prevouts(2);
let entry = entry_for(&tx, 200, 150);
let prefix = entry.txid_prefix();
store.insert(tx, entry);
assert!(!store.unresolved().contains(&prefix));
assert_eq!(store.len(), 1);
}
#[test]
fn remove_by_prefix_clears_unresolved_and_returns_record() {
let mut store = TxStore::default();
let tx = tx_without_prevouts(3);
let entry = entry_for(&tx, 300, 200);
let prefix = entry.txid_prefix();
store.insert(tx, entry);
assert!(store.unresolved().contains(&prefix));
let removed = store.remove_by_prefix(&prefix).expect("record present");
assert_eq!(removed.entry.txid_prefix(), prefix);
assert!(!store.unresolved().contains(&prefix));
assert_eq!(store.len(), 0);
assert!(store.remove_by_prefix(&prefix).is_none());
}
#[test]
fn apply_fills_writes_only_missing_inputs_and_refreshes_sigops() {
let mut store = TxStore::default();
let prev_present = TxOut::from((p2wpkh_script(4), Sats::from(7_000u64)));
let tx = fake_tx(
4,
&[None, Some(prev_present.clone())],
&[(p2wpkh_script(5), 1_000)],
);
let entry = entry_for(&tx, 400, 250);
let prefix = entry.txid_prefix();
store.insert(tx, entry);
assert!(store.unresolved().contains(&prefix));
let new_prevout = TxOut::from((p2wpkh_script(6), Sats::from(9_000u64)));
let overwrite_attempt = TxOut::from((p2wpkh_script(99), Sats::from(1u64)));
let applied = store.apply_fills(
&prefix,
vec![
(Vin::from(0u32), new_prevout.clone()),
(Vin::from(1u32), overwrite_attempt),
],
);
assert_eq!(applied.len(), 1);
assert_eq!(applied[0].value, new_prevout.value);
let record = store.record_by_prefix(&prefix).expect("record present");
assert_eq!(record.tx.input[0].prevout.as_ref().unwrap().value, new_prevout.value);
assert_eq!(
record.tx.input[1].prevout.as_ref().unwrap().value,
prev_present.value
);
assert!(!store.unresolved().contains(&prefix));
}
#[test]
fn apply_fills_unknown_prefix_is_noop() {
let mut store = TxStore::default();
let stray_prefix = TxidPrefix::from(&fake_txid(0xFF));
let applied = store.apply_fills(
&stray_prefix,
vec![(Vin::from(0u32), TxOut::from((ScriptBuf::new(), Sats::from(1u64))))],
);
assert!(applied.is_empty());
}
#[test]
fn apply_fills_partial_keeps_unresolved() {
let mut store = TxStore::default();
let tx = tx_without_prevouts(5);
let entry = entry_for(&tx, 500, 300);
let prefix = entry.txid_prefix();
store.insert(tx, entry);
let one = TxOut::from((p2wpkh_script(7), Sats::from(3_000u64)));
let applied = store.apply_fills(&prefix, vec![(Vin::from(0u32), one)]);
assert_eq!(applied.len(), 1);
assert!(
store.unresolved().contains(&prefix),
"input 1 still has None prevout"
);
}
#[test]
fn recent_is_capped_and_newest_first() {
let mut store = TxStore::default();
for i in 0..(RECENT_CAP as u8 + 5) {
let tx = tx_with_prevouts(i + 10);
let entry = entry_for(&tx, 100, 100);
store.insert(tx, entry);
}
assert_eq!(store.recent().len(), RECENT_CAP);
let newest = store.recent().first().expect("at least one");
let last_inserted_txid = fake_txid(RECENT_CAP as u8 + 5 + 10 - 1);
assert_eq!(newest.txid, last_inserted_txid);
}
#[test]
fn live_histogram_total_tracks_inserts_and_removes() {
let mut store = TxStore::default();
let tx_a = fake_tx(
20,
&[Some(TxOut::from((p2wpkh_script(8), Sats::from(1_234u64))))],
&[
(p2wpkh_script(9), 2_345),
(p2wpkh_script(10), 3_456),
],
);
let tx_b = fake_tx(
21,
&[Some(TxOut::from((p2wpkh_script(11), Sats::from(4_567u64))))],
&[(p2wpkh_script(12), 7_891)],
);
let entry_a = entry_for(&tx_a, 100, 100);
let entry_b = entry_for(&tx_b, 100, 100);
let prefix_a = entry_a.txid_prefix();
store.insert(tx_a, entry_a);
store.insert(tx_b, entry_b);
let total_after_both: u32 = store.live_histogram().iter().sum();
assert_eq!(total_after_both, 3, "two outputs + one output");
store.remove_by_prefix(&prefix_a);
let total_after_remove: u32 = store.live_histogram().iter().sum();
assert_eq!(total_after_remove, 1);
}
}
+119
View File
@@ -0,0 +1,119 @@
//! Tiny tx fixtures shared across the crate's unit tests. Keeps
//! constructor noise out of the test bodies so each test reads as
//! "set up, mutate, assert" without 20 lines of struct literals.
use bitcoin::{ScriptBuf, absolute::LockTime, hashes::Hash, transaction::Version};
use brk_types::{
MempoolEntryInfo, RawLockTime, Sats, SigOps, Timestamp, Transaction, TxIn, TxOut, TxStatus,
TxVersionRaw, Txid, VSize, Vout, Weight, Witness,
};
/// Deterministic `Txid` from a single seed byte. The first byte of the
/// hash is `seed`, the rest is zero, so tests can identify txs by eye
/// in debug output.
pub fn fake_txid(seed: u8) -> Txid {
let mut bytes = [0u8; 32];
bytes[0] = seed;
Txid::from(bitcoin::Txid::from_byte_array(bytes))
}
/// Minimal P2WPKH `script_pubkey` keyed off `seed` so distinct inputs
/// or outputs in the same test don't collide on `addr_bytes()`.
pub fn p2wpkh_script(seed: u8) -> ScriptBuf {
let mut bytes = [0u8; 20];
bytes[0] = seed;
ScriptBuf::new_p2wpkh(&bitcoin::WPubkeyHash::from_byte_array(bytes))
}
/// Build a `Transaction` with one input per entry in `prevouts` (each
/// either resolved or `None`) and one output per `(script, value)`
/// pair. Counterparty txids on the inputs are derived from the seed so
/// `addr_bytes` extraction sees distinct prev txids per input.
pub fn fake_tx(seed: u8, prevouts: &[Option<TxOut>], outputs: &[(ScriptBuf, u64)]) -> Transaction {
let input = prevouts
.iter()
.enumerate()
.map(|(i, p)| TxIn {
is_coinbase: false,
prevout: p.clone(),
txid: fake_txid(seed.wrapping_add(100 + i as u8)),
vout: Vout::ZERO,
script_sig: ScriptBuf::new(),
script_sig_asm: (),
witness: Witness::default(),
sequence: 0xffff_fffe,
inner_redeem_script_asm: (),
inner_witness_script_asm: (),
})
.collect();
let output = outputs
.iter()
.map(|(script, value)| TxOut::from((script.clone(), Sats::from(*value))))
.collect();
let mut tx = Transaction {
index: None,
txid: fake_txid(seed),
version: TxVersionRaw::from(Version::TWO),
lock_time: RawLockTime::from(LockTime::ZERO),
input,
output,
total_size: 200,
weight: Weight::from(800u64),
total_sigop_cost: SigOps::ZERO,
fee: Sats::ZERO,
status: TxStatus::UNCONFIRMED,
};
tx.refresh_sigops();
tx
}
/// Plain `MempoolEntryInfo` keyed off `txid`. Test bodies usually
/// already have the txid from `fake_tx`, so this just fills in the
/// non-essential fields with deterministic placeholders.
pub fn fake_entry_info(txid: Txid, fee: u64, vsize: u64) -> MempoolEntryInfo {
MempoolEntryInfo {
txid,
vsize: VSize::from(vsize),
weight: Weight::from(vsize * 4),
fee: Sats::from(fee),
first_seen: Timestamp::from(0u32),
depends: vec![],
}
}
/// Bitcoin-protocol `Transaction` matching `fake_tx`. Round-trippable
/// against a brk `Transaction`, lets the Preparer's `Fresh` path decode
/// it without a real RPC payload.
pub fn fake_bitcoin_tx(
prev_txid_seed: u8,
outputs: &[(ScriptBuf, u64)],
) -> bitcoin::Transaction {
let input = vec![bitcoin::TxIn {
previous_output: bitcoin::OutPoint {
txid: bitcoin::Txid::from_byte_array({
let mut b = [0u8; 32];
b[0] = prev_txid_seed;
b
}),
vout: 0,
},
script_sig: ScriptBuf::new(),
sequence: bitcoin::Sequence(0xffff_fffe),
witness: bitcoin::Witness::new(),
}];
let output = outputs
.iter()
.map(|(script, value)| bitcoin::TxOut {
value: bitcoin::Amount::from_sat(*value),
script_pubkey: script.clone(),
})
.collect();
bitcoin::Transaction {
version: Version::TWO,
lock_time: LockTime::ZERO,
input,
output,
}
}
+43 -31
View File
@@ -1,6 +1,8 @@
# brk_oracle
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 550,000 (November 2018) onward.
**Version 2**
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 525,000 (May 2018) onward.
Inspired by [UTXOracle](https://utxo.live/oracle/) by [@SteveSimple](https://x.com/SteveSimple), which proved the concept. brk_oracle takes the same core insight and redesigns the algorithm for per-block resolution and rolling operation. See [comparison](#comparison-with-utxoracle) below.
@@ -46,7 +48,7 @@ For each new block:
### 1. Filter outputs
Skip the coinbase transaction, then exclude noisy outputs: script types dominated by protocol activity (P2TR, P2WSH by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
### 2. Build a log-scale histogram
@@ -116,13 +118,11 @@ Parabolic interpolation between the best bin and its two neighbors refines the e
log-scale scoring interpolation
```
## Input formats
## Input
The oracle accepts three input formats:
The oracle consumes one pre-built histogram per block via `process_histogram(&hist)`, a `[u32; 2400]` bin-count array, and returns the updated reference bin.
- **Raw block**: `process_block(&block)` — filters and bins internally
- **Output pairs**: `process_outputs(iter)``(sats, output_type)` pairs, still applies configured filters
- **Histogram**: `process_histogram(&hist)` — pre-built `[u32; 2400]` array
The caller does the filtering when it builds the histogram. For each block it skips the coinbase, drops every output of a transaction carrying an `OP_RETURN`, then bins the rest. `default_eligible_bin(sats, output_type)` (or `Oracle::output_to_bin` for a non-default `Config`) applies the per-output rules: excluded script types, dust, and round-BTC values. It returns the bin index, or `None` for a filtered output.
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for every height up to 630,000 to derive a seed from.
@@ -137,7 +137,7 @@ All parameters via `Config` with sensible defaults:
| `search_below` / `search_above` | 9 / 11 | Search window around previous estimate (bins) |
| `min_sats` | 1,000 | Dust threshold |
| `exclude_common_round_values` | true | Filter d × 10ⁿ (d ∈ {1,2,3,5,6}) to prevent false stencil matches |
| `excluded_output_types` | P2TR, P2WSH | Script types dominated by protocol activity |
| `excluded_output_types` | P2TR | Script types dominated by protocol activity |
## Comparison with UTXOracle
@@ -150,30 +150,30 @@ All parameters via `Config` with sensible defaults:
| Algorithm | Single-pass stencil scoring with per-offset normalization | Multi-step: dual stencil → rough estimate → output-to-USD mapping → iterative convergence |
| Stencil | 19 round-USD offsets ($1 to $10k), each normalized to its own peak | 803-point Gaussian + weighted spike template targeting 17 round-USD amounts |
| Round BTC handling | Excluded from histogram entirely | Histogram bins smoothed by averaging neighbors |
| Output filtering | Per-output: script type, dust threshold, round BTC | Per-tx: exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
| Validated from | Height 550,000 (November 2018) | December 2023 |
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
| Validated from | Height 525,000 (May 2018) | December 2023 |
| Language | Rust | Python |
| Dependencies | None (pure computation, caller provides block data) | Bitcoin Core RPC |
| Bins per decade | 200 | 200 |
## Accuracy
Tested over 386,251 blocks (heights 550,000 to 937,447, as of February 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
Tested over 411,251 blocks (heights 525,000 to 949,800, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
### Per-block
| Metric | Value |
|--------|-------|
| Median error | 0.11% |
| 95th percentile | 0.66% |
| 99th percentile | 1.6% |
| 99.9th percentile | 6.2% |
| RMSE | 0.52% |
| 95th percentile | 0.67% |
| 99th percentile | 1.7% |
| 99.9th percentile | 5.4% |
| RMSE | 0.50% |
| Max error | 33.4% |
| Bias | +0.01 bins (essentially zero) |
| Blocks > 5% error | 519 (0.13%) |
| Blocks > 10% error | 203 |
| Blocks > 20% error | 5 |
| Bias | +0.00 bins (essentially zero) |
| Blocks > 5% error | 472 (0.11%) |
| Blocks > 10% error | 177 |
| Blocks > 20% error | 3 |
### Daily candles
@@ -181,26 +181,26 @@ Oracle daily OHLC built from per-block prices vs exchange daily OHLC:
| | Median | RMSE | Max |
|-------|--------|------|-----|
| Open | 0.21% | 0.59% | 15.4% |
| High | 0.53% | 1.18% | 28.0% |
| Low | 0.50% | 1.52% | 19.6% |
| Close | 0.24% | 0.74% | 15.5% |
| Open | 0.21% | 0.65% | 15.3% |
| High | 0.53% | 1.12% | 28.0% |
| Low | 0.51% | 1.38% | 19.7% |
| Close | 0.24% | 0.73% | 15.4% |
### By year
| Year | Blocks | Median | RMSE | Max | >5% | >10% | >20% | Price range |
|------|--------|--------|------|-----|-----|------|------|-------------|
| 2018 | 6,492 | 0.69% | 2.34% | 33.4% | 183 | 122 | 5 | $3,129$6,293 |
| 2019 | 54,272 | 0.16% | 0.74% | 17.4% | 195 | 69 | 0 | $3,338$13,868 |
| 2020 | 53,102 | 0.10% | 0.43% | 18.1% | 68 | 3 | 0 | $3,858$29,322 |
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 38 | 9 | 0 | $27,678$69,000 |
| 2018 | 31,492 | 0.21% | 1.11% | 33.4% | 169 | 109 | 3 | $3,129$8,488 |
| 2019 | 54,272 | 0.16% | 0.69% | 17.4% | 165 | 53 | 0 | $3,338$13,868 |
| 2020 | 53,102 | 0.10% | 0.44% | 12.6% | 70 | 6 | 0 | $3,858$29,322 |
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 42 | 9 | 0 | $27,678$69,000 |
| 2022 | 53,230 | 0.07% | 0.32% | 6.8% | 10 | 0 | 0 | $15,460$48,240 |
| 2023 | 54,032 | 0.10% | 0.25% | 6.7% | 5 | 0 | 0 | $16,490$44,700 |
| 2024 | 53,367 | 0.11% | 0.31% | 9.7% | 16 | 0 | 0 | $38,555$108,298 |
| 2023 | 54,032 | 0.10% | 0.25% | 6.6% | 5 | 0 | 0 | $16,490$44,700 |
| 2024 | 53,367 | 0.10% | 0.28% | 6.7% | 7 | 0 | 0 | $38,555$108,298 |
| 2025 | 53,113 | 0.11% | 0.25% | 5.8% | 4 | 0 | 0 | $74,409$126,198 |
| 2026 | 5,910 | 0.10% | 0.27% | 3.3% | 0 | 0 | 0 | $60,000$97,900 |
| 2026 | 5,910 | 0.11% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000$97,900 |
The oracle is only as good as the signal it reads. In late 2018 on-chain transaction volume was low and the round-dollar pattern was weak, so the first few thousand blocks are noisy (33% max error, 2.3% RMSE). By 2020 the signal is strong enough for 0.1% median accuracy. Since 2022, zero blocks exceed 10% error.
The oracle is only as good as the signal it reads. The largest errors cluster in late 2018: the November price crash fell faster than the narrow search window could follow (33% max error), and on-chain volume was lower then, so the round-dollar pattern was weaker (1.1% RMSE for the year). By 2020 the signal is strong enough for 0.1% median accuracy, and since 2022 no block exceeds 10% error.
### Why no outlier smoothing?
@@ -208,3 +208,15 @@ Post-hoc smoothing — for example, correcting any block whose price deviates mo
1. **Simplicity**: The oracle is a single forward pass with no lookback corrections. Adding smoothing means defining thresholds, neighbor windows, and replacement strategies, all of which add complexity for marginal gain.
2. **Finality**: Each block's price is produced once and never revised (unless the block itself is reorged). Downstream consumers can treat the oracle output as append-only. Smoothing would require retroactively changing already-published prices, breaking that property.
## Changelog
### v2
Changes from v1:
- **OP_RETURN filter**: every output of a transaction carrying an `OP_RETURN` is now dropped from the histogram. Such transactions are protocol machinery (cross-chain swaps, anchoring) whose payout amounts can form false round-dollar patterns. This was the trigger for the worst price glitches in v1.
- **P2WSH reactivated**: once the OP_RETURN filter removes the protocol noise, P2WSH outputs are usable round-dollar signal again, so they are no longer excluded. P2TR stays excluded.
- **Earlier start**: on-chain tracking begins at height 525,000 (May 2018) instead of 550,000, adding about 25,000 blocks of history.
`VERSION` is exposed as a crate constant so downstream consumers can invalidate prices computed by an earlier algorithm.
+7 -11
View File
@@ -6,7 +6,7 @@ use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin};
use brk_oracle::{Config, Histogram, NUM_BINS, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -159,7 +159,7 @@ fn main() {
let ref_config = Config::default();
let earliest_start = *start_heights.iter().min().unwrap();
for h in START_HEIGHT..total_heights {
for h in earliest_start..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
@@ -187,10 +187,6 @@ fn main() {
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
if h < earliest_start {
continue;
}
let values: Vec<Sats> = indexer
.vecs
.outputs
@@ -203,8 +199,8 @@ fn main() {
.collect_range_at(out_start, out_end);
// Build full histogram and per-digit histograms.
let mut full_hist = [0u32; NUM_BINS];
let mut digit_hist = [[0u32; NUM_BINS]; 9];
let mut full_hist = Histogram::zeros();
let mut digit_hist: [Histogram; 9] = std::array::from_fn(|_| Histogram::zeros());
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
@@ -214,11 +210,11 @@ fn main() {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
full_hist[bin] += 1;
full_hist.increment(bin);
if is_round(*sats) {
let d = leading_digit(*sats);
if (1..=9).contains(&d) {
digit_hist[(d - 1) as usize][bin] += 1;
digit_hist[(d - 1) as usize].increment(bin);
}
}
}
@@ -227,7 +223,7 @@ fn main() {
// Feed each (mask, start_height) combo.
for (mi, &(mask, _)) in masks.iter().enumerate() {
// Build filtered histogram for this mask.
let mut hist = full_hist;
let mut hist = full_hist.clone();
(0..9usize).for_each(|d| {
if mask & (1 << d) != 0 {
for bin in 0..NUM_BINS {
+6 -14
View File
@@ -11,7 +11,9 @@
use std::path::PathBuf;
use brk_indexer::Indexer;
use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin};
use brk_oracle::{
Config, Histogram, Oracle, PRICES, START_HEIGHT, cents_to_bin, default_eligible_bin,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -52,8 +54,6 @@ fn main() {
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let ref_config = Config::default();
// Reference oracle at 575k.
let ref_start = START_HEIGHT;
let mut ref_oracle = Oracle::new(seed_bin(ref_start), Config::default());
@@ -112,18 +112,10 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
let mut hist = [0u32; NUM_BINS];
let mut hist = Histogram::zeros();
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
continue;
}
if *sats < ref_config.min_sats
|| (ref_config.exclude_common_round_values && sats.is_common_round_value())
{
continue;
}
if let Some(bin) = sats_to_bin(sats) {
hist[bin] += 1;
if let Some(bin) = default_eligible_bin(sats, output_type) {
hist.increment(bin as usize);
}
}
+9 -20
View File
@@ -6,7 +6,7 @@ use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, default_eligible_bin};
use brk_types::{Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -19,7 +19,7 @@ fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / BPD) - 1.0) * 100.0
}
fn price_seed_bin(start_height: usize) -> f64 {
fn seed_bin(start_height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(start_height - 1)
@@ -30,9 +30,7 @@ fn price_seed_bin(start_height: usize) -> f64 {
}
/// Clamp the top N bins in `src` down to the (N+1)th highest value, writing into `dst`.
fn clamp_top_n(src: &[u32; NUM_BINS], dst: &mut [u32; NUM_BINS], n: usize) {
// Find the (n+1)th largest value.
// Collect non-zero counts, sort descending, take the (n+1)th.
fn clamp_top_n(src: &Histogram, dst: &mut Histogram, n: usize) {
let mut top: Vec<u32> = src.iter().copied().filter(|&v| v > 0).collect();
top.sort_unstable_by(|a, b| b.cmp(a));
let clamp_to = if top.len() > n { top[n] } else { 0 };
@@ -102,7 +100,7 @@ fn main() {
let total_blocks = total_heights - lowest;
struct BlockData {
hist: Box<[u32; NUM_BINS]>,
hist: Histogram,
high_bin: f64,
low_bin: f64,
}
@@ -144,21 +142,12 @@ fn main() {
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
let mut hist = Box::new([0u32; NUM_BINS]);
let mut hist = Histogram::zeros();
for i in out_start..out_end {
let sats: Sats = value_reader.get(i);
let output_type = output_type_reader.get(i);
if config.excluded_output_types.contains(&output_type) {
continue;
}
if *sats < config.min_sats {
continue;
}
if config.exclude_common_round_values && sats.is_common_round_value() {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
hist[bin] += 1;
if let Some(bin) = default_eligible_bin(sats, output_type) {
hist.increment(bin as usize);
}
}
@@ -206,7 +195,7 @@ fn main() {
println!("{}", "-".repeat(72));
for &start_height in &start_heights {
let mut oracle = Oracle::new(price_seed_bin(start_height), config.clone());
let mut oracle = Oracle::new(seed_bin(start_height), config.clone());
let block_offset = start_height - lowest;
let mut worst_err: f64 = 0.0;
@@ -217,7 +206,7 @@ fn main() {
let mut total_sq_err: f64 = 0.0;
let mut total_measured: u64 = 0;
let mut clamped_hist = [0u32; NUM_BINS];
let mut clamped_hist = Histogram::zeros();
for (i, bd) in blocks[block_offset..].iter().enumerate() {
if clamp_n > 0 {
clamp_top_n(&bd.hist, &mut clamped_hist, clamp_n);
+29 -29
View File
@@ -6,7 +6,8 @@ use std::path::PathBuf;
use brk_indexer::Indexer;
use brk_oracle::{
Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin, sats_to_bin,
Config, Histogram, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
default_eligible_bin,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -184,11 +185,12 @@ fn main() {
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
// Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too large.
// Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too
// large, so the tx-indexed first_txout_index is read through a forward cursor.
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let ref_config = Config::default();
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
let mut tx_starts: Vec<usize> = Vec::new();
let mut year_stats: Vec<YearStats> = Vec::new();
let mut overall = YearStats::new(0);
@@ -206,27 +208,22 @@ fn main() {
.copied()
.unwrap_or(TxIndex::from(total_txs));
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
indexer
.vecs
.transactions
.first_txout_index
.collect_one(ft + 1)
.unwrap()
.to_usize()
} else {
out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let block_first_tx = ft.to_usize() + 1;
let tx_count = next_ft.to_usize() - block_first_tx;
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
// First txout index of each non-coinbase tx, for per-tx grouping.
txout_cursor.advance(block_first_tx - txout_cursor.position());
tx_starts.clear();
for _ in 0..tx_count {
tx_starts.push(txout_cursor.next().unwrap().to_usize());
}
let out_start = tx_starts.first().copied().unwrap_or(out_end);
let values: Vec<Sats> = indexer
.vecs
.outputs
@@ -238,18 +235,21 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
let mut hist = [0u32; NUM_BINS];
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
// Drop every output of a tx carrying an OP_RETURN (protocol machinery).
let mut hist = Histogram::zeros();
for tx in 0..tx_count {
let lo = tx_starts[tx] - out_start;
let hi = tx_starts
.get(tx + 1)
.map(|s| s - out_start)
.unwrap_or(out_end - out_start);
if output_types[lo..hi].contains(&OutputType::OpReturn) {
continue;
}
if *sats < ref_config.min_sats
|| (ref_config.exclude_common_round_values && sats.is_common_round_value())
{
continue;
}
if let Some(bin) = sats_to_bin(sats) {
hist[bin] += 1;
for i in lo..hi {
if let Some(bin) = default_eligible_bin(values[i], output_types[i]) {
hist.increment(bin as usize);
}
}
}
+7 -11
View File
@@ -12,7 +12,7 @@ use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin};
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -117,7 +117,7 @@ impl Stats {
}
struct BlockData {
full_hist: Box<[u32; NUM_BINS]>,
full_hist: Histogram,
/// (bin_index, leading_digit) for outputs that are round values.
round_outputs: Vec<(u16, u8)>,
high_bin: f64,
@@ -173,7 +173,7 @@ fn main() {
let total_blocks = total_heights - sweep_start;
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
for h in START_HEIGHT..total_heights {
for h in sweep_start..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
@@ -201,10 +201,6 @@ fn main() {
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
if h < sweep_start {
continue;
}
let values: Vec<Sats> = indexer
.vecs
.outputs
@@ -216,7 +212,7 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
let mut full_hist = Box::new([0u32; NUM_BINS]);
let mut full_hist = Histogram::zeros();
let mut round_outputs = Vec::new();
for (sats, output_type) in values.into_iter().zip(output_types) {
@@ -227,7 +223,7 @@ fn main() {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
full_hist[bin] += 1;
full_hist.increment(bin);
if is_round(*sats) {
let d = leading_digit(*sats);
if (1..=9).contains(&d) {
@@ -260,7 +256,7 @@ fn main() {
}
}
let mem_hists = blocks.len() * std::mem::size_of::<[u32; NUM_BINS]>();
let mem_hists = blocks.len() * std::mem::size_of::<Histogram>();
let mem_rounds: usize = blocks.iter().map(|b| b.round_outputs.len() * 3).sum();
eprintln!(
"\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s",
@@ -308,7 +304,7 @@ fn main() {
let mut stats = Stats::new();
for bd in blocks.iter() {
let mut hist = *bd.full_hist;
let mut hist = bd.full_hist.clone();
for &(bin, digit) in &bd.round_outputs {
if mask & (1 << (digit - 1)) != 0 {
hist[bin as usize] -= 1;
+7 -11
View File
@@ -12,7 +12,7 @@ use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin};
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -114,7 +114,7 @@ struct RoundOutput {
}
struct BlockData {
full_hist: Box<[u32; NUM_BINS]>,
full_hist: Histogram,
round_outputs: Vec<RoundOutput>,
high_bin: f64,
low_bin: f64,
@@ -175,7 +175,7 @@ fn main() {
// Outputs beyond 5% relative error will never be filtered at any tolerance.
let max_tolerance: f64 = 0.05;
for h in START_HEIGHT..total_heights {
for h in sweep_start..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
@@ -203,10 +203,6 @@ fn main() {
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
if h < sweep_start {
continue;
}
let values: Vec<Sats> = indexer
.vecs
.outputs
@@ -218,7 +214,7 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
let mut full_hist = Box::new([0u32; NUM_BINS]);
let mut full_hist = Histogram::zeros();
let mut round_outputs = Vec::new();
for (sats, output_type) in values.into_iter().zip(output_types) {
@@ -229,7 +225,7 @@ fn main() {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
full_hist[bin] += 1;
full_hist.increment(bin);
let d = leading_digit(*sats);
if (1..=9).contains(&d) {
let rel_err = relative_roundness(*sats);
@@ -267,7 +263,7 @@ fn main() {
}
}
let mem_hists = blocks.len() * std::mem::size_of::<[u32; NUM_BINS]>();
let mem_hists = blocks.len() * std::mem::size_of::<Histogram>();
let mem_rounds: usize = blocks
.iter()
.map(|b| b.round_outputs.len() * std::mem::size_of::<RoundOutput>())
@@ -350,7 +346,7 @@ fn main() {
let mut stats = Stats::new();
for bd in blocks.iter() {
let mut hist = *bd.full_hist;
let mut hist = bd.full_hist.clone();
// Remove outputs matching this tolerance + mask.
let tol_f32 = tolerance as f32;
+6 -14
View File
@@ -9,7 +9,9 @@
use std::path::PathBuf;
use brk_indexer::Indexer;
use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin};
use brk_oracle::{
Config, Histogram, Oracle, PRICES, START_HEIGHT, cents_to_bin, default_eligible_bin,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -155,8 +157,6 @@ fn main() {
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let ref_config = Config::default();
for h in START_HEIGHT..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
@@ -197,18 +197,10 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
let mut hist = [0u32; NUM_BINS];
let mut hist = Histogram::zeros();
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
continue;
}
if *sats < ref_config.min_sats
|| (ref_config.exclude_common_round_values && sats.is_common_round_value())
{
continue;
}
if let Some(bin) = sats_to_bin(sats) {
hist[bin] += 1;
if let Some(bin) = default_eligible_bin(sats, output_type) {
hist.increment(bin as usize);
}
}
+39
View File
@@ -0,0 +1,39 @@
use brk_types::OutputType;
/// Dust floor used by `Config::default()` and `default_eligible_bin`.
pub(crate) const DEFAULT_MIN_SATS: u64 = 1000;
/// Output types skipped by `Config::default()` (protocol-dominated) and the
/// source of truth for `default_eligible_bin`'s precomputed exclusion mask.
pub(crate) const DEFAULT_EXCLUDED_OUTPUT_TYPES: &[OutputType] = &[OutputType::P2TR];
#[derive(Clone)]
pub struct Config {
/// EMA decay: 2/(N+1) where N is span in blocks. 2/7 = 6-block span.
pub alpha: f64,
/// Ring buffer depth. 12 blocks for deterministic convergence at any start height.
pub window_size: usize,
/// Search window bins below/above previous estimate. Asymmetric for log-scale.
pub search_below: usize,
pub search_above: usize,
/// Minimum output value in sats (dust filter).
pub min_sats: u64,
/// Exclude round BTC amounts that create false stencil matches.
pub exclude_common_round_values: bool,
/// Output types to ignore (e.g. P2TR, P2WSH are noisy).
pub excluded_output_types: Vec<OutputType>,
}
impl Default for Config {
fn default() -> Self {
Self {
alpha: 2.0 / 7.0,
window_size: 12,
search_below: 9,
search_above: 11,
min_sats: DEFAULT_MIN_SATS,
exclude_common_round_values: true,
excluded_output_types: DEFAULT_EXCLUDED_OUTPUT_TYPES.to_vec(),
}
}
}
+41
View File
@@ -0,0 +1,41 @@
use crate::NUM_BINS;
/// Per-block oracle histogram: count of eligible outputs per bin. Wraps
/// the raw `[u32; NUM_BINS]` so callers can't pass arbitrary bin-indexed
/// arrays to `Oracle::process_histogram`. Deref to the underlying array
/// gives indexing for read paths.
#[derive(Clone)]
pub struct Histogram([u32; NUM_BINS]);
impl Histogram {
#[inline]
pub fn zeros() -> Self {
Self([0; NUM_BINS])
}
#[inline]
pub fn increment(&mut self, bin: usize) {
self.0[bin] += 1;
}
}
impl Default for Histogram {
fn default() -> Self {
Self::zeros()
}
}
impl std::ops::Deref for Histogram {
type Target = [u32; NUM_BINS];
#[inline]
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for Histogram {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
+77 -101
View File
@@ -3,13 +3,24 @@
//! Detects round-dollar transaction patterns ($1, $5, $10, ... $10,000) in Bitcoin
//! block outputs to derive the current price without any exchange data.
use brk_types::{Block, Cents, Dollars, OutputType, Sats};
use brk_types::{Cents, Dollars, OutputType, Sats};
mod config;
mod histogram;
use config::{DEFAULT_EXCLUDED_OUTPUT_TYPES, DEFAULT_MIN_SATS};
pub use config::Config;
pub use histogram::Histogram;
/// Oracle algorithm version. Bump on any change that alters computed prices
/// so downstream consumers can invalidate cached results.
pub const VERSION: u32 = 2;
/// Pre-oracle dollar prices, one per line, heights 0..630_000.
pub const PRICES: &str = include_str!("prices.txt");
/// First height where the oracle computes from on-chain data.
pub const START_HEIGHT: usize = 550_000;
pub const START_HEIGHT: usize = 525_000;
pub const BINS_PER_DECADE: usize = 200;
const MIN_LOG_BTC: i32 = -8;
@@ -55,6 +66,33 @@ pub fn sats_to_bin(sats: Sats) -> Option<usize> {
}
}
/// Bitmask form of `DEFAULT_EXCLUDED_OUTPUT_TYPES`, evaluated at compile
/// time so `default_eligible_bin` checks membership with a single AND.
const DEFAULT_EXCLUDED_MASK: u16 = {
let mut mask = 0u16;
let mut i = 0;
while i < DEFAULT_EXCLUDED_OUTPUT_TYPES.len() {
mask |= 1u16 << DEFAULT_EXCLUDED_OUTPUT_TYPES[i] as u8;
i += 1;
}
mask
};
/// Bin index for `(sats, output_type)` under `Config::default()` rules.
/// Returns `None` for excluded types (P2TR/P2WSH), dust, round-BTC values,
/// or out-of-range bins. Mirror of `Oracle::output_to_bin` for callers that
/// can pre-bin outputs at write time and don't have an `Oracle` handle.
#[inline(always)]
pub fn default_eligible_bin(sats: Sats, output_type: OutputType) -> Option<u16> {
if DEFAULT_EXCLUDED_MASK & (1u16 << output_type as u8) != 0 {
return None;
}
if *sats < DEFAULT_MIN_SATS || sats.is_common_round_value() {
return None;
}
sats_to_bin(sats).map(|b| b as u16)
}
/// Converts a fractional bin to a USD price in cents.
/// For a $D output at price P: sats = D * 1e8 / P, so P = 10^(10 - bin/200) dollars,
/// where 10 = log10($100 reference * 1e8 sats/BTC).
@@ -140,40 +178,9 @@ fn find_best_bin(
best_bin as f64 + sub_bin
}
#[derive(Clone)]
pub struct Config {
/// EMA decay: 2/(N+1) where N is span in blocks. 2/7 = 6-block span.
pub alpha: f64,
/// Ring buffer depth. 12 blocks for deterministic convergence at any start height.
pub window_size: usize,
/// Search window bins below/above previous estimate. Asymmetric for log-scale.
pub search_below: usize,
pub search_above: usize,
/// Minimum output value in sats (dust filter).
pub min_sats: u64,
/// Exclude round BTC amounts that create false stencil matches.
pub exclude_common_round_values: bool,
/// Output types to ignore (e.g. P2TR, P2WSH are noisy).
pub excluded_output_types: Vec<OutputType>,
}
impl Default for Config {
fn default() -> Self {
Self {
alpha: 2.0 / 7.0,
window_size: 12,
search_below: 9,
search_above: 11,
min_sats: 1000,
exclude_common_round_values: true,
excluded_output_types: vec![OutputType::P2TR, OutputType::P2WSH],
}
}
}
#[derive(Clone)]
pub struct Oracle {
histograms: Vec<[u32; NUM_BINS]>,
histograms: Vec<Histogram>,
ema: Box<[f64; NUM_BINS]>,
cursor: usize,
filled: usize,
@@ -196,7 +203,7 @@ impl Oracle {
.iter()
.fold(0u16, |mask, ot| mask | (1 << *ot as u8));
Self {
histograms: vec![[0u32; NUM_BINS]; window_size],
histograms: vec![Histogram::zeros(); window_size],
ema: Box::new([0.0; NUM_BINS]),
cursor: 0,
filled: 0,
@@ -208,81 +215,21 @@ impl Oracle {
}
}
pub fn process_block(&mut self, block: &Block) -> f64 {
self.process_outputs(
block
.txdata
.iter()
.skip(1) // skip coinbase
.flat_map(|tx| &tx.output)
.map(|txout| {
(
Sats::from(txout.value),
OutputType::from(&txout.script_pubkey),
)
}),
)
}
pub fn process_outputs(&mut self, outputs: impl Iterator<Item = (Sats, OutputType)>) -> f64 {
let mut hist = [0u32; NUM_BINS];
for (sats, output_type) in outputs {
if let Some(bin) = self.eligible_bin(sats, output_type) {
hist[bin] += 1;
}
}
self.ingest(&hist)
}
/// Create an oracle restored from a known price.
/// `fill` should feed warmup blocks to populate the ring buffer.
/// ref_bin is anchored to the checkpoint regardless of warmup drift.
/// Create an oracle restored from a known price. `fill` should call
/// `process_histogram` for the warmup blocks; during warmup the ring
/// fills without recomputing EMA or searching, then we recompute once
/// at the end so the first non-warmup call has a primed EMA.
pub fn from_checkpoint(ref_bin: f64, config: Config, fill: impl FnOnce(&mut Self)) -> Self {
let mut oracle = Self::new(ref_bin, config);
oracle.warmup = true;
fill(&mut oracle);
oracle.warmup = false;
oracle.recompute_ema();
oracle.ref_bin = ref_bin;
oracle
}
pub fn process_histogram(&mut self, hist: &[u32; NUM_BINS]) -> f64 {
self.ingest(hist)
}
pub fn ref_bin(&self) -> f64 {
self.ref_bin
}
pub fn price_cents(&self) -> Cents {
bin_to_cents(self.ref_bin).into()
}
pub fn price_dollars(&self) -> Dollars {
self.price_cents().into()
}
#[inline(always)]
pub fn output_to_bin(&self, sats: Sats, output_type: OutputType) -> Option<usize> {
self.eligible_bin(sats, output_type)
}
#[inline(always)]
fn eligible_bin(&self, sats: Sats, output_type: OutputType) -> Option<usize> {
if self.excluded_mask & (1 << output_type as u8) != 0 {
return None;
}
if *sats < self.config.min_sats
|| (self.config.exclude_common_round_values && sats.is_common_round_value())
{
return None;
}
sats_to_bin(sats)
}
fn ingest(&mut self, hist: &[u32; NUM_BINS]) -> f64 {
self.histograms[self.cursor] = *hist;
pub fn process_histogram(&mut self, hist: &Histogram) -> f64 {
self.histograms[self.cursor] = hist.clone();
self.cursor = (self.cursor + 1) % self.config.window_size;
if self.filled < self.config.window_size {
self.filled += 1;
@@ -301,6 +248,35 @@ impl Oracle {
self.ref_bin
}
pub fn ref_bin(&self) -> f64 {
self.ref_bin
}
pub fn price_cents(&self) -> Cents {
bin_to_cents(self.ref_bin).into()
}
pub fn price_dollars(&self) -> Dollars {
self.price_cents().into()
}
/// Config-aware bin index for `(sats, output_type)`. Returns `None`
/// for excluded types, dust, round-BTC values, or out-of-range bins.
/// Callers under `Config::default()` should use `default_eligible_bin`
/// (free function) to skip the `&self` indirection.
#[inline(always)]
pub fn output_to_bin(&self, sats: Sats, output_type: OutputType) -> Option<usize> {
if self.excluded_mask & (1 << output_type as u8) != 0 {
return None;
}
if *sats < self.config.min_sats
|| (self.config.exclude_common_round_values && sats.is_common_round_value())
{
return None;
}
sats_to_bin(sats)
}
fn recompute_ema(&mut self) {
self.ema.fill(0.0);
for age in 0..self.filled {
+1
View File
@@ -17,6 +17,7 @@ brk_computer = { workspace = true }
brk_error = { workspace = true, features = ["jiff", "vecdb"] }
brk_indexer = { workspace = true }
brk_mempool = { workspace = true }
brk_oracle = { workspace = true }
brk_reader = { workspace = true }
brk_rpc = { workspace = true }
brk_traversable = { workspace = true }
+29 -28
View File
@@ -85,25 +85,6 @@ impl Query {
})
}
/// Esplora `/address/:address/txs` first page: up to `mempool_limit`
/// mempool entries (newest first), then chain entries fill the response
/// up to `total_limit`. Pagination is path-style via `/txs/chain/:after_txid`.
pub fn addr_txs(
&self,
addr: Addr,
total_limit: usize,
mempool_limit: usize,
) -> Result<Vec<Transaction>> {
let mut out = if self.mempool().is_some() {
self.addr_mempool_txs(&addr, mempool_limit)?
} else {
Vec::new()
};
let chain_limit = total_limit.saturating_sub(out.len());
out.extend(self.addr_txs_chain(&addr, None, chain_limit)?);
Ok(out)
}
pub fn addr_txs_chain(
&self,
addr: &Addr,
@@ -227,7 +208,7 @@ impl Query {
pub fn addr_mempool_hash(&self, addr: &Addr) -> Option<u64> {
let mempool = self.mempool()?;
let bytes = AddrBytes::from_str(addr).ok()?;
Some(mempool.addr_state_hash(&bytes))
mempool.addr_state_hash(&bytes)
}
pub fn addr_mempool_txs(&self, addr: &Addr, limit: usize) -> Result<Vec<Transaction>> {
@@ -236,8 +217,15 @@ impl Query {
Ok(mempool.addr_txs(&bytes, limit))
}
/// Height of the last on-chain activity for an address (last tx_index height).
pub fn addr_last_activity_height(&self, addr: &Addr) -> Result<Height> {
/// Height of the last on-chain activity for an address (last tx_index to height).
/// With `before_txid`, returns the newest activity strictly older than that
/// cursor. Used by paginated chain etags so a new tx above the cursor
/// doesn't invalidate deeper pages.
pub fn addr_last_activity_height(
&self,
addr: &Addr,
before_txid: Option<&Txid>,
) -> Result<Height> {
let (output_type, type_index) = self.resolve_addr(addr)?;
let store = self
.indexer()
@@ -246,12 +234,25 @@ impl Query {
.get(output_type)
.data()?;
let tx_index_len = self.safe_lengths().tx_index;
let last_tx_index = store
.prefix(type_index)
.rev()
.map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index())
.find(|tx_index| *tx_index < tx_index_len)
.ok_or(Error::UnknownAddr)?;
let last_tx_index = match before_txid {
Some(txid) => {
let before_tx_index = self.resolve_tx_index(txid)?;
let min = AddrIndexTxIndex::min_for_addr(type_index);
let cursor = AddrIndexTxIndex::from((type_index, before_tx_index));
store
.range(min..cursor)
.rev()
.map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index())
.find(|tx_index| *tx_index < tx_index_len)
.ok_or(Error::UnknownAddr)?
}
None => store
.prefix(type_index)
.rev()
.map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index())
.find(|tx_index| *tx_index < tx_index_len)
.ok_or(Error::UnknownAddr)?,
};
self.confirmed_status_height(last_tx_index)
}
+8 -4
View File
@@ -499,10 +499,11 @@ impl Query {
// === Helper methods ===
/// Hash to height. The prefix store keys on the first 8 bytes of
/// the hash, so the resolved height is verified against the full
/// `blockhash[height]` before being returned. Prefix collisions
/// (or unknown hashes) surface as `NotFound`.
/// Hash to height, clamped to the safe-lengths snapshot. The prefix
/// store keys on the first 8 bytes of the hash, so the resolved
/// height is verified against the full `blockhash[height]` before
/// being returned. Prefix collisions, unknown hashes, and hashes
/// past the snapshot all surface as `NotFound`.
pub fn height_by_hash(&self, hash: &BlockHash) -> Result<Height> {
let indexer = self.indexer();
let prefix = BlockHashPrefix::from(hash);
@@ -512,6 +513,9 @@ impl Query {
.get(&prefix)?
.map(|h| *h)
.ok_or(Error::NotFound("Block not found".into()))?;
if height >= self.safe_lengths().height {
return Err(Error::NotFound("Block not found".into()));
}
match indexer.vecs.blocks.blockhash.get(height) {
Some(stored) if &stored == hash => Ok(height),
_ => Err(Error::NotFound("Block not found".into())),
+13 -23
View File
@@ -10,10 +10,10 @@
//! carries the same chunk-rate semantics the live mempool produces.
use brk_error::{Error, OptionData, Result};
use brk_mempool::{ChunkInput, linearize};
use brk_types::{
CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, CpfpInfo, FeeRate, Height, Sats,
TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight,
CPFP_CHAIN_LIMIT, ChunkInput, CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry,
CpfpInfo, FeeRate, Height, Sats, TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight,
find_seed_chunk, linearize,
};
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
@@ -21,10 +21,6 @@ use vecdb::{ReadableVec, VecIndex};
use crate::Query;
/// Cap matches Bitcoin Core's default mempool ancestor/descendant
/// chain limits and mempool.space's truncation.
const MAX: usize = 25;
struct WalkResult {
/// Cluster members in `[ancestors..., seed, descendants...]` order,
/// each paired with its in-cluster parent edges resolved to the
@@ -58,13 +54,12 @@ impl Query {
/// Effective fee rate for `txid` using the same chunk-rate semantics
/// across paths:
///
/// - Live mempool: snapshot's per-tx `chunk_rate` (Core's
/// `fees.chunk` / `chunkweight`, or proxy fallback). If the tx is
/// in the pool but not in the latest snapshot (e.g. just added),
/// falls back to the entry's simple `fee/vsize`.
/// - Live mempool: snapshot's per-tx linearized `chunk_rate`. If
/// the tx is in the pool but not in the latest snapshot (e.g.
/// just added), falls back to the entry's simple `fee/vsize`.
/// - Confirmed: precomputed `effective_fee_rate.tx_index`.
/// - Graveyard-only RBF predecessor: simple `fee/vsize` snapshotted
/// at burial.
/// - Graveyard-only RBF predecessor: linearized chunk rate
/// captured at burial.
///
/// Returns `Error::UnknownTxid` for txids not seen in any of those.
pub fn effective_fee_rate(&self, txid: &Txid) -> Result<FeeRate> {
@@ -158,7 +153,7 @@ impl Query {
/// BFS the seed's same-block ancestors (via `outpoint`) and
/// descendants (via `spent.txin_index` -> `spending_tx`), capped
/// at `MAX` each side to match Core/mempool.space. Returns members
/// at `CPFP_CHAIN_LIMIT` each side to match Core/mempool.space. Returns members
/// laid out as `[ancestors..., seed, descendants...]` so the seed's
/// local index is `ancestors.len()`.
fn walk_same_block_cluster(&self, seed: TxIndex, height: Height) -> Result<WalkResult> {
@@ -202,7 +197,7 @@ impl Query {
};
let mut visited: FxHashMap<TxIndex, ()> =
FxHashMap::with_capacity_and_hasher(2 * MAX + 1, FxBuildHasher);
FxHashMap::with_capacity_and_hasher(2 * CPFP_CHAIN_LIMIT + 1, FxBuildHasher);
visited.insert(seed, ());
// Ancestor BFS: each push records (tx_index, raw parent tx_indices)
@@ -212,7 +207,7 @@ impl Query {
let mut stack: Vec<SmallVec<[TxIndex; 2]>> = vec![seed_inputs.clone()];
'a: while let Some(parents) = stack.pop() {
for parent in parents {
if ancestors.len() >= MAX {
if ancestors.len() >= CPFP_CHAIN_LIMIT {
break 'a;
}
if visited.insert(parent, ()).is_some() || !same_block(parent) {
@@ -249,7 +244,7 @@ impl Query {
}
descendants.push((child, walk_inputs(child)));
stack.push(child);
if descendants.len() >= MAX {
if descendants.len() >= CPFP_CHAIN_LIMIT {
break 'd;
}
}
@@ -336,12 +331,7 @@ fn build_cpfp_info(
})
.collect();
let chunks = linearize(&inputs);
let (chunk_index, seed_rate) = chunks
.iter()
.enumerate()
.find(|(_, ch)| ch.txs.contains(&seed_local))
.map(|(i, ch)| (i as u32, ch.feerate))
.unwrap_or((0, seed.rate));
let (chunk_index, seed_rate) = find_seed_chunk(&chunks, seed_local, seed.rate);
let cluster_txs: Vec<CpfpClusterTx> = members
.iter()
.map(|m| CpfpClusterTx {
+68 -66
View File
@@ -1,11 +1,12 @@
use crate::Query;
use brk_error::{Error, Result};
use brk_mempool::{Mempool, PrevoutResolver, RbfForTx, RbfNode};
use brk_mempool::{Mempool, RbfForTx, RbfNode};
use brk_types::{
CheckedSub, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx, OutputType, RbfResponse,
RbfTx, RecommendedFees, ReplacementNode, Sats, Timestamp, TxOut, TxOutIndex, Txid, TxidPrefix,
TypeIndex,
BlockTemplate, BlockTemplateDiff, CheckedSub, FeeRate, MempoolBlock, MempoolInfo,
MempoolRecentTx, NextBlockHash, OutputType, RbfResponse, RbfTx, RecommendedFees,
ReplacementNode, Sats, Timestamp, TxOut, TxOutIndex, Txid, TxidPrefix, TypeIndex, Vout,
};
use rustc_hash::FxHashMap;
const RECENT_REPLACEMENTS_LIMIT: usize = 25;
@@ -28,72 +29,58 @@ impl Query {
pub fn mempool_blocks(&self) -> Result<Vec<MempoolBlock>> {
let mempool = self.require_mempool()?;
let block_stats = mempool.block_stats();
let blocks = block_stats
.into_iter()
.map(|stats| {
MempoolBlock::new(
stats.tx_count,
stats.total_size,
stats.total_vsize,
stats.total_fee,
stats.fee_range,
)
})
.collect();
Ok(blocks)
Ok(mempool
.block_stats()
.iter()
.map(MempoolBlock::from)
.collect())
}
/// Indexer-backed resolver for confirmed-parent prevouts. Pass
/// the returned closure to `Mempool::start_with` /
/// `Mempool::update_with`; the mempool driver calls it post-apply
/// for every still-unfilled `prevout == None` input.
///
/// Reads go through `read_once` rather than a captured
/// `VecReader`: `VecReader::stored_len` is snapshotted at
/// construction, so a long-lived reader paired with fresh
/// `safe_lengths` would let `safe.tx_index` / `safe.txout_index`
/// advance past the reader's frozen length and panic in
/// `reader.get()`. `read_once` rebinds against the current vec
/// length per call and lets newly indexed parents become
/// resolvable on the next cycle.
pub fn indexer_prevout_resolver(&self) -> PrevoutResolver {
let query = self.clone();
/// Indexer-backed resolver for confirmed-parent prevouts. Boxed so
/// the caller (typically [`Mempool::start_with`]) can stash one
/// resolver behind a stable type for the lifetime of the loop.
#[allow(clippy::type_complexity)]
pub fn indexer_prevout_resolver(
&self,
) -> Box<dyn Fn(&[(Txid, Vout)]) -> FxHashMap<(Txid, Vout), TxOut> + Send + Sync> {
let indexer = self.0.indexer;
Box::new(move |prev_txid, vout| {
let safe = query.safe_lengths();
let prev_tx_index = indexer
.stores
.txid_prefix_to_tx_index
.get(&TxidPrefix::from(prev_txid))
.ok()??
.into_owned();
if prev_tx_index >= safe.tx_index {
return None;
Box::new(move |holes: &[(Txid, Vout)]| {
if holes.is_empty() {
return FxHashMap::default();
}
let first_txout: TxOutIndex = indexer
.vecs
.transactions
.first_txout_index
.read_once(prev_tx_index)
.ok()?;
let txout = first_txout + vout;
if txout >= safe.txout_index {
return None;
}
let output_type: OutputType = indexer.vecs.outputs.output_type.read_once(txout).ok()?;
let type_index: TypeIndex = indexer.vecs.outputs.type_index.read_once(txout).ok()?;
let value: Sats = indexer.vecs.outputs.value.read_once(txout).ok()?;
let script_pubkey = indexer
.vecs
.addrs
.addr_readers()
.script_pubkey(output_type, type_index);
Some(TxOut::from((script_pubkey, value)))
let safe = indexer.safe_lengths();
let first_txout_reader = indexer.vecs.transactions.first_txout_index.reader();
let output_type_reader = indexer.vecs.outputs.output_type.reader();
let type_index_reader = indexer.vecs.outputs.type_index.reader();
let value_reader = indexer.vecs.outputs.value.reader();
let addr_readers = indexer.vecs.addrs.addr_readers();
holes
.iter()
.filter_map(|(prev_txid, vout)| {
let prev_tx_index = indexer
.stores
.txid_prefix_to_tx_index
.get(&TxidPrefix::from(prev_txid))
.ok()??
.into_owned();
if prev_tx_index >= safe.tx_index {
return None;
}
let first_txout: TxOutIndex =
first_txout_reader.try_get(usize::from(prev_tx_index))?;
let txout = first_txout + *vout;
if txout >= safe.txout_index {
return None;
}
let txout_idx = usize::from(txout);
let output_type: OutputType = output_type_reader.try_get(txout_idx)?;
let type_index: TypeIndex = type_index_reader.try_get(txout_idx)?;
let value: Sats = value_reader.try_get(txout_idx)?;
let script_pubkey = addr_readers.script_pubkey(output_type, type_index);
Some(((*prev_txid, *vout), TxOut::from((script_pubkey, value))))
})
.collect()
})
}
@@ -172,7 +159,22 @@ impl Query {
/// Content hash of the projected next block. Same value as the
/// mempool ETag. Polling lets monitors detect a stalled sync.
pub fn mempool_hash(&self) -> Result<u64> {
pub fn mempool_hash(&self) -> Result<NextBlockHash> {
Ok(self.require_mempool()?.next_block_hash())
}
/// Full projected next block (Core's `getblocktemplate` selection)
/// with stats and full tx bodies in GBT order.
pub fn block_template(&self) -> Result<BlockTemplate> {
Ok(self.require_mempool()?.block_template())
}
/// Delta of the projected next block since `since`. `NotFound`
/// when `since` has aged out (client should fall back to
/// `block_template`).
pub fn block_template_diff(&self, since: NextBlockHash) -> Result<BlockTemplateDiff> {
self.require_mempool()?
.block_template_diff(since)
.ok_or_else(|| Error::NotFound(format!("unknown since hash: {since}")))
}
}
+54 -6
View File
@@ -1,20 +1,68 @@
use brk_error::Result;
use std::sync::Arc;
use brk_computer::prices::Vecs as PricesVecs;
use brk_error::{Error, Result};
use brk_oracle::{Config, Oracle, cents_to_bin};
use brk_types::{
Dollars, ExchangeRates, HistoricalPrice, HistoricalPriceEntry, Hour4, INDEX_EPOCH, Timestamp,
};
use vecdb::ReadableVec;
use vecdb::{AnyVec, ReadableVec, VecIndex};
use crate::Query;
impl Query {
pub fn live_price(&self) -> Result<Dollars> {
let mut oracle = self.computer().prices.live_oracle(self.indexer())?;
let base = self.cached_oracle()?;
Ok(match self.mempool() {
Some(mempool) => {
let mut oracle = (*base).clone();
oracle.process_histogram(&mempool.live_histogram());
oracle.price_dollars()
}
None => base.price_dollars(),
})
}
if let Some(mempool) = self.mempool() {
mempool.process_live_outputs(|iter| oracle.process_outputs(iter));
/// Oracle warmed by the last `window_size` committed blocks, seeded from
/// the last committed price. Cached per tip height; rebuilt on advance or
/// reorg. Reads are capped at `safe_lengths` so concurrent indexer writes
/// stay invisible.
fn cached_oracle(&self) -> Result<Arc<Oracle>> {
let safe_lengths = self.safe_lengths();
let height = safe_lengths.height;
if let Some(oracle) = self
.0
.live_oracle
.read()
.unwrap()
.as_ref()
.filter(|(h, _)| *h == height)
.map(|(_, o)| o.clone())
{
return Ok(oracle);
}
Ok(oracle.price_dollars())
let cents_height = &self.computer().prices.spot.cents.height;
let last_cents = cents_height
.len()
.checked_sub(1)
.and_then(|i| cents_height.collect_one_at(i))
.ok_or_else(|| Error::NotFound("oracle prices not yet computed".to_string()))?;
let config = Config::default();
let seed_bin = cents_to_bin(last_cents.inner() as f64);
let tip = height.to_usize();
let warmup_range = tip.saturating_sub(config.window_size)..tip;
let oracle = Arc::new(Oracle::from_checkpoint(seed_bin, config, |o| {
PricesVecs::feed_blocks(o, self.indexer(), warmup_range, Some(&safe_lengths));
}));
let mut cache = self.0.live_oracle.write().unwrap();
if cache.as_ref().is_none_or(|(h, _)| *h != height) {
*cache = Some((height, oracle.clone()));
}
Ok(oracle)
}
pub fn historical_price(&self, timestamp: Option<Timestamp>) -> Result<HistoricalPrice> {
+2 -2
View File
@@ -184,8 +184,8 @@ impl Query {
}
// Snapshot tip-derived state together so the historical-branch ETag stays
// self-consistent: stable_count is computed from tip_height, hash_prefix
// is the live tip.
// self-consistent: tip_height and hash_prefix both reflect the safe-bound
// tip, and stable_count is computed from tip_height.
let tip_height = self.height();
let hash_prefix = self.tip_hash_prefix();
let stable_count = self.stable_count(params.index, total, tip_height);
+8 -2
View File
@@ -1,12 +1,16 @@
#![doc = include_str!("../README.md")]
#![allow(clippy::module_inception)]
use std::{path::Path, sync::Arc};
use std::{
path::Path,
sync::{Arc, RwLock},
};
use brk_computer::Computer;
use brk_error::{OptionData, Result};
use brk_indexer::{Indexer, Lengths};
use brk_mempool::Mempool;
use brk_oracle::Oracle;
use brk_reader::Reader;
use brk_rpc::Client;
use brk_types::{BlockHash, BlockHashPrefix, Height, SyncStatus};
@@ -32,6 +36,7 @@ struct QueryInner<'a> {
indexer: &'a Indexer<Ro>,
computer: &'a Computer<Ro>,
mempool: Option<Mempool>,
live_oracle: RwLock<Option<(Height, Arc<Oracle>)>>,
}
impl Query {
@@ -54,6 +59,7 @@ impl Query {
indexer,
computer,
mempool,
live_oracle: RwLock::new(None),
}))
}
@@ -77,7 +83,7 @@ impl Query {
self.indexer().safe_lengths()
}
/// Tip block hash, cached in the indexer.
/// Tip block hash at the pipeline-safe ceiling.
#[inline]
pub fn tip_blockhash(&self) -> BlockHash {
self.indexer().tip_blockhash()
+1 -1
View File
@@ -14,7 +14,7 @@ pub struct CanonicalRange {
impl CanonicalRange {
pub fn walk(client: &Client, anchor: Option<&BlockHash>, tip: Height) -> Result<Self> {
let start = match anchor {
Some(hash) => Height::from(client.get_block_header_info(hash)?.height + 1),
Some(hash) => Height::from((client.get_block_header_info(hash)?.height + 1) as u64),
None => Height::ZERO,
};
let mut range = Self::between(client, start, tip)?;
+10 -30
View File
@@ -5,49 +5,29 @@ use std::{
time::Duration,
};
use bitcoin::ScriptBuf;
use brk_error::Result;
use brk_types::{BlockHash, Hex, Sats, Txid};
use brk_types::{Sats, Txid, Weight};
mod client;
mod methods;
use client::ClientInner;
pub use corepc_types::v17::{GetBlockHeaderVerbose, GetBlockVerboseOne, GetTxOut};
pub use methods::MempoolState;
#[derive(Debug, Clone)]
pub struct BlockInfo {
pub height: usize,
pub confirmations: i64,
}
#[derive(Debug, Clone)]
pub struct BlockHeaderInfo {
pub height: usize,
pub confirmations: i64,
pub previous_block_hash: Option<BlockHash>,
}
#[derive(Debug, Clone)]
pub struct TxOutInfo {
pub coinbase: bool,
pub value: Sats,
pub script_pub_key: ScriptBuf,
}
/// One transaction from `getblocktemplate`. Carries the full decoded
/// body and stats so block 0 can be projected without a follow-up
/// `getmempoolentry`/`getrawtransaction` per tx; that follow-up was the
/// source of the GBT/listing race that used to skip cycles.
#[derive(Debug, Clone)]
pub struct BlockTemplateTx {
pub txid: Txid,
pub fee: Sats,
}
/// A transaction fetched from Core alongside the exact hex bytes Core
/// returned, so downstream code can re-emit the raw tx without re-
/// serializing (which could diverge on segwit flag encoding, etc.).
#[derive(Debug, Clone)]
pub struct RawTx {
pub weight: Weight,
/// Parent txids also in this template (Core's own ancestor
/// accounting, resolved from the wire-level 1-based indices).
pub depends: Vec<Txid>,
pub tx: bitcoin::Transaction,
pub hex: Hex,
}
#[derive(Clone, Debug)]
+233 -188
View File
@@ -9,13 +9,14 @@ use brk_types::{
use corepc_jsonrpc::error::Error as JsonRpcError;
use corepc_types::{
v17::{
GetBlockCount, GetBlockHash, GetBlockHeader, GetBlockHeaderVerbose, GetBlockVerboseOne,
GetBlockVerboseZero, GetRawMempool, GetTxOut,
BlockTemplateTransaction, GetBlockCount, GetBlockHash, GetBlockHeader,
GetBlockHeaderVerbose, GetBlockTemplate, GetBlockVerboseOne, GetBlockVerboseZero,
GetRawMempool, GetTxOut,
},
v24::GetMempoolInfo,
v28::GetBlockchainInfo,
v24::{GetMempoolInfo, MempoolEntry},
};
use rustc_hash::FxHashMap;
use serde::Deserialize;
use serde_json::Value;
use tracing::{debug, info};
@@ -24,99 +25,92 @@ use tracing::{debug, info};
/// The mempool fetcher tolerates these per-item failures silently.
const RPC_NOT_FOUND: i32 = -5;
use crate::{BlockHeaderInfo, BlockInfo, BlockTemplateTx, Client, RawTx, TxOutInfo};
use crate::{BlockTemplateTx, Client};
/// Per-batch request count for `get_block_hashes_range`. Sized so the
/// JSON request body stays well under a megabyte and bitcoind doesn't
/// spend too long on a single batch before yielding results.
/// Per-batch request count for `get_block_hashes_range`,
/// `fetch_new_pool_data`, and `get_raw_transactions`. Sized so the JSON
/// request body stays well under a megabyte and bitcoind doesn't spend
/// too long on a single batch before yielding results. For the mixed
/// `getmempoolentry`+`getrawtransaction` batch this is the *txid* count;
/// the wire batch is twice that.
const BATCH_CHUNK: usize = 2000;
/// Live mempool state fetched in one batched bitcoind round-trip:
/// `getrawmempool verbose` + `getblocktemplate` + `getmempoolinfo`.
/// `gbt` is validated to be a subset of `entries` before construction;
/// callers that want strict consistency should rely on this fact.
/// Mempool snapshot data that survives one fetch cycle: the live
/// txid set, fee floor, and chain tip. Returned alongside the raw
/// `block_template` (which Fetcher consumes for GBT synthesis) by
/// `Client::fetch_mempool_state`.
pub struct MempoolState {
pub entries: Vec<MempoolEntryInfo>,
pub gbt: Vec<BlockTemplateTx>,
pub live_txids: Vec<Txid>,
pub min_fee: FeeRate,
/// Chain tip's hash (block-template's `previousblockhash`).
/// Compared between cycles to detect newly mined blocks.
pub tip_hash: BlockHash,
/// Chain tip's height (block-template's `height` minus one).
pub tip_height: Height,
}
#[derive(Deserialize)]
struct VerboseEntryRaw {
vsize: VSize,
weight: Weight,
time: Timestamp,
#[serde(rename = "ancestorcount")]
ancestor_count: u64,
#[serde(rename = "ancestorsize")]
ancestor_size: VSize,
#[serde(rename = "descendantsize")]
descendant_size: VSize,
fees: VerboseFeesRaw,
depends: Vec<String>,
#[serde(rename = "chunkweight", default)]
chunk_weight: Option<Weight>,
fn build_entry(txid: Txid, e: MempoolEntry) -> Result<MempoolEntryInfo> {
let depends = e
.depends
.iter()
.map(|s| Client::parse_txid(s, "depends txid"))
.collect::<Result<Vec<_>>>()?;
Ok(MempoolEntryInfo {
txid,
vsize: VSize::from(e.vsize as u64),
weight: Weight::from(e.weight as u64),
fee: Sats::from(Bitcoin::from(e.fees.base)),
first_seen: Timestamp::from(e.time),
depends,
})
}
#[derive(Deserialize)]
struct VerboseFeesRaw {
base: Bitcoin,
ancestor: Bitcoin,
descendant: Bitcoin,
#[serde(default)]
chunk: Option<Bitcoin>,
}
#[derive(Deserialize)]
struct GbtResponseRaw {
transactions: Vec<GbtTxRaw>,
}
#[derive(Deserialize)]
struct GbtTxRaw {
txid: bitcoin::Txid,
fee: u64,
}
fn build_verbose(raw: FxHashMap<String, VerboseEntryRaw>) -> Result<Vec<MempoolEntryInfo>> {
raw.into_iter()
.map(|(txid_str, e)| {
let depends = e
.depends
.iter()
.map(|s| Client::parse_txid(s, "depends txid"))
.collect::<Result<Vec<_>>>()?;
Ok(MempoolEntryInfo {
txid: Client::parse_txid(&txid_str, "mempool txid")?,
vsize: e.vsize,
weight: e.weight,
fee: Sats::from(e.fees.base),
first_seen: e.time,
ancestor_count: e.ancestor_count,
ancestor_size: e.ancestor_size,
ancestor_fee: Sats::from(e.fees.ancestor),
descendant_size: e.descendant_size,
descendant_fee: Sats::from(e.fees.descendant),
chunk_fee: e.fees.chunk.map(Sats::from),
chunk_weight: e.chunk_weight,
depends,
fn build_gbt(raw: GetBlockTemplate) -> Result<Vec<BlockTemplateTx>> {
// Pass 1: decode bodies and stash the 1-based GBT-array indices aside
// so each `data` hex string and `BlockTemplateTransaction` drops as
// soon as the tx is pushed.
let n = raw.transactions.len();
let mut depends_idx: Vec<Vec<i64>> = Vec::with_capacity(n);
let mut result: Vec<BlockTemplateTx> = Vec::with_capacity(n);
for t in raw.transactions {
let BlockTemplateTransaction {
data,
txid,
depends,
fee,
weight,
..
} = t;
depends_idx.push(depends);
result.push(BlockTemplateTx {
txid: Client::parse_txid(&txid, "gbt txid")?,
fee: Sats::from(fee as u64),
weight: Weight::from(weight),
depends: Vec::new(),
tx: encode::deserialize_hex(&data)?,
});
}
// Pass 2: resolve indices to txids now that the array is complete.
for (i, indices) in depends_idx.iter().enumerate() {
let resolved: Vec<Txid> = indices
.iter()
.filter_map(|d| {
let idx = usize::try_from(*d).ok()?.checked_sub(1)?;
result.get(idx).map(|t| t.txid)
})
})
.collect()
}
fn build_gbt(raw: GbtResponseRaw) -> Vec<BlockTemplateTx> {
raw.transactions
.into_iter()
.map(|t| BlockTemplateTx {
txid: Txid::from(t.txid),
fee: Sats::from(t.fee),
})
.collect()
.collect();
result[i].depends = resolved;
}
Ok(result)
}
/// Convert bitcoind's `mempoolminfee` (BTC/kvB f64) to sat/vB. Round-trip
/// via integer sat/kvB (bitcoind's native CFeeRate unit) so the f64 drift
/// in the JSON-decoded value can't push 1.0 sat/vB to 1.0...e-13 above 1.0
/// and trip `ceil_to(0.001)` downstream.
fn build_min_fee(raw: GetMempoolInfo) -> FeeRate {
FeeRate::from(raw.mempool_min_fee * 100_000.0)
let sat_per_kvb = (raw.mempool_min_fee * 100_000_000.0).round() as u64;
FeeRate::from(sat_per_kvb as f64 / 1000.0)
}
impl Client {
@@ -143,18 +137,13 @@ impl Client {
.map_err(|e| Error::Parse(format!("decode getblock: {e}")))
}
pub fn get_block_info<'a, H>(&self, hash: &'a H) -> Result<BlockInfo>
pub fn get_block_info<'a, H>(&self, hash: &'a H) -> Result<GetBlockVerboseOne>
where
&'a H: Into<&'a bitcoin::BlockHash>,
{
let hash: &bitcoin::BlockHash = hash.into();
let r: GetBlockVerboseOne = self
.0
.call_with_retry("getblock", &[serde_json::to_value(hash)?, Value::from(1u8)])?;
Ok(BlockInfo {
height: r.height as usize,
confirmations: r.confirmations,
})
self.0
.call_with_retry("getblock", &[serde_json::to_value(hash)?, Value::from(1u8)])
}
pub fn get_block_header<'a, H>(&self, hash: &'a H) -> Result<bitcoin::block::Header>
@@ -170,23 +159,13 @@ impl Client {
bitcoin::consensus::deserialize::<bitcoin::block::Header>(&bytes).map_err(Error::from)
}
pub fn get_block_header_info<'a, H>(&self, hash: &'a H) -> Result<BlockHeaderInfo>
pub fn get_block_header_info<'a, H>(&self, hash: &'a H) -> Result<GetBlockHeaderVerbose>
where
&'a H: Into<&'a bitcoin::BlockHash>,
{
let hash: &bitcoin::BlockHash = hash.into();
let r: GetBlockHeaderVerbose = self
.0
.call_with_retry("getblockheader", &[serde_json::to_value(hash)?])?;
let previous_block_hash = r
.previous_block_hash
.map(|s| Self::parse_block_hash(&s, "previousblockhash"))
.transpose()?;
Ok(BlockHeaderInfo {
height: r.height as usize,
confirmations: r.confirmations,
previous_block_hash,
})
self.0
.call_with_retry("getblockheader", &[serde_json::to_value(hash)?])
}
pub fn get_block_hash<H>(&self, height: H) -> Result<BlockHash>
@@ -237,7 +216,7 @@ impl Client {
txid: &Txid,
vout: Vout,
include_mempool: Option<bool>,
) -> Result<Option<TxOutInfo>> {
) -> Result<Option<GetTxOut>> {
let txid: &bitcoin::Txid = txid.into();
let mut args: Vec<Value> = vec![
serde_json::to_value(txid)?,
@@ -246,19 +225,7 @@ impl Client {
if let Some(mempool) = include_mempool {
args.push(Value::Bool(mempool));
}
let r: Option<GetTxOut> = self.0.call_with_retry("gettxout", &args)?;
match r {
Some(r) => {
let script_pub_key = bitcoin::ScriptBuf::from_hex(&r.script_pubkey.hex)
.map_err(|e| Error::Parse(format!("script hex: {e}")))?;
Ok(Some(TxOutInfo {
coinbase: r.coinbase,
value: Sats::from(Bitcoin::from(r.value)),
script_pub_key,
}))
}
None => Ok(None),
}
self.0.call_with_retry("gettxout", &args)
}
pub fn get_raw_mempool(&self) -> Result<Vec<Txid>> {
@@ -268,55 +235,70 @@ impl Client {
.collect()
}
pub fn get_raw_transaction<'a, T, H>(
pub fn get_raw_transaction<'a, T>(&self, txid: &'a T) -> Result<bitcoin::Transaction>
where
&'a T: Into<&'a bitcoin::Txid>,
{
let hex = self.get_raw_transaction_hex(txid)?;
Ok(encode::deserialize_hex::<bitcoin::Transaction>(&hex)?)
}
pub fn get_raw_transaction_from<'a, T, H>(
&self,
txid: &'a T,
block_hash: Option<&'a H>,
block_hash: &'a H,
) -> Result<bitcoin::Transaction>
where
&'a T: Into<&'a bitcoin::Txid>,
&'a H: Into<&'a bitcoin::BlockHash>,
{
let hex = self.get_raw_transaction_hex(txid, block_hash)?;
let tx = encode::deserialize_hex::<bitcoin::Transaction>(&hex)?;
Ok(tx)
let hex = self.get_raw_transaction_hex_from(txid, block_hash)?;
Ok(encode::deserialize_hex::<bitcoin::Transaction>(&hex)?)
}
pub fn get_raw_transaction_hex<'a, T, H>(
pub fn get_raw_transaction_hex<'a, T>(&self, txid: &'a T) -> Result<String>
where
&'a T: Into<&'a bitcoin::Txid>,
{
let txid: &bitcoin::Txid = txid.into();
let args = [serde_json::to_value(txid)?, Value::Bool(false)];
self.0.call_with_retry("getrawtransaction", &args)
}
pub fn get_raw_transaction_hex_from<'a, T, H>(
&self,
txid: &'a T,
block_hash: Option<&'a H>,
block_hash: &'a H,
) -> Result<String>
where
&'a T: Into<&'a bitcoin::Txid>,
&'a H: Into<&'a bitcoin::BlockHash>,
{
let txid: &bitcoin::Txid = txid.into();
let mut args: Vec<Value> = vec![serde_json::to_value(txid)?, Value::Bool(false)];
if let Some(bh) = block_hash {
let bh: &bitcoin::BlockHash = bh.into();
args.push(serde_json::to_value(bh)?);
}
let bh: &bitcoin::BlockHash = block_hash.into();
let args = [
serde_json::to_value(txid)?,
Value::Bool(false),
serde_json::to_value(bh)?,
];
self.0.call_with_retry("getrawtransaction", &args)
}
pub fn get_mempool_raw_tx(&self, txid: &Txid) -> Result<RawTx> {
let hex = self.get_raw_transaction_hex(txid, None as Option<&BlockHash>)?;
let tx = encode::deserialize_hex::<bitcoin::Transaction>(&hex)?;
Ok(RawTx {
tx,
hex: hex.into(),
})
pub fn get_mempool_raw_tx(&self, txid: &Txid) -> Result<bitcoin::Transaction> {
self.get_raw_transaction(txid)
}
/// Batched `getrawtransaction` over a slice of txids. Returns a map keyed
/// by txid containing the deserialized tx and its raw hex. Individual
/// failures (e.g. a tx that evicted between the listing and this call)
/// are logged and dropped so a single bad entry doesn't kill the batch.
/// by txid containing the deserialized tx. Individual failures (e.g. a
/// tx that evicted between the listing and this call) are logged and
/// dropped so a single bad entry doesn't kill the batch.
///
/// Chunked at `BATCH_CHUNK` requests per round-trip.
pub fn get_raw_transactions(&self, txids: &[Txid]) -> Result<FxHashMap<Txid, RawTx>> {
let mut out: FxHashMap<Txid, RawTx> =
pub fn get_raw_transactions(
&self,
txids: &[Txid],
) -> Result<FxHashMap<Txid, bitcoin::Transaction>> {
let mut out: FxHashMap<Txid, bitcoin::Transaction> =
FxHashMap::with_capacity_and_hasher(txids.len(), Default::default());
for chunk in txids.chunks(BATCH_CHUNK) {
@@ -332,14 +314,10 @@ impl Client {
for (txid, res) in chunk.iter().zip(results) {
match res.and_then(|hex| {
let tx = encode::deserialize_hex::<bitcoin::Transaction>(&hex)?;
Ok::<_, Error>(RawTx {
tx,
hex: hex.into(),
})
Ok(encode::deserialize_hex::<bitcoin::Transaction>(&hex)?)
}) {
Ok(raw) => {
out.insert(*txid, raw);
Ok(tx) => {
out.insert(*txid, tx);
}
Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc))) if rpc.code == RPC_NOT_FOUND => {}
Err(e) => {
@@ -371,50 +349,109 @@ impl Client {
Ok(Txid::from(txid))
}
/// Verbose mempool listing + Core's projected next block + live
/// `mempoolminfee`, fetched in a single bitcoind round-trip.
/// Validates that every GBT txid is present in the verbose listing
/// and returns `Ok(None)` on mismatch so the caller can skip the
/// cycle (within-batch races inside bitcoind are rare; persistent
/// drift is bug-shaped). Other failures bubble up as `Err`.
pub fn fetch_mempool_state(&self) -> Result<Option<MempoolState>> {
/// Core's projected next block + live mempool txid set +
/// `mempoolminfee`, fetched in a single bitcoind round-trip. GBT
/// carries each tx's full body and stats, so block 0 is exact even
/// when a tx vanishes from the mempool listing between the GBT and
/// `getrawmempool` calls; no follow-up entry fetch can race it.
/// Returns the passthrough `MempoolState` and the raw
/// `block_template` (consumed downstream by GBT synthesis), in one
/// batched round-trip: `getblocktemplate` + `getrawmempool false`
/// + `getmempoolinfo`.
pub fn fetch_mempool_state(&self) -> Result<(MempoolState, Vec<BlockTemplateTx>)> {
let requests: [(&str, Vec<Value>); 3] = [
("getrawmempool", vec![Value::Bool(true)]),
(
"getblocktemplate",
vec![serde_json::json!({ "rules": ["segwit"] })],
),
("getrawmempool", vec![Value::Bool(false)]),
("getmempoolinfo", vec![]),
];
let mut out = self.0.call_mixed_batch(&requests)?.into_iter();
let verbose_raw = out.next().ok_or(Error::Internal("missing verbose"))??;
let gbt_raw = out.next().ok_or(Error::Internal("missing gbt"))??;
let template_raw = out.next().ok_or(Error::Internal("missing gbt"))??;
let txids_raw = out.next().ok_or(Error::Internal("missing rawmempool"))??;
let info_raw = out.next().ok_or(Error::Internal("missing mempoolinfo"))??;
let verbose: FxHashMap<String, VerboseEntryRaw> = serde_json::from_str(verbose_raw.get())?;
let entries = build_verbose(verbose)?;
let gbt = build_gbt(serde_json::from_str(gbt_raw.get())?);
let txid_strs: Vec<String> = serde_json::from_str(txids_raw.get())?;
let live_txids: Vec<Txid> = txid_strs
.iter()
.map(|s| Self::parse_txid(s, "mempool txid"))
.collect::<Result<Vec<_>>>()?;
let template: GetBlockTemplate = serde_json::from_str(template_raw.get())?;
let tip_hash = Self::parse_block_hash(&template.previous_block_hash, "previousblockhash")?;
let tip_height = Height::from(u64::try_from(template.height - 1).map_err(|_| {
Error::Parse(format!("gbt height out of range: {}", template.height))
})?);
let block_template = build_gbt(template)?;
let min_fee = build_min_fee(serde_json::from_str(info_raw.get())?);
#[cfg(debug_assertions)]
{
let entry_set: rustc_hash::FxHashSet<Txid> = entries.iter().map(|e| e.txid).collect();
let missing = gbt.iter().filter(|t| !entry_set.contains(&t.txid)).count();
if missing > 0 {
tracing::warn!(
missing,
gbt_total = gbt.len(),
"getblocktemplate has {missing} txids not in verbose mempool; skipping cycle"
);
return Ok(None);
Ok((
MempoolState {
live_txids,
min_fee,
tip_hash,
tip_height,
},
block_template,
))
}
/// Mixed batch of `getmempoolentry` + `getrawtransaction` for the
/// same txid set in one round-trip. Returns the entries vec and the
/// raw-tx map keyed by txid. Per-item -5 (NOT_FOUND — tx evicted
/// between the listing and this call) drops silently for either leg;
/// transport-level failures still propagate. Chunked at `BATCH_CHUNK`
/// txids per round-trip (2× that on the wire).
pub fn fetch_new_pool_data(
&self,
txids: &[Txid],
) -> Result<(Vec<MempoolEntryInfo>, FxHashMap<Txid, bitcoin::Transaction>)> {
let mut entries: Vec<MempoolEntryInfo> = Vec::with_capacity(txids.len());
let mut txs: FxHashMap<Txid, bitcoin::Transaction> =
FxHashMap::with_capacity_and_hasher(txids.len(), Default::default());
for chunk in txids.chunks(BATCH_CHUNK) {
let mut requests: Vec<(&str, Vec<Value>)> = Vec::with_capacity(chunk.len() * 2);
for txid in chunk {
let bt: &bitcoin::Txid = txid.into();
let tv = serde_json::to_value(bt).unwrap_or(Value::Null);
requests.push(("getmempoolentry", vec![tv.clone()]));
requests.push(("getrawtransaction", vec![tv, Value::Bool(false)]));
}
let results = self.0.call_mixed_batch(&requests)?;
let mut iter = results.into_iter();
for txid in chunk {
let entry_res = iter.next().ok_or(Error::Internal("missing entry"))?;
let raw_res = iter.next().ok_or(Error::Internal("missing raw"))?;
match entry_res.and_then(|raw| {
let me: MempoolEntry = serde_json::from_str(raw.get())?;
build_entry(*txid, me)
}) {
Ok(info) => entries.push(info),
Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc))) if rpc.code == RPC_NOT_FOUND => {}
Err(e) => {
debug!(txid = %txid, error = %e, "getmempoolentry mixed batch: item failed")
}
}
match raw_res.and_then(|raw| {
let hex: String = serde_json::from_str(raw.get())?;
Ok(encode::deserialize_hex::<bitcoin::Transaction>(&hex)?)
}) {
Ok(tx) => {
txs.insert(*txid, tx);
}
Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc))) if rpc.code == RPC_NOT_FOUND => {}
Err(e) => {
debug!(txid = %txid, error = %e, "getrawtransaction mixed batch: item failed")
}
}
}
}
Ok(Some(MempoolState {
entries,
gbt,
min_fee,
}))
Ok((entries, txs))
}
pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> {
@@ -424,23 +461,31 @@ impl Client {
loop {
let info = self.get_block_header_info(&current)?;
if info.confirmations > 0 {
return Ok((info.height.into(), current));
return Ok((Height::from(info.height as u64), current));
}
current = info.previous_block_hash.ok_or(Error::NotFound(
let prev = info.previous_block_hash.ok_or(Error::NotFound(
"Reached genesis without finding main chain".into(),
))?;
current = Self::parse_block_hash(&prev, "previousblockhash")?;
}
}
pub fn get_blockchain_info(&self) -> Result<GetBlockchainInfo> {
self.0.call_with_retry("getblockchaininfo", &[])
}
/// Bitcoin network the connected node is running on, derived from
/// `getblockchaininfo.chain`.
pub fn get_network(&self) -> Result<bitcoin::Network> {
let chain = self.get_blockchain_info()?.chain;
bitcoin::Network::from_core_arg(&chain)
.map_err(|e| Error::Parse(format!("getblockchaininfo.chain '{chain}': {e}")))
}
pub fn wait_for_synced_node(&self) -> Result<()> {
#[derive(Deserialize)]
struct SyncProgress {
headers: u64,
blocks: u64,
}
let is_synced = || -> Result<bool> {
let p: SyncProgress = self.0.call_with_retry("getblockchaininfo", &[])?;
Ok(p.headers == p.blocks)
let info = self.get_blockchain_info()?;
Ok(info.headers == info.blocks)
};
if !is_synced()? {
+29 -10
View File
@@ -11,6 +11,14 @@ use crate::{
params::{AddrAfterTxidParam, AddrParam, Empty, ValidateAddrParam},
};
/// Esplora `/txs` and `/txs/chain` page sizes. Wire-protocol constants from
/// mempool.space/esplora, not deployment policy. `/txs` returns up to
/// `MEMPOOL_PAGE` mempool entries plus a chain page sized to reach
/// `TXS_TOTAL_TARGET` total, floored at `CHAIN_PAGE`.
const MEMPOOL_PAGE: usize = 50;
const CHAIN_PAGE: usize = 25;
const TXS_TOTAL_TARGET: usize = 50;
pub trait AddrRoutes {
fn add_addr_routes(self) -> Self;
}
@@ -26,7 +34,7 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_strategy(Version::ONE, &path.addr, false);
let strategy = state.addr_strategy(Version::ONE, &path.addr, false, None);
state.respond_json(&headers, strategy, &uri, move |q| q.addr(path.addr)).await
}, |op| op
.id("get_address")
@@ -49,13 +57,24 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_strategy(Version::ONE, &path.addr, false);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, 50, 50)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, false, None);
state.respond_json(&headers, strategy, &uri, move |q| {
let mempool_txs = if q.mempool().is_some() {
q.addr_mempool_txs(&path.addr, MEMPOOL_PAGE)?
} else {
Vec::new()
};
let chain_limit = TXS_TOTAL_TARGET.saturating_sub(mempool_txs.len()).max(CHAIN_PAGE);
let chain_txs = q.addr_txs_chain(&path.addr, None, chain_limit)?;
let mut out = mempool_txs;
out.extend(chain_txs);
Ok(out)
}).await
}, |op| op
.id("get_address_txs")
.addrs_tag()
.summary("Address transactions")
.description("Get transaction history for an address, sorted with newest first. Returns up to 50 entries: mempool transactions first, then confirmed transactions filling the remainder. To paginate further confirmed transactions, use `/address/{address}/txs/chain/{last_seen_txid}`.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-address-transactions)*")
.description("Get transaction history for an address, newest first. Returns up to 50 mempool transactions plus a confirmed page sized to fill the response to 50 total (chain floor of 25, so 25-50 confirmed depending on mempool weight). To paginate further confirmed history, use `/address/{address}/txs/chain/{last_seen_txid}`.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-address-transactions)*")
.json_response::<Vec<Transaction>>()
.not_modified()
.bad_request()
@@ -72,8 +91,8 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_strategy(Version::ONE, &path.addr, true);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs_chain(&path.addr, None, 25)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, true, None);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs_chain(&path.addr, None, CHAIN_PAGE)).await
}, |op| op
.id("get_address_confirmed_txs")
.addrs_tag()
@@ -95,8 +114,8 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_strategy(Version::ONE, &path.addr, true);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs_chain(&path.addr, Some(path.after_txid), 25)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, true, Some(&path.after_txid));
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs_chain(&path.addr, Some(path.after_txid), CHAIN_PAGE)).await
}, |op| op
.id("get_address_confirmed_txs_after")
.addrs_tag()
@@ -119,7 +138,7 @@ impl AddrRoutes for ApiRouter<AppState> {
State(state): State<AppState>
| {
let hash = state.sync(|q| q.addr_mempool_hash(&path.addr)).unwrap_or(0);
state.respond_json(&headers, CacheStrategy::MempoolHash(hash), &uri, move |q| q.addr_mempool_txs(&path.addr, 50)).await
state.respond_json(&headers, CacheStrategy::MempoolHash(hash), &uri, move |q| q.addr_mempool_txs(&path.addr, MEMPOOL_PAGE)).await
}, |op| op
.id("get_address_mempool_txs")
.addrs_tag()
@@ -141,7 +160,7 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_strategy(Version::ONE, &path.addr, false);
let strategy = state.addr_strategy(Version::ONE, &path.addr, false, None);
let max_utxos = state.max_utxos;
state.respond_json(&headers, strategy, &uri, move |q| q.addr_utxos(path.addr, max_utxos)).await
}, |op| op
+3 -3
View File
@@ -27,7 +27,7 @@ impl FeesRoutes for ApiRouter<AppState> {
op.id("get_mempool_blocks")
.fees_tag()
.summary("Projected mempool blocks")
.description("Get projected blocks from the mempool for fee estimation.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-mempool-blocks-fees)*")
.description("Projected blocks for fee estimation. Block 0 reflects Bitcoin Core's actual next-block selection; blocks 1+ are a fee-tier approximation.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-mempool-blocks-fees)*")
.json_response::<Vec<MempoolBlock>>()
.not_modified()
.server_error()
@@ -48,7 +48,7 @@ impl FeesRoutes for ApiRouter<AppState> {
op.id("get_recommended_fees")
.fees_tag()
.summary("Recommended fees")
.description("Get recommended fee rates for different confirmation targets.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-recommended-fees)*")
.description("Recommended fee rates by confirmation target.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-recommended-fees)*")
.json_response::<RecommendedFees>()
.not_modified()
.server_error()
@@ -69,7 +69,7 @@ impl FeesRoutes for ApiRouter<AppState> {
op.id("get_precise_fees")
.fees_tag()
.summary("Precise recommended fees")
.description("Get recommended fee rates with up to 3 decimal places.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-recommended-fees-precise)*")
.description("Recommended fee rates with sub-integer precision.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-recommended-fees-precise)*")
.json_response::<RecommendedFees>()
.not_modified()
.server_error()
+59 -5
View File
@@ -1,11 +1,18 @@
use aide::axum::{ApiRouter, routing::get_with};
use axum::{
extract::State,
extract::{Path, State},
http::{HeaderMap, Uri},
};
use brk_types::{Dollars, MempoolInfo, MempoolRecentTx, ReplacementNode, Txid};
use brk_types::{
BlockTemplate, BlockTemplateDiff, Dollars, MempoolInfo, MempoolRecentTx, NextBlockHash,
ReplacementNode, Txid,
};
use crate::{AppState, extended::TransformResponseExtended, params::Empty};
use crate::{
AppState,
extended::TransformResponseExtended,
params::{Empty, NextBlockHashParam},
};
pub trait MempoolRoutes {
fn add_mempool_routes(self) -> Self;
@@ -44,8 +51,8 @@ impl MempoolRoutes for ApiRouter<AppState> {
op.id("get_mempool_hash")
.mempool_tag()
.summary("Mempool content hash")
.description("Returns an opaque `u64` that changes whenever the projected next block changes. Same value as the mempool ETag. Useful as a freshness/liveness signal: if it stays constant for tens of seconds on a live network, the mempool sync loop has stalled.")
.json_response::<u64>()
.description("Returns an opaque hash that changes whenever the projected next block changes. Same value as the mempool ETag. Useful as a freshness/liveness signal: if it stays constant for tens of seconds on a live network, the mempool sync loop has stalled.")
.json_response::<NextBlockHash>()
.not_modified()
.server_error()
},
@@ -131,6 +138,53 @@ impl MempoolRoutes for ApiRouter<AppState> {
},
),
)
.api_route(
"/api/v1/mempool/block-template",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
q.block_template()
})
.await
},
|op| {
op.id("get_block_template")
.mempool_tag()
.summary("Projected next block template")
.description("Bitcoin Core's `getblocktemplate` selection: full transaction bodies in GBT order with aggregate stats. The returned `hash` is an opaque content token; pass it as `<hash>` on `/api/v1/mempool/block-template/diff/{hash}` to fetch deltas instead of refetching the whole template.")
.json_response::<BlockTemplate>()
.not_modified()
.server_error()
},
),
)
.api_route(
"/api/v1/mempool/block-template/diff/{hash}",
get_with(
async |uri: Uri,
headers: HeaderMap,
Path(path): Path<NextBlockHashParam>,
_: Empty,
State(state): State<AppState>| {
state
.respond_json(&headers, state.mempool_strategy(), &uri, move |q| {
q.block_template_diff(path.hash)
})
.await
},
|op| {
op.id("get_block_template_diff")
.mempool_tag()
.summary("Block template diff since hash")
.description("Delta of the projected next block since `<hash>`. `order` is the full new template in order: each entry is either a number (index into the prior template the client cached at `<hash>`) or a transaction object (new body to insert at this position). Walk `order` once to rebuild; `removed` is a convenience list of txids that left so clients can evict cached bodies. After applying, use the response `hash` as `<hash>` on the next call to keep iterating. Returns `404` when `<hash>` has aged out of server history; clients should fall back to `/api/v1/mempool/block-template`.")
.json_response::<BlockTemplateDiff>()
.not_modified()
.not_found()
.server_error()
},
),
)
.api_route(
"/api/mempool/price",
get_with(
+2 -1
View File
@@ -135,7 +135,8 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/pool/{slug}/blocks",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<PoolSlugParam>, _: Empty, State(state): State<AppState>| {
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_blocks(path.slug, None, POOL_BLOCKS_LIMIT)).await
let strategy = state.pool_blocks_strategy(Version::ONE, path.slug);
state.respond_json(&headers, strategy, &uri, move |q| q.pool_blocks(path.slug, None, POOL_BLOCKS_LIMIT)).await
},
|op| {
op.id("get_pool_blocks")
+2 -8
View File
@@ -29,13 +29,7 @@ impl ServerRoutes for ApiRouter<AppState> {
let uptime = state.started_instant.elapsed();
let started_at = state.started_at.to_string();
let sync = state
.run(move |q| {
let tip_height = q
.client()
.get_last_height()
.unwrap_or(q.height());
q.sync_status(tip_height)
})
.run(move |q| q.sync_status(q.height()))
.await
.expect("health sync task panicked");
let mut response = axum::Json(Health {
@@ -57,7 +51,7 @@ impl ServerRoutes for ApiRouter<AppState> {
op.id("get_health")
.server_tag()
.summary("Health check")
.description("Returns the health status of the API server, including uptime information.")
.description("Liveness probe. Returns server identity, uptime, and indexed/computed heights from local state only (no bitcoind round-trip). For real chain-tip catch-up, see `/api/server/sync`.")
.json_response::<Health>()
},
),
+15 -8
View File
@@ -102,9 +102,11 @@ impl Server {
let response_time_layer = axum::middleware::from_fn(
async |request: Request<Body>, next: Next| -> Response<Body> {
let uri = request.uri().clone();
let method = request.method().clone();
let start = Instant::now();
let mut response = next.run(request).await;
response.extensions_mut().insert(uri);
response.extensions_mut().insert(method);
response.headers_mut().insert(
"X-Response-Time",
format!("{}us", start.elapsed().as_micros())
@@ -182,14 +184,19 @@ impl Server {
.on_response(
|response: &Response<Body>, latency: Duration, _: &tracing::Span| {
let status = response.status().as_u16();
let unknown = Uri::from_static("/unknown");
let uri = response.extensions().get::<Uri>().unwrap_or(&unknown);
let unknown_uri = Uri::from_static("/unknown");
let unknown_method = axum::http::Method::default();
let uri = response.extensions().get::<Uri>().unwrap_or(&unknown_uri);
let method = response
.extensions()
.get::<axum::http::Method>()
.unwrap_or(&unknown_method);
match response.status() {
StatusCode::OK => info!(status, %uri, ?latency),
StatusCode::OK => info!(%method, status, %uri, ?latency),
StatusCode::NOT_MODIFIED
| StatusCode::TEMPORARY_REDIRECT
| StatusCode::PERMANENT_REDIRECT => info!(status, %uri, ?latency),
_ => error!(status, %uri, ?latency),
| StatusCode::PERMANENT_REDIRECT => info!(%method, status, %uri, ?latency),
_ => error!(%method, status, %uri, ?latency),
}
},
)
@@ -209,8 +216,6 @@ impl Server {
let router = router
.with_state(state)
.merge(website_router)
.layer(response_time_layer)
.layer(trace_layer)
.layer(TimeoutLayer::with_status_code(
StatusCode::GATEWAY_TIMEOUT,
REQUEST_TIMEOUT,
@@ -242,7 +247,9 @@ impl Server {
.or_else(|| panic.downcast_ref::<&str>().copied())
.unwrap_or("Unknown panic");
Error::internal(msg).into_response()
}));
}))
.layer(response_time_layer)
.layer(trace_layer);
let (listener, port) = match port {
Some(port) => {

Some files were not shown because too many files have changed in this diff Show More