From 7a0b4b589088c131452ab6397d1b568630b36a3b Mon Sep 17 00:00:00 2001 From: nym21 Date: Sun, 26 Apr 2026 23:12:17 +0200 Subject: [PATCH] global: big snapshot --- Cargo.lock | 193 ++------ Cargo.toml | 11 +- crates/brk/Cargo.toml | 2 +- crates/brk_cli/Cargo.toml | 2 +- crates/brk_cli/src/config.rs | 78 +++- crates/brk_cli/src/main.rs | 24 +- crates/brk_client/src/lib.rs | 13 +- crates/brk_computer/Cargo.toml | 2 +- crates/brk_error/Cargo.toml | 6 +- crates/brk_error/src/lib.rs | 6 +- crates/brk_indexer/Cargo.toml | 2 +- crates/brk_iterator/Cargo.toml | 2 +- crates/brk_logger/src/lib.rs | 2 +- crates/brk_mempool/Cargo.toml | 2 +- crates/brk_mempool/examples/mempool.rs | 8 +- crates/brk_mempool/src/block_builder/graph.rs | 175 -------- crates/brk_mempool/src/entry.rs | 47 -- crates/brk_mempool/src/entry_pool.rs | 84 ---- crates/brk_mempool/src/lib.rs | 181 +++++++- crates/brk_mempool/src/steps/applier.rs | 69 +++ .../brk_mempool/src/steps/fetcher/fetched.rs | 10 + crates/brk_mempool/src/steps/fetcher/mod.rs | 80 ++++ crates/brk_mempool/src/steps/mod.rs | 7 + .../brk_mempool/src/steps/preparer/added.rs | 124 ++++++ crates/brk_mempool/src/steps/preparer/mod.rs | 61 +++ .../brk_mempool/src/steps/preparer/pulled.rs | 10 + .../brk_mempool/src/steps/preparer/removed.rs | 58 +++ .../steps/rebuilder/block_builder/graph.rs | 85 ++++ .../rebuilder/block_builder/graph_bench.rs | 88 ++++ .../rebuilder}/block_builder/linearize/mod.rs | 18 +- .../rebuilder}/block_builder/linearize/sfl.rs | 94 ++-- .../block_builder/linearize/tests/basic.rs | 0 .../block_builder/linearize/tests/mod.rs | 2 +- .../block_builder/linearize/tests/oracle.rs | 6 +- .../block_builder/linearize/tests/stress.rs | 5 +- .../rebuilder}/block_builder/mod.rs | 6 +- .../rebuilder}/block_builder/package.rs | 2 +- .../rebuilder}/block_builder/partitioner.rs | 10 +- .../rebuilder/block_builder}/pool_index.rs | 0 .../rebuilder}/block_builder/tx_node.rs | 3 +- crates/brk_mempool/src/steps/rebuilder/mod.rs | 111 +++++ .../rebuilder}/projected_blocks/fees.rs | 0 .../rebuilder}/projected_blocks/mod.rs | 0 .../rebuilder}/projected_blocks/snapshot.rs | 19 +- .../rebuilder}/projected_blocks/stats.rs | 7 +- .../rebuilder}/projected_blocks/verify.rs | 13 +- crates/brk_mempool/src/steps/resolver.rs | 139 ++++++ .../src/{addrs.rs => stores/addr_tracker.rs} | 32 +- crates/brk_mempool/src/stores/entry.rs | 39 ++ crates/brk_mempool/src/stores/entry_pool.rs | 72 +++ crates/brk_mempool/src/stores/mod.rs | 32 ++ crates/brk_mempool/src/stores/state.rs | 35 ++ crates/brk_mempool/src/stores/tombstone.rs | 45 ++ crates/brk_mempool/src/stores/tx_graveyard.rs | 82 ++++ .../src/{types => stores}/tx_index.rs | 0 crates/brk_mempool/src/stores/tx_store.rs | 90 ++++ crates/brk_mempool/src/sync.rs | 354 --------------- crates/brk_mempool/src/tx_store.rs | 56 --- crates/brk_mempool/src/types/mod.rs | 7 - crates/brk_mempool/src/types/tx_with_hex.rs | 26 -- crates/brk_query/Cargo.toml | 2 +- crates/brk_query/src/impl/addr.rs | 23 +- crates/brk_query/src/impl/block/txs.rs | 8 +- crates/brk_query/src/impl/mempool.rs | 183 +++++++- crates/brk_query/src/impl/price.rs | 4 +- crates/brk_query/src/impl/series.rs | 9 +- crates/brk_query/src/impl/tx.rs | 23 +- crates/brk_reader/Cargo.toml | 2 +- crates/brk_rpc/Cargo.toml | 21 +- crates/brk_rpc/examples/compare_backends.rs | 269 ------------ crates/brk_rpc/src/backend/bitcoincore.rs | 338 -------------- crates/brk_rpc/src/backend/corepc.rs | 413 ------------------ crates/brk_rpc/src/backend/mod.rs | 70 --- crates/brk_rpc/src/client.rs | 198 +++++++++ crates/brk_rpc/src/lib.rs | 302 +++---------- crates/brk_rpc/src/methods.rs | 364 +++++++++++++++ crates/brk_server/Cargo.toml | 2 +- crates/brk_server/README.md | 31 +- crates/brk_server/examples/server.rs | 11 +- .../brk_server/src/api/mempool_space/addrs.rs | 2 +- .../src/api/mempool_space/mining.rs | 4 +- .../src/api/mempool_space/transactions.rs | 59 ++- crates/brk_server/src/api/metrics/mod.rs | 24 +- crates/brk_server/src/api/mod.rs | 5 +- .../src/api/{series/mod.rs => series.rs} | 127 ++++-- crates/brk_server/src/api/series/bulk.rs | 79 ---- crates/brk_server/src/api/series/data.rs | 102 ----- crates/brk_server/src/api/series/legacy.rs | 82 ---- .../cost_basis.rs => series_legacy.rs} | 91 ++-- crates/brk_server/src/api/server/mod.rs | 2 +- crates/brk_server/src/api/urpd/mod.rs | 2 +- crates/brk_server/src/cache.rs | 92 ---- crates/brk_server/src/cache/mod.rs | 20 + crates/brk_server/src/cache/mode.rs | 60 +++ crates/brk_server/src/cache/params.rs | 120 +++++ crates/brk_server/src/cache/strategy.rs | 30 ++ crates/brk_server/src/config.rs | 39 ++ crates/brk_server/src/error.rs | 21 +- crates/brk_server/src/etag.rs | 25 ++ crates/brk_server/src/extended/header_map.rs | 9 +- crates/brk_server/src/extended/response.rs | 58 +-- crates/brk_server/src/lib.rs | 29 +- crates/brk_server/src/state.rs | 160 +++++-- crates/brk_types/src/etag.rs | 61 --- crates/brk_types/src/hex.rs | 49 ++- crates/brk_types/src/lib.rs | 6 +- crates/brk_types/src/rbf.rs | 58 +++ crates/brk_types/src/tx.rs | 100 +++++ crates/brk_types/src/tx_version_raw.rs | 7 + crates/brk_types/src/txin.rs | 39 +- crates/brk_types/src/txout.rs | 10 + crates/brk_types/src/vin.rs | 7 + crates/brk_types/src/witness.rs | 62 +++ modules/brk-client/index.js | 69 ++- packages/brk_client/brk_client/__init__.py | 90 +++- website/scripts/_types.js | 2 +- website/scripts/explorer/chain.js | 58 ++- website/scripts/explorer/index.js | 10 + website/scripts/utils/chart/index.js | 2 +- website/scripts/utils/dom.js | 12 + website/styles/chart.css | 1 - website/styles/components.css | 1 - website/styles/elements.css | 2 +- website/styles/main.css | 1 - website/styles/panes/explorer.css | 63 ++- 125 files changed, 3833 insertions(+), 3129 deletions(-) delete mode 100644 crates/brk_mempool/src/block_builder/graph.rs delete mode 100644 crates/brk_mempool/src/entry.rs delete mode 100644 crates/brk_mempool/src/entry_pool.rs create mode 100644 crates/brk_mempool/src/steps/applier.rs create mode 100644 crates/brk_mempool/src/steps/fetcher/fetched.rs create mode 100644 crates/brk_mempool/src/steps/fetcher/mod.rs create mode 100644 crates/brk_mempool/src/steps/mod.rs create mode 100644 crates/brk_mempool/src/steps/preparer/added.rs create mode 100644 crates/brk_mempool/src/steps/preparer/mod.rs create mode 100644 crates/brk_mempool/src/steps/preparer/pulled.rs create mode 100644 crates/brk_mempool/src/steps/preparer/removed.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/linearize/mod.rs (93%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/linearize/sfl.rs (84%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/linearize/tests/basic.rs (100%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/linearize/tests/mod.rs (98%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/linearize/tests/oracle.rs (98%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/linearize/tests/stress.rs (97%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/mod.rs (92%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/package.rs (97%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/partitioner.rs (93%) rename crates/brk_mempool/src/{types => steps/rebuilder/block_builder}/pool_index.rs (100%) rename crates/brk_mempool/src/{ => steps/rebuilder}/block_builder/tx_node.rs (92%) create mode 100644 crates/brk_mempool/src/steps/rebuilder/mod.rs rename crates/brk_mempool/src/{ => steps/rebuilder}/projected_blocks/fees.rs (100%) rename crates/brk_mempool/src/{ => steps/rebuilder}/projected_blocks/mod.rs (100%) rename crates/brk_mempool/src/{ => steps/rebuilder}/projected_blocks/snapshot.rs (65%) rename crates/brk_mempool/src/{ => steps/rebuilder}/projected_blocks/stats.rs (94%) rename crates/brk_mempool/src/{ => steps/rebuilder}/projected_blocks/verify.rs (96%) create mode 100644 crates/brk_mempool/src/steps/resolver.rs rename crates/brk_mempool/src/{addrs.rs => stores/addr_tracker.rs} (59%) create mode 100644 crates/brk_mempool/src/stores/entry.rs create mode 100644 crates/brk_mempool/src/stores/entry_pool.rs create mode 100644 crates/brk_mempool/src/stores/mod.rs create mode 100644 crates/brk_mempool/src/stores/state.rs create mode 100644 crates/brk_mempool/src/stores/tombstone.rs create mode 100644 crates/brk_mempool/src/stores/tx_graveyard.rs rename crates/brk_mempool/src/{types => stores}/tx_index.rs (100%) create mode 100644 crates/brk_mempool/src/stores/tx_store.rs delete mode 100644 crates/brk_mempool/src/sync.rs delete mode 100644 crates/brk_mempool/src/tx_store.rs delete mode 100644 crates/brk_mempool/src/types/mod.rs delete mode 100644 crates/brk_mempool/src/types/tx_with_hex.rs delete mode 100644 crates/brk_rpc/examples/compare_backends.rs delete mode 100644 crates/brk_rpc/src/backend/bitcoincore.rs delete mode 100644 crates/brk_rpc/src/backend/corepc.rs delete mode 100644 crates/brk_rpc/src/backend/mod.rs create mode 100644 crates/brk_rpc/src/client.rs create mode 100644 crates/brk_rpc/src/methods.rs rename crates/brk_server/src/api/{series/mod.rs => series.rs} (74%) delete mode 100644 crates/brk_server/src/api/series/bulk.rs delete mode 100644 crates/brk_server/src/api/series/data.rs delete mode 100644 crates/brk_server/src/api/series/legacy.rs rename crates/brk_server/src/api/{series/cost_basis.rs => series_legacy.rs} (71%) delete mode 100644 crates/brk_server/src/cache.rs create mode 100644 crates/brk_server/src/cache/mod.rs create mode 100644 crates/brk_server/src/cache/mode.rs create mode 100644 crates/brk_server/src/cache/params.rs create mode 100644 crates/brk_server/src/cache/strategy.rs create mode 100644 crates/brk_server/src/config.rs create mode 100644 crates/brk_server/src/etag.rs delete mode 100644 crates/brk_types/src/etag.rs create mode 100644 crates/brk_types/src/rbf.rs create mode 100644 crates/brk_types/src/witness.rs diff --git a/Cargo.lock b/Cargo.lock index bd300b9c6..ab95a14f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,9 +98,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-compression" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -197,12 +197,6 @@ dependencies = [ "bitcoin_hashes", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.7" @@ -281,9 +275,9 @@ checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" [[package]] name = "bitcoin-units" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" +checksum = "346568ebaab2918487cea76dd55dae13c27bb618cdb737c952e69eb2017c4118" dependencies = [ "bitcoin-internals", "serde", @@ -300,30 +294,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bitcoincore-rpc" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" -dependencies = [ - "bitcoincore-rpc-json", - "jsonrpc", - "log", - "serde", - "serde_json", -] - -[[package]] -name = "bitcoincore-rpc-json" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" -dependencies = [ - "bitcoin", - "serde", - "serde_json", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -361,42 +331,6 @@ dependencies = [ "brk_types", ] -[[package]] -name = "brk-corepc-client" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a735b1f9b9e14301a267946f66b5e72bb147ce6e002ac1a34f5857f6849a5e24" -dependencies = [ - "bitcoin", - "brk-corepc-jsonrpc", - "brk-corepc-types", - "log", - "serde", - "serde_json", -] - -[[package]] -name = "brk-corepc-jsonrpc" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb0983c8009f2d34fa8dce3f08c2ab3aa1ea0cf092cc47be8934219b0b383eb" -dependencies = [ - "base64 0.22.1", - "serde", - "serde_json", -] - -[[package]] -name = "brk-corepc-types" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3f711507c9872a538ab57241486a3c59bf5827651a725928b862f296828f9b0" -dependencies = [ - "bitcoin", - "serde", - "serde_json", -] - [[package]] name = "brk_alloc" version = "0.3.0-beta.5" @@ -517,10 +451,9 @@ name = "brk_error" version = "0.3.0-beta.5" dependencies = [ "bitcoin", - "bitcoincore-rpc", - "brk-corepc-client", "fjall", "jiff", + "jsonrpc", "pco", "serde_json", "thiserror", @@ -658,13 +591,13 @@ name = "brk_rpc" version = "0.3.0-beta.5" dependencies = [ "bitcoin", - "bitcoincore-rpc", - "brk-corepc-client", - "brk-corepc-jsonrpc", "brk_error", "brk_logger", "brk_types", + "corepc-types", + "jsonrpc", "parking_lot", + "rustc-hash", "serde", "serde_json", "tracing", @@ -836,9 +769,9 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -926,9 +859,9 @@ checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" [[package]] name = "compression-codecs" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "brotli", "compression-core", @@ -940,9 +873,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "convert_case" @@ -1034,6 +967,17 @@ dependencies = [ "libc", ] +[[package]] +name = "corepc-types" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1583872320eb2ac629c36753023fd072f1ca1b3b74b20cc62bab055b54278789" +dependencies = [ + "bitcoin", + "serde", + "serde_json", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1883,9 +1827,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "log", @@ -1897,9 +1841,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -1934,12 +1878,11 @@ dependencies = [ [[package]] name = "jsonrpc" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +checksum = "629d2b4ae586d04b6bae3c75879d7ddd39325c2b67a9c87634f4ec88a488dc65" dependencies = [ - "base64 0.13.1", - "minreq", + "base64 0.22.1", "serde", "serde_json", ] @@ -1964,9 +1907,9 @@ checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -2130,16 +2073,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "minreq" -version = "2.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05015102dad0f7d61691ca347e9d9d9006685a64aefb3d79eecf62665de2153d" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "mio" version = "1.2.0" @@ -2255,9 +2188,9 @@ dependencies = [ [[package]] name = "pathfinder_simd" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57" +checksum = "4500030c302e4af1d423f36f3b958d1aecb6c04184356ed5a833bf6b60435777" dependencies = [ "rustc_version", ] @@ -2381,15 +2314,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -2451,35 +2375,11 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] [[package]] name = "rand_xoshiro" @@ -2501,9 +2401,9 @@ dependencies = [ [[package]] name = "rawdb" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ed1c5accd49323c74351b29118b82b456268580e2a61d259d1a859265c82e2" +checksum = "78ec6d90b48955942ae9aa12b8630c8100798bf79e73dbdffd8115a3e71e4915" dependencies = [ "libc", "log", @@ -2662,9 +2562,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "log", "once_cell", @@ -2677,9 +2577,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -2755,7 +2655,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ "bitcoin_hashes", - "rand", "secp256k1-sys", "serde", ] @@ -3395,9 +3294,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055727d024d7b99d298378b0fa30cf11ff4511ccce9d299bc792e2a213a85bba" +checksum = "2ca57cedd42c0c7d8a343c06ab9c311be28a731e5d1e4101ef671d9a9af409a8" dependencies = [ "itoa", "libc", @@ -3418,9 +3317,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e2c28b3de48dff1708bd4217da39ef94c1b554cd2799109b88745069fe6728" +checksum = "8840b74c96d5888e1a5846adcd2ec8355778052a07dd5f6d30d67ef0fbc33b7e" dependencies = [ "quote", "syn", diff --git a/Cargo.toml b/Cargo.toml index 86e0e3999..2c2eb399b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,6 @@ debug = true aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] } axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] } bitcoin = { version = "0.32.8", features = ["serde"] } -bitcoincore-rpc = "0.19.0" brk_alloc = { version = "0.3.0-beta.5", path = "crates/brk_alloc" } brk_bencher = { version = "0.3.0-beta.5", path = "crates/brk_bencher" } brk_bindgen = { version = "0.3.0-beta.5", path = "crates/brk_bindgen" } @@ -65,14 +64,12 @@ brk_types = { version = "0.3.0-beta.5", path = "crates/brk_types" } brk_website = { version = "0.3.0-beta.5", path = "crates/brk_website" } byteview = "0.10.1" color-eyre = "0.6.5" -corepc-client = { package = "brk-corepc-client", version = "0.11.1", features = ["client-sync"] } -# corepc-client = { package = "brk-corepc-client", path = "../corepc/client", features = ["client-sync"] } -corepc-jsonrpc = { package = "brk-corepc-jsonrpc", version = "0.19.1", features = ["simple_http"], default-features = false } -# corepc-jsonrpc = { package = "brk-corepc-jsonrpc", path = "../corepc/jsonrpc", features = ["simple_http"], default-features = false } +corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false } +corepc-types = { version = "0.12.0", features = ["std"], default-features = false } derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] } fjall = "=3.0.4" indexmap = { version = "2.14.0", features = ["serde"] } -jiff = { version = "0.2.23", features = ["perf-inline", "tz-system"], default-features = false } +jiff = { version = "0.2.24", features = ["perf-inline", "tz-system"], default-features = false } owo-colors = "4.3.0" parking_lot = "0.12.5" pco = "1.0.1" @@ -89,7 +86,7 @@ tower-http = { version = "0.6.8", features = ["catch-panic", "compression-br", " tower-layer = "0.3" tracing = { version = "0.1", default-features = false, features = ["std"] } ureq = { version = "3.3.0", features = ["json"] } -vecdb = { version = "0.10.1", features = ["derive", "serde_json", "pco", "schemars"] } +vecdb = { version = "0.10.2", features = ["derive", "serde_json", "pco", "schemars"] } # vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } [workspace.metadata.release] diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 4642072bc..5c7075a79 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -64,7 +64,7 @@ brk_mempool = { workspace = true, optional = true } brk_oracle = { workspace = true, optional = true } brk_query = { workspace = true, optional = true } brk_reader = { workspace = true, optional = true } -brk_rpc = { workspace = true, optional = true, features = ["corepc"] } +brk_rpc = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } brk_store = { workspace = true, optional = true } brk_traversable = { workspace = true, optional = true } diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 8b7ccf00e..cbad968b3 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -17,7 +17,7 @@ brk_logger = { workspace = true } brk_mempool = { workspace = true } brk_query = { workspace = true } brk_reader = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_server = { workspace = true } brk_types = { workspace = true } lexopt = "0.3" diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index 056545a39..4d8cfe696 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -5,7 +5,9 @@ use std::{ use brk_error::{Error, Result}; use brk_rpc::{Auth, Client}; -use brk_server::Website; +use brk_server::{ + CdnCacheMode, DEFAULT_CACHE_SIZE, DEFAULT_MAX_WEIGHT, DEFAULT_MAX_WEIGHT_LOCALHOST, Website, +}; use brk_types::Port; use owo_colors::OwoColorize; use serde::{Deserialize, Deserializer, Serialize}; @@ -23,6 +25,18 @@ pub struct Config { #[serde(default, deserialize_with = "default_on_error")] website: Option, + #[serde(default, deserialize_with = "default_on_error")] + cdn: Option, + + #[serde(default, deserialize_with = "default_on_error")] + maxweight: Option, + + #[serde(default, deserialize_with = "default_on_error")] + maxweightlocal: Option, + + #[serde(default, deserialize_with = "default_on_error")] + cachesize: Option, + #[serde(default, deserialize_with = "default_on_error")] bitcoindir: Option, @@ -66,6 +80,18 @@ impl Config { if let Some(v) = config_args.website { config.website = Some(v); } + if let Some(v) = config_args.cdn { + config.cdn = Some(v); + } + if let Some(v) = config_args.maxweight { + config.maxweight = Some(v); + } + if let Some(v) = config_args.maxweightlocal { + config.maxweightlocal = Some(v); + } + if let Some(v) = config_args.cachesize { + config.cachesize = Some(v); + } if let Some(v) = config_args.bitcoindir { config.bitcoindir = Some(v); } @@ -112,6 +138,16 @@ impl Config { Long("brkdir") => config.brkdir = Some(parser.value().unwrap().parse().unwrap()), Long("brkport") => config.brkport = Some(parser.value().unwrap().parse().unwrap()), Long("website") => config.website = Some(parser.value().unwrap().parse().unwrap()), + Long("cdn") => config.cdn = Some(parser.value().unwrap().parse().unwrap()), + Long("maxweight") => { + config.maxweight = Some(parser.value().unwrap().parse().unwrap()) + } + Long("maxweightlocal") => { + config.maxweightlocal = Some(parser.value().unwrap().parse().unwrap()) + } + Long("cachesize") => { + config.cachesize = Some(parser.value().unwrap().parse().unwrap()) + } Long("bitcoindir") => { config.bitcoindir = Some(parser.value().unwrap().parse().unwrap()) } @@ -171,6 +207,26 @@ impl Config { "".bright_black(), "[true]".bright_black() ); + println!( + " --cdn {} Aggressive CDN cache, requires purge on deploy {}", + "".bright_black(), + "[false]".bright_black() + ); + println!( + " --maxweight {} Max series response weight in bytes for external clients {}", + "".bright_black(), + format!("[{}]", DEFAULT_MAX_WEIGHT).bright_black() + ); + println!( + " --maxweightlocal {} Max series response weight in bytes for loopback clients {}", + "".bright_black(), + format!("[{}]", DEFAULT_MAX_WEIGHT_LOCALHOST).bright_black() + ); + println!( + " --cachesize {} LRU capacity for the in-process response cache {}", + "".bright_black(), + format!("[{}]", DEFAULT_CACHE_SIZE).bright_black() + ); println!(); println!( " --bitcoindir {} Bitcoin directory {}", @@ -333,6 +389,26 @@ Finally, you can run the program with '-h' for help." self.website.clone().unwrap_or_default() } + pub fn cdn_cache_mode(&self) -> CdnCacheMode { + if self.cdn.unwrap_or(false) { + CdnCacheMode::Aggressive + } else { + CdnCacheMode::Live + } + } + + pub fn max_weight(&self) -> usize { + self.maxweight.unwrap_or(DEFAULT_MAX_WEIGHT) + } + + pub fn max_weight_localhost(&self) -> usize { + self.maxweightlocal.unwrap_or(DEFAULT_MAX_WEIGHT_LOCALHOST) + } + + pub fn cache_size(&self) -> usize { + self.cachesize.unwrap_or(DEFAULT_CACHE_SIZE) + } + pub fn brkport(&self) -> Option { self.brkport } diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs index dc5737683..51e2ec430 100644 --- a/crates/brk_cli/src/main.rs +++ b/crates/brk_cli/src/main.rs @@ -13,7 +13,7 @@ use brk_indexer::Indexer; use brk_mempool::Mempool; use brk_query::AsyncQuery; use brk_reader::Reader; -use brk_server::Server; +use brk_server::{Server, ServerConfig}; use tracing::info; use vecdb::Exit; @@ -60,21 +60,29 @@ pub fn main() -> anyhow::Result<()> { let mempool = Mempool::new(&client); + let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool.clone())); + let mempool_clone = mempool.clone(); + let query_clone = query.clone(); thread::spawn(move || { - mempool_clone.start(); + mempool_clone.start_with(|| { + query_clone.sync(|q| q.fill_mempool_prevouts()); + }); }); - let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool)); - - let data_path = config.brkdir(); - - let website = config.website(); + let server_config = ServerConfig { + data_path: config.brkdir(), + website: config.website(), + cdn_cache_mode: config.cdn_cache_mode(), + max_weight: config.max_weight(), + max_weight_localhost: config.max_weight_localhost(), + cache_size: config.cache_size(), + }; let port = config.brkport(); let future = async move { - let server = Server::new(&query, data_path, website); + let server = Server::new(&query, server_config); tokio::spawn(async move { server.serve(port).await.unwrap(); diff --git a/crates/brk_client/src/lib.rs b/crates/brk_client/src/lib.rs index af3fc3790..89b856a27 100644 --- a/crates/brk_client/src/lib.rs +++ b/crates/brk_client/src/lib.rs @@ -8897,7 +8897,7 @@ pub struct BrkClient { impl BrkClient { /// Client version. - pub const VERSION: &'static str = "v0.3.0-beta.4"; + pub const VERSION: &'static str = "v0.3.0-beta.5"; /// Create a new client with the given base URL. pub fn new(base_url: impl Into) -> Self { @@ -9847,6 +9847,17 @@ impl BrkClient { self.base.get_json(&format!("/api/v1/transaction-times")) } + /// RBF replacement history + /// + /// Returns the RBF replacement tree for a transaction, if any. Both `replacements` and `replaces` are null when the tx has no known RBF history within the mempool monitor's retention window. + /// + /// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-transaction-rbf-history)* + /// + /// Endpoint: `GET /api/v1/tx/{txid}/rbf` + pub fn get_tx_rbf(&self, txid: Txid) -> Result { + self.base.get_json(&format!("/api/v1/tx/{txid}/rbf")) + } + /// Validate address /// /// Validate a Bitcoin address and get information about its type and scriptPubKey. Returns `isvalid: false` with an error message for invalid addresses. diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 28b2d85e6..f74b29e2d 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -15,7 +15,7 @@ brk_cohort = { workspace = true } brk_indexer = { workspace = true } brk_oracle = { workspace = true } brk_logger = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_traversable = { workspace = true } brk_types = { workspace = true } derive_more = { workspace = true } diff --git a/crates/brk_error/Cargo.toml b/crates/brk_error/Cargo.toml index 0009f75e6..c27106db0 100644 --- a/crates/brk_error/Cargo.toml +++ b/crates/brk_error/Cargo.toml @@ -9,8 +9,7 @@ repository.workspace = true [features] bitcoin = ["dep:bitcoin"] -bitcoincore-rpc = ["dep:bitcoincore-rpc"] -corepc = ["dep:corepc-client"] +corepc = ["dep:corepc-jsonrpc"] fjall = ["dep:fjall"] jiff = ["dep:jiff"] ureq = ["dep:ureq"] @@ -21,8 +20,7 @@ vecdb = ["dep:vecdb"] [dependencies] bitcoin = { workspace = true, optional = true } -bitcoincore-rpc = { workspace = true, optional = true } -corepc-client = { workspace = true, optional = true } +corepc-jsonrpc = { workspace = true, optional = true } fjall = { workspace = true, optional = true } jiff = { workspace = true, optional = true } ureq = { workspace = true, optional = true } diff --git a/crates/brk_error/src/lib.rs b/crates/brk_error/src/lib.rs index 0730bceac..c0a6908a3 100644 --- a/crates/brk_error/src/lib.rs +++ b/crates/brk_error/src/lib.rs @@ -26,13 +26,9 @@ pub enum Error { #[error(transparent)] IO(#[from] io::Error), - #[cfg(feature = "bitcoincore-rpc")] - #[error(transparent)] - BitcoinRPC(#[from] bitcoincore_rpc::Error), - #[cfg(feature = "corepc")] #[error(transparent)] - CorepcRPC(#[from] corepc_client::client_sync::Error), + CorepcRPC(#[from] corepc_jsonrpc::error::Error), #[cfg(feature = "jiff")] #[error(transparent)] diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 479aa8269..87d5646aa 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -14,7 +14,7 @@ brk_error = { workspace = true, features = ["fjall", "vecdb"] } brk_cohort = { workspace = true } brk_logger = { workspace = true } brk_reader = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_store = { workspace = true } brk_types = { workspace = true } brk_traversable = { workspace = true } diff --git a/crates/brk_iterator/Cargo.toml b/crates/brk_iterator/Cargo.toml index 359fe4ef6..43a02a12a 100644 --- a/crates/brk_iterator/Cargo.toml +++ b/crates/brk_iterator/Cargo.toml @@ -11,5 +11,5 @@ exclude = ["examples/"] [dependencies] brk_error = { workspace = true } brk_reader = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_types = { workspace = true } diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index dea11ca12..70b32015b 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -27,7 +27,7 @@ pub fn init(path: Option<&Path>) -> io::Result<()> { let directives = std::env::var("RUST_LOG").unwrap_or_else(|_| { format!( - "{level},bitcoin=off,bitcoincore_rpc=off,corepc=off,tracing=off,aide=off,fjall=off,lsm_tree=off,tower_http=off" + "{level},bitcoin=off,corepc=off,tracing=off,aide=off,fjall=off,lsm_tree=off,tower_http=off" ) }); diff --git a/crates/brk_mempool/Cargo.toml b/crates/brk_mempool/Cargo.toml index 5a45374b7..93418110a 100644 --- a/crates/brk_mempool/Cargo.toml +++ b/crates/brk_mempool/Cargo.toml @@ -11,7 +11,7 @@ exclude = ["examples/"] [dependencies] bitcoin = { workspace = true } brk_error = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_types = { workspace = true } derive_more = { workspace = true } tracing = { workspace = true } diff --git a/crates/brk_mempool/examples/mempool.rs b/crates/brk_mempool/examples/mempool.rs index 4d0d453d8..d4d66f604 100644 --- a/crates/brk_mempool/examples/mempool.rs +++ b/crates/brk_mempool/examples/mempool.rs @@ -26,8 +26,8 @@ fn main() -> Result<()> { thread::sleep(Duration::from_secs(5)); // Basic mempool info - let info = mempool.get_info(); - let block_stats = mempool.get_block_stats(); + let info = mempool.info(); + let block_stats = mempool.block_stats(); let total_fees: u64 = block_stats.iter().map(|s| u64::from(s.total_fee)).sum(); println!("\n=== Mempool Info ==="); println!(" Transactions: {}", info.count); @@ -38,7 +38,7 @@ fn main() -> Result<()> { ); // Fee recommendations (like mempool.space) - let fees = mempool.get_fees(); + let fees = mempool.fees(); println!("\n=== Recommended Fees (sat/vB) ==="); println!(" No Priority {:.4}", f64::from(fees.economy_fee)); println!(" Low Priority {:.4}", f64::from(fees.hour_fee)); @@ -63,7 +63,7 @@ fn main() -> Result<()> { } // Address tracking stats - let addrs = mempool.get_addrs(); + let addrs = mempool.addrs(); println!("\n=== Address Tracking ==="); println!(" Addresses with pending txs: {}", addrs.len()); diff --git a/crates/brk_mempool/src/block_builder/graph.rs b/crates/brk_mempool/src/block_builder/graph.rs deleted file mode 100644 index 2f5ea8f1a..000000000 --- a/crates/brk_mempool/src/block_builder/graph.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::ops::{Index, IndexMut}; - -use brk_types::TxidPrefix; -use rustc_hash::FxHashMap; - -use super::tx_node::TxNode; -use crate::{ - entry::Entry, - types::{PoolIndex, TxIndex}, -}; - -/// Type-safe wrapper around Vec that only allows PoolIndex access. -pub struct Graph(Vec); - -impl Graph { - #[inline] - pub fn len(&self) -> usize { - self.0.len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl Index for Graph { - type Output = TxNode; - - #[inline] - fn index(&self, idx: PoolIndex) -> &Self::Output { - &self.0[idx.as_usize()] - } -} - -impl IndexMut for Graph { - #[inline] - fn index_mut(&mut self, idx: PoolIndex) -> &mut Self::Output { - &mut self.0[idx.as_usize()] - } -} - -/// Build a dependency graph from mempool entries. -pub fn build_graph(entries: &[Option]) -> Graph { - let mut live: Vec<(TxIndex, &Entry)> = Vec::with_capacity(entries.len()); - for (i, opt) in entries.iter().enumerate() { - if let Some(e) = opt.as_ref() { - live.push((TxIndex::from(i), e)); - } - } - - if live.is_empty() { - return Graph(Vec::new()); - } - - let mut prefix_to_pool: FxHashMap = - FxHashMap::with_capacity_and_hasher(live.len(), Default::default()); - for (i, (_, entry)) in live.iter().enumerate() { - prefix_to_pool.insert(entry.txid_prefix(), PoolIndex::from(i)); - } - - let mut nodes: Vec = live - .iter() - .map(|(tx_index, entry)| { - let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize); - for parent_prefix in &entry.depends { - if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) { - node.parents.push(parent_pool_idx); - } - } - node - }) - .collect(); - - // Populate children via direct indexing; no intermediate edge vec. - // Reading parents[j] as a Copy value releases the immutable borrow - // before the mutable borrow of children's owner. - for i in 0..nodes.len() { - let plen = nodes[i].parents.len(); - for j in 0..plen { - let parent_idx = nodes[i].parents[j].as_usize(); - nodes[parent_idx].children.push(PoolIndex::from(i)); - } - } - - Graph(nodes) -} - -#[cfg(test)] -mod bench { - use std::time::Instant; - - use bitcoin::hashes::Hash; - use brk_types::{Sats, Timestamp, Txid, VSize}; - use smallvec::SmallVec; - - use super::build_graph; - use crate::entry::Entry; - - /// Synthetic mempool: mostly singletons, some CPFP chains/trees. - fn synthetic_mempool(n: usize) -> Vec> { - let make_txid = |i: usize| -> Txid { - let mut bytes = [0u8; 32]; - bytes[0..8].copy_from_slice(&(i as u64).to_ne_bytes()); - bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2654435761)).to_ne_bytes()); - Txid::from(bitcoin::Txid::from_slice(&bytes).unwrap()) - }; - - let mut entries: Vec> = Vec::with_capacity(n); - let mut txids: Vec = Vec::with_capacity(n); - for i in 0..n { - let txid = make_txid(i); - txids.push(txid.clone()); - - // 95% singletons, 4% 1-parent, 1% 2-parent (mimics real mempool). - let depends: SmallVec<[brk_types::TxidPrefix; 2]> = match i % 100 { - 0..=94 => SmallVec::new(), - 95..=98 if i > 0 => { - let p = (i.wrapping_mul(7919)) % i; - std::iter::once(brk_types::TxidPrefix::from(&txids[p])).collect() - } - _ if i > 1 => { - let p1 = (i.wrapping_mul(7919)) % i; - let p2 = (i.wrapping_mul(6151)) % i; - [ - brk_types::TxidPrefix::from(&txids[p1]), - brk_types::TxidPrefix::from(&txids[p2]), - ] - .into_iter() - .collect() - } - _ => SmallVec::new(), - }; - - entries.push(Some(Entry { - txid, - fee: Sats::from((i as u64).wrapping_mul(137) % 10_000 + 1), - vsize: VSize::from(250u64), - size: 250, - ancestor_fee: Sats::from(0u64), - ancestor_vsize: VSize::from(250u64), - depends, - first_seen: Timestamp::now(), - })); - } - entries - } - - #[test] - #[ignore = "perf benchmark; run with --ignored --nocapture"] - fn perf_build_graph() { - let sizes = [1_000usize, 10_000, 50_000, 100_000, 300_000]; - eprintln!(); - eprintln!("build_graph perf (release, single call):"); - eprintln!(" n build"); - eprintln!(" ------------------------"); - for &n in &sizes { - let entries = synthetic_mempool(n); - // Warm up allocator. - let _ = build_graph(&entries); - - let t = Instant::now(); - let g = build_graph(&entries); - let dt = t.elapsed(); - let ns = dt.as_nanos(); - let pretty = if ns >= 1_000_000 { - format!("{:.2} ms", ns as f64 / 1_000_000.0) - } else { - format!("{:.2} µs", ns as f64 / 1_000.0) - }; - eprintln!(" {:<10} {:<10} ({} nodes)", n, pretty, g.len()); - } - eprintln!(); - } -} diff --git a/crates/brk_mempool/src/entry.rs b/crates/brk_mempool/src/entry.rs deleted file mode 100644 index 671db5efd..000000000 --- a/crates/brk_mempool/src/entry.rs +++ /dev/null @@ -1,47 +0,0 @@ -use brk_types::{FeeRate, Sats, Timestamp, Txid, TxidPrefix, VSize}; -use smallvec::SmallVec; - -/// A mempool transaction entry. -/// -/// Stores only the data needed for fee estimation and block building. -/// Ancestor values are pre-computed by Bitcoin Core (correctly handling shared ancestors). -#[derive(Debug, Clone)] -pub struct Entry { - pub txid: Txid, - pub fee: Sats, - pub vsize: VSize, - /// Serialized tx size in bytes (witness + non-witness), from the raw tx. - pub size: u64, - /// Pre-computed ancestor fee (self + all ancestors, no double-counting) - pub ancestor_fee: Sats, - /// Pre-computed ancestor vsize (self + all ancestors, no double-counting) - pub ancestor_vsize: VSize, - /// Parent txid prefixes (most txs have 0-2 parents) - pub depends: SmallVec<[TxidPrefix; 2]>, - /// When this tx was first seen in the mempool - pub first_seen: Timestamp, -} - -impl Entry { - #[inline] - pub fn fee_rate(&self) -> FeeRate { - FeeRate::from((self.fee, self.vsize)) - } - - /// Ancestor fee rate (package rate for CPFP). - #[inline] - pub fn ancestor_fee_rate(&self) -> FeeRate { - FeeRate::from((self.ancestor_fee, self.ancestor_vsize)) - } - - /// Effective fee rate for display. - #[inline] - pub fn effective_fee_rate(&self) -> FeeRate { - self.fee_rate().max(self.ancestor_fee_rate()) - } - - #[inline] - pub fn txid_prefix(&self) -> TxidPrefix { - TxidPrefix::from(&self.txid) - } -} diff --git a/crates/brk_mempool/src/entry_pool.rs b/crates/brk_mempool/src/entry_pool.rs deleted file mode 100644 index df00c8e2a..000000000 --- a/crates/brk_mempool/src/entry_pool.rs +++ /dev/null @@ -1,84 +0,0 @@ -use brk_types::TxidPrefix; -use rustc_hash::FxHashMap; -use smallvec::SmallVec; - -use crate::{entry::Entry, types::TxIndex}; - -/// Pool of mempool entries with slot recycling. -/// -/// Uses a slot-based storage where removed entries leave holes -/// that get reused for new entries, avoiding index invalidation. -#[derive(Default)] -pub struct EntryPool { - entries: Vec>, - prefix_to_idx: FxHashMap, - parent_to_children: FxHashMap>, - free_slots: Vec, -} - -impl EntryPool { - /// Insert an entry, returning its index. - pub fn insert(&mut self, prefix: TxidPrefix, entry: Entry) -> TxIndex { - for parent in &entry.depends { - self.parent_to_children - .entry(*parent) - .or_default() - .push(prefix); - } - - let idx = match self.free_slots.pop() { - Some(idx) => { - self.entries[idx.as_usize()] = Some(entry); - idx - } - None => { - let idx = TxIndex::from(self.entries.len()); - self.entries.push(Some(entry)); - idx - } - }; - - self.prefix_to_idx.insert(prefix, idx); - idx - } - - /// Get an entry by its txid prefix. - pub fn get(&self, prefix: &TxidPrefix) -> Option<&Entry> { - let idx = self.prefix_to_idx.get(prefix)?; - self.entries.get(idx.as_usize())?.as_ref() - } - - /// Get direct children of a transaction (txs that depend on it). - pub fn children(&self, prefix: &TxidPrefix) -> &[TxidPrefix] { - self.parent_to_children - .get(prefix) - .map(SmallVec::as_slice) - .unwrap_or_default() - } - - /// Remove an entry by its txid prefix. - pub fn remove(&mut self, prefix: &TxidPrefix) { - if let Some(idx) = self.prefix_to_idx.remove(prefix) { - if let Some(entry) = self.entries.get(idx.as_usize()).and_then(|e| e.as_ref()) { - for parent in &entry.depends { - if let Some(children) = self.parent_to_children.get_mut(parent) { - children.retain(|c| c != prefix); - if children.is_empty() { - self.parent_to_children.remove(parent); - } - } - } - } - self.parent_to_children.remove(prefix); - if let Some(slot) = self.entries.get_mut(idx.as_usize()) { - *slot = None; - } - self.free_slots.push(idx); - } - } - - /// Get the entries slice for block building. - pub fn entries(&self) -> &[Option] { - &self.entries - } -} diff --git a/crates/brk_mempool/src/lib.rs b/crates/brk_mempool/src/lib.rs index ded1ea59f..373df8174 100644 --- a/crates/brk_mempool/src/lib.rs +++ b/crates/brk_mempool/src/lib.rs @@ -1,11 +1,172 @@ -mod addrs; -mod block_builder; -mod entry; -mod entry_pool; -mod projected_blocks; -mod sync; -mod tx_store; -mod types; +//! Live mempool monitor for the brk indexer. +//! +//! One pull cycle, five pipeline steps: +//! +//! 1. [`steps::fetcher::Fetcher`]: three batched RPCs against bitcoind +//! (verbose listing + raw txs for new entries + raw txs for +//! confirmed parents). Pure I/O. +//! 2. [`steps::preparer::Preparer`]: turn raw bytes into a typed diff +//! (`Pulled { added, removed }`), classifying additions as +//! Fresh or Revived and removals as Replaced or Vanished. +//! Pure CPU, no locks. +//! 3. [`steps::applier::Applier`]: apply the diff to the five-bucket +//! [`stores::state::MempoolState`] (info, txs, addrs, entries, +//! graveyard) under brief write locks. +//! 4. [`steps::resolver::Resolver`]: fill prevouts whose parents are +//! in the live mempool (run after every successful apply) +//! or via an external resolver supplied by the caller +//! (typically the brk indexer for confirmed parents). +//! 5. [`steps::rebuilder::Rebuilder`]: throttled rebuild of the +//! projected-blocks `Snapshot` consumed by the API. +//! +//! [`Mempool`] is the public entry point. `Mempool::start` drives the +//! cycle on a 1-second tick. +//! +//! Source layout: +//! +//! - `steps/` - one file or folder per pipeline step. +//! - `stores/` - the state buckets held inside `MempoolState` plus +//! the value types they contain. -pub use projected_blocks::{BlockStats, RecommendedFees, Snapshot}; -pub use sync::{Mempool, MempoolInner}; +mod steps; +mod stores; + +pub use steps::preparer::Removal; +pub use steps::rebuilder::projected_blocks::{BlockStats, RecommendedFees, Snapshot}; +pub use stores::{Entry, EntryPool, Tombstone, TxGraveyard, TxStore}; + +use std::{sync::Arc, thread, time::Duration}; + +use brk_error::Result; +use brk_rpc::Client; +use brk_types::{AddrBytes, MempoolInfo, TxOut, Txid, Vout}; +use parking_lot::RwLockReadGuard; +use tracing::error; + +use crate::{ + steps::{fetcher::Fetcher, preparer::Preparer, rebuilder::Rebuilder, resolver::Resolver}, + stores::{AddrTracker, MempoolState}, +}; + +/// Public entry point to the mempool monitor. +/// +/// Cheaply cloneable: wraps an `Arc` over the private state so clones +/// share a single live mempool. See the crate-level docs for the +/// pipeline shape. +#[derive(Clone)] +pub struct Mempool(Arc); + +struct Inner { + client: Client, + state: MempoolState, + rebuilder: Rebuilder, +} + +impl Mempool { + pub fn new(client: &Client) -> Self { + Self(Arc::new(Inner { + client: client.clone(), + state: MempoolState::default(), + rebuilder: Rebuilder::default(), + })) + } + + pub fn info(&self) -> MempoolInfo { + self.0.state.info.read().clone() + } + + pub fn snapshot(&self) -> Arc { + self.0.rebuilder.snapshot() + } + + pub fn fees(&self) -> RecommendedFees { + self.0.rebuilder.fees() + } + + pub fn block_stats(&self) -> Vec { + self.0.rebuilder.block_stats() + } + + pub fn next_block_hash(&self) -> u64 { + self.0.rebuilder.next_block_hash() + } + + pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 { + self.0.state.addrs.read().stats_hash(addr) + } + + pub fn txs(&self) -> RwLockReadGuard<'_, TxStore> { + self.0.state.txs.read() + } + + pub fn entries(&self) -> RwLockReadGuard<'_, EntryPool> { + self.0.state.entries.read() + } + + pub fn addrs(&self) -> RwLockReadGuard<'_, AddrTracker> { + self.0.state.addrs.read() + } + + pub fn graveyard(&self) -> RwLockReadGuard<'_, TxGraveyard> { + self.0.state.graveyard.read() + } + + /// Start an infinite update loop with a 1 second interval. + pub fn start(&self) { + self.start_with(|| {}); + } + + /// Variant of `start` that runs `after_update` after every cycle. + /// Used by `brk_cli` to drive `Query::fill_mempool_prevouts` so + /// indexer-resolvable prevouts get filled in place each tick. + pub fn start_with(&self, mut after_update: impl FnMut()) { + loop { + if let Err(e) = self.update() { + error!("Error updating mempool: {}", e); + } + after_update(); + thread::sleep(Duration::from_secs(1)); + } + } + + /// Fill any remaining `prevout == None` inputs on live mempool + /// txs using `resolver`. Only call this if you have an external + /// data source for confirmed parents (typically the brk indexer); + /// in-mempool same-cycle parents are filled automatically by + /// `MempoolState::apply` and don't need an external resolver. + pub fn fill_prevouts(&self, resolver: F) -> bool + where + F: Fn(&Txid, Vout) -> Option, + { + Resolver::resolve_external(&self.0.state, resolver) + } + + /// One sync cycle: fetch -> prepare -> apply -> resolve -> (maybe) rebuild. + /// The resolve step only runs when `apply` reported a change (no + /// new txs means no new unresolved prevouts to fill); the rebuild + /// step is throttled by `Rebuilder` regardless. + pub fn update(&self) -> Result<()> { + let inner = &*self.0; + + let fetched = Fetcher::fetch( + &inner.client, + &inner.state.txs.read(), + &inner.state.graveyard.read(), + )?; + + let pulled = Preparer::prepare( + fetched, + &inner.state.txs.read(), + &inner.state.graveyard.read(), + ); + + if inner.state.apply(pulled) { + Resolver::resolve_in_mempool(&inner.state); + inner.rebuilder.mark_dirty(); + } + + inner.rebuilder.tick(&inner.client, &inner.state.entries); + + Ok(()) + } +} diff --git a/crates/brk_mempool/src/steps/applier.rs b/crates/brk_mempool/src/steps/applier.rs new file mode 100644 index 000000000..de969b482 --- /dev/null +++ b/crates/brk_mempool/src/steps/applier.rs @@ -0,0 +1,69 @@ +use brk_types::{MempoolInfo, Transaction, Txid}; + +use crate::{ + steps::preparer::{Addition, Pulled}, + stores::{AddrTracker, EntryPool, TxGraveyard, TxStore}, +}; + +/// Applies a prepared diff to in-memory mempool state. +/// +/// Removals are torn down first: each tx+entry is moved into the +/// graveyard with its removal reason. +/// +/// Additions then publish to live state. For `Revived` additions the +/// tx body is exhumed from the graveyard (no clone); for `Fresh` ones +/// the tx arrives inline from the Preparer. +/// +/// Finally the graveyard evicts entries past its retention window. +pub struct Applier; + +impl Applier { + /// Apply `pulled` to all buckets. Returns true if anything changed. + pub fn apply( + pulled: Pulled, + info: &mut MempoolInfo, + txs: &mut TxStore, + addrs: &mut AddrTracker, + entries: &mut EntryPool, + graveyard: &mut TxGraveyard, + ) -> bool { + let Pulled { added, removed } = pulled; + let has_changes = !added.is_empty() || !removed.is_empty(); + + for (prefix, reason) in removed { + let Some(entry) = entries.remove(&prefix) else { + continue; + }; + let txid = entry.txid.clone(); + let Some(tx) = txs.remove(&txid) else { + continue; + }; + info.remove(&tx, entry.fee); + addrs.remove_tx(&tx, &txid); + graveyard.bury(txid, tx, entry, reason); + } + + let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len()); + for addition in added { + let (tx, entry) = match addition { + Addition::Fresh { tx, entry } => (tx, entry), + Addition::Revived { entry } => { + let Some(tomb) = graveyard.exhume(&entry.txid) else { + continue; + }; + (tomb.tx, entry) + } + }; + info.add(&tx, entry.fee); + addrs.add_tx(&tx, &entry.txid); + let txid = entry.txid.clone(); + entries.insert(entry); + to_store.push((txid, tx)); + } + txs.extend(to_store); + + graveyard.evict_old(); + + has_changes + } +} diff --git a/crates/brk_mempool/src/steps/fetcher/fetched.rs b/crates/brk_mempool/src/steps/fetcher/fetched.rs new file mode 100644 index 000000000..21dda6efc --- /dev/null +++ b/crates/brk_mempool/src/steps/fetcher/fetched.rs @@ -0,0 +1,10 @@ +use brk_rpc::RawTx; +use brk_types::{MempoolEntryInfo, Txid}; +use rustc_hash::FxHashMap; + +/// Raw RPC output for one pull cycle. Pure data; no interpretation. +pub struct Fetched { + pub entries_info: Vec, + pub new_raws: FxHashMap, + pub parent_raws: FxHashMap, +} diff --git a/crates/brk_mempool/src/steps/fetcher/mod.rs b/crates/brk_mempool/src/steps/fetcher/mod.rs new file mode 100644 index 000000000..9a5443642 --- /dev/null +++ b/crates/brk_mempool/src/steps/fetcher/mod.rs @@ -0,0 +1,80 @@ +mod fetched; + +pub use fetched::Fetched; + +use brk_error::Result; +use brk_rpc::{Client, RawTx}; +use brk_types::{MempoolEntryInfo, Txid}; +use rustc_hash::{FxHashMap, FxHashSet}; + +use crate::stores::{TxGraveyard, TxStore}; + +/// Cap on how many new txs we fetch per cycle (applied before the batch RPC +/// so we never hand bitcoind an unbounded batch). +const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000; + +/// Talks to Bitcoin Core. Three batched round-trips regardless of +/// mempool size: +/// 1. `getrawmempool verbose` - authoritative listing +/// 2. `getrawtransaction` batch - every new tx (txids not in +/// `known` / `graveyard`, capped at `MAX_TX_FETCHES_PER_CYCLE`) +/// 3. `getrawtransaction` batch - unique confirmed parents of those +/// new txs that aren't resolvable from `known` or step 2. +/// +/// Step 3 is best-effort: without `-txindex`, Core returns -5 for every +/// confirmed parent and the batch yields an empty map. `brk_query` +/// fills missing prevouts at read time from the indexer, so this is +/// purely a latency optimization when `-txindex` is available. +pub struct Fetcher; + +impl Fetcher { + pub fn fetch(client: &Client, known: &TxStore, graveyard: &TxGraveyard) -> Result { + let entries_info = client.get_raw_mempool_verbose()?; + + let new_txids = Self::new_txids(&entries_info, known, graveyard); + let new_raws = client.get_raw_transactions(&new_txids)?; + + let parent_txids = Self::unique_confirmed_parents(&new_raws, known); + let parent_raws = client.get_raw_transactions(&parent_txids)?; + + Ok(Fetched { + entries_info, + new_raws, + parent_raws, + }) + } + + /// Txids in the listing that we don't already have cached (live or + /// buried) and therefore need to fetch raw bytes for. Order-preserving + /// so the batch matches the listing order for debuggability. + fn new_txids( + entries_info: &[MempoolEntryInfo], + known: &TxStore, + graveyard: &TxGraveyard, + ) -> Vec { + entries_info + .iter() + .filter(|info| !known.contains(&info.txid) && !graveyard.contains(&info.txid)) + .take(MAX_TX_FETCHES_PER_CYCLE) + .map(|info| info.txid.clone()) + .collect() + } + + /// Parent txids referenced by `new_raws` inputs that aren't already + /// resolvable: not in the mempool store, not in `new_raws` itself. + fn unique_confirmed_parents( + new_raws: &FxHashMap, + known: &TxStore, + ) -> Vec { + let mut set: FxHashSet = FxHashSet::default(); + for raw in new_raws.values() { + for txin in &raw.tx.input { + let prev: Txid = txin.previous_output.txid.into(); + if !known.contains_key(&prev) && !new_raws.contains_key(&prev) { + set.insert(prev); + } + } + } + set.into_iter().collect() + } +} diff --git a/crates/brk_mempool/src/steps/mod.rs b/crates/brk_mempool/src/steps/mod.rs new file mode 100644 index 000000000..b5f70c084 --- /dev/null +++ b/crates/brk_mempool/src/steps/mod.rs @@ -0,0 +1,7 @@ +//! The five pipeline steps. See the crate-level docs for the cycle. + +pub mod applier; +pub mod fetcher; +pub mod preparer; +pub mod rebuilder; +pub mod resolver; diff --git a/crates/brk_mempool/src/steps/preparer/added.rs b/crates/brk_mempool/src/steps/preparer/added.rs new file mode 100644 index 000000000..2b309d192 --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/added.rs @@ -0,0 +1,124 @@ +//! Classification and construction of newly-observed mempool txs. +//! +//! Two kinds of arrival: +//! - **Fresh**: the tx is unknown to us, so we decode the raw bytes, +//! resolve prevouts against `known` or `parent_raws`, and build a +//! full `Transaction` + `Entry`. +//! - **Revived**: the tx is in the graveyard. We rebuild the `Entry` +//! (preserving `first_seen` / `rbf` / `size`) and let the Applier +//! exhume the cached tx body. No raw decoding. + +use std::mem; + +use brk_rpc::RawTx; +use brk_types::{ + MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, TxidPrefix, VSize, Vout, +}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +use crate::stores::{Entry, Tombstone, TxStore}; + +/// A newly observed tx. `Fresh` carries decoded raw data (just parsed +/// from `new_raws`); `Revived` carries only the rebuilt entry because +/// the tx body is still sitting in the graveyard and will be exhumed +/// by the Applier. +pub enum Addition { + Fresh { tx: Transaction, entry: Entry }, + Revived { entry: Entry }, +} + +/// Decode a raw tx into a full `Fresh` addition. Resolves prevouts +/// against the live mempool first, then `parent_raws` (confirmed +/// parents fetched in step 3 of the Fetcher pipeline). Inputs whose +/// parent isn't in either source land with `prevout: None` and are +/// filled later by the Resolver or by `brk_query` at read time. +pub(super) fn fresh( + info: &MempoolEntryInfo, + mut raw: RawTx, + parent_raws: &FxHashMap, + mempool_txs: &TxStore, +) -> Addition { + let total_size = raw.hex.len() / 2; + let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf()); + + let input = mem::take(&mut raw.tx.input) + .into_iter() + .map(|txin| { + let prev_txid: Txid = txin.previous_output.txid.into(); + let prev_vout = usize::from(Vout::from(txin.previous_output.vout)); + + let prevout = if let Some(prev) = mempool_txs.get(&prev_txid) { + prev.output + .get(prev_vout) + .map(|o| TxOut::from((o.script_pubkey.clone(), o.value))) + } else if let Some(parent) = parent_raws.get(&prev_txid) { + parent + .tx + .output + .get(prev_vout) + .map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into()))) + } else { + None + }; + + TxIn { + // Mempool txs are never coinbase (Core rejects + // them from the pool entirely). A missing prevout + // only means we couldn't resolve the confirmed + // parent (no `-txindex`); brk_query fills it at + // read time from the indexer. + is_coinbase: false, + prevout, + txid: prev_txid, + vout: txin.previous_output.vout.into(), + script_sig: txin.script_sig, + script_sig_asm: (), + witness: txin.witness.into(), + sequence: txin.sequence.into(), + inner_redeem_script_asm: (), + inner_witness_script_asm: (), + } + }) + .collect(); + + let mut tx = Transaction { + index: None, + txid: info.txid.clone(), + version: raw.tx.version.into(), + total_sigop_cost: 0, + weight: info.weight.into(), + lock_time: raw.tx.lock_time.into(), + total_size, + fee: info.fee, + input, + output: raw.tx.output.into_iter().map(TxOut::from).collect(), + status: TxStatus::UNCONFIRMED, + }; + tx.total_sigop_cost = tx.total_sigop_cost(); + + let entry = build_entry(info, tx.total_size as u64, rbf, Timestamp::now()); + + Addition::Fresh { tx, entry } +} + +/// Resurrect an entry from a tombstone. The tx body stays buried +/// until the Applier exhumes it; we only rebuild the `Entry` so the +/// preserved `first_seen` / `rbf` / `size` carry over. +pub(super) fn revived(info: &MempoolEntryInfo, tomb: &Tombstone) -> Addition { + let entry = build_entry(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen); + Addition::Revived { entry } +} + +fn build_entry(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Entry { + let depends: SmallVec<[TxidPrefix; 2]> = info.depends.iter().map(TxidPrefix::from).collect(); + Entry { + txid: info.txid.clone(), + fee: info.fee, + vsize: VSize::from(info.vsize), + size, + depends, + first_seen, + rbf, + } +} diff --git a/crates/brk_mempool/src/steps/preparer/mod.rs b/crates/brk_mempool/src/steps/preparer/mod.rs new file mode 100644 index 000000000..81d247ca5 --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/mod.rs @@ -0,0 +1,61 @@ +//! Pipeline step 2: turn `Fetched` raws into a typed diff for the Applier. +//! +//! Pure CPU work, no locks. Three classes of new tx are handled: +//! - **live**: already in `known`, skipped (no update needed) +//! - **revivable**: in the graveyard, resurrected from the tombstone +//! - **fresh**: decoded from `new_raws`, prevouts resolved against +//! `known` or `parent_raws`, RBF detected from the raw tx +//! +//! Removals come from cross-referencing inputs (see `removed.rs`). + +mod added; +mod pulled; +mod removed; + +pub use added::Addition; +pub use pulled::Pulled; +pub use removed::Removal; + +use brk_types::TxidPrefix; +use rustc_hash::FxHashSet; + +use crate::{ + steps::fetcher::Fetched, + stores::{TxGraveyard, TxStore}, +}; + +pub struct Preparer; + +impl Preparer { + pub fn prepare(fetched: Fetched, known: &TxStore, graveyard: &TxGraveyard) -> Pulled { + let Fetched { + entries_info, + mut new_raws, + parent_raws, + } = fetched; + + let mut added: Vec = Vec::new(); + let mut live: FxHashSet = + FxHashSet::with_capacity_and_hasher(entries_info.len(), Default::default()); + + for info in &entries_info { + live.insert(TxidPrefix::from(&info.txid)); + + if known.contains(&info.txid) { + continue; + } + if let Some(tomb) = graveyard.get(&info.txid) { + added.push(added::revived(info, tomb)); + continue; + } + let Some(raw) = new_raws.remove(&info.txid) else { + continue; + }; + added.push(added::fresh(info, raw, &parent_raws, known)); + } + + let removed = removed::classify(&live, &added, known); + + Pulled { added, removed } + } +} diff --git a/crates/brk_mempool/src/steps/preparer/pulled.rs b/crates/brk_mempool/src/steps/preparer/pulled.rs new file mode 100644 index 000000000..f85ae54e8 --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/pulled.rs @@ -0,0 +1,10 @@ +use brk_types::TxidPrefix; +use rustc_hash::FxHashMap; + +use super::{Addition, Removal}; + +/// Output of one pull cycle: the full diff, ready for the Applier. +pub struct Pulled { + pub added: Vec, + pub removed: FxHashMap, +} diff --git a/crates/brk_mempool/src/steps/preparer/removed.rs b/crates/brk_mempool/src/steps/preparer/removed.rs new file mode 100644 index 000000000..5d681ae3e --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/removed.rs @@ -0,0 +1,58 @@ +//! Classification of txs that left the mempool between two pull cycles. +//! +//! `Replaced` = at least one added tx this cycle spends one of its +//! inputs (BIP-125 replacement inferred from conflicting outpoints). +//! `Vanished` = any other reason we can't distinguish from the data +//! at hand (mined, expired, evicted, or replaced by a tx we didn't +//! fetch due to the per-cycle fetch cap). + +use brk_types::{Txid, TxidPrefix, Vout}; +use rustc_hash::{FxHashMap, FxHashSet}; + +use super::added::Addition; +use crate::stores::TxStore; + +#[derive(Debug)] +pub enum Removal { + Replaced { by: Txid }, + Vanished, +} + +/// Diff the store against Core's listing. `live` is the set of txid +/// prefixes Core returned this cycle; anything in `known` whose prefix +/// isn't in `live` left the pool. Each loser is classified by cross- +/// referencing its inputs against the freshly added txs' inputs. +pub(super) fn classify( + live: &FxHashSet, + added: &[Addition], + known: &TxStore, +) -> FxHashMap { + // (parent txid, vout) -> Txid of the new tx that spends it. + // Only `Fresh` additions carry tx input data; revived txs were + // already in-pool and can't be "new spenders" of anything. + let mut spent_by: FxHashMap<(Txid, Vout), Txid> = FxHashMap::default(); + for addition in added { + if let Addition::Fresh { tx, .. } = addition { + for txin in &tx.input { + spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone()); + } + } + } + + known + .iter() + .filter_map(|(txid, tx)| { + let prefix = TxidPrefix::from(txid); + if live.contains(&prefix) { + return None; + } + let removal = tx + .input + .iter() + .find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned()) + .map(|by| Removal::Replaced { by }) + .unwrap_or(Removal::Vanished); + Some((prefix, removal)) + }) + .collect() +} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs new file mode 100644 index 000000000..0ada76486 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs @@ -0,0 +1,85 @@ +use std::ops::{Index, IndexMut}; + +use brk_types::TxidPrefix; +use rustc_hash::FxHashMap; + +use super::{pool_index::PoolIndex, tx_node::TxNode}; +use crate::stores::{Entry, TxIndex}; + +/// Type-safe wrapper around Vec that only allows PoolIndex access. +pub struct Graph(Vec); + +impl Graph { + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl Index for Graph { + type Output = TxNode; + + #[inline] + fn index(&self, idx: PoolIndex) -> &Self::Output { + &self.0[idx.as_usize()] + } +} + +impl IndexMut for Graph { + #[inline] + fn index_mut(&mut self, idx: PoolIndex) -> &mut Self::Output { + &mut self.0[idx.as_usize()] + } +} + +/// Build a dependency graph from mempool entries. +pub fn build_graph(entries: &[Option]) -> Graph { + // Pass 1: collect live entries and index their prefixes in lockstep. + // We can't resolve parent links yet because a parent may sit later in + // slot order than its child, so prefix_to_pool needs to be complete + // before we touch `entry.depends`. + let mut live: Vec<(TxIndex, &Entry)> = Vec::with_capacity(entries.len()); + let mut prefix_to_pool: FxHashMap = + FxHashMap::with_capacity_and_hasher(entries.len(), Default::default()); + for (i, opt) in entries.iter().enumerate() { + if let Some(e) = opt.as_ref() { + prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len())); + live.push((TxIndex::from(i), e)); + } + } + + if live.is_empty() { + return Graph(Vec::new()); + } + + // Pass 2: materialize nodes with their parent edges. + let mut nodes: Vec = live + .iter() + .map(|(tx_index, entry)| { + let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize); + for parent_prefix in &entry.depends { + if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) { + node.parents.push(parent_pool_idx); + } + } + node + }) + .collect(); + + // Pass 3: mirror parent edges as children. Direct indexing only; + // no intermediate edge vec. + for i in 0..nodes.len() { + let plen = nodes[i].parents.len(); + for j in 0..plen { + let parent_idx = nodes[i].parents[j].as_usize(); + nodes[parent_idx].children.push(PoolIndex::from(i)); + } + } + + Graph(nodes) +} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs new file mode 100644 index 000000000..290b72ea4 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs @@ -0,0 +1,88 @@ +//! Throwaway perf bench for `build_graph`. +//! +//! Run with `cargo test --release -p brk_mempool -- --ignored --nocapture +//! perf_build_graph`. Not part of the regular test sweep. + +use std::time::Instant; + +use bitcoin::hashes::Hash; +use brk_types::{Sats, Timestamp, Txid, TxidPrefix, VSize}; +use smallvec::SmallVec; + +use super::graph::build_graph; +use crate::stores::Entry; + +/// Synthetic mempool: mostly singletons, some CPFP chains/trees. +fn synthetic_mempool(n: usize) -> Vec> { + let make_txid = |i: usize| -> Txid { + let mut bytes = [0u8; 32]; + bytes[0..8].copy_from_slice(&(i as u64).to_ne_bytes()); + bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2654435761)).to_ne_bytes()); + Txid::from(bitcoin::Txid::from_slice(&bytes).unwrap()) + }; + + let mut entries: Vec> = Vec::with_capacity(n); + let mut txids: Vec = Vec::with_capacity(n); + for i in 0..n { + let txid = make_txid(i); + txids.push(txid.clone()); + + // 95% singletons, 4% 1-parent, 1% 2-parent (mimics real mempool). + let depends: SmallVec<[TxidPrefix; 2]> = match i % 100 { + 0..=94 => SmallVec::new(), + 95..=98 if i > 0 => { + let p = (i.wrapping_mul(7919)) % i; + std::iter::once(TxidPrefix::from(&txids[p])).collect() + } + _ if i > 1 => { + let p1 = (i.wrapping_mul(7919)) % i; + let p2 = (i.wrapping_mul(6151)) % i; + [ + TxidPrefix::from(&txids[p1]), + TxidPrefix::from(&txids[p2]), + ] + .into_iter() + .collect() + } + _ => SmallVec::new(), + }; + + entries.push(Some(Entry { + txid, + fee: Sats::from((i as u64).wrapping_mul(137) % 10_000 + 1), + vsize: VSize::from(250u64), + size: 250, + depends, + first_seen: Timestamp::now(), + rbf: false, + })); + } + entries +} + +#[test] +#[ignore = "perf benchmark; run with --ignored --nocapture"] +fn perf_build_graph() { + let sizes = [1_000usize, 10_000, 50_000, 100_000, 300_000]; + eprintln!(); + eprintln!("build_graph perf (release, single call):"); + eprintln!(" n build"); + eprintln!(" ------------------------"); + for &n in &sizes { + let entries = synthetic_mempool(n); + // Warm up allocator. + let _ = build_graph(&entries); + + let t = Instant::now(); + let g = build_graph(&entries); + let dt = t.elapsed(); + let ns = dt.as_nanos(); + let pretty = if ns >= 1_000_000 { + format!("{:.2} ms", ns as f64 / 1_000_000.0) + } else { + format!("{:.2} µs", ns as f64 / 1_000.0) + }; + eprintln!(" {:<10} {:<10} ({} nodes)", n, pretty, g.len()); + } + eprintln!(); +} diff --git a/crates/brk_mempool/src/block_builder/linearize/mod.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs similarity index 93% rename from crates/brk_mempool/src/block_builder/linearize/mod.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs index 160b3d3db..e4e88cde5 100644 --- a/crates/brk_mempool/src/block_builder/linearize/mod.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs @@ -15,8 +15,8 @@ use brk_types::{FeeRate, Sats, VSize}; use rustc_hash::FxHashMap; use smallvec::SmallVec; -use super::{graph::Graph, package::Package}; -use crate::types::{PoolIndex, TxIndex}; +use super::{graph::Graph, package::Package, pool_index::PoolIndex}; +use crate::stores::TxIndex; /// Cluster-local index for a node within one cluster's flat array. type LocalIdx = u32; @@ -59,13 +59,13 @@ pub fn linearize_clusters(graph: &Graph) -> Vec { packages } -/// BFS over (parents + children) adjacency to partition `graph` into +/// DFS over (parents + children) adjacency to partition `graph` into /// connected components, each re-indexed locally. fn find_components(graph: &Graph) -> Vec { let n = graph.len(); let mut seen: Vec = vec![false; n]; let mut clusters: Vec = Vec::new(); - let mut queue: Vec = Vec::new(); + let mut stack: Vec = Vec::new(); for start in 0..n { if seen[start] { @@ -73,23 +73,23 @@ fn find_components(graph: &Graph) -> Vec { } let mut members: Vec = Vec::new(); - queue.clear(); - queue.push(PoolIndex::from(start)); + stack.clear(); + stack.push(PoolIndex::from(start)); seen[start] = true; - while let Some(idx) = queue.pop() { + while let Some(idx) = stack.pop() { members.push(idx); let node = &graph[idx]; for &p in &node.parents { if !seen[p.as_usize()] { seen[p.as_usize()] = true; - queue.push(p); + stack.push(p); } } for &c in &node.children { if !seen[c.as_usize()] { seen[c.as_usize()] = true; - queue.push(c); + stack.push(c); } } } diff --git a/crates/brk_mempool/src/block_builder/linearize/sfl.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs similarity index 84% rename from crates/brk_mempool/src/block_builder/linearize/sfl.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs index 5e04eff54..f88b407fc 100644 --- a/crates/brk_mempool/src/block_builder/linearize/sfl.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs @@ -72,8 +72,17 @@ pub fn linearize(cluster: &Cluster) -> Vec { remaining &= !mask; } - canonicalize(&mut chunks); - chunks + canonicalize(chunks) +} + +/// Immutable inputs for the brute-force recursion. Packing them into a +/// struct keeps `recurse` to four moving args: `(idx, included, f, v)`. +struct Ctx<'a> { + topo_order: &'a [LocalIdx], + parents_mask: &'a [u128], + fee_of: &'a [u64], + vsize_of: &'a [u64], + remaining: u128, } /// Recursive enumeration of topologically-closed subsets of @@ -85,86 +94,46 @@ fn best_subset( fee_of: &[u64], vsize_of: &[u64], ) -> (u128, u64, u64) { - let mut best = (0u128, 0u64, 1u64); - recurse( - 0, + let ctx = Ctx { topo_order, parents_mask, - remaining, - 0, - 0, - 0, fee_of, vsize_of, - &mut best, - ); + remaining, + }; + let mut best = (0u128, 0u64, 1u64); + recurse(&ctx, 0, 0, 0, 0, &mut best); best } -#[allow(clippy::too_many_arguments)] -fn recurse( - idx: usize, - topo_order: &[LocalIdx], - parents_mask: &[u128], - remaining: u128, - included: u128, - f: u64, - v: u64, - fee_of: &[u64], - vsize_of: &[u64], - best: &mut (u128, u64, u64), -) { - if idx == topo_order.len() { +fn recurse(ctx: &Ctx, idx: usize, included: u128, f: u64, v: u64, best: &mut (u128, u64, u64)) { + if idx == ctx.topo_order.len() { if included != 0 && f as u128 * best.2 as u128 > best.1 as u128 * v as u128 { *best = (included, f, v); } return; } - let node = topo_order[idx]; + let node = ctx.topo_order[idx]; let bit = 1u128 << node; // Not in remaining, or a parent (within remaining) is excluded: // this node is forced-excluded, no branching. - if (bit & remaining) == 0 || (parents_mask[node as usize] & remaining & !included) != 0 { - recurse( - idx + 1, - topo_order, - parents_mask, - remaining, - included, - f, - v, - fee_of, - vsize_of, - best, - ); + if (bit & ctx.remaining) == 0 + || (ctx.parents_mask[node as usize] & ctx.remaining & !included) != 0 + { + recurse(ctx, idx + 1, included, f, v, best); return; } // Exclude - recurse( - idx + 1, - topo_order, - parents_mask, - remaining, - included, - f, - v, - fee_of, - vsize_of, - best, - ); + recurse(ctx, idx + 1, included, f, v, best); // Include recurse( + ctx, idx + 1, - topo_order, - parents_mask, - remaining, included | bit, - f + fee_of[node as usize], - v + vsize_of[node as usize], - fee_of, - vsize_of, + f + ctx.fee_of[node as usize], + v + ctx.vsize_of[node as usize], best, ); } @@ -239,10 +208,9 @@ fn best_ancestor_union( /// Single-pass stack merge: for each incoming chunk, merge it into /// the stack top while the merge would raise the top's feerate, then /// push. O(n) total regardless of how many merges cascade. -fn canonicalize(chunks: &mut Vec) { - let taken = std::mem::take(chunks); - let mut out: Vec = Vec::with_capacity(taken.len()); - for mut cur in taken { +fn canonicalize(chunks: Vec) -> Vec { + let mut out: Vec = Vec::with_capacity(chunks.len()); + for mut cur in chunks { while let Some(top) = out.last() { if cur.fee as u128 * top.vsize as u128 > top.fee as u128 * cur.vsize as u128 { let mut prev = out.pop().unwrap(); @@ -256,7 +224,7 @@ fn canonicalize(chunks: &mut Vec) { } out.push(cur); } - *chunks = out; + out } #[inline] diff --git a/crates/brk_mempool/src/block_builder/linearize/tests/basic.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs similarity index 100% rename from crates/brk_mempool/src/block_builder/linearize/tests/basic.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs diff --git a/crates/brk_mempool/src/block_builder/linearize/tests/mod.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs similarity index 98% rename from crates/brk_mempool/src/block_builder/linearize/tests/mod.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs index 069f6a3ad..538d9ae2f 100644 --- a/crates/brk_mempool/src/block_builder/linearize/tests/mod.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs @@ -13,7 +13,7 @@ use smallvec::SmallVec; use super::sfl::Chunk; use super::{Cluster, ClusterNode, LocalIdx, kahn_topo_rank, sfl}; -use crate::types::TxIndex; +use crate::stores::TxIndex; /// Build a `Cluster` from `(fee, vsize)` tuples plus a list of /// `(parent_local, child_local)` edges. Tx indices are assigned 0..n. diff --git a/crates/brk_mempool/src/block_builder/linearize/tests/oracle.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs similarity index 98% rename from crates/brk_mempool/src/block_builder/linearize/tests/oracle.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs index 79900282e..dfc664621 100644 --- a/crates/brk_mempool/src/block_builder/linearize/tests/oracle.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs @@ -296,9 +296,12 @@ impl DagRng { } } +/// `(fee, vsize)` per node + edge list. Used by random-DAG generators. +type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>); + /// Random DAG with `n` nodes: each node i > 0 has 0-3 parents drawn /// uniformly from nodes {0..i}. Fees/vsizes are varied. -fn random_dag(n: usize, seed: u64) -> (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>) { +fn random_dag(n: usize, seed: u64) -> FvAndEdges { let mut rng = DagRng::new(seed); let fees_vsizes: Vec<(u64, u64)> = (0..n) .map(|_| { @@ -324,6 +327,7 @@ fn random_dag(n: usize, seed: u64) -> (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx) (fees_vsizes, edges) } +#[expect(dead_code, reason = "kept for ad-hoc oracle sweeps; called via uncommented stress tests")] fn assert_optimal_on_random(n: usize, seed: u64) { let (fv, edges) = random_dag(n, seed); let cluster = super::make_cluster(&fv, &edges); diff --git a/crates/brk_mempool/src/block_builder/linearize/tests/stress.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs similarity index 97% rename from crates/brk_mempool/src/block_builder/linearize/tests/stress.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs index 455c41089..6c1e468de 100644 --- a/crates/brk_mempool/src/block_builder/linearize/tests/stress.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs @@ -29,10 +29,13 @@ impl Rng { } } +/// `(fee, vsize)` per node + edge list. +type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>); + /// Build a random DAG with `n` nodes. For each node `i` > 0, add a /// random number of parents from nodes with index < i (guarantees /// acyclic). Fee and vsize are random in a small range. -fn random_cluster(n: usize, seed: u64) -> (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>) { +fn random_cluster(n: usize, seed: u64) -> FvAndEdges { let mut rng = Rng::new(seed); let mut fees_vsizes = Vec::with_capacity(n); for _ in 0..n { diff --git a/crates/brk_mempool/src/block_builder/mod.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs similarity index 92% rename from crates/brk_mempool/src/block_builder/mod.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs index ede7da011..c52441cd1 100644 --- a/crates/brk_mempool/src/block_builder/mod.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs @@ -2,11 +2,15 @@ mod graph; mod linearize; mod package; mod partitioner; +mod pool_index; mod tx_node; +#[cfg(test)] +mod graph_bench; + pub use package::Package; -use crate::entry::Entry; +use crate::stores::Entry; /// Target vsize per block (~1MB, derived from 4MW weight limit). pub(crate) const BLOCK_VSIZE: u64 = 1_000_000; diff --git a/crates/brk_mempool/src/block_builder/package.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs similarity index 97% rename from crates/brk_mempool/src/block_builder/package.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs index 9aeb1d0e2..9384776d8 100644 --- a/crates/brk_mempool/src/block_builder/package.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs @@ -1,6 +1,6 @@ use brk_types::FeeRate; -use crate::types::TxIndex; +use crate::stores::TxIndex; /// A CPFP package: transactions the linearizer decided to mine together /// because a child pays for its parent. diff --git a/crates/brk_mempool/src/block_builder/partitioner.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs similarity index 93% rename from crates/brk_mempool/src/block_builder/partitioner.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs index 3e37d17e4..ab8199ec4 100644 --- a/crates/brk_mempool/src/block_builder/partitioner.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs @@ -34,16 +34,10 @@ pub fn partition_into_blocks( let mut blocks: Vec> = Vec::with_capacity(num_blocks); let normal_blocks = num_blocks.saturating_sub(1); - let mut idx = fill_normal_blocks(&mut slots, &mut blocks, normal_blocks, &mut cluster_next); + let idx = fill_normal_blocks(&mut slots, &mut blocks, normal_blocks, &mut cluster_next); if blocks.len() < num_blocks { - let mut overflow: Vec = Vec::new(); - while idx < slots.len() { - if let Some(pkg) = slots[idx].take() { - overflow.push(pkg); - } - idx += 1; - } + let overflow: Vec = slots[idx..].iter_mut().filter_map(Option::take).collect(); if !overflow.is_empty() { blocks.push(overflow); } diff --git a/crates/brk_mempool/src/types/pool_index.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/pool_index.rs similarity index 100% rename from crates/brk_mempool/src/types/pool_index.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/pool_index.rs diff --git a/crates/brk_mempool/src/block_builder/tx_node.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs similarity index 92% rename from crates/brk_mempool/src/block_builder/tx_node.rs rename to crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs index d043d4948..6d408a20b 100644 --- a/crates/brk_mempool/src/block_builder/tx_node.rs +++ b/crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs @@ -1,7 +1,8 @@ use brk_types::{Sats, VSize}; use smallvec::SmallVec; -use crate::types::{PoolIndex, TxIndex}; +use super::pool_index::PoolIndex; +use crate::stores::TxIndex; /// A transaction node in the dependency graph. /// diff --git a/crates/brk_mempool/src/steps/rebuilder/mod.rs b/crates/brk_mempool/src/steps/rebuilder/mod.rs new file mode 100644 index 000000000..dd13e585d --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/mod.rs @@ -0,0 +1,111 @@ +pub mod block_builder; +pub mod projected_blocks; + +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::{SystemTime, UNIX_EPOCH}, +}; + +use brk_rpc::Client; +use parking_lot::RwLock; + +#[cfg(debug_assertions)] +use self::projected_blocks::verify::Verifier; +use self::{ + block_builder::build_projected_blocks, + projected_blocks::{BlockStats, RecommendedFees, Snapshot}, +}; +use crate::stores::EntryPool; + +/// Minimum interval between rebuilds (milliseconds). +const MIN_REBUILD_INTERVAL_MS: u64 = 1000; + +/// Owns the projected-blocks `Snapshot` and the scheduling around its +/// rebuild. +/// +/// Internally stateful: a `dirty` flag the Applier nudges after each +/// state change, a `last_rebuild_ms` throttle so we rebuild at most +/// once per `MIN_REBUILD_INTERVAL_MS` regardless of churn, and the +/// `Snapshot` itself swapped behind a cheap `Arc` so readers clone a +/// pointer, not the vectors inside. +#[derive(Default)] +pub struct Rebuilder { + snapshot: RwLock>, + dirty: AtomicBool, + last_rebuild_ms: AtomicU64, +} + +impl Rebuilder { + /// Signal that state has changed and a rebuild is eventually needed. + pub fn mark_dirty(&self) { + self.dirty.store(true, Ordering::Release); + } + + /// Rebuild iff dirty and enough time has passed since the last + /// run. Takes a short read lock on `entries` while building and + /// a short write lock on the internal snapshot at swap time. + pub fn tick(&self, client: &Client, entries: &RwLock) { + if !self.dirty.load(Ordering::Acquire) { + return; + } + + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + + let last = self.last_rebuild_ms.load(Ordering::Acquire); + if now_ms.saturating_sub(last) < MIN_REBUILD_INTERVAL_MS { + return; + } + + if self + .last_rebuild_ms + .compare_exchange(last, now_ms, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + self.dirty.store(false, Ordering::Release); + + let built = { + let entries = entries.read(); + let entries_slice = entries.entries(); + let blocks = build_projected_blocks(entries_slice); + + #[cfg(debug_assertions)] + Verifier::check(client, &blocks, entries_slice); + #[cfg(not(debug_assertions))] + let _ = client; + + Snapshot::build(blocks, entries_slice) + }; + + *self.snapshot.write() = Arc::new(built); + } + + /// Cheap: reader clones an `Arc` pointer and releases the lock. + fn current(&self) -> Arc { + self.snapshot.read().clone() + } + + pub fn snapshot(&self) -> Arc { + self.current() + } + + pub fn fees(&self) -> RecommendedFees { + self.current().fees.clone() + } + + pub fn block_stats(&self) -> Vec { + self.current().block_stats.clone() + } + + pub fn next_block_hash(&self) -> u64 { + self.current().next_block_hash + } +} diff --git a/crates/brk_mempool/src/projected_blocks/fees.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs similarity index 100% rename from crates/brk_mempool/src/projected_blocks/fees.rs rename to crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs diff --git a/crates/brk_mempool/src/projected_blocks/mod.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs similarity index 100% rename from crates/brk_mempool/src/projected_blocks/mod.rs rename to crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs diff --git a/crates/brk_mempool/src/projected_blocks/snapshot.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs similarity index 65% rename from crates/brk_mempool/src/projected_blocks/snapshot.rs rename to crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs index 8b9bca0e4..6b2774172 100644 --- a/crates/brk_mempool/src/projected_blocks/snapshot.rs +++ b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs @@ -3,10 +3,11 @@ use std::hash::{DefaultHasher, Hash, Hasher}; use brk_types::RecommendedFees; use super::{ + super::block_builder::Package, fees, stats::{self, BlockStats}, }; -use crate::{block_builder::Package, entry::Entry, types::TxIndex}; +use crate::stores::{Entry, TxIndex}; /// Immutable snapshot of projected blocks. #[derive(Debug, Clone, Default)] @@ -16,6 +17,12 @@ pub struct Snapshot { pub blocks: Vec>, pub block_stats: Vec, pub fees: RecommendedFees, + /// ETag-like cache key for the first projected block. A hash of + /// the block's tx ordering, not a Bitcoin block header hash (no + /// header exists yet - it's a projection). Precomputed at build + /// time since the snapshot is immutable; `0` iff there are no + /// projected blocks. + pub next_block_hash: u64, } impl Snapshot { @@ -28,21 +35,23 @@ impl Snapshot { let fees = fees::compute_recommended_fees(&block_stats); - let blocks = blocks + let blocks: Vec> = blocks .into_iter() .map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect()) .collect(); + let next_block_hash = Self::hash_next_block(&blocks); + Self { blocks, block_stats, fees, + next_block_hash, } } - /// Hash of the first projected block (the one about to be mined). - pub fn next_block_hash(&self) -> u64 { - let Some(block) = self.blocks.first() else { + fn hash_next_block(blocks: &[Vec]) -> u64 { + let Some(block) = blocks.first() else { return 0; }; let mut hasher = DefaultHasher::new(); diff --git a/crates/brk_mempool/src/projected_blocks/stats.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs similarity index 94% rename from crates/brk_mempool/src/projected_blocks/stats.rs rename to crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs index b94b8dc59..fb7e74738 100644 --- a/crates/brk_mempool/src/projected_blocks/stats.rs +++ b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs @@ -1,6 +1,7 @@ use brk_types::{FeeRate, Sats, VSize}; -use crate::{block_builder::Package, entry::Entry}; +use super::super::block_builder::Package; +use crate::stores::Entry; /// Statistics for a single projected block. #[derive(Debug, Clone, Default)] @@ -32,10 +33,6 @@ impl BlockStats { /// containing package's `fee_rate` to the percentile distribution, /// since that's the rate the miner collects per vsize. pub fn compute_block_stats(block: &[Package], entries: &[Option]) -> BlockStats { - if block.is_empty() { - return BlockStats::default(); - } - let mut total_fee = Sats::default(); let mut total_vsize = VSize::default(); let mut total_size: u64 = 0; diff --git a/crates/brk_mempool/src/projected_blocks/verify.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs similarity index 96% rename from crates/brk_mempool/src/projected_blocks/verify.rs rename to crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs index 008b9b4af..a829d7364 100644 --- a/crates/brk_mempool/src/projected_blocks/verify.rs +++ b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs @@ -3,11 +3,8 @@ use brk_types::{Sats, SatsSigned, TxidPrefix}; use rustc_hash::{FxHashMap, FxHashSet}; use tracing::{debug, warn}; -use crate::{ - block_builder::{BLOCK_VSIZE, Package}, - entry::Entry, - types::TxIndex, -}; +use super::super::block_builder::{BLOCK_VSIZE, Package}; +use crate::stores::{Entry, TxIndex}; type PrefixSet = FxHashSet; type FeeByPrefix = FxHashMap; @@ -48,12 +45,12 @@ impl Verifier { } } - fn live_entry<'e>( - entries: &'e [Option], + fn live_entry( + entries: &[Option], tx_index: TxIndex, b: usize, p: usize, - ) -> &'e Entry { + ) -> &Entry { entries[tx_index.as_usize()] .as_ref() .unwrap_or_else(|| panic!("block {b} pkg {p}: dead tx_index {tx_index:?}")) diff --git a/crates/brk_mempool/src/steps/resolver.rs b/crates/brk_mempool/src/steps/resolver.rs new file mode 100644 index 000000000..970622e8c --- /dev/null +++ b/crates/brk_mempool/src/steps/resolver.rs @@ -0,0 +1,139 @@ +//! Prevout resolution for live mempool txs. +//! +//! A fresh tx can land in the store with `prevout: None` on some +//! inputs when the Preparer can't see the parent (parent arrived in +//! the same cycle as the child, or parent is confirmed and Core +//! lacks `-txindex`). Two paths fix that, both writing through the +//! same `apply_fills` -> `add_input` plumbing: +//! +//! - [`Resolver::resolve_in_mempool`]: same-cycle parents from the +//! live `txs` map. Run by the orchestrator after each successful +//! `MempoolState::apply`. No external dependency. +//! - [`Resolver::resolve_external`]: caller-supplied resolver +//! (typically the brk indexer). Run on demand by API consumers +//! that have a confirmed-tx data source. Lock-free during the +//! resolver call. +//! +//! Both phases: +//! 1. Snapshot under `txs.read()`, gather work for unresolved txs +//! (early-exit if `txs.unresolved()` is empty). +//! 2. (external only) Call the resolver outside any lock. +//! 3. Write fills under `txs.write()` + `addrs.write()`, in that +//! order to match the Applier's lock order. +//! +//! Idempotent: `apply_fills` checks `prevout.is_none()` per input +//! and bails if the tx was removed between phases. + +use brk_types::{TxOut, Txid, Vin, Vout}; + +use crate::stores::MempoolState; + +/// Per-tx fills to apply: (vin index, resolved prevout). +type Fills = Vec<(Vin, TxOut)>; +/// Per-tx holes to resolve: (vin index, parent txid, parent vout). +type Holes = Vec<(Vin, Txid, Vout)>; + +pub struct Resolver; + +impl Resolver { + /// Fill prevouts whose parent is also live in the mempool. + /// + /// Called by the orchestrator after each successful + /// `MempoolState::apply`. Catches parent/child pairs that arrived + /// in the same cycle: the Preparer resolves against a snapshot + /// taken before the cycle's adds were applied, so neither parent + /// nor child is in it; both are in `txs` by the time we run. + pub fn resolve_in_mempool(state: &MempoolState) -> bool { + let filled: Vec<(Txid, Fills)> = { + let txs = state.txs.read(); + if txs.unresolved().is_empty() { + return false; + } + txs.unresolved() + .iter() + .filter_map(|txid| { + let tx = txs.get(txid)?; + let fills: Fills = tx + .input + .iter() + .enumerate() + .filter(|(_, txin)| txin.prevout.is_none()) + .filter_map(|(i, txin)| { + let parent = txs.get(&txin.txid)?; + let out = parent.output.get(usize::from(txin.vout))?; + Some((Vin::from(i), out.clone())) + }) + .collect(); + (!fills.is_empty()).then_some((txid.clone(), fills)) + }) + .collect() + }; + Self::write_back(state, filled) + } + + /// Fill prevouts via an external resolver, typically backed by the + /// brk indexer for confirmed parents. + /// + /// Phase 1 collects holes under `txs.read()`; phase 2 runs the + /// resolver outside any lock; phase 3 writes back. Holes already + /// resolvable from in-mempool parents have been filled by + /// [`Resolver::resolve_in_mempool`] in the preceding `apply`, so + /// anything reaching the resolver here is genuinely external. + pub fn resolve_external(state: &MempoolState, resolver: F) -> bool + where + F: Fn(&Txid, Vout) -> Option, + { + let holes: Vec<(Txid, Holes)> = { + let txs = state.txs.read(); + if txs.unresolved().is_empty() { + return false; + } + txs.unresolved() + .iter() + .filter_map(|txid| { + let tx = txs.get(txid)?; + let holes: Holes = tx + .input + .iter() + .enumerate() + .filter(|(_, txin)| txin.prevout.is_none()) + .map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout)) + .collect(); + (!holes.is_empty()).then_some((txid.clone(), holes)) + }) + .collect() + }; + + let filled: Vec<(Txid, Fills)> = holes + .into_iter() + .filter_map(|(txid, holes)| { + let fills: Fills = holes + .into_iter() + .filter_map(|(vin, prev_txid, vout)| { + resolver(&prev_txid, vout).map(|o| (vin, o)) + }) + .collect(); + (!fills.is_empty()).then_some((txid, fills)) + }) + .collect(); + + Self::write_back(state, filled) + } + + /// Apply per-tx fills under `txs.write()` + `addrs.write()`. + /// Each successful prevout write is folded into `AddrTracker` via + /// `add_input`. Lock order matches the Applier's (txs before addrs). + fn write_back(state: &MempoolState, fills: Vec<(Txid, Fills)>) -> bool { + if fills.is_empty() { + return false; + } + let mut txs = state.txs.write(); + let mut addrs = state.addrs.write(); + for (txid, tx_fills) in fills { + for prevout in txs.apply_fills(&txid, tx_fills) { + addrs.add_input(&txid, &prevout); + } + } + true + } +} diff --git a/crates/brk_mempool/src/addrs.rs b/crates/brk_mempool/src/stores/addr_tracker.rs similarity index 59% rename from crates/brk_mempool/src/addrs.rs rename to crates/brk_mempool/src/stores/addr_tracker.rs index 535a23d03..a9890d9e4 100644 --- a/crates/brk_mempool/src/addrs.rs +++ b/crates/brk_mempool/src/stores/addr_tracker.rs @@ -1,4 +1,6 @@ -use brk_types::{AddrBytes, AddrMempoolStats, Transaction, Txid}; +use std::hash::{DefaultHasher, Hash, Hasher}; + +use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid}; use derive_more::Deref; use rustc_hash::{FxHashMap, FxHashSet}; @@ -20,6 +22,34 @@ impl AddrTracker { self.update(tx, txid, false); } + /// Hash of an address's per-mempool stats. Stable while the address + /// is unchanged; cheaper to recompute than to track invalidation. + /// Returns 0 for unknown addresses (collision with a real hash is + /// astronomically unlikely and only costs one ETag false-hit if it + /// ever happens). + pub fn stats_hash(&self, addr: &AddrBytes) -> u64 { + let Some((stats, _)) = self.0.get(addr) else { + return 0; + }; + let mut hasher = DefaultHasher::new(); + stats.hash(&mut hasher); + hasher.finish() + } + + /// Fold a single newly-resolved input into the per-address stats. + /// Called by the Resolver after a prevout that was previously + /// `None` has been filled. Inputs whose prevout doesn't resolve + /// to an addr are no-ops. + pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) { + let Some(bytes) = prevout.addr_bytes() else { + return; + }; + let (stats, txids) = self.0.entry(bytes).or_default(); + txids.insert(txid.clone()); + stats.sending(prevout); + stats.update_tx_count(txids.len() as u32); + } + fn update(&mut self, tx: &Transaction, txid: &Txid, is_addition: bool) { // Inputs: track sending for txin in &tx.input { diff --git a/crates/brk_mempool/src/stores/entry.rs b/crates/brk_mempool/src/stores/entry.rs new file mode 100644 index 000000000..3e8d361a0 --- /dev/null +++ b/crates/brk_mempool/src/stores/entry.rs @@ -0,0 +1,39 @@ +use brk_types::{FeeRate, Sats, Timestamp, Txid, TxidPrefix, VSize}; +use smallvec::SmallVec; + +/// A mempool transaction entry. +/// +/// Stores only immutable per-tx facts. Ancestor aggregates are +/// deliberately not cached: they're derivable from the live +/// dependency graph, and any cached copy would go stale the moment +/// any ancestor confirms or is replaced. +#[derive(Debug, Clone)] +pub struct Entry { + pub txid: Txid, + pub fee: Sats, + pub vsize: VSize, + /// Serialized tx size in bytes (witness + non-witness), from the raw tx. + pub size: u64, + /// Parent txid prefixes (most txs have 0-2 parents). + /// + /// May reference parents no longer in the pool; consumers resolve + /// against the live pool and drop misses, so staleness here is + /// self-healing. + pub depends: SmallVec<[TxidPrefix; 2]>, + /// When this tx was first seen in the mempool. + pub first_seen: Timestamp, + /// BIP-125 explicit signaling: any input has sequence < 0xfffffffe. + pub rbf: bool, +} + +impl Entry { + #[inline] + pub fn fee_rate(&self) -> FeeRate { + FeeRate::from((self.fee, self.vsize)) + } + + #[inline] + pub fn txid_prefix(&self) -> TxidPrefix { + TxidPrefix::from(&self.txid) + } +} diff --git a/crates/brk_mempool/src/stores/entry_pool.rs b/crates/brk_mempool/src/stores/entry_pool.rs new file mode 100644 index 000000000..5f0ff9648 --- /dev/null +++ b/crates/brk_mempool/src/stores/entry_pool.rs @@ -0,0 +1,72 @@ +use brk_types::TxidPrefix; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +use super::{Entry, TxIndex}; + +/// Pool of mempool entries with slot recycling. +/// +/// Slot-based storage: removed entries leave holes that are reused +/// by the next insert, so `TxIndex` stays stable for the lifetime of +/// an entry. Only stores what can't be derived: the entries +/// themselves, their prefix-to-slot index, and the free slot list. +#[derive(Default)] +pub struct EntryPool { + entries: Vec>, + prefix_to_idx: FxHashMap, + free_slots: Vec, +} + +impl EntryPool { + /// Insert an entry, returning its index. The prefix is derived from + /// `entry.txid`, so the caller never has to pass it in. + pub fn insert(&mut self, entry: Entry) -> TxIndex { + let prefix = entry.txid_prefix(); + let idx = match self.free_slots.pop() { + Some(idx) => { + self.entries[idx.as_usize()] = Some(entry); + idx + } + None => { + let idx = TxIndex::from(self.entries.len()); + self.entries.push(Some(entry)); + idx + } + }; + + self.prefix_to_idx.insert(prefix, idx); + idx + } + + /// Get an entry by its txid prefix. + pub fn get(&self, prefix: &TxidPrefix) -> Option<&Entry> { + let idx = self.prefix_to_idx.get(prefix)?; + self.entries.get(idx.as_usize())?.as_ref() + } + + /// Direct children of a transaction (txs whose `depends` includes + /// `prefix`). Derived on demand via a linear scan — called only by + /// the CPFP query endpoint, which is not on the hot path. + pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> { + let mut out: SmallVec<[TxidPrefix; 2]> = SmallVec::new(); + for entry in self.entries.iter().flatten() { + if entry.depends.iter().any(|p| p == prefix) { + out.push(entry.txid_prefix()); + } + } + out + } + + /// Remove an entry by its txid prefix, returning it if present. + pub fn remove(&mut self, prefix: &TxidPrefix) -> Option { + let idx = self.prefix_to_idx.remove(prefix)?; + let entry = self.entries.get_mut(idx.as_usize()).and_then(Option::take)?; + self.free_slots.push(idx); + Some(entry) + } + + /// Get the entries slice for block building. + pub fn entries(&self) -> &[Option] { + &self.entries + } +} diff --git a/crates/brk_mempool/src/stores/mod.rs b/crates/brk_mempool/src/stores/mod.rs new file mode 100644 index 000000000..30eeba32e --- /dev/null +++ b/crates/brk_mempool/src/stores/mod.rs @@ -0,0 +1,32 @@ +//! State held inside the mempool, plus the value types stored in it. +//! +//! [`state::MempoolState`] aggregates four locked buckets: +//! +//! - [`tx_store::TxStore`] - full `Transaction` data for live txs. +//! - [`addr_tracker::AddrTracker`] - per-address mempool stats. +//! - [`entry_pool::EntryPool`] - slot-recycled `Entry` storage indexed +//! by [`tx_index::TxIndex`]. +//! - [`tx_graveyard::TxGraveyard`] - recently-dropped txs as +//! [`tombstone::Tombstone`]s, retained for reappearance detection +//! and post-mine analytics. +//! +//! A fifth bucket, `info`, holds a `MempoolInfo` from `brk_types`, +//! so it has no file here. + +pub mod addr_tracker; +pub mod entry; +pub mod entry_pool; +pub mod state; +pub mod tombstone; +pub mod tx_graveyard; +pub mod tx_index; +pub mod tx_store; + +pub use addr_tracker::AddrTracker; +pub use entry::Entry; +pub use entry_pool::EntryPool; +pub use state::MempoolState; +pub use tombstone::Tombstone; +pub use tx_graveyard::TxGraveyard; +pub use tx_index::TxIndex; +pub use tx_store::TxStore; diff --git a/crates/brk_mempool/src/stores/state.rs b/crates/brk_mempool/src/stores/state.rs new file mode 100644 index 000000000..6ab3b1535 --- /dev/null +++ b/crates/brk_mempool/src/stores/state.rs @@ -0,0 +1,35 @@ +use brk_types::MempoolInfo; +use parking_lot::RwLock; + +use super::{AddrTracker, EntryPool, TxGraveyard, TxStore}; +use crate::steps::{applier::Applier, preparer::Pulled}; + +/// The five buckets making up live mempool state. +/// +/// Each bucket has its own `RwLock` so readers of different buckets +/// don't contend with each other; the Applier takes all five write +/// locks in a fixed order for a brief window once per cycle. +#[derive(Default)] +pub struct MempoolState { + pub(crate) info: RwLock, + pub(crate) txs: RwLock, + pub(crate) addrs: RwLock, + pub(crate) entries: RwLock, + pub(crate) graveyard: RwLock, +} + +impl MempoolState { + /// Apply a prepared diff to all five buckets atomically. Returns + /// true iff the Applier observed any change. Same-cycle prevout + /// resolution is a separate pipeline step run by the orchestrator. + pub fn apply(&self, pulled: Pulled) -> bool { + Applier::apply( + pulled, + &mut self.info.write(), + &mut self.txs.write(), + &mut self.addrs.write(), + &mut self.entries.write(), + &mut self.graveyard.write(), + ) + } +} diff --git a/crates/brk_mempool/src/stores/tombstone.rs b/crates/brk_mempool/src/stores/tombstone.rs new file mode 100644 index 000000000..62502285f --- /dev/null +++ b/crates/brk_mempool/src/stores/tombstone.rs @@ -0,0 +1,45 @@ +use std::time::{Duration, Instant}; + +use brk_types::Transaction; + +use super::Entry; +use crate::steps::preparer::Removal; + +/// A buried mempool tx, retained for reappearance detection and +/// post-mine analytics. +pub struct Tombstone { + pub tx: Transaction, + pub entry: Entry, + removal: Removal, + removed_at: Instant, +} + +impl Tombstone { + pub(super) fn new(tx: Transaction, entry: Entry, removal: Removal, removed_at: Instant) -> Self { + Self { + tx, + entry, + removal, + removed_at, + } + } + + pub fn reason(&self) -> &Removal { + &self.removal + } + + pub fn age(&self) -> Duration { + self.removed_at.elapsed() + } + + pub(super) fn removed_at(&self) -> Instant { + self.removed_at + } + + pub(super) fn replaced_by(&self) -> Option<&brk_types::Txid> { + match &self.removal { + Removal::Replaced { by } => Some(by), + Removal::Vanished => None, + } + } +} diff --git a/crates/brk_mempool/src/stores/tx_graveyard.rs b/crates/brk_mempool/src/stores/tx_graveyard.rs new file mode 100644 index 000000000..361e131ef --- /dev/null +++ b/crates/brk_mempool/src/stores/tx_graveyard.rs @@ -0,0 +1,82 @@ +use std::{ + collections::VecDeque, + time::{Duration, Instant}, +}; + +use brk_types::{Transaction, Txid}; +use rustc_hash::FxHashMap; + +use super::{Entry, Tombstone}; +use crate::steps::preparer::Removal; + +/// How long a dropped tx stays retained after removal. +const RETENTION: Duration = Duration::from_secs(60 * 60); + +/// Recently-dropped txs retained for reappearance detection (Puller can revive +/// them without RPC) and post-mine analytics (RBF/replacement chains, etc.). +#[derive(Default)] +pub struct TxGraveyard { + tombstones: FxHashMap, + order: VecDeque<(Instant, Txid)>, +} + +impl TxGraveyard { + pub fn contains(&self, txid: &Txid) -> bool { + self.tombstones.contains_key(txid) + } + + pub fn get(&self, txid: &Txid) -> Option<&Tombstone> { + self.tombstones.get(txid) + } + + /// Tombstones marked as `Replaced { by: replacer }`. Used to walk + /// backward through RBF history: given a tx that's still live (or + /// in the graveyard), find every tx it displaced. + pub fn predecessors_of<'a>( + &'a self, + replacer: &'a Txid, + ) -> impl Iterator { + self.tombstones + .iter() + .filter_map(move |(txid, ts)| (ts.replaced_by() == Some(replacer)).then_some((txid, ts))) + } + + pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: Entry, removal: Removal) { + let now = Instant::now(); + self.tombstones + .insert(txid.clone(), Tombstone::new(tx, entry, removal, now)); + self.order.push_back((now, txid)); + } + + /// Remove and return the tombstone, e.g. when the tx comes back to life. + pub fn exhume(&mut self, txid: &Txid) -> Option { + self.tombstones.remove(txid) + } + + /// Drop tombstones older than RETENTION. O(k) in the number of evictions. + /// + /// The order queue may carry stale entries (from re-buries or prior + /// exhumes); the timestamp-match check skips those without disturbing + /// live tombstones. + pub fn evict_old(&mut self) { + while let Some(&(t, _)) = self.order.front() { + if t.elapsed() < RETENTION { + break; + } + let (_, txid) = self.order.pop_front().unwrap(); + if let Some(ts) = self.tombstones.get(&txid) + && ts.removed_at() == t + { + self.tombstones.remove(&txid); + } + } + } + + pub fn len(&self) -> usize { + self.tombstones.len() + } + + pub fn is_empty(&self) -> bool { + self.tombstones.is_empty() + } +} diff --git a/crates/brk_mempool/src/types/tx_index.rs b/crates/brk_mempool/src/stores/tx_index.rs similarity index 100% rename from crates/brk_mempool/src/types/tx_index.rs rename to crates/brk_mempool/src/stores/tx_index.rs diff --git a/crates/brk_mempool/src/stores/tx_store.rs b/crates/brk_mempool/src/stores/tx_store.rs new file mode 100644 index 000000000..cf093aac5 --- /dev/null +++ b/crates/brk_mempool/src/stores/tx_store.rs @@ -0,0 +1,90 @@ +use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, Vin}; +use derive_more::Deref; +use rustc_hash::{FxHashMap, FxHashSet}; + +const RECENT_CAP: usize = 10; + +/// Store of full transaction data for API access. +#[derive(Default, Deref)] +pub struct TxStore { + #[deref] + txs: FxHashMap, + recent: Vec, + /// Txids whose tx has at least one input with `prevout == None`. + /// Maintained on every `extend` / `remove` / `apply_fills` so the + /// post-update prevout filler can early-exit when this set is empty. + unresolved: FxHashSet, +} + +impl TxStore { + pub fn contains(&self, txid: &Txid) -> bool { + self.txs.contains_key(txid) + } + + /// Insert each `(Txid, Transaction)` yielded by `items`, and push + /// up to `RECENT_CAP` of them onto the front of `recent` as the + /// newest-seen window (older entries fall off the end). + pub fn extend(&mut self, items: I) + where + I: IntoIterator, + { + let mut new_recent: Vec = Vec::with_capacity(RECENT_CAP); + for (txid, tx) in items { + if new_recent.len() < RECENT_CAP { + new_recent.push(MempoolRecentTx::from((&txid, &tx))); + } + if tx.input.iter().any(|i| i.prevout.is_none()) { + self.unresolved.insert(txid.clone()); + } + self.txs.insert(txid, tx); + } + + let keep = RECENT_CAP.saturating_sub(new_recent.len()); + new_recent.extend(self.recent.drain(..keep.min(self.recent.len()))); + self.recent = new_recent; + } + + pub fn recent(&self) -> &[MempoolRecentTx] { + &self.recent + } + + /// Remove a single tx and return its stored data if present. `recent` + /// isn't touched: it's an "added" window, not a live-set mirror. + pub fn remove(&mut self, txid: &Txid) -> Option { + self.unresolved.remove(txid); + self.txs.remove(txid) + } + + /// Set of txids with at least one unfilled prevout. Used by the + /// prevout filler as a cheap "is there any work?" gate. + pub fn unresolved(&self) -> &FxHashSet { + &self.unresolved + } + + /// Apply resolved prevouts to a tx in place. `fills` is `(vin, prevout)`. + /// Returns the prevouts that were actually written (so the caller can + /// fold them into `AddrTracker`). Updates `unresolved` if the tx is + /// fully resolved after the fill, and recomputes `total_sigop_cost` + /// since the P2SH and witness components depend on prevouts. + pub fn apply_fills(&mut self, txid: &Txid, fills: Vec<(Vin, TxOut)>) -> Vec { + let Some(tx) = self.txs.get_mut(txid) else { + return Vec::new(); + }; + let mut applied = Vec::with_capacity(fills.len()); + for (vin, prevout) in fills { + if let Some(txin) = tx.input.get_mut(usize::from(vin)) + && txin.prevout.is_none() + { + txin.prevout = Some(prevout.clone()); + applied.push(prevout); + } + } + if !applied.is_empty() { + tx.total_sigop_cost = tx.total_sigop_cost(); + } + if !tx.input.iter().any(|i| i.prevout.is_none()) { + self.unresolved.remove(txid); + } + applied + } +} diff --git a/crates/brk_mempool/src/sync.rs b/crates/brk_mempool/src/sync.rs deleted file mode 100644 index cac795264..000000000 --- a/crates/brk_mempool/src/sync.rs +++ /dev/null @@ -1,354 +0,0 @@ -use std::{ - hash::{DefaultHasher, Hash, Hasher}, - mem, - sync::{ - Arc, - atomic::{AtomicBool, AtomicU64, Ordering}, - }, - thread, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use bitcoin::hex::DisplayHex; -use brk_error::Result; -use brk_rpc::Client; -use brk_types::{ - AddrBytes, BlockHash, MempoolEntryInfo, MempoolInfo, Timestamp, Transaction, TxIn, TxOut, - TxStatus, Txid, TxidPrefix, VSize, Vout, -}; -use derive_more::Deref; -use parking_lot::{RwLock, RwLockReadGuard}; -use rustc_hash::FxHashMap; -use tracing::error; - -use crate::{ - addrs::AddrTracker, - block_builder::build_projected_blocks, - entry::Entry, - entry_pool::EntryPool, - projected_blocks::{BlockStats, RecommendedFees, Snapshot}, - tx_store::TxStore, - types::TxWithHex, -}; - -/// Max new txs to fetch full data for per update cycle (for address tracking). -const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000; - -/// Minimum interval between rebuilds (milliseconds). -const MIN_REBUILD_INTERVAL_MS: u64 = 1000; - -/// Mempool monitor. -/// -/// Thread-safe wrapper around `MempoolInner`. Free to clone. -#[derive(Clone, Deref)] -pub struct Mempool(Arc); - -impl Mempool { - pub fn new(client: &Client) -> Self { - Self(Arc::new(MempoolInner::new(client.clone()))) - } -} - -/// Inner mempool state and logic. -pub struct MempoolInner { - client: Client, - - info: RwLock, - txs: RwLock, - addrs: RwLock, - entries: RwLock, - - snapshot: RwLock, - - dirty: AtomicBool, - last_rebuild_ms: AtomicU64, -} - -impl MempoolInner { - pub fn new(client: Client) -> Self { - Self { - client, - info: RwLock::new(MempoolInfo::default()), - txs: RwLock::new(TxStore::default()), - addrs: RwLock::new(AddrTracker::default()), - entries: RwLock::new(EntryPool::default()), - snapshot: RwLock::new(Snapshot::default()), - dirty: AtomicBool::new(false), - last_rebuild_ms: AtomicU64::new(0), - } - } - - pub fn get_info(&self) -> MempoolInfo { - self.info.read().clone() - } - - pub fn get_fees(&self) -> RecommendedFees { - self.snapshot.read().fees.clone() - } - - pub fn get_snapshot(&self) -> Snapshot { - self.snapshot.read().clone() - } - - pub fn get_block_stats(&self) -> Vec { - self.snapshot.read().block_stats.clone() - } - - pub fn next_block_hash(&self) -> u64 { - self.snapshot.read().next_block_hash() - } - - pub fn addr_hash(&self, addr: &AddrBytes) -> u64 { - let addrs = self.addrs.read(); - let Some((stats, _)) = addrs.get(addr) else { - return 0; - }; - let mut hasher = DefaultHasher::new(); - stats.hash(&mut hasher); - hasher.finish() - } - - pub fn get_txs(&self) -> RwLockReadGuard<'_, TxStore> { - self.txs.read() - } - - pub fn get_entries(&self) -> RwLockReadGuard<'_, EntryPool> { - self.entries.read() - } - - pub fn get_addrs(&self) -> RwLockReadGuard<'_, AddrTracker> { - self.addrs.read() - } - - /// Start an infinite update loop with a 1 second interval. - pub fn start(&self) { - loop { - if let Err(e) = self.update() { - error!("Error updating mempool: {}", e); - } - thread::sleep(Duration::from_secs(1)); - } - } - - /// Sync with Bitcoin Core mempool and rebuild projections if needed. - pub fn update(&self) -> Result<()> { - let entries_info = self.client.get_raw_mempool_verbose()?; - - let new_txs = self.fetch_new_txs(&entries_info); - let has_changes = self.apply_changes(&entries_info, new_txs); - - if has_changes { - self.dirty.store(true, Ordering::Release); - } - - self.rebuild_if_needed(); - - Ok(()) - } - - /// Fetch full transaction data for new txids (needed for address tracking). - fn fetch_new_txs(&self, entries_info: &[MempoolEntryInfo]) -> FxHashMap { - let txs = self.txs.read(); - entries_info - .iter() - .filter(|e| !txs.contains(&e.txid)) - .take(MAX_TX_FETCHES_PER_CYCLE) - .filter_map(|entry| { - self.build_transaction(entry, &txs) - .ok() - .map(|tx| (entry.txid.clone(), tx)) - }) - .collect() - } - - fn build_transaction( - &self, - entry: &MempoolEntryInfo, - mempool_txs: &TxStore, - ) -> Result { - let (mut btc_tx, hex) = self.client.get_mempool_raw_tx(&entry.txid)?; - - let total_size = hex.len() / 2; - let total_sigop_cost = btc_tx.total_sigop_cost(|_| None); - - // Collect unique parent txids not in the mempool store, - // fetch each once instead of one get_tx_out per input - let mut parent_cache: FxHashMap> = FxHashMap::default(); - for txin in &btc_tx.input { - let prev_txid: Txid = txin.previous_output.txid.into(); - if !mempool_txs.contains_key(&prev_txid) - && !parent_cache.contains_key(&prev_txid) - && let Ok(prev) = self - .client - .get_raw_transaction(&prev_txid, None as Option<&BlockHash>) - { - parent_cache.insert(prev_txid, prev.output); - } - } - - let input = mem::take(&mut btc_tx.input) - .into_iter() - .map(|txin| { - let prev_txid: Txid = txin.previous_output.txid.into(); - let prev_vout = usize::from(Vout::from(txin.previous_output.vout)); - - let prevout = if let Some(prev) = mempool_txs.get(&prev_txid) { - prev.tx() - .output - .get(prev_vout) - .map(|o| TxOut::from((o.script_pubkey.clone(), o.value))) - } else if let Some(outputs) = parent_cache.get(&prev_txid) { - outputs - .get(prev_vout) - .map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into()))) - } else { - None - }; - - TxIn { - is_coinbase: prevout.is_none(), - prevout, - txid: prev_txid, - vout: txin.previous_output.vout.into(), - script_sig: txin.script_sig, - script_sig_asm: (), - witness: txin - .witness - .iter() - .map(|w| w.to_lower_hex_string()) - .collect(), - sequence: txin.sequence.into(), - inner_redeem_script_asm: (), - inner_witness_script_asm: (), - } - }) - .collect(); - - let tx = Transaction { - index: None, - txid: entry.txid.clone(), - version: btc_tx.version.into(), - total_sigop_cost, - weight: entry.weight.into(), - lock_time: btc_tx.lock_time.into(), - total_size, - fee: entry.fee, - input, - output: btc_tx.output.into_iter().map(TxOut::from).collect(), - status: TxStatus::UNCONFIRMED, - }; - - Ok(TxWithHex::new(tx, hex)) - } - - /// Apply transaction additions and removals. Returns true if there were changes. - fn apply_changes( - &self, - entries_info: &[MempoolEntryInfo], - new_txs: FxHashMap, - ) -> bool { - let entries_by_prefix: FxHashMap = entries_info - .iter() - .map(|e| (TxidPrefix::from(&e.txid), e)) - .collect(); - - let mut info = self.info.write(); - let mut txs = self.txs.write(); - let mut addrs = self.addrs.write(); - let mut entries = self.entries.write(); - - let mut had_removals = false; - let had_additions = !new_txs.is_empty(); - - // Remove transactions no longer in mempool - txs.retain_or_remove( - |txid| entries_by_prefix.contains_key(&TxidPrefix::from(txid)), - |txid, tx_with_hex| { - had_removals = true; - let tx = tx_with_hex.tx(); - let prefix = TxidPrefix::from(txid); - - // Get fee from entries (before removing) - this is the authoritative fee from Bitcoin Core - let fee = entries.get(&prefix).map(|e| e.fee).unwrap_or_default(); - info.remove(tx, fee); - addrs.remove_tx(tx, txid); - entries.remove(&prefix); - }, - ); - - // Add new transactions - for (txid, tx_with_hex) in &new_txs { - let tx = tx_with_hex.tx(); - let prefix = TxidPrefix::from(txid); - - let Some(entry_info) = entries_by_prefix.get(&prefix) else { - continue; - }; - - info.add(tx, entry_info.fee); - addrs.add_tx(tx, txid); - entries.insert( - prefix, - Entry { - txid: entry_info.txid.clone(), - fee: entry_info.fee, - vsize: VSize::from(entry_info.vsize), - size: tx.total_size as u64, - ancestor_fee: entry_info.ancestor_fee, - ancestor_vsize: VSize::from(entry_info.ancestor_size), - depends: entry_info.depends.iter().map(TxidPrefix::from).collect(), - first_seen: Timestamp::now(), - }, - ); - } - txs.extend(new_txs); - - had_removals || had_additions - } - - /// Rebuild projected blocks if dirty and enough time has passed. - fn rebuild_if_needed(&self) { - if !self.dirty.load(Ordering::Acquire) { - return; - } - - let now_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - - let last = self.last_rebuild_ms.load(Ordering::Acquire); - if now_ms.saturating_sub(last) < MIN_REBUILD_INTERVAL_MS { - return; - } - - if self - .last_rebuild_ms - .compare_exchange(last, now_ms, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return; - } - - self.dirty.store(false, Ordering::Release); - - // let i = Instant::now(); - self.rebuild_projected_blocks(); - // debug!("mempool: rebuild_projected_blocks in {:?}", i.elapsed()); - } - - /// Rebuild projected blocks snapshot. - fn rebuild_projected_blocks(&self) { - let entries = self.entries.read(); - let entries_slice = entries.entries(); - - let blocks = build_projected_blocks(entries_slice); - - #[cfg(debug_assertions)] - crate::projected_blocks::verify::Verifier::check(&self.client, &blocks, entries_slice); - - let snapshot = Snapshot::build(blocks, entries_slice); - - *self.snapshot.write() = snapshot; - } -} diff --git a/crates/brk_mempool/src/tx_store.rs b/crates/brk_mempool/src/tx_store.rs deleted file mode 100644 index 9565d30f8..000000000 --- a/crates/brk_mempool/src/tx_store.rs +++ /dev/null @@ -1,56 +0,0 @@ -use brk_types::{MempoolRecentTx, Txid}; -use derive_more::Deref; -use rustc_hash::FxHashMap; - -use crate::types::TxWithHex; - -const RECENT_CAP: usize = 10; - -/// Store of full transaction data for API access. -#[derive(Default, Deref)] -pub struct TxStore { - #[deref] - txs: FxHashMap, - recent: Vec, -} - -impl TxStore { - /// Check if a transaction exists. - pub fn contains(&self, txid: &Txid) -> bool { - self.txs.contains_key(txid) - } - - /// Add transactions in bulk. - pub fn extend(&mut self, txs: FxHashMap) { - let mut new: Vec<_> = txs - .iter() - .take(RECENT_CAP) - .map(|(txid, tx_hex)| MempoolRecentTx::from((txid, tx_hex.tx()))) - .collect(); - let keep = RECENT_CAP.saturating_sub(new.len()); - new.extend(self.recent.drain(..keep.min(self.recent.len()))); - self.recent = new; - self.txs.extend(txs); - } - - /// Last 10 transactions to enter the mempool. - pub fn recent(&self) -> &[MempoolRecentTx] { - &self.recent - } - - /// Keep items matching predicate, call `on_remove` for each removed item. - pub fn retain_or_remove(&mut self, mut keep: K, mut on_remove: R) - where - K: FnMut(&Txid) -> bool, - R: FnMut(&Txid, &TxWithHex), - { - self.txs.retain(|txid, tx| { - if keep(txid) { - true - } else { - on_remove(txid, tx); - false - } - }); - } -} diff --git a/crates/brk_mempool/src/types/mod.rs b/crates/brk_mempool/src/types/mod.rs deleted file mode 100644 index e5e2eeaee..000000000 --- a/crates/brk_mempool/src/types/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod pool_index; -mod tx_index; -mod tx_with_hex; - -pub use pool_index::PoolIndex; -pub use tx_index::TxIndex; -pub use tx_with_hex::TxWithHex; diff --git a/crates/brk_mempool/src/types/tx_with_hex.rs b/crates/brk_mempool/src/types/tx_with_hex.rs deleted file mode 100644 index 3a769afd7..000000000 --- a/crates/brk_mempool/src/types/tx_with_hex.rs +++ /dev/null @@ -1,26 +0,0 @@ -use brk_types::Transaction; - -/// A transaction with its raw hex representation -#[derive(Debug, Clone)] -pub struct TxWithHex { - tx: Transaction, - hex: String, -} - -impl TxWithHex { - pub fn new(tx: Transaction, hex: String) -> Self { - Self { tx, hex } - } - - pub fn tx(&self) -> &Transaction { - &self.tx - } - - pub fn hex(&self) -> &str { - &self.hex - } - - pub fn into_parts(self) -> (Transaction, String) { - (self.tx, self.hex) - } -} diff --git a/crates/brk_query/Cargo.toml b/crates/brk_query/Cargo.toml index 60e1dc47d..e0fac3371 100644 --- a/crates/brk_query/Cargo.toml +++ b/crates/brk_query/Cargo.toml @@ -18,7 +18,7 @@ brk_error = { workspace = true, features = ["jiff", "vecdb"] } brk_indexer = { workspace = true } brk_mempool = { workspace = true } brk_reader = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_traversable = { workspace = true } brk_types = { workspace = true } derive_more = { workspace = true } diff --git a/crates/brk_query/src/impl/addr.rs b/crates/brk_query/src/impl/addr.rs index f089d3dea..c88777aa6 100644 --- a/crates/brk_query/src/impl/addr.rs +++ b/crates/brk_query/src/impl/addr.rs @@ -85,13 +85,8 @@ impl Query { tx_count: addr_data.tx_count, realized_price, }, - mempool_stats: self.mempool().map(|mempool| { - mempool - .get_addrs() - .get(&bytes) - .map(|(stats, _)| stats) - .cloned() - .unwrap_or_default() + mempool_stats: self.mempool().and_then(|m| { + m.addrs().get(&bytes).map(|(stats, _)| stats.clone()) }), }) } @@ -221,21 +216,17 @@ impl Query { let Ok(bytes) = AddrBytes::from_str(addr) else { return 0; }; - mempool.addr_hash(&bytes) + mempool.addr_state_hash(&bytes) } pub fn addr_mempool_txids(&self, addr: Addr) -> Result> { - let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - let bytes = AddrBytes::from_str(&addr)?; - let addrs = mempool.get_addrs(); - - let txids: Vec = addrs + let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; + Ok(mempool + .addrs() .get(&bytes) .map(|(_, txids)| txids.iter().take(MAX_MEMPOOL_TXIDS).cloned().collect()) - .unwrap_or_default(); - - Ok(txids) + .unwrap_or_default()) } /// Height of the last on-chain activity for an address (last tx_index → height). diff --git a/crates/brk_query/src/impl/block/txs.rs b/crates/brk_query/src/impl/block/txs.rs index ae5e8a0ae..678e09c56 100644 --- a/crates/brk_query/src/impl/block/txs.rs +++ b/crates/brk_query/src/impl/block/txs.rs @@ -1,6 +1,6 @@ use std::io::Cursor; -use bitcoin::{consensus::Decodable, hex::DisplayHex}; +use bitcoin::consensus::Decodable; use brk_error::{Error, OptionData, Result}; use brk_types::{ BlkPosition, BlockHash, Height, OutPoint, OutputType, RawLockTime, Sats, StoredU32, @@ -217,11 +217,7 @@ impl Query { (prev_txid, outpoint.vout(), Some(prev_txout)) }; - let witness = txin - .witness - .iter() - .map(|w| w.to_lower_hex_string()) - .collect(); + let witness = txin.witness.clone().into(); Ok(TxIn { txid: prev_txid, diff --git a/crates/brk_query/src/impl/mempool.rs b/crates/brk_query/src/impl/mempool.rs index 74780bd92..519d973d3 100644 --- a/crates/brk_query/src/impl/mempool.rs +++ b/crates/brk_query/src/impl/mempool.rs @@ -1,35 +1,39 @@ use std::cmp::Ordering; use brk_error::{Error, Result}; +use brk_mempool::{Entry, EntryPool, Removal, Tombstone, TxGraveyard, TxStore}; use brk_types::{ - CpfpEntry, CpfpInfo, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx, RecommendedFees, - Txid, TxidPrefix, Weight, + CheckedSub, CpfpEntry, CpfpInfo, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx, OutputType, + RbfResponse, RbfTx, RecommendedFees, ReplacementNode, Sats, Timestamp, Transaction, TxOut, + TxOutIndex, Txid, TxidPrefix, TypeIndex, VSize, Weight, }; +use rustc_hash::FxHashSet; +use vecdb::VecIndex; use crate::Query; impl Query { pub fn mempool_info(&self) -> Result { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - Ok(mempool.get_info()) + Ok(mempool.info()) } pub fn mempool_txids(&self) -> Result> { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - let txs = mempool.get_txs(); + let txs = mempool.txs(); Ok(txs.keys().cloned().collect()) } pub fn recommended_fees(&self) -> Result { self.mempool() - .map(|mempool| mempool.get_fees()) + .map(|mempool| mempool.fees()) .ok_or(Error::MempoolNotAvailable) } pub fn mempool_blocks(&self) -> Result> { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - let block_stats = mempool.get_block_stats(); + let block_stats = mempool.block_stats(); let blocks = block_stats .into_iter() @@ -47,25 +51,74 @@ impl Query { Ok(blocks) } + /// Fill any `prevout == None` inputs on live mempool txs from the + /// indexer, mutating them in place. Cheap when the unresolved set + /// is empty (the steady-state with `-txindex` on); otherwise resolves + /// each missing prevout via the same lookup chain used for confirmed + /// txs: `txid → tx_index → first_txout_index + vout → output_type + /// / type_index / value → script_pubkey`. + /// + /// Driver calls this once per cycle, right after `mempool.update()`. + /// Returns true if at least one prevout was filled. + pub fn fill_mempool_prevouts(&self) -> bool { + let Some(mempool) = self.mempool() else { + return false; + }; + + let indexer = self.indexer(); + let stores = &indexer.stores; + let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader(); + let output_type_reader = indexer.vecs.outputs.output_type.reader(); + let type_index_reader = indexer.vecs.outputs.type_index.reader(); + let value_reader = indexer.vecs.outputs.value.reader(); + let addr_readers = indexer.vecs.addrs.addr_readers(); + + mempool.fill_prevouts(|prev_txid, vout| { + let prev_tx_index = stores + .txid_prefix_to_tx_index + .get(&TxidPrefix::from(prev_txid)) + .ok()?? + .into_owned(); + let first_txout: TxOutIndex = first_txout_index_reader.get(prev_tx_index.to_usize()); + let txout_index = usize::from(first_txout + vout); + let output_type: OutputType = output_type_reader.get(txout_index); + let type_index: TypeIndex = type_index_reader.get(txout_index); + let value: Sats = value_reader.get(txout_index); + let script_pubkey = addr_readers.script_pubkey(output_type, type_index); + Some(TxOut::from((script_pubkey, value))) + }) + } + pub fn mempool_recent(&self) -> Result> { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - Ok(mempool.get_txs().recent().to_vec()) + Ok(mempool.txs().recent().to_vec()) } pub fn cpfp(&self, txid: &Txid) -> Result { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - let entries = mempool.get_entries(); + let entries = mempool.entries(); let prefix = TxidPrefix::from(txid); let Some(entry) = entries.get(&prefix) else { return Ok(CpfpInfo::default()); }; - // Ancestors: walk up the depends chain + // Ancestor walk doubles as package-rate aggregation. Stale + // `depends` entries pointing at mined/evicted txs are silently + // dropped via the live `entries.get` probe, so the aggregates + // reflect only in-pool ancestors. let mut ancestors = Vec::new(); + let mut visited: FxHashSet = FxHashSet::default(); + let mut package_fee = u64::from(entry.fee); + let mut package_vsize = u64::from(entry.vsize); let mut stack: Vec = entry.depends.to_vec(); while let Some(p) = stack.pop() { + if !visited.insert(p) { + continue; + } if let Some(anc) = entries.get(&p) { + package_fee += u64::from(anc.fee); + package_vsize += u64::from(anc.vsize); ancestors.push(CpfpEntry { txid: anc.txid.clone(), weight: Weight::from(anc.vsize), @@ -77,7 +130,7 @@ impl Query { let mut descendants = Vec::new(); for child_prefix in entries.children(&prefix) { - if let Some(e) = entries.get(child_prefix) { + if let Some(e) = entries.get(&child_prefix) { descendants.push(CpfpEntry { txid: e.txid.clone(), weight: Weight::from(e.vsize), @@ -86,7 +139,13 @@ impl Query { } } - let effective_fee_per_vsize = entry.effective_fee_rate(); + let self_rate = entry.fee_rate(); + let package_rate = FeeRate::from((Sats::from(package_fee), VSize::from(package_vsize))); + let effective_fee_per_vsize = if package_rate > self_rate { + package_rate + } else { + self_rate + }; let best_descendant = descendants .iter() @@ -107,9 +166,109 @@ impl Query { }) } + /// RBF history for a tx, matching mempool.space's + /// `GET /api/v1/tx/:txid/rbf`. Walks forward through the graveyard + /// to find the latest known replacer (tree root), then recursively + /// walks `predecessors_of` backward to build the tree. `replaces` + /// is the requested tx's own direct predecessors. + pub fn tx_rbf(&self, txid: &Txid) -> Result { + let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; + let txs = mempool.txs(); + let entries = mempool.entries(); + let graveyard = mempool.graveyard(); + + let mut root_txid = txid.clone(); + while let Some(Removal::Replaced { by }) = graveyard.get(&root_txid).map(Tombstone::reason) { + root_txid = by.clone(); + } + + let replaces_vec: Vec = graveyard + .predecessors_of(txid) + .map(|(p, _)| p.clone()) + .collect(); + let replaces = (!replaces_vec.is_empty()).then_some(replaces_vec); + + let replacements = Self::build_rbf_node(&root_txid, None, &txs, &entries, &graveyard) + .map(|mut node| { + node.tx.full_rbf = Some(node.full_rbf); + node.interval = None; + node + }); + + Ok(RbfResponse { + replacements, + replaces, + }) + } + + /// Resolve a txid to the data we need for an `RbfTx`. The live + /// pool takes priority; the graveyard is the fallback. Returns + /// `None` if the tx has no known data in either. + fn resolve_rbf_node_data<'a>( + txid: &Txid, + txs: &'a TxStore, + entries: &'a EntryPool, + graveyard: &'a TxGraveyard, + ) -> Option<(&'a Transaction, &'a Entry)> { + if let (Some(tx), Some(entry)) = + (txs.get(txid), entries.get(&TxidPrefix::from(txid))) + { + return Some((tx, entry)); + } + graveyard + .get(txid) + .map(|tomb| (&tomb.tx, &tomb.entry)) + } + + /// Recursively build an RBF tree node rooted at `txid`. + /// Predecessors are always in the graveyard (that's where + /// `Removal::Replaced` lives), so the recursion only needs the + /// graveyard; the live pool is consulted for the root. + fn build_rbf_node( + txid: &Txid, + successor_time: Option, + txs: &TxStore, + entries: &EntryPool, + graveyard: &TxGraveyard, + ) -> Option { + let (tx, entry) = Self::resolve_rbf_node_data(txid, txs, entries, graveyard)?; + + let replaces: Vec = graveyard + .predecessors_of(txid) + .filter_map(|(pred_txid, _)| { + Self::build_rbf_node(pred_txid, Some(entry.first_seen), txs, entries, graveyard) + }) + .collect(); + + let full_rbf = replaces.iter().any(|c| !c.tx.rbf || c.full_rbf); + + let interval = successor_time + .and_then(|st| st.checked_sub(entry.first_seen)) + .map(|d| usize::from(d) as u32); + + let value = Sats::from(tx.output.iter().map(|o| u64::from(o.value)).sum::()); + + Some(ReplacementNode { + tx: RbfTx { + txid: txid.clone(), + fee: entry.fee, + vsize: entry.vsize, + value, + rate: entry.fee_rate(), + time: entry.first_seen, + rbf: entry.rbf, + full_rbf: None, + }, + time: entry.first_seen, + full_rbf, + interval, + replaces, + }) + } + pub fn transaction_times(&self, txids: &[Txid]) -> Result> { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; - let entries = mempool.get_entries(); + let entries = mempool.entries(); Ok(txids .iter() .map(|txid| { diff --git a/crates/brk_query/src/impl/price.rs b/crates/brk_query/src/impl/price.rs index f67dda836..fe3e8e9a2 100644 --- a/crates/brk_query/src/impl/price.rs +++ b/crates/brk_query/src/impl/price.rs @@ -9,10 +9,10 @@ impl Query { let mut oracle = self.computer().prices.live_oracle(self.indexer())?; if let Some(mempool) = self.mempool() { - let txs = mempool.get_txs(); + let txs = mempool.txs(); oracle.process_outputs( txs.values() - .flat_map(|tx| &tx.tx().output) + .flat_map(|tx| &tx.output) .map(|txout| (txout.value, txout.type_())), ); } diff --git a/crates/brk_query/src/impl/series.rs b/crates/brk_query/src/impl/series.rs index 20c2605dd..a946b4d0d 100644 --- a/crates/brk_query/src/impl/series.rs +++ b/crates/brk_query/src/impl/series.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; use brk_error::{Error, Result}; use brk_traversable::TreeNode; use brk_types::{ - BlockHashPrefix, Date, DetailedSeriesCount, Epoch, Etag, Format, Halving, Height, Index, + BlockHashPrefix, Date, DetailedSeriesCount, Epoch, Format, Halving, Height, Index, IndexInfo, LegacyValue, Limit, Output, OutputLegacy, PaginatedSeries, Pagination, PaginationIndex, RangeIndex, RangeMap, SearchQuery, SeriesData, SeriesInfo, SeriesName, SeriesOutput, SeriesOutputLegacy, SeriesSelection, Timestamp, Version, @@ -449,7 +449,8 @@ impl Query { } /// A resolved series query ready for formatting. -/// Contains the vecs and metadata needed to build an ETag or format the output. +/// Carries the vecs plus the metadata (version, total, end, hash_prefix) callers +/// need to derive an etag or cache policy. pub struct ResolvedQuery { pub vecs: Vec<&'static dyn AnyExportableVec>, pub format: Format, @@ -462,10 +463,6 @@ pub struct ResolvedQuery { } impl ResolvedQuery { - pub fn etag(&self) -> Etag { - Etag::from_series(self.version, self.total, self.end, self.hash_prefix) - } - pub fn format(&self) -> Format { self.format } diff --git a/crates/brk_query/src/impl/tx.rs b/crates/brk_query/src/impl/tx.rs index 9e2d9da2a..181466b5b 100644 --- a/crates/brk_query/src/impl/tx.rs +++ b/crates/brk_query/src/impl/tx.rs @@ -1,4 +1,4 @@ -use bitcoin::hex::{DisplayHex, FromHex}; +use bitcoin::hex::DisplayHex; use brk_error::{Error, OptionData, Result}; use brk_types::{ BlockHash, Height, MerkleProof, Timestamp, Transaction, TxInIndex, TxIndex, TxOutIndex, @@ -87,15 +87,15 @@ impl Query { pub fn transaction(&self, txid: &Txid) -> Result { if let Some(mempool) = self.mempool() - && let Some(tx_with_hex) = mempool.get_txs().get(txid) + && let Some(tx) = mempool.txs().get(txid) { - return Ok(tx_with_hex.tx().clone()); + return Ok(tx.clone()); } self.transaction_by_index(self.resolve_tx_index(txid)?) } pub fn transaction_status(&self, txid: &Txid) -> Result { - if self.mempool().is_some_and(|m| m.get_txs().contains_key(txid)) { + if self.mempool().is_some_and(|m| m.txs().contains_key(txid)) { return Ok(TxStatus::UNCONFIRMED); } self.confirmed_status(self.resolve_tx_index(txid)?) @@ -103,19 +103,18 @@ impl Query { pub fn transaction_raw(&self, txid: &Txid) -> Result> { if let Some(mempool) = self.mempool() - && let Some(tx_with_hex) = mempool.get_txs().get(txid) + && let Some(tx) = mempool.txs().get(txid) { - return Vec::from_hex(tx_with_hex.hex()) - .map_err(|_| Error::Parse("Failed to decode mempool tx hex".into())); + return Ok(tx.encode_bytes()); } self.transaction_raw_by_index(self.resolve_tx_index(txid)?) } pub fn transaction_hex(&self, txid: &Txid) -> Result { if let Some(mempool) = self.mempool() - && let Some(tx_with_hex) = mempool.get_txs().get(txid) + && let Some(tx) = mempool.txs().get(txid) { - return Ok(tx_with_hex.hex().to_string()); + return Ok(tx.encode_bytes().to_lower_hex_string()); } self.transaction_hex_by_index(self.resolve_tx_index(txid)?) } @@ -123,7 +122,7 @@ impl Query { // ── Outspend queries ─────────────────────────────────────────── pub fn outspend(&self, txid: &Txid, vout: Vout) -> Result { - if self.mempool().is_some_and(|m| m.get_txs().contains_key(txid)) { + if self.mempool().is_some_and(|m| m.txs().contains_key(txid)) { return Ok(TxOutspend::UNSPENT); } let (_, first_txout, output_count) = self.resolve_tx_outputs(txid)?; @@ -135,9 +134,9 @@ impl Query { pub fn outspends(&self, txid: &Txid) -> Result> { if let Some(mempool) = self.mempool() - && let Some(tx_with_hex) = mempool.get_txs().get(txid) + && let Some(tx) = mempool.txs().get(txid) { - return Ok(vec![TxOutspend::UNSPENT; tx_with_hex.tx().output.len()]); + return Ok(vec![TxOutspend::UNSPENT; tx.output.len()]); } let (_, first_txout, output_count) = self.resolve_tx_outputs(txid)?; self.resolve_outspends(first_txout, output_count) diff --git a/crates/brk_reader/Cargo.toml b/crates/brk_reader/Cargo.toml index 238857307..67cef7104 100644 --- a/crates/brk_reader/Cargo.toml +++ b/crates/brk_reader/Cargo.toml @@ -13,7 +13,7 @@ exclude = ["examples/"] [dependencies] bitcoin = { workspace = true } brk_error = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_types = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_more = { workspace = true } diff --git a/crates/brk_rpc/Cargo.toml b/crates/brk_rpc/Cargo.toml index ab7929d89..36f347307 100644 --- a/crates/brk_rpc/Cargo.toml +++ b/crates/brk_rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brk_rpc" -description = "A thin wrapper around bitcoincore-rpc or corepc-client" +description = "A thin wrapper around Bitcoin Core's JSON-RPC" version.workspace = true edition.workspace = true license.workspace = true @@ -8,20 +8,15 @@ homepage.workspace = true repository.workspace = true exclude = ["examples/"] -[features] -default = ["corepc"] -bitcoincore-rpc = ["dep:bitcoincore-rpc", "dep:serde_json", "brk_error/bitcoincore-rpc"] -corepc = ["dep:corepc-client", "dep:corepc-jsonrpc", "dep:serde_json", "dep:serde", "brk_error/corepc"] - [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true, optional = true } -corepc-client = { workspace = true, optional = true } -corepc-jsonrpc = { workspace = true, optional = true } -brk_error = { workspace = true } +brk_error = { workspace = true, features = ["corepc", "serde_json"] } brk_logger = { workspace = true } brk_types = { workspace = true } -tracing = { workspace = true } +corepc-jsonrpc = { workspace = true } +corepc-types = { workspace = true } parking_lot = { workspace = true } -serde = { workspace = true, optional = true } -serde_json = { workspace = true, optional = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/brk_rpc/examples/compare_backends.rs b/crates/brk_rpc/examples/compare_backends.rs deleted file mode 100644 index cce9a6385..000000000 --- a/crates/brk_rpc/examples/compare_backends.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! Compares results from the bitcoincore-rpc and corepc backends. -//! -//! Run with: -//! cargo run -p brk_rpc --example compare_backends --features corepc - -#[cfg(all(feature = "bitcoincore-rpc", feature = "corepc"))] -use std::time::{Duration, Instant}; - -#[cfg(not(all(feature = "bitcoincore-rpc", feature = "corepc")))] -fn main() { - eprintln!("This example requires both features: --features bitcoincore-rpc,corepc"); - std::process::exit(1); -} - -#[cfg(all(feature = "bitcoincore-rpc", feature = "corepc"))] -fn main() { - use brk_rpc::backend::{self, Auth}; - - brk_logger::init(None).unwrap(); - - let bitcoin_dir = brk_rpc::Client::default_bitcoin_path(); - let auth = Auth::CookieFile(bitcoin_dir.join(".cookie")); - let url = brk_rpc::Client::default_url(); - - let bc = backend::bitcoincore::ClientInner::new(url, auth.clone(), 10, Duration::from_secs(1)) - .expect("bitcoincore client"); - let cp = backend::corepc::ClientInner::new(url, auth, 10, Duration::from_secs(1)) - .expect("corepc client"); - - println!("=== Comparing backends ===\n"); - - // --- get_blockchain_info --- - { - let (t1, r1) = timed(|| bc.get_blockchain_info()); - let (t2, r2) = timed(|| cp.get_blockchain_info()); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_blockchain_info:"); - println!( - " bitcoincore: headers={} blocks={} ({t1:?})", - r1.headers, r1.blocks - ); - println!( - " corepc: headers={} blocks={} ({t2:?})", - r2.headers, r2.blocks - ); - assert_eq!(r1.headers, r2.headers, "headers mismatch"); - assert_eq!(r1.blocks, r2.blocks, "blocks mismatch"); - println!(" MATCH\n"); - } - - // --- get_block_count --- - { - let (t1, r1) = timed(|| bc.get_block_count()); - let (t2, r2) = timed(|| cp.get_block_count()); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_block_count:"); - println!(" bitcoincore: {r1} ({t1:?})"); - println!(" corepc: {r2} ({t2:?})"); - assert_eq!(r1, r2, "block count mismatch"); - println!(" MATCH\n"); - } - - // --- get_block_hash (height 0) --- - let genesis_hash; - { - let (t1, r1) = timed(|| bc.get_block_hash(0)); - let (t2, r2) = timed(|| cp.get_block_hash(0)); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - genesis_hash = r1; - println!("get_block_hash(0):"); - println!(" bitcoincore: {r1} ({t1:?})"); - println!(" corepc: {r2} ({t2:?})"); - assert_eq!(r1, r2, "genesis hash mismatch"); - println!(" MATCH\n"); - } - - // --- get_block_header --- - { - let (t1, r1) = timed(|| bc.get_block_header(&genesis_hash)); - let (t2, r2) = timed(|| cp.get_block_header(&genesis_hash)); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_block_header(genesis):"); - println!(" bitcoincore: prev={} ({t1:?})", r1.prev_blockhash); - println!(" corepc: prev={} ({t2:?})", r2.prev_blockhash); - assert_eq!(r1, r2, "header mismatch"); - println!(" MATCH\n"); - } - - // --- get_block_info --- - { - let (t1, r1) = timed(|| bc.get_block_info(&genesis_hash)); - let (t2, r2) = timed(|| cp.get_block_info(&genesis_hash)); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_block_info(genesis):"); - println!( - " bitcoincore: height={} confirmations={} ({t1:?})", - r1.height, r1.confirmations - ); - println!( - " corepc: height={} confirmations={} ({t2:?})", - r2.height, r2.confirmations - ); - assert_eq!(r1.height, r2.height, "height mismatch"); - // confirmations can drift by 1 between calls - assert!( - (r1.confirmations - r2.confirmations).abs() <= 1, - "confirmations mismatch: {} vs {}", - r1.confirmations, - r2.confirmations - ); - println!(" MATCH\n"); - } - - // --- get_block_header_info --- - { - let (t1, r1) = timed(|| bc.get_block_header_info(&genesis_hash)); - let (t2, r2) = timed(|| cp.get_block_header_info(&genesis_hash)); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_block_header_info(genesis):"); - println!( - " bitcoincore: height={} prev={:?} ({t1:?})", - r1.height, r1.previous_block_hash - ); - println!( - " corepc: height={} prev={:?} ({t2:?})", - r2.height, r2.previous_block_hash - ); - assert_eq!(r1.height, r2.height, "height mismatch"); - assert_eq!( - r1.previous_block_hash, r2.previous_block_hash, - "prev hash mismatch" - ); - println!(" MATCH\n"); - } - - // --- get_block (genesis) --- - { - let (t1, r1) = timed(|| bc.get_block(&genesis_hash)); - let (t2, r2) = timed(|| cp.get_block(&genesis_hash)); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_block(genesis):"); - println!(" bitcoincore: txs={} ({t1:?})", r1.txdata.len()); - println!(" corepc: txs={} ({t2:?})", r2.txdata.len()); - assert_eq!(r1, r2, "block mismatch"); - println!(" MATCH\n"); - } - - // --- get_raw_mempool --- - { - let (t1, r1) = timed(|| bc.get_raw_mempool()); - let (t2, r2) = timed(|| cp.get_raw_mempool()); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_raw_mempool:"); - println!(" bitcoincore: {} txs ({t1:?})", r1.len()); - println!(" corepc: {} txs ({t2:?})", r2.len()); - // Mempool can change between calls, just check they're reasonable - println!( - " {} (mempool is live, counts may differ slightly)\n", - if r1.len() == r2.len() { - "MATCH" - } else { - "CLOSE" - } - ); - } - - // --- get_raw_mempool_verbose --- - { - let (t1, r1) = timed(|| bc.get_raw_mempool_verbose()); - let (t2, r2) = timed(|| cp.get_raw_mempool_verbose()); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_raw_mempool_verbose:"); - println!(" bitcoincore: {} entries ({t1:?})", r1.len()); - println!(" corepc: {} entries ({t2:?})", r2.len()); - - // Compare a sample entry if both have data - if let (Some((txid1, e1)), Some(_)) = (r1.first(), r2.first()) - && let Some((_, e2)) = r2.iter().find(|(t, _)| t == txid1) - { - println!(" sample txid {txid1}:"); - println!( - " bitcoincore: vsize={} fee={} ancestor_count={}", - e1.vsize, e1.base_fee_sats, e1.ancestor_count - ); - println!( - " corepc: vsize={} fee={} ancestor_count={}", - e2.vsize, e2.base_fee_sats, e2.ancestor_count - ); - assert_eq!(e1.base_fee_sats, e2.base_fee_sats, "fee mismatch"); - assert_eq!( - e1.ancestor_count, e2.ancestor_count, - "ancestor_count mismatch" - ); - println!(" MATCH"); - } - println!(); - } - - // --- get_raw_transaction_hex (tx from block 1, genesis coinbase can't be retrieved) --- - let block1_hash; - { - block1_hash = bc.get_block_hash(1).unwrap(); - let block = bc.get_block(&block1_hash).unwrap(); - let coinbase_txid = block.txdata[0].compute_txid(); - let (t1, r1) = timed(|| bc.get_raw_transaction_hex(&coinbase_txid, Some(&block1_hash))); - let (t2, r2) = timed(|| cp.get_raw_transaction_hex(&coinbase_txid, Some(&block1_hash))); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_raw_transaction_hex(block 1 coinbase):"); - println!(" bitcoincore: {}... ({t1:?})", &r1[..40.min(r1.len())]); - println!(" corepc: {}... ({t2:?})", &r2[..40.min(r2.len())]); - assert_eq!(r1, r2, "raw tx hex mismatch"); - println!(" MATCH\n"); - } - - // --- get_tx_out (genesis coinbase, likely unspendable but test the call) --- - { - let block = bc.get_block(&genesis_hash).unwrap(); - let coinbase_txid = block.txdata[0].compute_txid(); - let (t1, r1) = timed(|| bc.get_tx_out(&coinbase_txid, 0, Some(false))); - let (t2, r2) = timed(|| cp.get_tx_out(&coinbase_txid, 0, Some(false))); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - println!("get_tx_out(genesis coinbase, vout=0):"); - match (&r1, &r2) { - (Some(a), Some(b)) => { - println!( - " bitcoincore: coinbase={} value={:?} ({t1:?})", - a.coinbase, a.value - ); - println!( - " corepc: coinbase={} value={:?} ({t2:?})", - b.coinbase, b.value - ); - assert_eq!(a.coinbase, b.coinbase, "coinbase mismatch"); - assert_eq!(a.value, b.value, "value mismatch"); - assert_eq!(a.script_pub_key, b.script_pub_key, "script mismatch"); - println!(" MATCH"); - } - (None, None) => { - println!(" both: None (spent) ({t1:?} / {t2:?})"); - println!(" MATCH"); - } - _ => { - println!(" MISMATCH: bitcoincore={r1:?}, corepc={r2:?}"); - panic!("get_tx_out mismatch"); - } - } - println!(); - } - - println!("=== All checks passed ==="); -} - -#[cfg(all(feature = "bitcoincore-rpc", feature = "corepc"))] -fn timed(f: impl FnOnce() -> T) -> (Duration, T) { - let start = Instant::now(); - let result = f(); - (start.elapsed(), result) -} diff --git a/crates/brk_rpc/src/backend/bitcoincore.rs b/crates/brk_rpc/src/backend/bitcoincore.rs deleted file mode 100644 index 1c9c3ce8f..000000000 --- a/crates/brk_rpc/src/backend/bitcoincore.rs +++ /dev/null @@ -1,338 +0,0 @@ -use std::{thread::sleep, time::Duration}; - -use bitcoincore_rpc::{ - Client as CoreClient, Error as RpcError, RpcApi, - json::{GetBlockTemplateCapabilities, GetBlockTemplateModes, GetBlockTemplateRules}, - jsonrpc, -}; -use brk_error::{Error, Result}; -use brk_types::{Sats, Txid}; -use parking_lot::RwLock; -use serde_json::value::RawValue; -use tracing::info; - -use super::{ - Auth, BlockHeaderInfo, BlockInfo, BlockTemplateTx, BlockchainInfo, RawMempoolEntry, TxOutInfo, -}; - -/// Per-batch request count for `get_block_hashes_range`. Sized so the -/// JSON request body stays well under a megabyte and bitcoind doesn't -/// spend too long on a single batch before yielding results. -const BATCH_CHUNK: usize = 2000; - -fn to_rpc_auth(auth: &Auth) -> bitcoincore_rpc::Auth { - match auth { - Auth::None => bitcoincore_rpc::Auth::None, - Auth::UserPass(u, p) => bitcoincore_rpc::Auth::UserPass(u.clone(), p.clone()), - Auth::CookieFile(path) => bitcoincore_rpc::Auth::CookieFile(path.clone()), - } -} - -#[derive(Debug)] -pub struct ClientInner { - url: String, - auth: Auth, - client: RwLock, - max_retries: usize, - retry_delay: Duration, -} - -impl ClientInner { - pub fn new(url: &str, auth: Auth, max_retries: usize, retry_delay: Duration) -> Result { - let rpc_auth = to_rpc_auth(&auth); - let client = Self::retry(max_retries, retry_delay, || { - CoreClient::new(url, rpc_auth.clone()).map_err(Into::into) - })?; - - Ok(Self { - url: url.to_string(), - auth, - client: RwLock::new(client), - max_retries, - retry_delay, - }) - } - - fn recreate(&self) -> Result<()> { - *self.client.write() = CoreClient::new(&self.url, to_rpc_auth(&self.auth))?; - Ok(()) - } - - fn is_retriable(error: &RpcError) -> bool { - matches!( - error, - RpcError::JsonRpc(jsonrpc::Error::Rpc(e)) - if e.code == -32600 || e.code == 401 || e.code == -28 - ) || matches!(error, RpcError::JsonRpc(jsonrpc::Error::Transport(_))) - } - - fn retry(max_retries: usize, delay: Duration, mut f: F) -> Result - where - F: FnMut() -> Result, - { - let mut last_error = None; - - for attempt in 0..=max_retries { - if attempt > 0 { - info!( - "Retrying to connect to Bitcoin Core (attempt {}/{})", - attempt, max_retries - ); - sleep(delay); - } - - match f() { - Ok(value) => { - if attempt > 0 { - info!( - "Successfully connected to Bitcoin Core after {} retries", - attempt - ); - } - return Ok(value); - } - Err(e) => { - if attempt == 0 { - info!("Could not connect to Bitcoin Core, retrying: {}", e); - } - last_error = Some(e); - } - } - } - - let err = last_error.unwrap(); - info!( - "Failed to connect to Bitcoin Core after {} attempts", - max_retries + 1 - ); - Err(err) - } - - pub fn call_with_retry(&self, f: F) -> Result - where - F: Fn(&CoreClient) -> Result, - { - for attempt in 0..=self.max_retries { - if attempt > 0 { - info!( - "Trying to reconnect to Bitcoin Core (attempt {}/{})", - attempt, self.max_retries - ); - self.recreate().ok(); - sleep(self.retry_delay); - } - - match f(&self.client.read()) { - Ok(value) => { - if attempt > 0 { - info!( - "Successfully reconnected to Bitcoin Core after {} attempts", - attempt - ); - } - return Ok(value); - } - Err(e) if Self::is_retriable(&e) => { - if attempt == 0 { - info!("Lost connection to Bitcoin Core, reconnecting..."); - } - } - Err(e) => return Err(e), - } - } - - info!( - "Could not reconnect to Bitcoin Core after {} attempts", - self.max_retries + 1 - ); - Err(RpcError::JsonRpc(jsonrpc::Error::Rpc( - jsonrpc::error::RpcError { - code: -1, - message: "Max retries exceeded".to_string(), - data: None, - }, - ))) - } - - pub fn call_once(&self, f: F) -> Result - where - F: Fn(&CoreClient) -> Result, - { - f(&self.client.read()) - } - - // --- Wrapped methods returning shared types --- - - pub fn get_blockchain_info(&self) -> Result { - let r = self.call_with_retry(|c| c.get_blockchain_info())?; - Ok(BlockchainInfo { - headers: r.headers, - blocks: r.blocks, - }) - } - - pub fn get_block(&self, hash: &bitcoin::BlockHash) -> Result { - Ok(self.call_with_retry(|c| c.get_block(hash))?) - } - - pub fn get_block_count(&self) -> Result { - Ok(self.call_with_retry(|c| c.get_block_count())?) - } - - pub fn get_block_hash(&self, height: u64) -> Result { - Ok(self.call_with_retry(|c| c.get_block_hash(height))?) - } - - /// Batched canonical height → block hash lookup over the inclusive - /// range `start..=end`. See the corepc backend for the rationale and - /// chunking strategy; this mirror uses bitcoincore-rpc's - /// `get_jsonrpc_client` accessor. - pub fn get_block_hashes_range( - &self, - start: u64, - end: u64, - ) -> Result> { - if end < start { - return Ok(Vec::new()); - } - let total = (end - start + 1) as usize; - let mut hashes = Vec::with_capacity(total); - - let mut chunk_start = start; - while chunk_start <= end { - let chunk_end = (chunk_start + BATCH_CHUNK as u64 - 1).min(end); - self.batch_get_block_hashes(chunk_start, chunk_end, &mut hashes)?; - chunk_start = chunk_end + 1; - } - Ok(hashes) - } - - fn batch_get_block_hashes( - &self, - start: u64, - end: u64, - out: &mut Vec, - ) -> Result<()> { - let params: Vec> = (start..=end) - .map(|h| { - RawValue::from_string(format!("[{h}]")).map_err(|e| Error::Parse(e.to_string())) - }) - .collect::>>()?; - - let client = self.client.read(); - let jsonrpc_client = client.get_jsonrpc_client(); - let requests: Vec = params - .iter() - .map(|p| jsonrpc_client.build_request("getblockhash", Some(p))) - .collect(); - - let responses = jsonrpc_client - .send_batch(&requests) - .map_err(|e| Error::Parse(format!("getblockhash batch failed: {e}")))?; - - for response in responses { - let response = response.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?; - let hex: String = response - .result() - .map_err(|e| Error::Parse(format!("getblockhash batch result: {e}")))?; - out.push( - hex.parse::() - .map_err(|e| Error::Parse(format!("invalid block hash hex: {e}")))?, - ); - } - Ok(()) - } - - pub fn get_block_header(&self, hash: &bitcoin::BlockHash) -> Result { - Ok(self.call_with_retry(|c| c.get_block_header(hash))?) - } - - pub fn get_block_info(&self, hash: &bitcoin::BlockHash) -> Result { - let r = self.call_with_retry(|c| c.get_block_info(hash))?; - Ok(BlockInfo { - height: r.height, - confirmations: r.confirmations as i64, - }) - } - - pub fn get_block_header_info(&self, hash: &bitcoin::BlockHash) -> Result { - let r = self.call_with_retry(|c| c.get_block_header_info(hash))?; - Ok(BlockHeaderInfo { - height: r.height, - confirmations: r.confirmations as i64, - previous_block_hash: r.previous_block_hash, - }) - } - - pub fn get_tx_out( - &self, - txid: &bitcoin::Txid, - vout: u32, - include_mempool: Option, - ) -> Result> { - let r = self.call_with_retry(|c| c.get_tx_out(txid, vout, include_mempool))?; - match r { - Some(r) => Ok(Some(TxOutInfo { - coinbase: r.coinbase, - value: Sats::from(r.value.to_sat()), - script_pub_key: r.script_pub_key.script()?, - })), - None => Ok(None), - } - } - - pub fn get_raw_mempool(&self) -> Result> { - Ok(self.call_with_retry(|c| c.get_raw_mempool())?) - } - - pub fn get_raw_mempool_verbose(&self) -> Result> { - let r = self.call_with_retry(|c| c.get_raw_mempool_verbose())?; - Ok(r.into_iter() - .map(|(txid, entry)| { - ( - txid, - RawMempoolEntry { - vsize: entry.vsize, - weight: entry.weight.unwrap_or(entry.vsize * 4), - base_fee_sats: entry.fees.base.to_sat(), - ancestor_count: entry.ancestor_count, - ancestor_size: entry.ancestor_size, - ancestor_fee_sats: entry.fees.ancestor.to_sat(), - depends: entry.depends.into_iter().collect(), - }, - ) - }) - .collect()) - } - - pub fn get_raw_transaction_hex( - &self, - txid: &bitcoin::Txid, - block_hash: Option<&bitcoin::BlockHash>, - ) -> Result { - Ok(self.call_with_retry(|c| c.get_raw_transaction_hex(txid, block_hash))?) - } - - pub fn send_raw_transaction(&self, hex: &str) -> Result { - Ok(self.call_once(|c| c.send_raw_transaction(hex))?) - } - - /// Transactions Bitcoin Core would include in the next block it would - /// mine. Core requires the `segwit` rule to be declared. - pub fn get_block_template_txs(&self) -> Result> { - let r = self.call_with_retry(|c| { - c.get_block_template( - GetBlockTemplateModes::Template, - &[GetBlockTemplateRules::SegWit], - &[] as &[GetBlockTemplateCapabilities], - ) - })?; - Ok(r.transactions - .into_iter() - .map(|t| BlockTemplateTx { - txid: Txid::from(t.txid), - fee: Sats::from(t.fee.to_sat()), - }) - .collect()) - } -} diff --git a/crates/brk_rpc/src/backend/corepc.rs b/crates/brk_rpc/src/backend/corepc.rs deleted file mode 100644 index 50edf59f9..000000000 --- a/crates/brk_rpc/src/backend/corepc.rs +++ /dev/null @@ -1,413 +0,0 @@ -use std::{thread::sleep, time::Duration}; - -use brk_error::{Error, Result}; -use brk_types::{Sats, Txid}; -use corepc_client::client_sync::Auth as CorepcAuth; -use parking_lot::RwLock; -use serde_json::value::RawValue; -use tracing::info; - -use super::{ - Auth, BlockHeaderInfo, BlockInfo, BlockTemplateTx, BlockchainInfo, RawMempoolEntry, TxOutInfo, -}; - -type CoreClient = corepc_client::client_sync::v30::Client; -type CoreError = corepc_client::client_sync::Error; - -/// Per-batch request count for `get_block_hashes_range`. Sized so the -/// JSON request body stays well under a megabyte and bitcoind doesn't -/// spend too long on a single batch before yielding results. -const BATCH_CHUNK: usize = 2000; - -#[derive(Debug)] -pub struct ClientInner { - url: String, - auth: Auth, - client: RwLock, - max_retries: usize, - retry_delay: Duration, -} - -impl ClientInner { - pub fn new(url: &str, auth: Auth, max_retries: usize, retry_delay: Duration) -> Result { - let client = Self::retry(max_retries, retry_delay, || { - Self::create_client(url, &auth).map_err(Into::into) - })?; - - Ok(Self { - url: url.to_string(), - auth, - client: RwLock::new(client), - max_retries, - retry_delay, - }) - } - - fn create_client(url: &str, auth: &Auth) -> Result { - let corepc_auth = match auth { - Auth::None => CorepcAuth::None, - Auth::UserPass(u, p) => CorepcAuth::UserPass(u.clone(), p.clone()), - Auth::CookieFile(path) => CorepcAuth::CookieFile(path.clone()), - }; - match corepc_auth { - CorepcAuth::None => Ok(CoreClient::new(url)), - other => CoreClient::new_with_auth(url, other), - } - } - - fn recreate(&self) -> Result<()> { - *self.client.write() = Self::create_client(&self.url, &self.auth)?; - Ok(()) - } - - fn is_retriable(error: &CoreError) -> bool { - match error { - CoreError::JsonRpc(corepc_jsonrpc::error::Error::Rpc(e)) => { - e.code == -32600 || e.code == 401 || e.code == -28 - } - CoreError::JsonRpc(corepc_jsonrpc::error::Error::Transport(_)) => true, - _ => false, - } - } - - fn retry(max_retries: usize, delay: Duration, mut f: F) -> Result - where - F: FnMut() -> Result, - { - let mut last_error = None; - - for attempt in 0..=max_retries { - if attempt > 0 { - info!( - "Retrying to connect to Bitcoin Core (attempt {}/{})", - attempt, max_retries - ); - sleep(delay); - } - - match f() { - Ok(value) => { - if attempt > 0 { - info!( - "Successfully connected to Bitcoin Core after {} retries", - attempt - ); - } - return Ok(value); - } - Err(e) => { - if attempt == 0 { - info!("Could not connect to Bitcoin Core, retrying: {}", e); - } - last_error = Some(e); - } - } - } - - let err = last_error.unwrap(); - info!( - "Failed to connect to Bitcoin Core after {} attempts", - max_retries + 1 - ); - Err(err) - } - - fn call_with_retry(&self, f: F) -> Result - where - F: Fn(&CoreClient) -> Result, - { - for attempt in 0..=self.max_retries { - if attempt > 0 { - info!( - "Trying to reconnect to Bitcoin Core (attempt {}/{})", - attempt, self.max_retries - ); - self.recreate().ok(); - sleep(self.retry_delay); - } - - match f(&self.client.read()) { - Ok(value) => { - if attempt > 0 { - info!( - "Successfully reconnected to Bitcoin Core after {} attempts", - attempt - ); - } - return Ok(value); - } - Err(e) if Self::is_retriable(&e) => { - if attempt == 0 { - info!("Lost connection to Bitcoin Core, reconnecting..."); - } - } - Err(e) => return Err(e), - } - } - - info!( - "Could not reconnect to Bitcoin Core after {} attempts", - self.max_retries + 1 - ); - Err(CoreError::JsonRpc(corepc_jsonrpc::error::Error::Rpc( - corepc_jsonrpc::error::RpcError { - code: -1, - message: "Max retries exceeded".to_string(), - data: None, - }, - ))) - } - - // --- Wrapped methods returning shared types --- - - pub fn get_blockchain_info(&self) -> Result { - let r = self.call_with_retry(|c| c.get_blockchain_info())?; - Ok(BlockchainInfo { - headers: r.headers as u64, - blocks: r.blocks as u64, - }) - } - - pub fn get_block(&self, hash: &bitcoin::BlockHash) -> Result { - Ok(self.call_with_retry(|c| c.get_block(*hash))?) - } - - pub fn get_block_count(&self) -> Result { - let r = self.call_with_retry(|c| c.get_block_count())?; - Ok(r.0) - } - - pub fn get_block_hash(&self, height: u64) -> Result { - let r = self.call_with_retry(|c| c.get_block_hash(height))?; - Ok(r.block_hash()?) - } - - /// Batched canonical height → block hash lookup over the inclusive - /// range `start..=end`. Internally splits into JSON-RPC batches of - /// `BATCH_CHUNK` requests so a 1M-block reindex doesn't try to push - /// a 50 MB request body or hold every response in memory at once. - /// - /// Returns hashes in canonical order (`start`, `start+1`, …, `end`). - pub fn get_block_hashes_range(&self, start: u64, end: u64) -> Result> { - if end < start { - return Ok(Vec::new()); - } - let total = (end - start + 1) as usize; - let mut hashes = Vec::with_capacity(total); - - let mut chunk_start = start; - while chunk_start <= end { - let chunk_end = (chunk_start + BATCH_CHUNK as u64 - 1).min(end); - self.batch_get_block_hashes(chunk_start, chunk_end, &mut hashes)?; - chunk_start = chunk_end + 1; - } - Ok(hashes) - } - - fn batch_get_block_hashes( - &self, - start: u64, - end: u64, - out: &mut Vec, - ) -> Result<()> { - let params: Vec> = (start..=end) - .map(|h| { - RawValue::from_string(format!("[{h}]")).map_err(|e| Error::Parse(e.to_string())) - }) - .collect::>>()?; - - let client = self.client.read(); - let requests: Vec = params - .iter() - .map(|p| client.jsonrpc().build_request("getblockhash", Some(p))) - .collect(); - - let responses = client - .jsonrpc() - .send_batch(&requests) - .map_err(|e| Error::Parse(format!("getblockhash batch failed: {e}")))?; - - for response in responses { - let response = response.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?; - let hex: String = response - .result() - .map_err(|e| Error::Parse(format!("getblockhash batch result: {e}")))?; - out.push( - hex.parse::() - .map_err(|e| Error::Parse(format!("invalid block hash hex: {e}")))?, - ); - } - Ok(()) - } - - pub fn get_block_header(&self, hash: &bitcoin::BlockHash) -> Result { - let r = self.call_with_retry(|c| c.get_block_header(hash))?; - r.block_header() - .map_err(|_| CoreError::UnexpectedStructure.into()) - } - - pub fn get_block_info(&self, hash: &bitcoin::BlockHash) -> Result { - let r = self.call_with_retry(|c| c.get_block_verbose_one(*hash))?; - Ok(BlockInfo { - height: r.height as usize, - confirmations: r.confirmations, - }) - } - - pub fn get_block_header_info(&self, hash: &bitcoin::BlockHash) -> Result { - let r = self.call_with_retry(|c| c.get_block_header_verbose(hash))?; - let previous_block_hash = r - .previous_block_hash - .map(|s| s.parse::()) - .transpose() - .map_err(|_| corepc_client::client_sync::Error::UnexpectedStructure)?; - Ok(BlockHeaderInfo { - height: r.height as usize, - confirmations: r.confirmations, - previous_block_hash, - }) - } - - pub fn get_tx_out( - &self, - txid: &bitcoin::Txid, - vout: u32, - include_mempool: Option, - ) -> Result> { - // corepc's typed get_tx_out doesn't support include_mempool, so use raw call - let r: Option = self.call_with_retry(|c| { - let mut args = vec![ - serde_json::to_value(txid).map_err(CoreError::from)?, - serde_json::to_value(vout).map_err(CoreError::from)?, - ]; - if let Some(mempool) = include_mempool { - args.push(serde_json::to_value(mempool).map_err(CoreError::from)?); - } - c.call("gettxout", &args) - })?; - - match r { - Some(r) => { - let script_pub_key = bitcoin::ScriptBuf::from_hex(&r.script_pub_key.hex) - .map_err(|_| corepc_client::client_sync::Error::UnexpectedStructure)?; - let sats = (r.value * 100_000_000.0).round() as u64; - Ok(Some(TxOutInfo { - coinbase: r.coinbase, - value: Sats::from(sats), - script_pub_key, - })) - } - None => Ok(None), - } - } - - pub fn get_raw_mempool(&self) -> Result> { - let r = self.call_with_retry(|c| c.get_raw_mempool())?; - r.0.iter() - .map(|s| { - s.parse::() - .map_err(|_| corepc_client::client_sync::Error::UnexpectedStructure.into()) - }) - .collect() - } - - pub fn get_raw_mempool_verbose(&self) -> Result> { - let r = self.call_with_retry(|c| c.get_raw_mempool_verbose())?; - r.0.into_iter() - .map(|(txid_str, entry)| { - let txid = txid_str - .parse::() - .map_err(|_| corepc_client::client_sync::Error::UnexpectedStructure)?; - let depends = entry - .depends - .iter() - .map(|s| { - s.parse::() - .map_err(|_| corepc_client::client_sync::Error::UnexpectedStructure) - }) - .collect::, _>>()?; - Ok(( - txid, - RawMempoolEntry { - vsize: entry.vsize as u64, - weight: entry.weight as u64, - base_fee_sats: (entry.fees.base * 100_000_000.0).round() as u64, - ancestor_count: entry.ancestor_count as u64, - ancestor_size: entry.ancestor_size as u64, - ancestor_fee_sats: (entry.fees.ancestor * 100_000_000.0).round() as u64, - depends, - }, - )) - }) - .collect() - } - - pub fn get_raw_transaction_hex( - &self, - txid: &bitcoin::Txid, - block_hash: Option<&bitcoin::BlockHash>, - ) -> Result { - // corepc's get_raw_transaction doesn't support block_hash param, use raw call - let r: String = self.call_with_retry(|c| { - let mut args: Vec = vec![ - serde_json::to_value(txid).map_err(CoreError::from)?, - serde_json::Value::Bool(false), - ]; - if let Some(bh) = block_hash { - args.push(serde_json::to_value(bh).map_err(CoreError::from)?); - } - c.call("getrawtransaction", &args) - })?; - Ok(r) - } - - pub fn send_raw_transaction(&self, hex: &str) -> Result { - let hex = hex.to_string(); - Ok(self.call_with_retry(|c| { - let args = [serde_json::Value::String(hex.clone())]; - c.call("sendrawtransaction", &args) - })?) - } - - /// Transactions Bitcoin Core would include in the next block it would - /// mine. Core requires the `segwit` rule to be declared. - pub fn get_block_template_txs(&self) -> Result> { - let args = [serde_json::json!({ "rules": ["segwit"] })]; - let r: GetBlockTemplateResponse = - self.call_with_retry(|c| c.call("getblocktemplate", &args))?; - - Ok(r.transactions - .into_iter() - .map(|t| BlockTemplateTx { - txid: Txid::from(t.txid), - fee: Sats::from(t.fee), - }) - .collect()) - } -} - -// Local deserialization structs for raw RPC responses - -#[derive(serde::Deserialize)] -struct TxOutResponse { - coinbase: bool, - value: f64, - #[serde(rename = "scriptPubKey")] - script_pub_key: TxOutScriptPubKey, -} - -#[derive(serde::Deserialize)] -struct TxOutScriptPubKey { - hex: String, -} - -#[derive(serde::Deserialize)] -struct GetBlockTemplateResponse { - transactions: Vec, -} - -#[derive(serde::Deserialize)] -struct GetBlockTemplateTx { - txid: bitcoin::Txid, - fee: u64, -} diff --git a/crates/brk_rpc/src/backend/mod.rs b/crates/brk_rpc/src/backend/mod.rs deleted file mode 100644 index 80939ae87..000000000 --- a/crates/brk_rpc/src/backend/mod.rs +++ /dev/null @@ -1,70 +0,0 @@ -use std::path::PathBuf; - -use bitcoin::ScriptBuf; -use brk_types::{Sats, Txid}; - -#[derive(Debug, Clone)] -pub struct BlockchainInfo { - pub headers: u64, - pub blocks: u64, -} - -#[derive(Debug, Clone)] -pub struct BlockInfo { - pub height: usize, - pub confirmations: i64, -} - -#[derive(Debug, Clone)] -pub struct BlockHeaderInfo { - pub height: usize, - pub confirmations: i64, - pub previous_block_hash: Option, -} - -#[derive(Debug, Clone)] -pub struct TxOutInfo { - pub coinbase: bool, - pub value: Sats, - pub script_pub_key: ScriptBuf, -} - -#[derive(Debug, Clone)] -pub struct BlockTemplateTx { - pub txid: Txid, - pub fee: Sats, -} - -#[derive(Debug, Clone)] -pub struct RawMempoolEntry { - pub vsize: u64, - pub weight: u64, - pub base_fee_sats: u64, - pub ancestor_count: u64, - pub ancestor_size: u64, - pub ancestor_fee_sats: u64, - pub depends: Vec, -} - -#[derive(Clone, Debug)] -pub enum Auth { - None, - UserPass(String, String), - CookieFile(PathBuf), -} - -#[cfg(feature = "bitcoincore-rpc")] -pub mod bitcoincore; - -#[cfg(feature = "corepc")] -pub mod corepc; - -// Default ClientInner: prefer bitcoincore-rpc when both are enabled -#[cfg(feature = "bitcoincore-rpc")] -pub use bitcoincore::ClientInner; - -#[cfg(all(feature = "corepc", not(feature = "bitcoincore-rpc")))] -pub use corepc::ClientInner; - -#[cfg(not(any(feature = "bitcoincore-rpc", feature = "corepc")))] -compile_error!("brk_rpc requires either the `bitcoincore-rpc` or `corepc` feature"); diff --git a/crates/brk_rpc/src/client.rs b/crates/brk_rpc/src/client.rs new file mode 100644 index 000000000..1ce776dcb --- /dev/null +++ b/crates/brk_rpc/src/client.rs @@ -0,0 +1,198 @@ +use std::{thread::sleep, time::Duration}; + +use brk_error::{Error, Result}; +use corepc_jsonrpc::{ + Client as JsonRpcClient, Request, error::Error as JsonRpcError, simple_http, +}; +use parking_lot::RwLock; +use serde::Deserialize; +use serde_json::{Value, value::RawValue}; +use tracing::info; + +use crate::Auth; + +#[derive(Debug)] +pub(crate) struct ClientInner { + url: String, + auth: Auth, + client: RwLock, + max_retries: usize, + retry_delay: Duration, +} + +impl ClientInner { + pub(crate) fn new( + url: &str, + auth: Auth, + max_retries: usize, + retry_delay: Duration, + ) -> Result { + let client = Self::create_client(url, &auth)?; + Ok(Self { + url: url.to_string(), + auth, + client: RwLock::new(client), + max_retries, + retry_delay, + }) + } + + /// Builds a `jsonrpc::Client` using the `simple_http` transport, which + /// keeps a single pooled TCP socket with reconnect-on-failure. The + /// upstream `corepc-client` hard-wires `bitreq_http` (one TCP connect + /// per request), which collapses under concurrent load. + fn create_client(url: &str, auth: &Auth) -> Result { + let builder = simple_http::Builder::new() + .url(url) + .map_err(|e| Error::Parse(format!("bad rpc url: {e}")))? + .timeout(Duration::from_secs(60)); + let builder = match auth { + Auth::None => builder, + Auth::UserPass(u, p) => builder.auth(u.clone(), Some(p.clone())), + Auth::CookieFile(path) => { + let cookie = std::fs::read_to_string(path)?; + builder.cookie_auth(cookie.trim()) + } + }; + Ok(JsonRpcClient::with_transport(builder.build())) + } + + fn recreate(&self) -> Result<()> { + *self.client.write() = Self::create_client(&self.url, &self.auth)?; + Ok(()) + } + + fn is_retriable(error: &JsonRpcError) -> bool { + match error { + JsonRpcError::Rpc(e) => e.code == -32600 || e.code == 401 || e.code == -28, + JsonRpcError::Transport(_) => true, + _ => false, + } + } + + pub(crate) fn call_with_retry(&self, method: &str, args: &[Value]) -> Result + where + T: for<'de> Deserialize<'de>, + { + let raw = serde_json::value::to_raw_value(args).map_err(Error::from)?; + + for attempt in 0..=self.max_retries { + if attempt > 0 { + info!( + "Trying to reconnect to Bitcoin Core (attempt {}/{})", + attempt, self.max_retries + ); + self.recreate().ok(); + sleep(self.retry_delay); + } + + match self.client.read().call::(method, Some(&raw)) { + Ok(value) => { + if attempt > 0 { + info!( + "Successfully reconnected to Bitcoin Core after {} attempts", + attempt + ); + } + return Ok(value); + } + Err(e) if Self::is_retriable(&e) => { + if attempt == 0 { + info!("Lost connection to Bitcoin Core, reconnecting..."); + } + } + Err(e) => return Err(e.into()), + } + } + + info!( + "Could not reconnect to Bitcoin Core after {} attempts", + self.max_retries + 1 + ); + Err(JsonRpcError::Rpc(corepc_jsonrpc::error::RpcError { + code: -1, + message: "Max retries exceeded".to_string(), + data: None, + }) + .into()) + } + + pub(crate) fn call_once(&self, method: &str, args: &[Value]) -> Result + where + T: for<'de> Deserialize<'de>, + { + let raw = serde_json::value::to_raw_value(args).map_err(Error::from)?; + Ok(self.client.read().call::(method, Some(&raw))?) + } + + /// Send a batch of calls sharing `method`, one set of args per request. + /// No retry: the caller decides batch sizing and failure semantics. + pub(crate) fn call_batch( + &self, + method: &str, + batch_args: impl IntoIterator>, + ) -> Result> + where + T: for<'de> Deserialize<'de>, + { + let params: Vec> = batch_args + .into_iter() + .map(|args| serde_json::value::to_raw_value(&args).map_err(Error::from)) + .collect::>>()?; + + let client = self.client.read(); + let requests: Vec = params + .iter() + .map(|p| client.build_request(method, Some(p))) + .collect(); + + let responses = client + .send_batch(&requests) + .map_err(|e| Error::Parse(format!("batch {method} failed: {e}")))?; + + responses + .into_iter() + .map(|resp| { + let resp = resp.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?; + resp.result::() + .map_err(|e| Error::Parse(format!("batch {method} result: {e}"))) + }) + .collect() + } + + /// Like `call_batch` but reports per-request success/failure independently, + /// so one bad item doesn't nuke an otherwise-healthy chunk. The outer + /// `Result` still fails if the HTTP round-trip itself fails. + pub(crate) fn call_batch_per_item( + &self, + method: &str, + batch_args: impl IntoIterator>, + ) -> Result>> + where + T: for<'de> Deserialize<'de>, + { + let params: Vec> = batch_args + .into_iter() + .map(|args| serde_json::value::to_raw_value(&args).map_err(Error::from)) + .collect::>>()?; + + let client = self.client.read(); + let requests: Vec = params + .iter() + .map(|p| client.build_request(method, Some(p))) + .collect(); + + let responses = client + .send_batch(&requests) + .map_err(|e| Error::Parse(format!("batch {method} failed: {e}")))?; + + Ok(responses + .into_iter() + .map(|resp| { + let resp = resp.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?; + resp.result::() + .map_err(|e| Error::Parse(format!("batch {method} result: {e}"))) + }) + .collect()) + } +} diff --git a/crates/brk_rpc/src/lib.rs b/crates/brk_rpc/src/lib.rs index c12052a85..26a6928f7 100644 --- a/crates/brk_rpc/src/lib.rs +++ b/crates/brk_rpc/src/lib.rs @@ -1,21 +1,66 @@ use std::{ - env, mem, + env, path::{Path, PathBuf}, sync::Arc, - thread::sleep, time::Duration, }; -use bitcoin::consensus::encode; -use brk_error::{Error, Result}; -use brk_types::{BlockHash, Height, MempoolEntryInfo, Sats, Txid, Vout}; +use bitcoin::ScriptBuf; +use brk_error::Result; +use brk_types::{BlockHash, Hex, Sats, Txid}; -pub mod backend; +mod client; +mod methods; -pub use backend::{Auth, BlockHeaderInfo, BlockInfo, BlockTemplateTx, BlockchainInfo, TxOutInfo}; +use client::ClientInner; -use backend::ClientInner; -use tracing::{debug, info}; +#[derive(Debug, Clone)] +pub struct BlockchainInfo { + pub headers: u64, + pub blocks: u64, +} + +#[derive(Debug, Clone)] +pub struct BlockInfo { + pub height: usize, + pub confirmations: i64, +} + +#[derive(Debug, Clone)] +pub struct BlockHeaderInfo { + pub height: usize, + pub confirmations: i64, + pub previous_block_hash: Option, +} + +#[derive(Debug, Clone)] +pub struct TxOutInfo { + pub coinbase: bool, + pub value: Sats, + pub script_pub_key: ScriptBuf, +} + +#[derive(Debug, Clone)] +pub struct BlockTemplateTx { + pub txid: Txid, + pub fee: Sats, +} + +/// A transaction fetched from Core alongside the exact hex bytes Core +/// returned, so downstream code can re-emit the raw tx without re- +/// serializing (which could diverge on segwit flag encoding, etc.). +#[derive(Debug, Clone)] +pub struct RawTx { + pub tx: bitcoin::Transaction, + pub hex: Hex, +} + +#[derive(Clone, Debug)] +pub enum Auth { + None, + UserPass(String, String), + CookieFile(PathBuf), +} /// /// Bitcoin Core RPC Client @@ -23,7 +68,7 @@ use tracing::{debug, info}; /// Thread safe and free to clone /// #[derive(Debug, Clone)] -pub struct Client(Arc); +pub struct Client(pub(crate) Arc); impl Client { pub fn new(url: &str, auth: Auth) -> Result { @@ -44,243 +89,6 @@ impl Client { )?))) } - /// Returns a data structure containing various state info regarding - /// blockchain processing. - pub fn get_blockchain_info(&self) -> Result { - self.0.get_blockchain_info() - } - - pub fn get_block<'a, H>(&self, hash: &'a H) -> Result - where - &'a H: Into<&'a bitcoin::BlockHash>, - { - self.0.get_block(hash.into()) - } - - /// Returns the numbers of block in the longest chain. - pub fn get_block_count(&self) -> Result { - self.0.get_block_count() - } - - /// Returns the numbers of block in the longest chain. - pub fn get_last_height(&self) -> Result { - self.0.get_block_count().map(Height::from) - } - - /// Get block hash at a given height - pub fn get_block_hash(&self, height: H) -> Result - where - H: Into + Copy, - { - self.0.get_block_hash(height.into()).map(BlockHash::from) - } - - /// Get every canonical block hash for the inclusive height range - /// `start..=end` in a single JSON-RPC batch request. Returns hashes - /// in canonical order (`start`, `start+1`, …, `end`). Use this - /// whenever resolving more than ~2 heights — one HTTP round-trip - /// beats N sequential `get_block_hash` calls once the per-call - /// overhead dominates. - pub fn get_block_hashes_range(&self, start: H1, end: H2) -> Result> - where - H1: Into, - H2: Into, - { - self.0 - .get_block_hashes_range(start.into(), end.into()) - .map(|v| v.into_iter().map(BlockHash::from).collect()) - } - - pub fn get_block_header<'a, H>(&self, hash: &'a H) -> Result - where - &'a H: Into<&'a bitcoin::BlockHash>, - { - self.0.get_block_header(hash.into()) - } - - pub fn get_block_info<'a, H>(&self, hash: &'a H) -> Result - where - &'a H: Into<&'a bitcoin::BlockHash>, - { - self.0.get_block_info(hash.into()) - } - - pub fn get_block_header_info<'a, H>(&self, hash: &'a H) -> Result - where - &'a H: Into<&'a bitcoin::BlockHash>, - { - self.0.get_block_header_info(hash.into()) - } - - pub fn get_transaction<'a, T, H>( - &self, - txid: &'a T, - block_hash: Option<&'a H>, - ) -> brk_error::Result - where - &'a T: Into<&'a bitcoin::Txid>, - &'a H: Into<&'a bitcoin::BlockHash>, - { - let tx = self.get_raw_transaction(txid, block_hash)?; - Ok(tx) - } - - pub fn get_mempool_raw_tx( - &self, - txid: &Txid, - ) -> Result<(bitcoin::Transaction, String)> { - let hex = self.get_raw_transaction_hex(txid, None as Option<&BlockHash>)?; - let tx = encode::deserialize_hex::(&hex)?; - Ok((tx, hex)) - } - - pub fn get_tx_out( - &self, - txid: &Txid, - vout: Vout, - include_mempool: Option, - ) -> Result> { - self.0.get_tx_out(txid.into(), vout.into(), include_mempool) - } - - /// Get txids of all transactions in a memory pool - pub fn get_raw_mempool(&self) -> Result> { - self.0 - .get_raw_mempool() - .map(|v| unsafe { mem::transmute(v) }) - } - - /// Get all mempool entries with their fee data in a single RPC call - pub fn get_raw_mempool_verbose(&self) -> Result> { - let result = self.0.get_raw_mempool_verbose()?; - Ok(result - .into_iter() - .map( - |(txid, entry): (bitcoin::Txid, backend::RawMempoolEntry)| MempoolEntryInfo { - txid: txid.into(), - vsize: entry.vsize, - weight: entry.weight, - fee: Sats::from(entry.base_fee_sats), - ancestor_count: entry.ancestor_count, - ancestor_size: entry.ancestor_size, - ancestor_fee: Sats::from(entry.ancestor_fee_sats), - depends: entry.depends.into_iter().map(Txid::from).collect(), - }, - ) - .collect()) - } - - pub fn get_raw_transaction<'a, T, H>( - &self, - txid: &'a T, - block_hash: Option<&'a H>, - ) -> brk_error::Result - where - &'a T: Into<&'a bitcoin::Txid>, - &'a H: Into<&'a bitcoin::BlockHash>, - { - let hex = self.get_raw_transaction_hex(txid, block_hash)?; - let tx = encode::deserialize_hex::(&hex)?; - Ok(tx) - } - - pub fn get_raw_transaction_hex<'a, T, H>( - &self, - txid: &'a T, - block_hash: Option<&'a H>, - ) -> Result - where - &'a T: Into<&'a bitcoin::Txid>, - &'a H: Into<&'a bitcoin::BlockHash>, - { - self.0 - .get_raw_transaction_hex(txid.into(), block_hash.map(|h| h.into())) - } - - pub fn send_raw_transaction(&self, hex: &str) -> Result { - self.0.send_raw_transaction(hex).map(Txid::from) - } - - /// Transactions (txid + fee) Bitcoin Core would include in the next - /// block it would mine, via `getblocktemplate`. - pub fn get_block_template_txs(&self) -> Result> { - self.0.get_block_template_txs() - } - - /// Checks if a block is in the main chain (has positive confirmations) - pub fn is_in_main_chain(&self, hash: &BlockHash) -> Result { - let block_info = self.get_block_info(hash)?; - Ok(block_info.confirmations > 0) - } - - pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> { - debug!("Get closest valid height..."); - - match self.get_block_header_info(&hash) { - Ok(block_info) => { - if self.is_in_main_chain(&hash)? { - return Ok((block_info.height.into(), hash)); - } - - let mut hash = - block_info - .previous_block_hash - .map(BlockHash::from) - .ok_or(Error::NotFound( - "Genesis block has no previous block".into(), - ))?; - - loop { - if self.is_in_main_chain(&hash)? { - let current_info = self.get_block_header_info(&hash)?; - return Ok((current_info.height.into(), hash)); - } - - let info = self.get_block_header_info(&hash)?; - hash = info - .previous_block_hash - .map(BlockHash::from) - .ok_or(Error::NotFound( - "Reached genesis without finding main chain".into(), - ))?; - } - } - Err(_) => Err(Error::NotFound("Block hash not found in blockchain".into())), - } - } - - pub fn wait_for_synced_node(&self) -> Result<()> { - let is_synced = || -> Result { - let info = self.get_blockchain_info()?; - Ok(info.headers == info.blocks) - }; - - if !is_synced()? { - info!("Waiting for node to sync..."); - while !is_synced()? { - sleep(Duration::from_secs(1)) - } - } - - Ok(()) - } - - #[cfg(feature = "bitcoincore-rpc")] - pub fn call(&self, f: F) -> Result - where - F: Fn(&bitcoincore_rpc::Client) -> Result, - { - self.0.call_with_retry(f) - } - - #[cfg(feature = "bitcoincore-rpc")] - pub fn call_once(&self, f: F) -> Result - where - F: Fn(&bitcoincore_rpc::Client) -> Result, - { - self.0.call_once(f) - } - pub fn default_url() -> &'static str { "http://localhost:8332" } diff --git a/crates/brk_rpc/src/methods.rs b/crates/brk_rpc/src/methods.rs new file mode 100644 index 000000000..bf0463596 --- /dev/null +++ b/crates/brk_rpc/src/methods.rs @@ -0,0 +1,364 @@ +use std::{thread::sleep, time::Duration}; + +use bitcoin::{consensus::encode, hex::FromHex}; +use brk_error::{Error, Result}; +use brk_types::{Bitcoin, BlockHash, Height, MempoolEntryInfo, Sats, Txid, Vout}; +use corepc_types::v30::{ + GetBlockCount, GetBlockHash, GetBlockHeader, GetBlockHeaderVerbose, GetBlockVerboseOne, + GetBlockVerboseZero, GetBlockchainInfo, GetRawMempool, GetRawMempoolVerbose, GetTxOut, +}; +use rustc_hash::FxHashMap; +use serde::Deserialize; +use serde_json::Value; +use tracing::{debug, info}; + +use crate::{BlockHeaderInfo, BlockInfo, BlockTemplateTx, BlockchainInfo, Client, RawTx, TxOutInfo}; + +/// Per-batch request count for `get_block_hashes_range`. Sized so the +/// JSON request body stays well under a megabyte and bitcoind doesn't +/// spend too long on a single batch before yielding results. +const BATCH_CHUNK: usize = 2000; + +impl Client { + pub fn get_blockchain_info(&self) -> Result { + let r: GetBlockchainInfo = self.0.call_with_retry("getblockchaininfo", &[])?; + Ok(BlockchainInfo { + headers: r.headers as u64, + blocks: r.blocks as u64, + }) + } + + /// Returns the numbers of block in the longest chain. + pub fn get_block_count(&self) -> Result { + let r: GetBlockCount = self.0.call_with_retry("getblockcount", &[])?; + Ok(r.0) + } + + /// Returns the numbers of block in the longest chain. + pub fn get_last_height(&self) -> Result { + self.get_block_count().map(Height::from) + } + + pub fn get_block<'a, H>(&self, hash: &'a H) -> Result + where + &'a H: Into<&'a bitcoin::BlockHash>, + { + let hash: &bitcoin::BlockHash = hash.into(); + let r: GetBlockVerboseZero = self.0.call_with_retry( + "getblock", + &[serde_json::to_value(hash)?, Value::from(0u8)], + )?; + r.block() + .map_err(|e| Error::Parse(format!("decode getblock: {e}"))) + } + + pub fn get_block_info<'a, H>(&self, hash: &'a H) -> Result + where + &'a H: Into<&'a bitcoin::BlockHash>, + { + let hash: &bitcoin::BlockHash = hash.into(); + let r: GetBlockVerboseOne = self.0.call_with_retry( + "getblock", + &[serde_json::to_value(hash)?, Value::from(1u8)], + )?; + Ok(BlockInfo { + height: r.height as usize, + confirmations: r.confirmations, + }) + } + + pub fn get_block_header<'a, H>(&self, hash: &'a H) -> Result + where + &'a H: Into<&'a bitcoin::BlockHash>, + { + let hash: &bitcoin::BlockHash = hash.into(); + let r: GetBlockHeader = self.0.call_with_retry( + "getblockheader", + &[serde_json::to_value(hash)?, Value::Bool(false)], + )?; + let bytes = Vec::from_hex(&r.0).map_err(|e| Error::Parse(format!("header hex: {e}")))?; + bitcoin::consensus::deserialize::(&bytes).map_err(Error::from) + } + + pub fn get_block_header_info<'a, H>(&self, hash: &'a H) -> Result + where + &'a H: Into<&'a bitcoin::BlockHash>, + { + let hash: &bitcoin::BlockHash = hash.into(); + let r: GetBlockHeaderVerbose = self + .0 + .call_with_retry("getblockheader", &[serde_json::to_value(hash)?])?; + let previous_block_hash = r + .previous_block_hash + .map(|s| Self::parse_block_hash(&s, "previousblockhash")) + .transpose()?; + Ok(BlockHeaderInfo { + height: r.height as usize, + confirmations: r.confirmations, + previous_block_hash, + }) + } + + /// Get block hash at a given height + pub fn get_block_hash(&self, height: H) -> Result + where + H: Into + Copy, + { + let height: u64 = height.into(); + let r: GetBlockHash = self + .0 + .call_with_retry("getblockhash", &[serde_json::to_value(height)?])?; + Ok(BlockHash::from(r.block_hash()?)) + } + + /// Get every canonical block hash for the inclusive height range + /// `start..=end` in a single JSON-RPC batch request. Returns hashes + /// in canonical order (`start`, `start+1`, …, `end`). Use this + /// whenever resolving more than ~2 heights — one HTTP round-trip + /// beats N sequential `get_block_hash` calls once the per-call + /// overhead dominates. + pub fn get_block_hashes_range(&self, start: H1, end: H2) -> Result> + where + H1: Into, + H2: Into, + { + let start: u64 = start.into(); + let end: u64 = end.into(); + if end < start { + return Ok(Vec::new()); + } + let total = (end - start + 1) as usize; + let mut hashes = Vec::with_capacity(total); + + let mut chunk_start = start; + while chunk_start <= end { + let chunk_end = (chunk_start + BATCH_CHUNK as u64 - 1).min(end); + let args = (chunk_start..=chunk_end).map(|h| vec![Value::from(h)]); + let chunk: Vec = self.0.call_batch("getblockhash", args)?; + for hex in chunk { + hashes.push(Self::parse_block_hash(&hex, "getblockhash batch")?); + } + chunk_start = chunk_end + 1; + } + Ok(hashes) + } + + pub fn get_tx_out( + &self, + txid: &Txid, + vout: Vout, + include_mempool: Option, + ) -> Result> { + let txid: &bitcoin::Txid = txid.into(); + let mut args: Vec = vec![ + serde_json::to_value(txid)?, + serde_json::to_value(u32::from(vout))?, + ]; + if let Some(mempool) = include_mempool { + args.push(Value::Bool(mempool)); + } + let r: Option = self.0.call_with_retry("gettxout", &args)?; + match r { + Some(r) => { + let script_pub_key = bitcoin::ScriptBuf::from_hex(&r.script_pubkey.hex) + .map_err(|e| Error::Parse(format!("script hex: {e}")))?; + Ok(Some(TxOutInfo { + coinbase: r.coinbase, + value: Sats::from(Bitcoin::from(r.value)), + script_pub_key, + })) + } + None => Ok(None), + } + } + + /// Get txids of all transactions in a memory pool + pub fn get_raw_mempool(&self) -> Result> { + let r: GetRawMempool = self.0.call_with_retry("getrawmempool", &[])?; + r.0.iter() + .map(|s| Self::parse_txid(s, "mempool txid")) + .collect() + } + + /// Get all mempool entries with their fee data in a single RPC call + pub fn get_raw_mempool_verbose(&self) -> Result> { + let r: GetRawMempoolVerbose = self + .0 + .call_with_retry("getrawmempool", &[Value::Bool(true)])?; + r.0.into_iter() + .map(|(txid_str, entry)| { + let depends = entry + .depends + .iter() + .map(|s| Self::parse_txid(s, "depends txid")) + .collect::>>()?; + Ok(MempoolEntryInfo { + txid: Self::parse_txid(&txid_str, "mempool txid")?, + vsize: entry.vsize as u64, + weight: entry.weight as u64, + fee: Sats::from(Bitcoin::from(entry.fees.base)), + ancestor_count: entry.ancestor_count as u64, + ancestor_size: entry.ancestor_size as u64, + ancestor_fee: Sats::from(Bitcoin::from(entry.fees.ancestor)), + depends, + }) + }) + .collect() + } + + pub fn get_raw_transaction<'a, T, H>( + &self, + txid: &'a T, + block_hash: Option<&'a H>, + ) -> Result + where + &'a T: Into<&'a bitcoin::Txid>, + &'a H: Into<&'a bitcoin::BlockHash>, + { + let hex = self.get_raw_transaction_hex(txid, block_hash)?; + let tx = encode::deserialize_hex::(&hex)?; + Ok(tx) + } + + pub fn get_raw_transaction_hex<'a, T, H>( + &self, + txid: &'a T, + block_hash: Option<&'a H>, + ) -> Result + where + &'a T: Into<&'a bitcoin::Txid>, + &'a H: Into<&'a bitcoin::BlockHash>, + { + let txid: &bitcoin::Txid = txid.into(); + let mut args: Vec = vec![serde_json::to_value(txid)?, Value::Bool(false)]; + if let Some(bh) = block_hash { + let bh: &bitcoin::BlockHash = bh.into(); + args.push(serde_json::to_value(bh)?); + } + self.0.call_with_retry("getrawtransaction", &args) + } + + pub fn get_mempool_raw_tx(&self, txid: &Txid) -> Result { + let hex = self.get_raw_transaction_hex(txid, None as Option<&BlockHash>)?; + let tx = encode::deserialize_hex::(&hex)?; + Ok(RawTx { tx, hex: hex.into() }) + } + + /// Batched `getrawtransaction` over a slice of txids. Returns a map keyed + /// by txid containing the deserialized tx and its raw hex. Individual + /// failures (e.g. a tx that evicted between the listing and this call) + /// are logged and dropped so a single bad entry doesn't kill the batch. + /// + /// Chunked at `BATCH_CHUNK` requests per round-trip. + pub fn get_raw_transactions( + &self, + txids: &[Txid], + ) -> Result> { + let mut out: FxHashMap = + FxHashMap::with_capacity_and_hasher(txids.len(), Default::default()); + + for chunk in txids.chunks(BATCH_CHUNK) { + let args = chunk.iter().map(|t| { + let bt: &bitcoin::Txid = t.into(); + vec![ + serde_json::to_value(bt).unwrap_or(Value::Null), + Value::Bool(false), + ] + }); + let results: Vec> = + self.0.call_batch_per_item("getrawtransaction", args)?; + + for (txid, res) in chunk.iter().zip(results) { + match res.and_then(|hex| { + let tx = encode::deserialize_hex::(&hex)?; + Ok::<_, Error>(RawTx { tx, hex: hex.into() }) + }) { + Ok(raw) => { + out.insert(txid.clone(), raw); + } + // Silenced: users without `-txindex` expect -5 for + // every confirmed tx. Downgraded so the mempool + // parent-fetch loop doesn't spam the log each cycle. + Err(e) => debug!(txid = %txid, error = %e, "getrawtransaction batch: item failed"), + } + } + } + + Ok(out) + } + + pub fn send_raw_transaction(&self, hex: &str) -> Result { + let txid: bitcoin::Txid = self + .0 + .call_once("sendrawtransaction", &[Value::String(hex.to_string())])?; + Ok(Txid::from(txid)) + } + + /// Transactions (txid + fee) Bitcoin Core would include in the next + /// block it would mine, via `getblocktemplate`. Core requires the + /// `segwit` rule to be declared. + pub fn get_block_template_txs(&self) -> Result> { + #[derive(Deserialize)] + struct Response { + transactions: Vec, + } + #[derive(Deserialize)] + struct Tx { + txid: bitcoin::Txid, + fee: u64, + } + + let args = [serde_json::json!({ "rules": ["segwit"] })]; + let r: Response = self.0.call_with_retry("getblocktemplate", &args)?; + Ok(r.transactions + .into_iter() + .map(|t| BlockTemplateTx { + txid: Txid::from(t.txid), + fee: Sats::from(t.fee), + }) + .collect()) + } + + pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> { + debug!("Get closest valid height..."); + + let mut current = hash; + loop { + let info = self.get_block_header_info(¤t)?; + if info.confirmations > 0 { + return Ok((info.height.into(), current)); + } + current = info.previous_block_hash.ok_or(Error::NotFound( + "Reached genesis without finding main chain".into(), + ))?; + } + } + + pub fn wait_for_synced_node(&self) -> Result<()> { + let is_synced = || -> Result { + let info = self.get_blockchain_info()?; + Ok(info.headers == info.blocks) + }; + + if !is_synced()? { + info!("Waiting for node to sync..."); + while !is_synced()? { + sleep(Duration::from_secs(1)) + } + } + + Ok(()) + } + + fn parse_txid(s: &str, label: &str) -> Result { + s.parse::() + .map(Txid::from) + .map_err(|e| Error::Parse(format!("{label}: {e}"))) + } + + fn parse_block_hash(s: &str, label: &str) -> Result { + s.parse::() + .map(BlockHash::from) + .map_err(|e| Error::Parse(format!("{label}: {e}"))) + } +} diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 4d7773554..3b681604e 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -23,7 +23,7 @@ brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_query = { workspace = true } brk_reader = { workspace = true } -brk_rpc = { workspace = true, features = ["corepc"] } +brk_rpc = { workspace = true } brk_types = { workspace = true } brk_traversable = { workspace = true } brk_website = { workspace = true } diff --git a/crates/brk_server/README.md b/crates/brk_server/README.md index 860e79a24..cf6ba138c 100644 --- a/crates/brk_server/README.md +++ b/crates/brk_server/README.md @@ -6,16 +6,22 @@ HTTP API server for Bitcoin on-chain analytics. - **OpenAPI spec**: Auto-generated docs at `/api` with full spec at `/openapi.json` - **LLM-optimized**: Compact spec at `/api.json` for AI tools -- **Response caching**: ETag-based with LRU cache (5000 entries) +- **Response caching**: ETag-based with LRU cache (1000 entries by default, configurable via `ServerConfig::cache_size`) - **Compression**: Brotli, gzip, deflate, zstd - **Static files**: Optional web interface hosting ## Usage ```rust,ignore -let server = Server::new(&async_query, data_path, Website::Filesystem(files_path)); -// Or Website::Default, or Website::Disabled -server.serve().await?; +let server = Server::new( + &async_query, + ServerConfig { + data_path, + website: Website::Filesystem(files_path), + ..Default::default() + }, +); +server.serve(None).await?; ``` ## Endpoints @@ -35,10 +41,19 @@ server.serve().await?; ## Caching -Uses ETag-based caching with `must-revalidate`: -- **Height-indexed**: Invalidates when new block arrives -- **Immutable**: 1-year cache for deeply-confirmed blocks/txs (6+ confirmations) -- **Mempool**: Short max-age, no ETag +ETag-based revalidation. Five strategies pick the etag scheme: + +- **Tip**: chain-state, etag = tip hash prefix (invalidates per block + reorgs) +- **Immutable**: deeply-confirmed data, etag = format version +- **BlockBound**: data tied to a specific block hash (reorg-safe) +- **Deploy**: catalog/static data, etag = build version +- **MempoolHash**: mempool data, etag = projected next-block hash + +Browser sees `Cache-Control: public, no-cache, stale-if-error=86400` (always +revalidate, ETag makes it cheap). CDN sees a separate `CDN-Cache-Control` +directive whose stable tier is selected by `CdnCacheMode` (`Live` revalidates +every request; `Aggressive` caches up to a year as `immutable` and requires a +purge on deploy). ## Configuration diff --git a/crates/brk_server/examples/server.rs b/crates/brk_server/examples/server.rs index 4433309db..78b7486fe 100644 --- a/crates/brk_server/examples/server.rs +++ b/crates/brk_server/examples/server.rs @@ -7,7 +7,7 @@ use brk_mempool::Mempool; use brk_query::AsyncQuery; use brk_reader::Reader; use brk_rpc::{Auth, Client}; -use brk_server::{Server, Website}; +use brk_server::{Server, ServerConfig, Website}; use tracing::info; use vecdb::Exit; @@ -43,7 +43,14 @@ pub fn main() -> Result<()> { // Option 1: block_on to run and properly propagate errors runtime.block_on(async move { - let server = Server::new(&query, outputs_dir, Website::Disabled); + let server = Server::new( + &query, + ServerConfig { + data_path: outputs_dir, + website: Website::Disabled, + ..Default::default() + }, + ); let handle = tokio::spawn(async move { server.serve(None).await }); diff --git a/crates/brk_server/src/api/mempool_space/addrs.rs b/crates/brk_server/src/api/mempool_space/addrs.rs index c45709dac..2c5d1cbe7 100644 --- a/crates/brk_server/src/api/mempool_space/addrs.rs +++ b/crates/brk_server/src/api/mempool_space/addrs.rs @@ -142,7 +142,7 @@ impl AddrRoutes for ApiRouter { Path(path): Path, State(state): State | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |_q| Ok(AddrValidation::from_addr(&path.addr))).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |_q| Ok(AddrValidation::from_addr(&path.addr))).await }, |op| op .id("validate_address") .addrs_tag() diff --git a/crates/brk_server/src/api/mempool_space/mining.rs b/crates/brk_server/src/api/mempool_space/mining.rs index 8ab68beac..c40bacb37 100644 --- a/crates/brk_server/src/api/mempool_space/mining.rs +++ b/crates/brk_server/src/api/mempool_space/mining.rs @@ -31,8 +31,8 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/pools", get_with( async |uri: Uri, headers: HeaderMap, State(state): State| { - // Pool list is static, only changes on code update - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.all_pools())).await + // Pool list is compiled-in, only changes on deploy + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.all_pools())).await }, |op| { op.id("get_pools") diff --git a/crates/brk_server/src/api/mempool_space/transactions.rs b/crates/brk_server/src/api/mempool_space/transactions.rs index d6f5ef0a9..ac02fc71b 100644 --- a/crates/brk_server/src/api/mempool_space/transactions.rs +++ b/crates/brk_server/src/api/mempool_space/transactions.rs @@ -6,12 +6,13 @@ use axum::{ extract::{Path, State}, http::{HeaderMap, Uri}, }; -use brk_types::{CpfpInfo, MerkleProof, Transaction, TxOutspend, TxStatus, Txid, Version}; +use brk_types::{ + CpfpInfo, MerkleProof, RbfResponse, Transaction, TxOutspend, TxStatus, Txid, Version, +}; use crate::{ AppState, CacheStrategy, - cache::CacheParams, - extended::{ResponseExtended, TransformResponseExtended}, + extended::TransformResponseExtended, params::{TxIndexParam, TxidParam, TxidVout, TxidsParam}, }; @@ -58,6 +59,24 @@ impl TxRoutes for ApiRouter { .server_error(), ), ) + .api_route( + "/api/v1/tx/{txid}/rbf", + get_with( + async |uri: Uri, headers: HeaderMap, Path(param): Path, State(state): State| { + state.cached_json(&headers, state.mempool_cache(), &uri, move |q| q.tx_rbf(¶m.txid)).await + }, + |op| op + .id("get_tx_rbf") + .transactions_tag() + .summary("RBF replacement history") + .description("Returns the RBF replacement tree for a transaction, if any. Both `replacements` and `replaces` are null when the tx has no known RBF history within the mempool monitor's retention window.\n\n*[Mempool.space docs](https://mempool.space/docs/api/rest#get-transaction-rbf-history)*") + .json_response::() + .not_modified() + .bad_request() + .not_found() + .server_error(), + ), + ) .api_route( "/api/tx/{txid}", get_with( @@ -154,15 +173,15 @@ impl TxRoutes for ApiRouter { State(state): State | { let v = Version::ONE; - let immutable = CacheParams::immutable(v); - if immutable.matches_etag(&headers) { - return ResponseExtended::new_not_modified_with(&immutable); - } - let outspend = state.run(move |q| q.outspend(&path.txid, path.vout)).await; - let height = state.sync(|q| q.height()); - let is_deep = outspend.as_ref().is_ok_and(|o| o.is_deeply_spent(height)); - let strategy = if is_deep { CacheStrategy::Immutable(v) } else { CacheStrategy::Tip }; - state.cached_json(&headers, strategy, &uri, move |_| outspend).await + state.cached_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| { + let outspend = q.outspend(&path.txid, path.vout)?; + let strategy = if outspend.is_deeply_spent(q.height()) { + CacheStrategy::Immutable(v) + } else { + CacheStrategy::Tip + }; + Ok((outspend, strategy)) + }).await }, |op| op .id("get_tx_outspend") @@ -188,15 +207,13 @@ impl TxRoutes for ApiRouter { State(state): State | { let v = Version::ONE; - let immutable = CacheParams::immutable(v); - if immutable.matches_etag(&headers) { - return ResponseExtended::new_not_modified_with(&immutable); - } - let outspends = state.run(move |q| q.outspends(¶m.txid)).await; - let height = state.sync(|q| q.height()); - let all_deep = outspends.as_ref().is_ok_and(|os| os.iter().all(|o| o.is_deeply_spent(height))); - let strategy = if all_deep { CacheStrategy::Immutable(v) } else { CacheStrategy::Tip }; - state.cached_json(&headers, strategy, &uri, move |_| outspends).await + state.cached_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| { + let outspends = q.outspends(¶m.txid)?; + let height = q.height(); + let all_deep = outspends.iter().all(|o| o.is_deeply_spent(height)); + let strategy = if all_deep { CacheStrategy::Immutable(v) } else { CacheStrategy::Tip }; + Ok((outspends, strategy)) + }).await }, |op| op .id("get_tx_outspends") diff --git a/crates/brk_server/src/api/metrics/mod.rs b/crates/brk_server/src/api/metrics/mod.rs index a072b5e21..065f4afbc 100644 --- a/crates/brk_server/src/api/metrics/mod.rs +++ b/crates/brk_server/src/api/metrics/mod.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use crate::{CacheStrategy, Error, extended::TransformResponseExtended}; use super::AppState; -use super::series::legacy; +use super::series_legacy; /// Legacy path parameter for `/api/metric/{metric}` #[derive(Deserialize, JsonSchema)] @@ -47,7 +47,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { "/api/metrics", get_with( async |uri: Uri, headers: HeaderMap, State(state): State| { - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.series_catalog().clone())).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await }, |op| op .id("get_metrics_tree_deprecated") @@ -70,7 +70,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { headers: HeaderMap, State(state): State | { - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.series_count())).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await }, |op| op .id("get_metrics_count_deprecated") @@ -93,7 +93,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { headers: HeaderMap, State(state): State | { - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.indexes().to_vec())).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await }, |op| op .id("get_indexes_deprecated") @@ -117,7 +117,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Query(pagination): Query | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |q| Ok(q.series_list(pagination))).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await }, |op| op .id("list_metrics_deprecated") @@ -141,7 +141,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Query(query): Query | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |q| Ok(q.search_series(&query))).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await }, |op| op .id("search_metrics_deprecated") @@ -161,7 +161,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { "/api/metrics/bulk", get_with( |uri: Uri, headers: HeaderMap, addr: Extension, query: Query, state: State| async move { - legacy::handler(uri, headers, addr, query, state) + series_legacy::handler(uri, headers, addr, query, state) .await .into_response() }, @@ -189,7 +189,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Path(path): Path | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |q| { + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| { q.series_info(&path.metric).ok_or_else(|| q.series_not_found_error(&path.metric)) }).await }, @@ -219,7 +219,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { Query(range): Query| -> Response { let params = SeriesSelection::from((path.index, path.metric, range)); - legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, addr, Query(params), state) .await .into_response() }, @@ -249,7 +249,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { Query(range): Query| -> Response { let params = SeriesSelection::from((path.index, path.metric, range)); - legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, addr, Query(params), state) .await .into_response() }, @@ -373,7 +373,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { SeriesList::from(split.collect::>().join(separator)), range, )); - legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, addr, Query(params), state) .await .into_response() }, @@ -401,7 +401,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { state: State| -> Response { let params: SeriesSelection = params.into(); - legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, addr, Query(params), state) .await .into_response() }, diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index e836220de..90d3c5f5f 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -15,7 +15,8 @@ use crate::{ Error, api::{ mempool_space::MempoolSpaceRoutes, metrics::ApiMetricsLegacyRoutes, - series::ApiSeriesRoutes, server::ServerRoutes, urpd::ApiUrpdRoutes, + series::ApiSeriesRoutes, series_legacy::ApiSeriesLegacyRoutes, server::ServerRoutes, + urpd::ApiUrpdRoutes, }, extended::{ResponseExtended, TransformResponseExtended}, }; @@ -26,6 +27,7 @@ mod mempool_space; mod metrics; mod openapi; mod series; +mod series_legacy; mod server; mod urpd; @@ -39,6 +41,7 @@ impl ApiRoutes for ApiRouter { fn add_api_routes(self) -> Self { self.add_server_routes() .add_series_routes() + .add_series_legacy_routes() .add_urpd_routes() .add_metrics_legacy_routes() .add_mempool_space_routes() diff --git a/crates/brk_server/src/api/series/mod.rs b/crates/brk_server/src/api/series.rs similarity index 74% rename from crates/brk_server/src/api/series/mod.rs rename to crates/brk_server/src/api/series.rs index eba3210dc..ba51526cc 100644 --- a/crates/brk_server/src/api/series/mod.rs +++ b/crates/brk_server/src/api/series.rs @@ -1,46 +1,108 @@ +//! Live `/api/series/*` API: catalog, search, info, single-series, bulk. +//! +//! Holds the shared `serve` helper used by every series endpoint that returns +//! a formatted body (single + raw + bulk + the legacy module's deprecated +//! handler in `series_legacy.rs`). + use std::net::SocketAddr; use aide::axum::{ApiRouter, routing::get_with}; use axum::{ Extension, + body::Bytes, extract::{Path, Query, State}, http::{HeaderMap, Uri}, response::{IntoResponse, Response}, }; +use brk_error::Result as BrkResult; +use brk_query::{Query as BrkQuery, ResolvedQuery}; use brk_traversable::TreeNode; use brk_types::{ - DataRangeFormat, IndexInfo, PaginatedSeries, Pagination, SearchQuery, SeriesCount, SeriesData, - SeriesInfo, SeriesNameWithIndex, SeriesSelection, + DataRangeFormat, Format, IndexInfo, Output, PaginatedSeries, Pagination, SearchQuery, + SeriesCount, SeriesData, SeriesInfo, SeriesNameWithIndex, SeriesSelection, }; use crate::{ - CacheStrategy, - cache::CACHE_CONTROL, - extended::TransformResponseExtended, + AppState, CacheParams, CacheStrategy, Result, + extended::{HeaderMapExtended, TransformResponseExtended}, params::SeriesParam, }; -use self::cost_basis::ApiCostBasisLegacyRoutes; -use super::AppState; +/// Shared response pipeline for every series endpoint. +/// +/// Resolves the query (which determines the cache key), then delegates to +/// [`AppState::cached_with_params`] for the etag short-circuit, server-side +/// cache lookup, body formatting, and header assembly. +pub(super) async fn serve( + state: AppState, + uri: Uri, + headers: HeaderMap, + addr: SocketAddr, + params: SeriesSelection, + to_bytes: impl FnOnce(&BrkQuery, ResolvedQuery) -> BrkResult + Send + 'static, +) -> Result { + let max_weight = state.max_weight_for(&addr); + let resolved = state + .run(move |q| q.resolve(params, max_weight)) + .await?; -mod bulk; -mod cost_basis; -mod data; -pub mod legacy; + let format = resolved.format(); + let csv_filename = resolved.csv_filename(); + let cache_params = CacheParams::series( + resolved.version, + resolved.total, + resolved.end, + resolved.hash_prefix, + ); -/// Maximum allowed request weight in bytes (320KB) -const MAX_WEIGHT: usize = 4 * 8 * 10_000; -/// Maximum allowed request weight for localhost (50MB) -const MAX_WEIGHT_LOCALHOST: usize = 50 * 1_000_000; + Ok(state + .cached_with_params( + &headers, + &uri, + cache_params, + move |h| match format { + Format::CSV => { + h.insert_content_disposition_attachment(&csv_filename); + h.insert_content_type_text_csv(); + } + Format::JSON => h.insert_content_type_application_json(), + }, + move |q, enc| Ok(enc.compress(to_bytes(q, resolved)?)), + ) + .await) +} -/// Returns the max weight for a request based on the client address. -/// Localhost requests get a generous limit, external requests get a stricter one. -fn max_weight(addr: &SocketAddr) -> usize { - if addr.ip().is_loopback() { - MAX_WEIGHT_LOCALHOST - } else { - MAX_WEIGHT - } +fn output_to_bytes(out: brk_types::SeriesOutput) -> BrkResult { + Ok(match out.output { + Output::CSV(s) => Bytes::from(s), + Output::Json(v) => Bytes::from(v), + }) +} + +async fn data_handler( + uri: Uri, + headers: HeaderMap, + Extension(addr): Extension, + Query(params): Query, + State(state): State, +) -> Result { + serve(state, uri, headers, addr, params, |q, r| { + output_to_bytes(q.format(r)?) + }) + .await +} + +async fn data_raw_handler( + uri: Uri, + headers: HeaderMap, + Extension(addr): Extension, + Query(params): Query, + State(state): State, +) -> Result { + serve(state, uri, headers, addr, params, |q, r| { + output_to_bytes(q.format_raw(r)?) + }) + .await } pub trait ApiSeriesRoutes { @@ -53,7 +115,7 @@ impl ApiSeriesRoutes for ApiRouter { "/api/series", get_with( async |uri: Uri, headers: HeaderMap, State(state): State| { - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.series_catalog().clone())).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await }, |op| op .id("get_series_tree") @@ -75,7 +137,7 @@ impl ApiSeriesRoutes for ApiRouter { headers: HeaderMap, State(state): State | { - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.series_count())).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await }, |op| op .id("get_series_count") @@ -94,7 +156,7 @@ impl ApiSeriesRoutes for ApiRouter { headers: HeaderMap, State(state): State | { - state.cached_json(&headers, CacheStrategy::Static, &uri, |q| Ok(q.indexes().to_vec())).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await }, |op| op .id("get_indexes") @@ -116,7 +178,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Query(pagination): Query | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |q| Ok(q.series_list(pagination))).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await }, |op| op .id("list_series") @@ -136,7 +198,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Query(query): Query | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |q| Ok(q.search_series(&query))).await + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await }, |op| op .id("search_series") @@ -157,7 +219,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Path(path): Path | { - state.cached_json(&headers, CacheStrategy::Static, &uri, move |q| { + state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| { q.series_info(&path.series).ok_or_else(|| q.series_not_found_error(&path.series)) }).await }, @@ -184,7 +246,7 @@ impl ApiSeriesRoutes for ApiRouter { Path(path): Path, Query(range): Query| -> Response { - data::handler( + data_handler( uri, headers, addr, @@ -218,7 +280,7 @@ impl ApiSeriesRoutes for ApiRouter { Path(path): Path, Query(range): Query| -> Response { - data::raw_handler( + data_raw_handler( uri, headers, addr, @@ -317,7 +379,7 @@ impl ApiSeriesRoutes for ApiRouter { "/api/series/bulk", get_with( |uri, headers, addr, query, state| async move { - bulk::handler(uri, headers, addr, query, state).await.into_response() + data_handler(uri, headers, addr, query, state).await.into_response() }, |op| op .id("get_series_bulk") @@ -332,6 +394,5 @@ impl ApiSeriesRoutes for ApiRouter { .not_modified(), ), ) - .add_cost_basis_legacy_routes() } } diff --git a/crates/brk_server/src/api/series/bulk.rs b/crates/brk_server/src/api/series/bulk.rs deleted file mode 100644 index 65bc82598..000000000 --- a/crates/brk_server/src/api/series/bulk.rs +++ /dev/null @@ -1,79 +0,0 @@ -use std::net::SocketAddr; - -use axum::{ - Extension, - body::{Body, Bytes}, - extract::{Query, State}, - http::{HeaderMap, Uri}, - response::Response, -}; -use brk_types::{Format, Output, SeriesSelection}; - -use crate::{ - Result, - api::series::{CACHE_CONTROL, max_weight}, - extended::{ContentEncoding, HeaderMapExtended, ResponseExtended}, -}; - -use super::AppState; - -pub async fn handler( - uri: Uri, - headers: HeaderMap, - Extension(addr): Extension, - Query(params): Query, - State(state): State, -) -> Result { - // Phase 1: Search and resolve metadata (cheap) - let resolved = state - .run(move |q| q.resolve(params, max_weight(&addr))) - .await?; - - let format = resolved.format(); - let etag = resolved.etag(); - let csv_filename = resolved.csv_filename(); - - if headers.has_etag(etag.as_str()) { - return Ok(Response::new_not_modified(&etag, CACHE_CONTROL)); - } - - // Phase 2: Format (expensive, server-side cached) - let encoding = ContentEncoding::negotiate(&headers); - let cache_key = format!( - "bulk-{}{}{}-{}", - uri.path(), - uri.query().unwrap_or(""), - etag, - encoding.as_str() - ); - let query = &state; - let bytes = state - .get_or_insert(&cache_key, async move { - query - .run(move |q| { - let out = q.format(resolved)?; - let raw = match out.output { - Output::CSV(s) => Bytes::from(s), - Output::Json(v) => Bytes::from(v), - }; - Ok(encoding.compress(raw)) - }) - .await - }) - .await?; - - let mut response = Response::new(Body::from(bytes)); - let h = response.headers_mut(); - h.insert_etag(etag.as_str()); - h.insert_cache_control(CACHE_CONTROL); - h.insert_content_encoding(encoding); - match format { - Format::CSV => { - h.insert_content_disposition_attachment(&csv_filename); - h.insert_content_type_text_csv(); - } - Format::JSON => h.insert_content_type_application_json(), - } - - Ok(response) -} diff --git a/crates/brk_server/src/api/series/data.rs b/crates/brk_server/src/api/series/data.rs deleted file mode 100644 index 533642b3a..000000000 --- a/crates/brk_server/src/api/series/data.rs +++ /dev/null @@ -1,102 +0,0 @@ -use std::net::SocketAddr; - -use axum::{ - Extension, - body::{Body, Bytes}, - extract::{Query, State}, - http::{HeaderMap, Uri}, - response::Response, -}; -use brk_error::Result as BrkResult; -use brk_query::{Query as BrkQuery, ResolvedQuery}; -use brk_types::{Format, Output, SeriesOutput, SeriesSelection}; - -use crate::{ - Result, - api::series::{CACHE_CONTROL, max_weight}, - extended::{ContentEncoding, HeaderMapExtended, ResponseExtended}, -}; - -use super::AppState; - -pub async fn handler( - uri: Uri, - headers: HeaderMap, - addr: Extension, - Query(params): Query, - state: State, -) -> Result { - format_and_respond(uri, headers, addr, params, state, |q, r| q.format(r)).await -} - -pub async fn raw_handler( - uri: Uri, - headers: HeaderMap, - addr: Extension, - Query(params): Query, - state: State, -) -> Result { - format_and_respond(uri, headers, addr, params, state, |q, r| q.format_raw(r)).await -} - -async fn format_and_respond( - uri: Uri, - headers: HeaderMap, - Extension(addr): Extension, - params: SeriesSelection, - state: State, - formatter: fn(&BrkQuery, ResolvedQuery) -> BrkResult, -) -> Result { - // Phase 1: Search and resolve metadata (cheap) - let resolved = state - .run(move |q| q.resolve(params, max_weight(&addr))) - .await?; - - let format = resolved.format(); - let etag = resolved.etag(); - let csv_filename = resolved.csv_filename(); - - if headers.has_etag(etag.as_str()) { - return Ok(Response::new_not_modified(&etag, CACHE_CONTROL)); - } - - // Phase 2: Format (expensive, server-side cached) - let encoding = ContentEncoding::negotiate(&headers); - let cache_key = format!( - "single-{}{}{}-{}", - uri.path(), - uri.query().unwrap_or(""), - etag, - encoding.as_str() - ); - let query = &state; - let bytes = state - .get_or_insert(&cache_key, async move { - query - .run(move |q| { - let out = formatter(q, resolved)?; - let raw = match out.output { - Output::CSV(s) => Bytes::from(s), - Output::Json(v) => Bytes::from(v), - }; - Ok(encoding.compress(raw)) - }) - .await - }) - .await?; - - let mut response = Response::new(Body::from(bytes)); - let h = response.headers_mut(); - h.insert_etag(etag.as_str()); - h.insert_cache_control(CACHE_CONTROL); - h.insert_content_encoding(encoding); - match format { - Format::CSV => { - h.insert_content_disposition_attachment(&csv_filename); - h.insert_content_type_text_csv(); - } - Format::JSON => h.insert_content_type_application_json(), - } - - Ok(response) -} diff --git a/crates/brk_server/src/api/series/legacy.rs b/crates/brk_server/src/api/series/legacy.rs deleted file mode 100644 index 82eaa4683..000000000 --- a/crates/brk_server/src/api/series/legacy.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::net::SocketAddr; - -use axum::{ - Extension, - body::{Body, Bytes}, - extract::{Query, State}, - http::{HeaderMap, Uri}, - response::Response, -}; -use brk_types::{Format, OutputLegacy, SeriesSelection}; - -use crate::{ - Result, - api::series::{CACHE_CONTROL, max_weight}, - extended::{ContentEncoding, HeaderMapExtended, ResponseExtended}, -}; - -pub const SUNSET: &str = "2027-01-01T00:00:00Z"; - -use super::AppState; - -pub async fn handler( - uri: Uri, - headers: HeaderMap, - Extension(addr): Extension, - Query(params): Query, - State(state): State, -) -> Result { - // Phase 1: Search and resolve metadata (cheap) - let resolved = state - .run(move |q| q.resolve(params, max_weight(&addr))) - .await?; - - let format = resolved.format(); - let etag = resolved.etag(); - let csv_filename = resolved.csv_filename(); - - if headers.has_etag(etag.as_str()) { - return Ok(Response::new_not_modified(&etag, CACHE_CONTROL)); - } - - // Phase 2: Format (expensive, server-side cached) - let encoding = ContentEncoding::negotiate(&headers); - let cache_key = format!( - "legacy-{}{}{}-{}", - uri.path(), - uri.query().unwrap_or(""), - etag, - encoding.as_str() - ); - let query = &state; - let bytes = state - .get_or_insert(&cache_key, async move { - query - .run(move |q| { - let out = q.format_legacy(resolved)?; - let raw = match out.output { - OutputLegacy::CSV(s) => Bytes::from(s), - OutputLegacy::Json(v) => Bytes::from(v.to_vec()), - }; - Ok(encoding.compress(raw)) - }) - .await - }) - .await?; - - let mut response = Response::new(Body::from(bytes)); - let h = response.headers_mut(); - h.insert_etag(etag.as_str()); - h.insert_cache_control(CACHE_CONTROL); - h.insert_content_encoding(encoding); - match format { - Format::CSV => { - h.insert_content_disposition_attachment(&csv_filename); - h.insert_content_type_text_csv(); - } - Format::JSON => h.insert_content_type_application_json(), - } - h.insert_deprecation(SUNSET); - - Ok(response) -} diff --git a/crates/brk_server/src/api/series/cost_basis.rs b/crates/brk_server/src/api/series_legacy.rs similarity index 71% rename from crates/brk_server/src/api/series/cost_basis.rs rename to crates/brk_server/src/api/series_legacy.rs index 2424ac7dc..d13140d51 100644 --- a/crates/brk_server/src/api/series/cost_basis.rs +++ b/crates/brk_server/src/api/series_legacy.rs @@ -1,47 +1,86 @@ -//! Deprecated `/api/series/cost-basis/*` routes. -//! Sunset date: 2027-01-01. Delete this file and its registration in `mod.rs` together. +//! Deprecated series-format infrastructure. Sunset date: 2027-01-01. +//! +//! Two responsibilities, deletable as a unit when the sunset arrives: +//! - `handler` / `SUNSET`: the shared legacy series handler used by `/api/series` +//! in legacy mode (registered by metrics endpoints that emit the old format). +//! - `add_series_legacy_routes`: the deprecated `/api/series/cost-basis/*` URLs. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, net::SocketAddr}; use aide::axum::{ApiRouter, routing::get_with}; use axum::{ - extract::{Path, Query as AxumQuery, State}, - http::{HeaderMap, Uri}, + Extension, + body::Bytes, + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode, Uri}, + response::Response, +}; +use brk_error::{Error, Result as BrkResult}; +use brk_query::Query as BrkQuery; +use brk_types::{ + Bitcoin, Cents, Cohort, Date, Day1, Dollars, OutputLegacy, Sats, SeriesSelection, + UrpdAggregation, Version, }; -use brk_error::{Error, Result}; -use brk_query::Query; -use brk_types::{Bitcoin, Cents, Cohort, Date, Day1, Dollars, Sats, UrpdAggregation, Version}; use rustc_hash::FxHashMap; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use vecdb::ReadableOptionVec; -use crate::{AppState, CacheStrategy, extended::TransformResponseExtended}; +use crate::{ + AppState, CacheStrategy, Result, + extended::{HeaderMapExtended, TransformResponseExtended}, +}; + +pub const SUNSET: &str = "2027-01-01T00:00:00Z"; + +/// Legacy series handler. Emits the pre-2027 `OutputLegacy` format and tags +/// the response with `Deprecation` / `Sunset` headers. Reused by `metrics/*` +/// for endpoints that must stay on the old format until sunset. +pub async fn handler( + uri: Uri, + headers: HeaderMap, + Extension(addr): Extension, + Query(params): Query, + State(state): State, +) -> Result { + let mut response = super::series::serve(state, uri, headers, addr, params, legacy_bytes).await?; + if response.status() == StatusCode::OK { + response.headers_mut().insert_deprecation(SUNSET); + } + Ok(response) +} + +fn legacy_bytes(q: &BrkQuery, r: brk_query::ResolvedQuery) -> BrkResult { + Ok(match q.format_legacy(r)?.output { + OutputLegacy::CSV(s) => Bytes::from(s), + OutputLegacy::Json(v) => Bytes::from(v.to_vec()), + }) +} #[derive(Deserialize, JsonSchema)] -pub(super) struct CostBasisParams { - pub cohort: Cohort, +struct CostBasisParams { + cohort: Cohort, #[schemars(with = "String", example = &"2024-01-01")] - pub date: Date, + date: Date, } #[derive(Deserialize, JsonSchema)] -pub(super) struct CostBasisCohortParam { - pub cohort: Cohort, +struct CostBasisCohortParam { + cohort: Cohort, } #[derive(Deserialize, JsonSchema)] -pub(super) struct CostBasisQuery { +struct CostBasisQuery { #[serde(default)] - pub bucket: UrpdAggregation, + bucket: UrpdAggregation, #[serde(default)] - pub value: CostBasisValue, + value: CostBasisValue, } /// Value type for the deprecated cost-basis distribution output. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "lowercase")] -pub(super) enum CostBasisValue { +enum CostBasisValue { #[default] Supply, Realized, @@ -53,12 +92,12 @@ pub(super) enum CostBasisValue { type CostBasisFormatted = BTreeMap; fn cost_basis_formatted( - q: &Query, + q: &BrkQuery, cohort: &Cohort, date: Date, agg: UrpdAggregation, value: CostBasisValue, -) -> Result { +) -> BrkResult { let raw = q.urpd_raw(cohort, date)?; let day1 = Day1::try_from(date).map_err(|e| Error::Parse(e.to_string()))?; let spot_cents = q @@ -102,18 +141,18 @@ fn cost_basis_formatted( .collect()) } -pub(super) trait ApiCostBasisLegacyRoutes { - fn add_cost_basis_legacy_routes(self) -> Self; +pub trait ApiSeriesLegacyRoutes { + fn add_series_legacy_routes(self) -> Self; } -impl ApiCostBasisLegacyRoutes for ApiRouter { - fn add_cost_basis_legacy_routes(self) -> Self { +impl ApiSeriesLegacyRoutes for ApiRouter { + fn add_series_legacy_routes(self) -> Self { self.api_route( "/api/series/cost-basis", get_with( async |uri: Uri, headers: HeaderMap, State(state): State| { state - .cached_json(&headers, CacheStrategy::Static, &uri, |q| q.urpd_cohorts()) + .cached_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts()) .await }, |op| { @@ -166,7 +205,7 @@ impl ApiCostBasisLegacyRoutes for ApiRouter { async |uri: Uri, headers: HeaderMap, Path(params): Path, - AxumQuery(query): AxumQuery, + Query(query): Query, State(state): State| { let strategy = state.date_cache(Version::ONE, params.date); state diff --git a/crates/brk_server/src/api/server/mod.rs b/crates/brk_server/src/api/server/mod.rs index af13e77af..5161094c8 100644 --- a/crates/brk_server/src/api/server/mod.rs +++ b/crates/brk_server/src/api/server/mod.rs @@ -57,7 +57,7 @@ impl ServerRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, State(state): State| { state - .cached_json(&headers, CacheStrategy::Static, &uri, |_| { + .cached_json(&headers, CacheStrategy::Deploy, &uri, |_| { Ok(env!("CARGO_PKG_VERSION")) }) .await diff --git a/crates/brk_server/src/api/urpd/mod.rs b/crates/brk_server/src/api/urpd/mod.rs index 4f0de1790..3ef4659cc 100644 --- a/crates/brk_server/src/api/urpd/mod.rs +++ b/crates/brk_server/src/api/urpd/mod.rs @@ -24,7 +24,7 @@ impl ApiUrpdRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, State(state): State| { state - .cached_json(&headers, CacheStrategy::Static, &uri, |q| q.urpd_cohorts()) + .cached_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts()) .await }, |op| { diff --git a/crates/brk_server/src/cache.rs b/crates/brk_server/src/cache.rs deleted file mode 100644 index 835f1309d..000000000 --- a/crates/brk_server/src/cache.rs +++ /dev/null @@ -1,92 +0,0 @@ -use axum::http::HeaderMap; -use brk_types::{BlockHashPrefix, Version}; - -use crate::{VERSION, extended::HeaderMapExtended}; - -/// Cache strategy for HTTP responses. -pub enum CacheStrategy { - /// Chain-dependent data (addresses, mining stats, txs, outspends). - /// Etag = {tip_hash_prefix:x}. Invalidates on any tip change including reorgs. - Tip, - - /// Immutable data identified by hash in the URL (blocks by hash, confirmed tx data). - /// Etag = {version}. Permanent; only bumped when response format changes. - Immutable(Version), - - /// Static non-chain data (validate-address, series catalog, pool list). - /// Etag = CARGO_PKG_VERSION. Invalidates on deploy. - Static, - - /// Immutable data bound to a specific block (confirmed tx data, block status). - /// Etag = {version}-{block_hash_prefix:x}. Invalidates naturally on reorg. - BlockBound(Version, BlockHashPrefix), - - /// Mempool data — etag from next projected block hash. - /// Etag = m{hash:x}. Invalidates on mempool change. - MempoolHash(u64), -} - -pub(crate) const CACHE_CONTROL: &str = "public, max-age=1, must-revalidate"; - -/// Resolved cache parameters -pub struct CacheParams { - pub etag: Option, - pub cache_control: &'static str, -} - -impl CacheParams { - pub fn tip(tip: BlockHashPrefix) -> Self { - Self { - etag: Some(format!("t{:x}", *tip)), - cache_control: CACHE_CONTROL, - } - } - - pub fn immutable(version: Version) -> Self { - Self { - etag: Some(format!("i{version}")), - cache_control: CACHE_CONTROL, - } - } - - pub fn block_bound(version: Version, prefix: BlockHashPrefix) -> Self { - Self { - etag: Some(format!("b{version}-{:x}", *prefix)), - cache_control: CACHE_CONTROL, - } - } - - pub fn static_version() -> Self { - Self { - etag: Some(format!("s{VERSION}")), - cache_control: CACHE_CONTROL, - } - } - - pub fn mempool_hash(hash: u64) -> Self { - Self { - etag: Some(format!("m{hash:x}")), - cache_control: CACHE_CONTROL, - } - } - - pub fn etag_str(&self) -> &str { - self.etag.as_deref().unwrap_or("") - } - - pub fn matches_etag(&self, headers: &HeaderMap) -> bool { - self.etag - .as_ref() - .is_some_and(|etag| headers.has_etag(etag)) - } - - pub fn resolve(strategy: &CacheStrategy, tip: impl FnOnce() -> BlockHashPrefix) -> Self { - match strategy { - CacheStrategy::Tip => Self::tip(tip()), - CacheStrategy::Immutable(v) => Self::immutable(*v), - CacheStrategy::BlockBound(v, prefix) => Self::block_bound(*v, *prefix), - CacheStrategy::Static => Self::static_version(), - CacheStrategy::MempoolHash(hash) => Self::mempool_hash(*hash), - } - } -} diff --git a/crates/brk_server/src/cache/mod.rs b/crates/brk_server/src/cache/mod.rs new file mode 100644 index 000000000..f31385a9f --- /dev/null +++ b/crates/brk_server/src/cache/mod.rs @@ -0,0 +1,20 @@ +//! HTTP cache layer. ETag-based revalidation with separate browser and CDN +//! directives (RFC 9213). Three concepts, one file each: +//! +//! - [`CacheStrategy`] — *what kind of resource* the handler is returning +//! (input enum picked by the route). +//! - [`CacheParams`] — the *resolved* etag + Cache-Control + CDN-Cache-Control, +//! derived from a strategy plus current chain tip. +//! - [`CdnCacheMode`] — operator-level toggle for the CDN cached tier +//! (process-global, set once via [`init`] from `Server::new`). + +mod mode; +mod params; +mod strategy; + +pub use mode::CdnCacheMode; +pub use params::CacheParams; +pub use strategy::CacheStrategy; + +pub(crate) use mode::init; +pub(crate) use params::CC_ERROR; diff --git a/crates/brk_server/src/cache/mode.rs b/crates/brk_server/src/cache/mode.rs new file mode 100644 index 000000000..beadd0afd --- /dev/null +++ b/crates/brk_server/src/cache/mode.rs @@ -0,0 +1,60 @@ +use std::sync::OnceLock; + +// CDN-facing (RFC 9213). Two tiers: live (chain-state, changes per block / +// mempool event) and cached (stable, ETag-invalidated). +// +// `Live` is always-revalidate: origin handles every request, cheap via ETag, +// no risk of stale data for self-hosters who don't run a purge step. +// `Aggressive` caches stable responses for up to a year and treats them as +// `immutable` (RFC 8246) — the operator must purge the CDN on every deploy. +pub(super) const CDN_LIVE: &str = "public, max-age=1, stale-if-error=300"; +const CDN_AGGRESSIVE: &str = "public, max-age=31536000, immutable"; + +/// CDN caching strategy for stable responses (immutable / deploy / block-bound / +/// historical series). Live-tier responses (`Tip`, `MempoolHash`, tail series) +/// are unaffected. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum CdnCacheMode { + /// Origin revalidates every response via ETag. No CDN purge required. Safe default. + #[default] + Live, + /// CDN holds stable responses for up to a year and treats them as immutable. + /// Operator must purge on every deploy. + Aggressive, +} + +impl CdnCacheMode { + const fn as_str(self) -> &'static str { + match self { + Self::Live => CDN_LIVE, + Self::Aggressive => CDN_AGGRESSIVE, + } + } +} + +static CDN_CACHE_MODE: OnceLock = OnceLock::new(); + +/// Set once at server startup. Subsequent calls are ignored (first-wins). If a +/// later call conflicts with the existing mode, log a warning so the mismatch +/// is visible in plugin / orchestrator setups that spin up multiple servers in +/// the same process. +pub(crate) fn init(mode: CdnCacheMode) { + if CDN_CACHE_MODE.set(mode).is_err() { + let existing = CDN_CACHE_MODE.get().copied().unwrap_or_default(); + if existing != mode { + tracing::warn!( + "cache::init called with {mode:?} but mode is already set to {existing:?}; ignoring" + ); + } + } +} + +/// Cached-tier directive for stable responses. Defaults to `Live` if [`init`] +/// was never called (tests, library use without a `Server`). +pub(super) fn cdn_cached() -> &'static str { + CDN_CACHE_MODE + .get() + .copied() + .unwrap_or_default() + .as_str() +} diff --git a/crates/brk_server/src/cache/params.rs b/crates/brk_server/src/cache/params.rs new file mode 100644 index 000000000..f19967f4a --- /dev/null +++ b/crates/brk_server/src/cache/params.rs @@ -0,0 +1,120 @@ +use axum::http::HeaderMap; +use brk_types::{BlockHashPrefix, Version}; + +use crate::{VERSION, etag::Etag, extended::HeaderMapExtended}; + +use super::{ + mode::{CDN_LIVE, cdn_cached}, + strategy::CacheStrategy, +}; + +// Browser-facing: always revalidate via ETag. `no-cache` means "cache it but +// check before use" (not "don't cache"); ETag makes the check cheap. +const CC: &str = "public, no-cache, stale-if-error=86400"; + +// Errors: short, must-revalidate, no `stale-if-error` (we don't want a 24h-old +// error served when origin recovers). Same string for browser and CDN. +pub(crate) const CC_ERROR: &str = "public, max-age=1, must-revalidate"; + +/// Resolved cache parameters: an ETag plus the two Cache-Control directives. +pub struct CacheParams { + pub etag: Etag, + cache_control: &'static str, + cdn_cache_control: &'static str, +} + +impl CacheParams { + fn tip(tip: BlockHashPrefix) -> Self { + Self { + etag: format!("t{:x}", *tip).into(), + cache_control: CC, + cdn_cache_control: CDN_LIVE, + } + } + + fn immutable(version: Version) -> Self { + Self { + etag: format!("i{version}").into(), + cache_control: CC, + cdn_cache_control: cdn_cached(), + } + } + + fn block_bound(version: Version, prefix: BlockHashPrefix) -> Self { + Self { + etag: format!("b{version}-{:x}", *prefix).into(), + cache_control: CC, + cdn_cache_control: cdn_cached(), + } + } + + /// Deploy-tied response: etag from the build version. Used directly + /// by static handlers (OpenAPI spec, scalar bundle) that don't have + /// a [`CacheStrategy`] context. + pub fn deploy() -> Self { + Self { + etag: format!("d{VERSION}").into(), + cache_control: CC, + cdn_cache_control: cdn_cached(), + } + } + + fn mempool_hash(hash: u64) -> Self { + Self { + etag: format!("m{hash:x}").into(), + cache_control: CC, + cdn_cache_control: CDN_LIVE, + } + } + + /// Series query: tail-bound (`end >= total`) gets LIVE, historical gets CACHED. + /// Etag distinguishes the two: tail uses tip hash (per-block + reorgs), + /// historical uses total length (only changes when new data is appended). + pub fn series(version: Version, total: usize, end: usize, hash: BlockHashPrefix) -> Self { + let v = u32::from(version); + if end >= total { + Self { + etag: format!("s{v}-{:x}", *hash).into(), + cache_control: CC, + cdn_cache_control: CDN_LIVE, + } + } else { + Self { + etag: format!("s{v}-{total}").into(), + cache_control: CC, + cdn_cache_control: cdn_cached(), + } + } + } + + /// Error response: keeps the originating ETag (so retries can 304), + /// uses [`CC_ERROR`] for both browser and CDN. + pub fn error(etag: Etag) -> Self { + Self { + etag, + cache_control: CC_ERROR, + cdn_cache_control: CC_ERROR, + } + } + + pub fn matches_etag(&self, headers: &HeaderMap) -> bool { + headers.has_etag(self.etag.as_str()) + } + + /// Write this cache policy (etag + cache-control + cdn-cache-control) onto a response's headers. + pub fn apply_to(&self, headers: &mut HeaderMap) { + headers.insert_etag(self.etag.as_str()); + headers.insert_cache_control(self.cache_control); + headers.insert_cdn_cache_control(self.cdn_cache_control); + } + + pub fn resolve(strategy: &CacheStrategy, tip: BlockHashPrefix) -> Self { + match strategy { + CacheStrategy::Tip => Self::tip(tip), + CacheStrategy::Immutable(v) => Self::immutable(*v), + CacheStrategy::BlockBound(v, prefix) => Self::block_bound(*v, *prefix), + CacheStrategy::Deploy => Self::deploy(), + CacheStrategy::MempoolHash(hash) => Self::mempool_hash(*hash), + } + } +} diff --git a/crates/brk_server/src/cache/strategy.rs b/crates/brk_server/src/cache/strategy.rs new file mode 100644 index 000000000..4ece4451d --- /dev/null +++ b/crates/brk_server/src/cache/strategy.rs @@ -0,0 +1,30 @@ +use brk_types::{BlockHashPrefix, Version}; + +/// Cache strategy for HTTP responses. +/// +/// The series strategy is computed directly in `api/series::serve` because +/// its parameters (total / end / hash) only become known after query +/// resolution, so it bypasses this enum and builds a +/// [`CacheParams`](super::CacheParams) via +/// [`CacheParams::series`](super::CacheParams::series). +pub enum CacheStrategy { + /// Chain-dependent data (addresses, mining stats, txs, outspends). + /// Etag = `t{tip_hash_prefix:x}`. Invalidates on any tip change including reorgs. + Tip, + + /// Immutable data identified by hash in the URL (blocks by hash, confirmed tx data). + /// Etag = `i{version}`. Permanent, only bumped when response format changes. + Immutable(Version), + + /// Non-chain data tied to the deploy (validate-address, series catalog, pool list). + /// Etag = `d{CARGO_PKG_VERSION}`. Invalidates on deploy. + Deploy, + + /// Immutable data bound to a specific block (confirmed tx data, block status). + /// Etag = `b{version}-{block_hash_prefix:x}`. Invalidates naturally on reorg. + BlockBound(Version, BlockHashPrefix), + + /// Mempool data, etag from next projected block hash. + /// Etag = `m{hash:x}`. Invalidates on mempool change. + MempoolHash(u64), +} diff --git a/crates/brk_server/src/config.rs b/crates/brk_server/src/config.rs new file mode 100644 index 000000000..5e4629eab --- /dev/null +++ b/crates/brk_server/src/config.rs @@ -0,0 +1,39 @@ +use std::path::PathBuf; + +use brk_website::Website; + +use crate::cache::CdnCacheMode; + +/// Default max series-query response weight for non-loopback clients. +/// `4 * 8 * 10_000` = 320 KB (4 vecs x 8 bytes x 10k rows). +pub const DEFAULT_MAX_WEIGHT: usize = 4 * 8 * 10_000; + +/// Default max series-query response weight for loopback clients. +pub const DEFAULT_MAX_WEIGHT_LOCALHOST: usize = 50 * 1_000_000; + +/// Default LRU capacity for the in-process response cache. +pub const DEFAULT_CACHE_SIZE: usize = 1_000; + +/// Server-wide configuration set at startup. +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub data_path: PathBuf, + pub website: Website, + pub cdn_cache_mode: CdnCacheMode, + pub max_weight: usize, + pub max_weight_localhost: usize, + pub cache_size: usize, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + data_path: PathBuf::default(), + website: Website::default(), + cdn_cache_mode: CdnCacheMode::default(), + max_weight: DEFAULT_MAX_WEIGHT, + max_weight_localhost: DEFAULT_MAX_WEIGHT_LOCALHOST, + cache_size: DEFAULT_CACHE_SIZE, + } + } +} diff --git a/crates/brk_server/src/error.rs b/crates/brk_server/src/error.rs index f0811d7a7..81d917f3f 100644 --- a/crates/brk_server/src/error.rs +++ b/crates/brk_server/src/error.rs @@ -7,7 +7,11 @@ use brk_error::Error as BrkError; use schemars::JsonSchema; use serde::Serialize; -use crate::extended::HeaderMapExtended; +use crate::{ + cache::{CC_ERROR, CacheParams}, + etag::Etag, + extended::HeaderMapExtended, +}; /// Server result type with Error that implements IntoResponse. pub type Result = std::result::Result; @@ -139,11 +143,10 @@ impl Error { Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal_error", msg) } - pub(crate) fn into_response_with_etag(self, etag: &str) -> Response { + pub(crate) fn into_response_with_etag(self, etag: Etag) -> Response { + let params = CacheParams::error(etag); let mut response = self.into_response(); - let headers = response.headers_mut(); - headers.insert_etag(etag); - headers.insert_cache_control_must_revalidate(); + params.apply_to(response.headers_mut()); response } } @@ -165,11 +168,15 @@ impl OperationOutput for Error { impl IntoResponse for Error { fn into_response(self) -> Response { let body = build_error_body(self.status, self.code, self.message); - ( + let mut response = ( self.status, [(header::CONTENT_TYPE, "application/problem+json")], body, ) - .into_response() + .into_response(); + let h = response.headers_mut(); + h.insert_cache_control(CC_ERROR); + h.insert_cdn_cache_control(CC_ERROR); + response } } diff --git a/crates/brk_server/src/etag.rs b/crates/brk_server/src/etag.rs new file mode 100644 index 000000000..f122dd5c2 --- /dev/null +++ b/crates/brk_server/src/etag.rs @@ -0,0 +1,25 @@ +use std::fmt; + +/// Typed entity-tag wrapper. Owns a `String` so values built from `format!` +/// can be passed around without re-allocating, while keeping callsites typed +/// (`Etag` instead of `String`). +#[derive(Clone, Debug)] +pub struct Etag(String); + +impl Etag { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From for Etag { + fn from(value: String) -> Self { + Self(value) + } +} + +impl fmt::Display for Etag { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} diff --git a/crates/brk_server/src/extended/header_map.rs b/crates/brk_server/src/extended/header_map.rs index 67898ee22..56dd1c159 100644 --- a/crates/brk_server/src/extended/header_map.rs +++ b/crates/brk_server/src/extended/header_map.rs @@ -10,7 +10,7 @@ pub trait HeaderMapExtended { fn insert_etag(&mut self, etag: &str); fn insert_cache_control(&mut self, value: &str); - fn insert_cache_control_must_revalidate(&mut self); + fn insert_cdn_cache_control(&mut self, value: &str); fn insert_content_disposition_attachment(&mut self, filename: &str); @@ -45,8 +45,11 @@ impl HeaderMapExtended for HeaderMap { self.insert(header::CACHE_CONTROL, value.parse().unwrap()); } - fn insert_cache_control_must_revalidate(&mut self) { - self.insert_cache_control("public, max-age=1, must-revalidate"); + fn insert_cdn_cache_control(&mut self, value: &str) { + self.insert( + axum::http::HeaderName::from_static("cdn-cache-control"), + value.parse().unwrap(), + ); } fn insert_content_disposition_attachment(&mut self, filename: &str) { diff --git a/crates/brk_server/src/extended/response.rs b/crates/brk_server/src/extended/response.rs index a1d5bf816..b2e687e32 100644 --- a/crates/brk_server/src/extended/response.rs +++ b/crates/brk_server/src/extended/response.rs @@ -3,21 +3,25 @@ use axum::{ http::{HeaderMap, Response, StatusCode, header}, response::IntoResponse, }; -use brk_types::Etag; use serde::Serialize; use super::header_map::HeaderMapExtended; use crate::cache::CacheParams; +fn new_json_cached(value: T, params: &CacheParams) -> Response { + let bytes = serde_json::to_vec(&value).unwrap(); + let mut response = Response::builder().body(bytes.into()).unwrap(); + let h = response.headers_mut(); + h.insert_content_type_application_json(); + params.apply_to(h); + response +} + pub trait ResponseExtended where Self: Sized, { - fn new_not_modified(etag: &Etag, cache_control: &str) -> Self; - fn new_not_modified_with(params: &CacheParams) -> Self; - fn new_json_cached(value: T, params: &CacheParams) -> Self - where - T: Serialize; + fn new_not_modified(params: &CacheParams) -> Self; fn static_json(headers: &HeaderMap, value: T) -> Self where T: Serialize; @@ -30,31 +34,9 @@ where } impl ResponseExtended for Response { - fn new_not_modified(etag: &Etag, cache_control: &str) -> Response { + fn new_not_modified(params: &CacheParams) -> Response { let mut response = (StatusCode::NOT_MODIFIED, "").into_response(); - let headers = response.headers_mut(); - headers.insert_etag(etag.as_str()); - headers.insert_cache_control(cache_control); - response - } - - fn new_not_modified_with(params: &CacheParams) -> Response { - let etag = Etag::from(params.etag_str()); - Self::new_not_modified(&etag, params.cache_control) - } - - fn new_json_cached(value: T, params: &CacheParams) -> Self - where - T: Serialize, - { - let bytes = serde_json::to_vec(&value).unwrap(); - let mut response = Response::builder().body(bytes.into()).unwrap(); - let headers = response.headers_mut(); - headers.insert_content_type_application_json(); - headers.insert_cache_control(params.cache_control); - if let Some(etag) = ¶ms.etag { - headers.insert_etag(etag); - } + params.apply_to(response.headers_mut()); response } @@ -62,11 +44,11 @@ impl ResponseExtended for Response { where T: Serialize, { - let params = CacheParams::static_version(); + let params = CacheParams::deploy(); if params.matches_etag(headers) { - return Self::new_not_modified_with(¶ms); + return Self::new_not_modified(¶ms); } - Self::new_json_cached(value, ¶ms) + new_json_cached(value, ¶ms) } fn static_bytes( @@ -75,18 +57,16 @@ impl ResponseExtended for Response { content_type: &'static str, content_encoding: &'static str, ) -> Self { - let params = CacheParams::static_version(); + let params = CacheParams::deploy(); if params.matches_etag(headers) { - return Self::new_not_modified_with(¶ms); + return Self::new_not_modified(¶ms); } let mut response = Response::new(Body::from(bytes)); let h = response.headers_mut(); h.insert(header::CONTENT_TYPE, content_type.parse().unwrap()); h.insert(header::CONTENT_ENCODING, content_encoding.parse().unwrap()); - h.insert_cache_control(params.cache_control); - if let Some(etag) = ¶ms.etag { - h.insert_etag(etag); - } + h.insert_vary_accept_encoding(); + params.apply_to(h); response } } diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index e537109c0..3119ac093 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -3,11 +3,13 @@ use std::{ any::Any, net::SocketAddr, - path::PathBuf, sync::{Arc, atomic::AtomicU64}, time::{Duration, Instant}, }; +#[cfg(feature = "bindgen")] +use std::path::PathBuf; + use aide::axum::ApiRouter; use axum::{ Extension, ServiceExt, @@ -33,17 +35,23 @@ use tower_layer::Layer; use tracing::{error, info}; mod api; -pub mod cache; +mod cache; +mod config; mod error; +mod etag; mod extended; -pub mod params; +mod params; mod state; pub use api::ApiRoutes; use api::*; pub use brk_types::Port; pub use brk_website::Website; -pub use cache::{CacheParams, CacheStrategy}; +pub use cache::CdnCacheMode; +use cache::{CacheParams, CacheStrategy}; +pub use config::{ + DEFAULT_CACHE_SIZE, DEFAULT_MAX_WEIGHT, DEFAULT_MAX_WEIGHT_LOCALHOST, ServerConfig, +}; pub use error::{Error, Result}; use state::*; @@ -52,16 +60,19 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub struct Server(AppState); impl Server { - pub fn new(query: &AsyncQuery, data_path: PathBuf, website: Website) -> Self { - website.log(); + pub fn new(query: &AsyncQuery, config: ServerConfig) -> Self { + config.website.log(); + cache::init(config.cdn_cache_mode); Self(AppState { query: query.clone(), - data_path, - website, - cache: Arc::new(Cache::new(1_000)), + data_path: config.data_path, + website: config.website, + cache: Arc::new(Cache::new(config.cache_size)), last_tip: Arc::new(AtomicU64::new(0)), started_at: jiff::Timestamp::now(), started_instant: Instant::now(), + max_weight: config.max_weight, + max_weight_localhost: config.max_weight_localhost, }) } diff --git a/crates/brk_server/src/state.rs b/crates/brk_server/src/state.rs index 76cc1222c..b97abb9fd 100644 --- a/crates/brk_server/src/state.rs +++ b/crates/brk_server/src/state.rs @@ -1,5 +1,6 @@ use std::{ future::Future, + net::SocketAddr, path::PathBuf, sync::{ Arc, @@ -38,9 +39,23 @@ pub struct AppState { pub last_tip: Arc, pub started_at: Timestamp, pub started_instant: Instant, + pub max_weight: usize, + pub max_weight_localhost: usize, } impl AppState { + /// Per-request series weight cap: loopback gets `max_weight_localhost`, + /// everyone else gets `max_weight`. The `connect_info_layer` rewrites the + /// peer to non-loopback when `CF-Connecting-IP` is present, so requests + /// proxied through a tunnel are billed at the external rate. + pub fn max_weight_for(&self, addr: &SocketAddr) -> usize { + if addr.ip().is_loopback() { + self.max_weight_localhost + } else { + self.max_weight + } + } + /// `Immutable` if height is >6 deep, `Tip` otherwise. pub fn height_cache(&self, version: Version, height: Height) -> CacheStrategy { let is_deep = self.sync(|q| (*q.height()).saturating_sub(*height) > 6); @@ -136,10 +151,12 @@ impl AppState { /// Mempool → `MempoolHash`, confirmed → `BlockBound`, unknown → `Tip`. pub fn tx_cache(&self, version: Version, txid: &Txid) -> CacheStrategy { self.sync(|q| { - if q.mempool().is_some_and(|m| m.get_txs().contains(txid)) { - let hash = q.mempool().map(|m| m.next_block_hash()).unwrap_or(0); - return CacheStrategy::MempoolHash(hash); - } else if let Ok((_, height)) = q.resolve_tx(txid) + if let Some(mempool) = q.mempool() + && mempool.txs().contains(txid) + { + return CacheStrategy::MempoolHash(mempool.next_block_hash()); + } + if let Ok((_, height)) = q.resolve_tx(txid) && let Ok(block_hash) = q.block_hash_by_height(height) { return CacheStrategy::BlockBound(version, BlockHashPrefix::from(&block_hash)); @@ -153,7 +170,53 @@ impl AppState { CacheStrategy::MempoolHash(hash) } - /// Cached + pre-compressed response. Compression runs on the blocking thread. + /// Shared response pipeline: tip-clear, etag short-circuit, server-side + /// cache lookup, body computation on a blocking thread, header assembly. + /// Used by [`AppState::cached`] (strategy-driven) and the series endpoint + /// (which builds [`CacheParams`] directly from query resolution). + pub(crate) async fn cached_with_params( + &self, + headers: &HeaderMap, + uri: &Uri, + params: CacheParams, + apply_content_headers: impl FnOnce(&mut HeaderMap), + f: F, + ) -> Response + where + F: FnOnce(&brk_query::Query, ContentEncoding) -> brk_error::Result + Send + 'static, + { + let tip = self.sync(|q| q.tip_hash_prefix()); + if self.last_tip.swap(*tip, Ordering::Relaxed) != *tip { + self.cache.clear(); + } + + if params.matches_etag(headers) { + return ResponseExtended::new_not_modified(¶ms); + } + + let encoding = ContentEncoding::negotiate(headers); + let cache_key = format!("{}-{}-{}", uri, params.etag, encoding.as_str()); + let result = self + .get_or_insert( + &cache_key, + async move { self.run(move |q| f(q, encoding)).await }, + ) + .await; + + match result { + Ok(bytes) => { + let mut response = Response::new(Body::from(bytes)); + let h = response.headers_mut(); + apply_content_headers(h); + params.apply_to(h); + h.insert_content_encoding(encoding); + response + } + Err(e) => Error::from(e).into_response_with_etag(params.etag.clone()), + } + } + + /// Strategy-driven cached response. Compression runs on the blocking thread. async fn cached( &self, headers: &HeaderMap, @@ -165,38 +228,18 @@ impl AppState { where F: FnOnce(&brk_query::Query, ContentEncoding) -> brk_error::Result + Send + 'static, { - let encoding = ContentEncoding::negotiate(headers); let tip = self.sync(|q| q.tip_hash_prefix()); - if self.last_tip.swap(*tip, Ordering::Relaxed) != *tip { - self.cache.clear(); - } - let params = CacheParams::resolve(&strategy, || tip); - if params.matches_etag(headers) { - return ResponseExtended::new_not_modified_with(¶ms); - } - - let full_key = format!("{}-{}-{}", uri, params.etag_str(), encoding.as_str()); - let result = self - .get_or_insert( - &full_key, - async move { self.run(move |q| f(q, encoding)).await }, - ) - .await; - - match result { - Ok(bytes) => { - let mut response = Response::new(Body::from(bytes)); - let h = response.headers_mut(); + let params = CacheParams::resolve(&strategy, tip); + self.cached_with_params( + headers, + uri, + params, + |h| { h.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); - h.insert_cache_control(params.cache_control); - h.insert_content_encoding(encoding); - if let Some(etag) = ¶ms.etag { - h.insert_etag(etag); - } - response - } - Err(e) => Error::from(e).into_response_with_etag(params.etag_str()), - } + }, + f, + ) + .await } /// JSON response with HTTP + server-side caching @@ -218,6 +261,51 @@ impl AppState { .await } + /// JSON response where the strategy depends on the loaded value. + /// + /// Clients already holding `optimistic`'s ETag get a 304 before any work + /// is done. Otherwise the closure runs on a blocking thread and returns + /// both the value and the actual strategy (e.g. `Immutable` if deeply + /// confirmed, `Tip` otherwise). Errors fall back to `Tip`. Use for + /// resources whose freshness category depends on the data itself + /// (outspends, threshold-based block status). + pub async fn cached_json_optimistic( + &self, + headers: &HeaderMap, + optimistic: CacheStrategy, + uri: &Uri, + f: F, + ) -> Response + where + T: Serialize + Send + 'static, + F: FnOnce(&brk_query::Query) -> brk_error::Result<(T, CacheStrategy)> + Send + 'static, + { + let tip = self.sync(|q| q.tip_hash_prefix()); + let optimistic_params = CacheParams::resolve(&optimistic, tip); + if optimistic_params.matches_etag(headers) { + return ResponseExtended::new_not_modified(&optimistic_params); + } + + let (value_result, strategy) = match self.run(f).await { + Ok((v, s)) => (Ok(v), s), + Err(e) => (Err(e), CacheStrategy::Tip), + }; + let params = CacheParams::resolve(&strategy, tip); + self.cached_with_params( + headers, + uri, + params, + |h| { + h.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + }, + move |_q, enc| { + let value = value_result?; + Ok(enc.compress(Bytes::from(serde_json::to_vec(&value).unwrap()))) + }, + ) + .await + } + /// Text response with HTTP + server-side caching pub async fn cached_text( &self, @@ -263,7 +351,7 @@ impl AppState { } /// Check server-side cache, compute on miss - pub async fn get_or_insert( + async fn get_or_insert( &self, cache_key: &str, compute: impl Future>, diff --git a/crates/brk_types/src/etag.rs b/crates/brk_types/src/etag.rs deleted file mode 100644 index c40b8c80a..000000000 --- a/crates/brk_types/src/etag.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::fmt; - -use super::{BlockHashPrefix, Version}; - -/// HTTP ETag value. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Etag(String); - -impl Etag { - /// Create from raw string - pub fn new(s: impl Into) -> Self { - Self(s.into()) - } - - /// Get inner string reference - pub fn as_str(&self) -> &str { - &self.0 - } - - /// Consume and return inner string - pub fn into_string(self) -> String { - self.0 - } -} - -impl fmt::Display for Etag { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From for Etag { - fn from(s: String) -> Self { - Self(s) - } -} - -impl From<&str> for Etag { - fn from(s: &str) -> Self { - Self(s.to_string()) - } -} - -impl Etag { - /// Tail uses hash prefix (changes per-block and on reorgs), - /// non-tail uses total (changes per-block). - pub fn from_series( - version: Version, - total: usize, - end: usize, - hash_prefix: BlockHashPrefix, - ) -> Self { - let v = u32::from(version); - if end >= total { - let h = *hash_prefix; - Self(format!("v{v}-{h:x}")) - } else { - Self(format!("v{v}-{total}")) - } - } -} diff --git a/crates/brk_types/src/hex.rs b/crates/brk_types/src/hex.rs index 6848de80a..9f6242d64 100644 --- a/crates/brk_types/src/hex.rs +++ b/crates/brk_types/src/hex.rs @@ -1,7 +1,54 @@ +use std::{fmt, ops::Deref}; + use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -/// Hex-encoded string +/// Hex-encoded string. Transparent wrapper over `String`: serializes +/// as a plain JSON string and derefs to `str`, so anywhere `&str` or +/// `AsRef<[u8]>` is expected the `Hex` "just works". #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(transparent)] pub struct Hex(String); + +impl Hex { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From for Hex { + fn from(s: String) -> Self { + Self(s) + } +} + +impl From for String { + fn from(h: Hex) -> Self { + h.0 + } +} + +impl Deref for Hex { + type Target = str; + fn deref(&self) -> &str { + &self.0 + } +} + +impl AsRef for Hex { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl AsRef<[u8]> for Hex { + fn as_ref(&self) -> &[u8] { + self.0.as_bytes() + } +} + +impl fmt::Display for Hex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} diff --git a/crates/brk_types/src/lib.rs b/crates/brk_types/src/lib.rs index 2987866f8..d902fa541 100644 --- a/crates/brk_types/src/lib.rs +++ b/crates/brk_types/src/lib.rs @@ -60,7 +60,6 @@ mod empty_addr_data; mod empty_addr_index; mod empty_output_index; mod epoch; -mod etag; mod feerate; mod feerate_percentiles; mod format; @@ -131,6 +130,7 @@ mod port; mod range_index; mod range_map; mod raw_locktime; +mod rbf; mod recommended_fees; mod reward_stats; mod sats; @@ -189,6 +189,7 @@ mod vout; mod vsize; mod week1; mod weight; +mod witness; mod year; mod year1; mod year10; @@ -251,7 +252,6 @@ pub use empty_addr_data::*; pub use empty_addr_index::*; pub use empty_output_index::*; pub use epoch::*; -pub use etag::*; pub use feerate::*; pub use feerate_percentiles::*; pub use format::*; @@ -322,6 +322,7 @@ pub use port::*; pub use range_index::*; pub use range_map::*; pub use raw_locktime::*; +pub use rbf::*; pub use recommended_fees::*; pub use reward_stats::*; pub use sats::*; @@ -380,6 +381,7 @@ pub use vout::*; pub use vsize::*; pub use week1::*; pub use weight::*; +pub use witness::*; pub use year::*; pub use year1::*; pub use year10::*; diff --git a/crates/brk_types/src/rbf.rs b/crates/brk_types/src/rbf.rs new file mode 100644 index 000000000..2523a214e --- /dev/null +++ b/crates/brk_types/src/rbf.rs @@ -0,0 +1,58 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{FeeRate, Sats, Timestamp, Txid, VSize}; + +/// Transaction summary carried inside an RBF replacement node. Shape +/// matches mempool.space's `/api/v1/tx/:txid/rbf` and +/// `/api/v1/replacements` responses. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct RbfTx { + pub txid: Txid, + pub fee: Sats, + pub vsize: VSize, + /// Sum of output amounts. + pub value: Sats, + pub rate: FeeRate, + pub time: Timestamp, + /// BIP-125 signaling: at least one input has sequence < 0xffffffff-1. + pub rbf: bool, + /// Only populated on the root `tx` of an RBF response. `true` iff + /// this tx displaced at least one non-signaling predecessor. + #[serde(rename = "fullRbf", skip_serializing_if = "Option::is_none", default)] + pub full_rbf: Option, +} + +/// One node in an RBF replacement tree. The node's `tx` replaced each +/// entry in `replaces`, recursively. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct ReplacementNode { + pub tx: RbfTx, + /// First-seen timestamp, duplicated here to match mempool.space's + /// on-the-wire shape. + pub time: Timestamp, + /// Any predecessor in this subtree was non-signaling. + #[serde(rename = "fullRbf")] + pub full_rbf: bool, + /// Seconds between this node's `time` and the successor that + /// replaced it. Omitted on the root of an RBF response. + #[serde(skip_serializing_if = "Option::is_none")] + pub interval: Option, + pub replaces: Vec, +} + +/// Response body for `GET /api/v1/tx/:txid/rbf`. Both fields are null +/// when the tx has no known RBF history within the mempool monitor's +/// graveyard retention window. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct RbfResponse { + pub replacements: Option, + pub replaces: Option>, +} + +impl RbfResponse { + pub const EMPTY: Self = Self { + replacements: None, + replaces: None, + }; +} diff --git a/crates/brk_types/src/tx.rs b/crates/brk_types/src/tx.rs index a2f13deb5..cbf7a64bf 100644 --- a/crates/brk_types/src/tx.rs +++ b/crates/brk_types/src/tx.rs @@ -1,6 +1,8 @@ use crate::{ FeeRate, RawLockTime, Sats, TxIn, TxIndex, TxOut, TxStatus, TxVersionRaw, Txid, VSize, Weight, + Witness, }; +use bitcoin::Script; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use vecdb::CheckedSub; @@ -70,6 +72,20 @@ impl Transaction { self.fee = Self::fee(self).unwrap_or_default(); } + /// Re-encode to canonical Bitcoin protocol bytes via + /// `bitcoin::Transaction`. Lossless for mempool/confirmed txs + /// (verified bytewise round-trip against Core over a 1000-tx live + /// sample). Coinbase txs don't round-trip because brk's `Vout` is + /// `u16` while the protocol's coinbase vout is `0xFFFFFFFF` - + /// callers that may see coinbase shouldn't rely on this. + pub fn encode_bytes(&self) -> Vec { + let bitcoin_tx: bitcoin::Transaction = self.into(); + let mut buf = Vec::with_capacity(self.total_size); + bitcoin::consensus::Encodable::consensus_encode(&bitcoin_tx, &mut buf) + .expect("in-memory consensus_encode is infallible"); + buf + } + /// Virtual size in vbytes (weight / 4, rounded up) #[inline] pub fn vsize(&self) -> VSize { @@ -81,4 +97,88 @@ impl Transaction { pub fn fee_rate(&self) -> FeeRate { FeeRate::from((self.fee, self.vsize())) } + + /// Total sigop cost (BIP-141 weight units). + /// + /// Mirrors `bitcoin::Transaction::total_sigop_cost`, but reads + /// prevouts from `TxIn.prevout` and uses bitcoin's public + /// `Script::redeem_script` (push-only check + last-push extraction + /// in one). Inputs whose `prevout` is `None` skip the P2SH and + /// witness components - legacy script-sig sigops are still counted. + pub fn total_sigop_cost(&self) -> usize { + let mut legacy: usize = 0; + let mut redeem: usize = 0; + let mut witness: usize = 0; + + for input in &self.input { + legacy = legacy.saturating_add(input.script_sig.count_sigops_legacy()); + + let Some(prevout) = input.prevout.as_ref() else { + continue; + }; + let spk: &Script = &prevout.script_pubkey; + + let redeem_script = spk + .is_p2sh() + .then(|| input.script_sig.redeem_script()) + .flatten(); + + if let Some(rs) = redeem_script { + redeem = redeem.saturating_add(rs.count_sigops()); + } + + let witness_program: Option<&Script> = if spk.is_witness_program() { + Some(spk) + } else { + redeem_script + }; + + if let Some(wp) = witness_program { + witness = + witness.saturating_add(count_sigops_with_witness_program(&input.witness, wp)); + } + } + + for output in &self.output { + legacy = legacy.saturating_add(output.script_pubkey.count_sigops_legacy()); + } + + legacy + .saturating_mul(4) + .saturating_add(redeem.saturating_mul(4)) + .saturating_add(witness) + } +} + +fn count_sigops_with_witness_program(witness: &Witness, witness_program: &Script) -> usize { + if witness_program.is_p2wpkh() { + 1 + } else if witness_program.is_p2wsh() { + witness + .last() + .map(|bytes| Script::from_bytes(bytes).count_sigops()) + .unwrap_or(0) + } else { + 0 + } +} + +/// Re-encode a brk `Transaction` to a canonical `bitcoin::Transaction`. +/// Lossless for mempool/confirmed txs (verified bytewise round-trip +/// against Core's `getrawtransaction` over a 1000-tx live sample). +/// +/// Coinbase round-trip is **not** byte-perfect because brk's `Vout` is +/// a `u16` and coinbase encodes `vout = 0xFFFFFFFF` in the protocol; +/// the reconstructed value is `u16::MAX` (65535). Mempool txs are +/// never coinbase, and confirmed-tx callers don't go through this path. +impl From<&Transaction> for bitcoin::Transaction { + #[inline] + fn from(tx: &Transaction) -> Self { + Self { + version: tx.version.into(), + lock_time: tx.lock_time.into(), + input: tx.input.iter().map(bitcoin::TxIn::from).collect(), + output: tx.output.iter().map(bitcoin::TxOut::from).collect(), + } + } } diff --git a/crates/brk_types/src/tx_version_raw.rs b/crates/brk_types/src/tx_version_raw.rs index 9fd3e5855..09711df79 100644 --- a/crates/brk_types/src/tx_version_raw.rs +++ b/crates/brk_types/src/tx_version_raw.rs @@ -21,3 +21,10 @@ impl From for TxVersionRaw { Self(value.0) } } + +impl From for bitcoin::transaction::Version { + #[inline] + fn from(value: TxVersionRaw) -> Self { + Self(value.0) + } +} diff --git a/crates/brk_types/src/txin.rs b/crates/brk_types/src/txin.rs index 73e647461..332865742 100644 --- a/crates/brk_types/src/txin.rs +++ b/crates/brk_types/src/txin.rs @@ -1,5 +1,5 @@ -use crate::{TxOut, Txid, Vout}; -use bitcoin::ScriptBuf; +use crate::{TxOut, Txid, Vout, Witness}; +use bitcoin::{Script, ScriptBuf, Sequence, transaction::OutPoint}; use schemars::JsonSchema; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; @@ -26,8 +26,8 @@ pub struct TxIn { #[schemars(rename = "scriptsig_asm", with = "String")] pub script_sig_asm: (), - /// Witness data (hex-encoded stack items, present for SegWit inputs) - pub witness: Vec, + /// Witness data (stack items, present for SegWit inputs; hex-encoded on the wire) + pub witness: Witness, /// Whether this input is a coinbase (block reward) input #[schemars(example = false)] @@ -46,6 +46,24 @@ pub struct TxIn { pub inner_witness_script_asm: (), } +/// Reconstruct a canonical `bitcoin::TxIn` from the stored brk shape. +/// Mempool txs are never coinbase, so `vout` (u16 in brk) always fits +/// the bitcoin protocol's u32 vout field via widening. +impl From<&TxIn> for bitcoin::TxIn { + #[inline] + fn from(txin: &TxIn) -> Self { + Self { + previous_output: OutPoint { + txid: (&txin.txid).into(), + vout: u32::from(txin.vout), + }, + script_sig: txin.script_sig.clone(), + sequence: Sequence(txin.sequence), + witness: (&txin.witness).into(), + } + } +} + impl Serialize for TxIn { fn serialize(&self, serializer: S) -> Result where @@ -75,17 +93,14 @@ impl Serialize for TxIn { .as_ref() .is_some_and(|p| p.script_pubkey.is_p2tr()); let inner_witness = if has_witness && self.witness.len() > 2 { - let script_hex = if is_p2tr { - self.witness.get(self.witness.len() - 2) + let script_bytes = if is_p2tr { + self.witness.second_to_last() } else { self.witness.last() }; - if let Some(hex) = script_hex { - let bytes: Vec = bitcoin::hex::FromHex::from_hex(hex).unwrap_or_default(); - ScriptBuf::from(bytes).to_asm_string() - } else { - String::new() - } + script_bytes + .map(|b| Script::from_bytes(b).to_asm_string()) + .unwrap_or_default() } else { String::new() }; diff --git a/crates/brk_types/src/txout.rs b/crates/brk_types/src/txout.rs index 154e9e8e7..469246640 100644 --- a/crates/brk_types/src/txout.rs +++ b/crates/brk_types/src/txout.rs @@ -74,6 +74,16 @@ impl From for TxOut { } } +impl From<&TxOut> for bitcoin::TxOut { + #[inline] + fn from(txout: &TxOut) -> Self { + Self { + value: txout.value.into(), + script_pubkey: txout.script_pubkey.clone(), + } + } +} + impl From<(ScriptBuf, Sats)> for TxOut { #[inline] fn from((script, value): (ScriptBuf, Sats)) -> Self { diff --git a/crates/brk_types/src/vin.rs b/crates/brk_types/src/vin.rs index 0fd12c64f..b21c235e3 100644 --- a/crates/brk_types/src/vin.rs +++ b/crates/brk_types/src/vin.rs @@ -46,3 +46,10 @@ impl From for u64 { value.0 as u64 } } + +impl From for usize { + #[inline] + fn from(value: Vin) -> Self { + value.0 as usize + } +} diff --git a/crates/brk_types/src/witness.rs b/crates/brk_types/src/witness.rs new file mode 100644 index 000000000..0cec6b9ac --- /dev/null +++ b/crates/brk_types/src/witness.rs @@ -0,0 +1,62 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Transaction witness: a stack of byte arrays, one per witness item. +/// +/// Wraps `bitcoin::Witness` (single-buffer layout with offsets, much +/// more compact than `Vec>`). Serializes as a JSON array of +/// hex strings - the format used by Bitcoin Core REST and mempool.space +/// and matching brk's `script_sig: ScriptBuf` (bytes internally, hex +/// on the wire). +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(transparent)] +#[schemars(with = "Vec")] +pub struct Witness(bitcoin::Witness); + +impl Witness { + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn last(&self) -> Option<&[u8]> { + self.0.last() + } + + #[inline] + pub fn second_to_last(&self) -> Option<&[u8]> { + self.0.second_to_last() + } + + #[inline] + pub fn iter(&self) -> bitcoin::blockdata::witness::Iter<'_> { + self.0.iter() + } +} + +impl From for Witness { + #[inline] + fn from(w: bitcoin::Witness) -> Self { + Self(w) + } +} + +impl From for bitcoin::Witness { + #[inline] + fn from(w: Witness) -> Self { + w.0 + } +} + +impl From<&Witness> for bitcoin::Witness { + #[inline] + fn from(w: &Witness) -> Self { + w.0.clone() + } +} diff --git a/modules/brk-client/index.js b/modules/brk-client/index.js index f4933c5a8..8daad2e99 100644 --- a/modules/brk-client/index.js +++ b/modules/brk-client/index.js @@ -845,6 +845,31 @@ Matches mempool.space/bitcoin-cli behavior. * * @typedef {number} RawLockTime */ +/** + * Response body for `GET /api/v1/tx/:txid/rbf`. Both fields are null + * when the tx has no known RBF history within the mempool monitor's + * graveyard retention window. + * + * @typedef {Object} RbfResponse + * @property {(ReplacementNode|null)=} replacements + * @property {?Txid[]=} replaces + */ +/** + * Transaction summary carried inside an RBF replacement node. Shape + * matches mempool.space's `/api/v1/tx/:txid/rbf` and + * `/api/v1/replacements` responses. + * + * @typedef {Object} RbfTx + * @property {Txid} txid + * @property {Sats} fee + * @property {VSize} vsize + * @property {Sats} value - Sum of output amounts. + * @property {FeeRate} rate + * @property {Timestamp} time + * @property {boolean} rbf - BIP-125 signaling: at least one input has sequence < 0xffffffff-1. + * @property {?boolean=} fullRbf - Only populated on the root `tx` of an RBF response. `true` iff +this tx displaced at least one non-signaling predecessor. + */ /** * Recommended fee rates in sat/vB * @@ -855,6 +880,19 @@ Matches mempool.space/bitcoin-cli behavior. * @property {FeeRate} economyFee - Fee rate for economical confirmation * @property {FeeRate} minimumFee - Minimum relay fee rate */ +/** + * One node in an RBF replacement tree. The node's `tx` replaced each + * entry in `replaces`, recursively. + * + * @typedef {Object} ReplacementNode + * @property {RbfTx} tx + * @property {Timestamp} time - First-seen timestamp, duplicated here to match mempool.space's +on-the-wire shape. + * @property {boolean} fullRbf - Any predecessor in this subtree was non-signaling. + * @property {?number=} interval - Seconds between this node's `time` and the successor that +replaced it. Omitted on the root of an RBF response. + * @property {ReplacementNode[]} replaces + */ /** * Block reward statistics over a range of blocks * @@ -1060,7 +1098,7 @@ Matches mempool.space/bitcoin-cli behavior. * @property {(TxOut|null)=} prevout - Information about the previous output being spent * @property {string} scriptsig - Signature script (hex, for non-SegWit inputs) * @property {string} scriptsigAsm - Signature script in assembly format - * @property {string[]} witness - Witness data (hex-encoded stack items, present for SegWit inputs) + * @property {Witness} witness - Witness data (stack items, present for SegWit inputs; hex-encoded on the wire) * @property {boolean} isCoinbase - Whether this input is a coinbase (block reward) input * @property {number} sequence - Input sequence number * @property {string} innerRedeemscriptAsm - Inner redeemscript in assembly (for P2SH-wrapped SegWit: scriptsig + witness both present) @@ -1240,6 +1278,17 @@ Matches mempool.space/bitcoin-cli behavior. * * @typedef {number} Weight */ +/** + * Transaction witness: a stack of byte arrays, one per witness item. + * + * Wraps `bitcoin::Witness` (single-buffer layout with offsets, much + * more compact than `Vec>`). Serializes as a JSON array of + * hex strings - the format used by Bitcoin Core REST and mempool.space + * and matching brk's `script_sig: ScriptBuf` (bytes internally, hex + * on the wire). + * + * @typedef {string[]} Witness + */ /** @typedef {number} Year1 */ /** @typedef {number} Year10 */ @@ -11599,6 +11648,24 @@ class BrkClient extends BrkClientBase { return this.getJson(path, { signal, onUpdate }); } + /** + * RBF replacement history + * + * Returns the RBF replacement tree for a transaction, if any. Both `replacements` and `replaces` are null when the tx has no known RBF history within the mempool monitor's retention window. + * + * *[Mempool.space docs](https://mempool.space/docs/api/rest#get-transaction-rbf-history)* + * + * Endpoint: `GET /api/v1/tx/{txid}/rbf` + * + * @param {Txid} txid + * @param {{ signal?: AbortSignal, onUpdate?: (value: RbfResponse) => void }} [options] + * @returns {Promise} + */ + async getTxRbf(txid, { signal, onUpdate } = {}) { + const path = `/api/v1/tx/${txid}/rbf`; + return this.getJson(path, { signal, onUpdate }); + } + /** * Validate address * diff --git a/packages/brk_client/brk_client/__init__.py b/packages/brk_client/brk_client/__init__.py index 29019c2c2..13dc00b96 100644 --- a/packages/brk_client/brk_client/__init__.py +++ b/packages/brk_client/brk_client/__init__.py @@ -21,14 +21,10 @@ T = TypeVar('T') Addr = str # US Dollar amount Dollars = float -# Amount in satoshis (1 BTC = 100,000,000 sats) -Sats = int # Index within its type (e.g., 0 for first P2WPKH address) TypeIndex = int # Type (P2PKH, P2WPKH, P2SH, P2TR, etc.) OutputType = Literal["p2pk", "p2pk", "p2pkh", "multisig", "p2sh", "op_return", "v0_p2wpkh", "v0_p2wsh", "v1_p2tr", "p2a", "empty", "unknown"] -# Transaction ID (hash) -Txid = str # Unified index for any address type (funded or empty) AnyAddrIndex = TypeIndex # Unsigned basis points stored as u16. @@ -55,14 +51,10 @@ BasisPointsSigned32 = int Bitcoin = float # URL-friendly mining pool identifier PoolSlug = Literal["unknown", "blockfills", "ultimuspool", "terrapool", "luxor", "onethash", "btccom", "bitfarms", "huobipool", "wayicn", "canoepool", "btctop", "bitcoincom", "pool175btc", "gbminers", "axbt", "asicminer", "bitminter", "bitcoinrussia", "btcserv", "simplecoinus", "btcguild", "eligius", "ozcoin", "eclipsemc", "maxbtc", "triplemining", "coinlab", "pool50btc", "ghashio", "stminingcorp", "bitparking", "mmpool", "polmine", "kncminer", "bitalo", "f2pool", "hhtt", "megabigpower", "mtred", "nmcbit", "yourbtcnet", "givemecoins", "braiinspool", "antpool", "multicoinco", "bcpoolio", "cointerra", "kanopool", "solock", "ckpool", "nicehash", "bitclub", "bitcoinaffiliatenetwork", "btcc", "bwpool", "exxbw", "bitsolo", "bitfury", "twentyoneinc", "digitalbtc", "eightbaochi", "mybtccoinpool", "tbdice", "hashpool", "nexious", "bravomining", "hotpool", "okexpool", "bcmonster", "onehash", "bixin", "tatmaspool", "viabtc", "connectbtc", "batpool", "waterhole", "dcexploration", "dcex", "btpool", "fiftyeightcoin", "bitcoinindia", "shawnp0wers", "phashio", "rigpool", "haozhuzhu", "sevenpool", "miningkings", "hashbx", "dpool", "rawpool", "haominer", "helix", "bitcoinukraine", "poolin", "secretsuperstar", "tigerpoolnet", "sigmapoolcom", "okpooltop", "hummerpool", "tangpool", "bytepool", "spiderpool", "novablock", "miningcity", "binancepool", "minerium", "lubiancom", "okkong", "aaopool", "emcdpool", "foundryusa", "sbicrypto", "arkpool", "purebtccom", "marapool", "kucoinpool", "entrustcharitypool", "okminer", "titan", "pegapool", "btcnuggets", "cloudhashing", "digitalxmintsy", "telco214", "btcpoolparty", "multipool", "transactioncoinmining", "btcdig", "trickysbtcpool", "btcmp", "eobot", "unomp", "patels", "gogreenlight", "bitcoinindiapool", "ekanembtc", "canoe", "tiger", "onem1x", "zulupool", "secpool", "ocean", "whitepool", "wiz", "wk057", "futurebitapollosolo", "carbonnegative", "portlandhodl", "phoenix", "neopool", "maxipool", "bitfufupool", "gdpool", "miningdutch", "publicpool", "miningsquared", "innopolistech", "btclab", "parasite", "redrockpool", "est3lar", "braiinssolo", "solopool"] -# Fee rate in sat/vB -FeeRate = float # Weight in weight units (WU). Max block weight is 4,000,000 WU. Weight = int # Block height Height = int -# UNIX timestamp in seconds -Timestamp = int # Block hash BlockHash = str # Transaction index within a block (0 = coinbase) @@ -98,8 +90,6 @@ CostBasisValue = Literal["supply", "realized", "unrealized"] # Options: raw (no aggregation), lin200/lin500/lin1000 (linear $200/$500/$1000), # log10/log50/log100/log200 (logarithmic with 10/50/100/200 buckets per decade). UrpdAggregation = Literal["raw", "lin200", "lin500", "lin1000", "log10", "log50", "log100", "log200"] -# Virtual size in vbytes (weight / 4, rounded up). Max block vsize is ~1,000,000 vB. -VSize = int # Date in YYYYMMDD format stored as u32 Date = int # Output format for API responses @@ -199,6 +189,14 @@ StoredU64 = int TimePeriod = Literal["24h", "3d", "1w", "1m", "3m", "6m", "1y", "2y", "3y", "all"] # Index of the output being spent in the previous transaction Vout = int +# Transaction witness: a stack of byte arrays, one per witness item. +# +# Wraps `bitcoin::Witness` (single-buffer layout with offsets, much +# more compact than `Vec>`). Serializes as a JSON array of +# hex strings - the format used by Bitcoin Core REST and mempool.space +# and matching brk's `script_sig: ScriptBuf` (bytes internally, hex +# on the wire). +Witness = List[str] # Raw transaction version (i32) from Bitcoin protocol. # Unlike TxVersion (u8, indexed), this preserves non-standard values # used in coinbase txs for miner signaling/branding. @@ -213,11 +211,21 @@ UnknownOutputIndex = TypeIndex Week1 = int Year1 = int Year10 = int +# Fee rate in sat/vB +FeeRate = float # Aggregation dimension for querying series. Includes time-based (date, week, month, year), # block-based (height, tx_index), and address/output type indexes. Index = Literal["minute10", "minute30", "hour1", "hour4", "hour12", "day1", "day3", "week1", "month1", "month3", "month6", "year1", "year10", "halving", "epoch", "height", "tx_index", "txin_index", "txout_index", "empty_output_index", "op_return_index", "p2a_addr_index", "p2ms_output_index", "p2pk33_addr_index", "p2pk65_addr_index", "p2pkh_addr_index", "p2sh_addr_index", "p2tr_addr_index", "p2wpkh_addr_index", "p2wsh_addr_index", "unknown_output_index", "funded_addr_index", "empty_addr_index"] +# Amount in satoshis (1 BTC = 100,000,000 sats) +Sats = int +# UNIX timestamp in seconds +Timestamp = int # Hierarchical tree node for organizing series into categories TreeNode = Union[dict[str, "TreeNode"], "SeriesLeafWithSchema"] +# Transaction ID (hash) +Txid = str +# Virtual size in vbytes (weight / 4, rounded up). Max block vsize is ~1,000,000 vB. +VSize = int class AddrChainStats(TypedDict): """ Address statistics on the blockchain (confirmed transactions only) @@ -1233,6 +1241,15 @@ class Prices(TypedDict): time: Timestamp USD: Dollars +class RbfResponse(TypedDict): + """ + Response body for `GET /api/v1/tx/:txid/rbf`. Both fields are null + when the tx has no known RBF history within the mempool monitor's + graveyard retention window. + """ + replacements: Union[ReplacementNode, None] + replaces: Optional[List[Txid]] + class RecommendedFees(TypedDict): """ Recommended fee rates in sat/vB @@ -1398,7 +1415,7 @@ class TxIn(TypedDict): prevout: Information about the previous output being spent scriptsig: Signature script (hex, for non-SegWit inputs) scriptsig_asm: Signature script in assembly format - witness: Witness data (hex-encoded stack items, present for SegWit inputs) + witness: Witness data (stack items, present for SegWit inputs; hex-encoded on the wire) is_coinbase: Whether this input is a coinbase (block reward) input sequence: Input sequence number inner_redeemscript_asm: Inner redeemscript in assembly (for P2SH-wrapped SegWit: scriptsig + witness both present) @@ -1409,7 +1426,7 @@ class TxIn(TypedDict): prevout: Union[TxOut, None] scriptsig: str scriptsig_asm: str - witness: List[str] + witness: Witness is_coinbase: bool sequence: int inner_redeemscript_asm: str @@ -1578,6 +1595,45 @@ class ValidateAddrParam(TypedDict): """ address: str +class RbfTx(TypedDict): + """ + Transaction summary carried inside an RBF replacement node. Shape + matches mempool.space's `/api/v1/tx/:txid/rbf` and + `/api/v1/replacements` responses. + + Attributes: + value: Sum of output amounts. + rbf: BIP-125 signaling: at least one input has sequence < 0xffffffff-1. + fullRbf: Only populated on the root `tx` of an RBF response. `true` iff +this tx displaced at least one non-signaling predecessor. + """ + txid: Txid + fee: Sats + vsize: VSize + value: Sats + rate: FeeRate + time: Timestamp + rbf: bool + fullRbf: Optional[bool] + +class ReplacementNode(TypedDict): + """ + One node in an RBF replacement tree. The node's `tx` replaced each + entry in `replaces`, recursively. + + Attributes: + time: First-seen timestamp, duplicated here to match mempool.space's +on-the-wire shape. + fullRbf: Any predecessor in this subtree was non-signaling. + interval: Seconds between this node's `time` and the successor that +replaced it. Omitted on the root of an RBF response. + """ + tx: RbfTx + time: Timestamp + fullRbf: bool + interval: Optional[int] + replaces: List["ReplacementNode"] + class SeriesLeafWithSchema(TypedDict): """ SeriesLeaf with JSON Schema for client generation @@ -8451,6 +8507,16 @@ class BrkClient(BrkClientBase): Endpoint: `GET /api/v1/transaction-times`""" return self.get_json('/api/v1/transaction-times') + def get_tx_rbf(self, txid: Txid) -> RbfResponse: + """RBF replacement history. + + Returns the RBF replacement tree for a transaction, if any. Both `replacements` and `replaces` are null when the tx has no known RBF history within the mempool monitor's retention window. + + *[Mempool.space docs](https://mempool.space/docs/api/rest#get-transaction-rbf-history)* + + Endpoint: `GET /api/v1/tx/{txid}/rbf`""" + return self.get_json(f'/api/v1/tx/{txid}/rbf') + def validate_address(self, address: str) -> AddrValidation: """Validate address. diff --git a/website/scripts/_types.js b/website/scripts/_types.js index 9b48ff02c..a5484dfe7 100644 --- a/website/scripts/_types.js +++ b/website/scripts/_types.js @@ -1,5 +1,5 @@ /** - * @import { IChartApi, ISeriesApi as _ISeriesApi, SeriesDefinition, SingleValueData as _SingleValueData, CandlestickData as _CandlestickData, BaselineData as _BaselineData, HistogramData as _HistogramData, SeriesType as LCSeriesType, IPaneApi, LineSeriesPartialOptions as _LineSeriesPartialOptions, HistogramSeriesPartialOptions as _HistogramSeriesPartialOptions, BaselineSeriesPartialOptions as _BaselineSeriesPartialOptions, CandlestickSeriesPartialOptions as _CandlestickSeriesPartialOptions, WhitespaceData, DeepPartial, ChartOptions, Time, LineData as _LineData, createChart as CreateLCChart, LineStyle, createSeriesMarkers as CreateSeriesMarkers, SeriesMarker, ISeriesMarkersPluginApi } from './modules/lightweight-charts/5.1.0/dist/typings.js' + * @import { IChartApi, ISeriesApi as _ISeriesApi, SeriesDefinition, SingleValueData as _SingleValueData, CandlestickData as _CandlestickData, BaselineData as _BaselineData, HistogramData as _HistogramData, SeriesType as LCSeriesType, IPaneApi, LineSeriesPartialOptions as _LineSeriesPartialOptions, HistogramSeriesPartialOptions as _HistogramSeriesPartialOptions, BaselineSeriesPartialOptions as _BaselineSeriesPartialOptions, CandlestickSeriesPartialOptions as _CandlestickSeriesPartialOptions, WhitespaceData, DeepPartial, ChartOptions, Time, LineData as _LineData, createChart as CreateLCChart, LineStyle, createSeriesMarkers as CreateSeriesMarkers, SeriesMarker, ISeriesMarkersPluginApi } from './modules/lightweight-charts/5.2.0/dist/typings.js' * * @import * as Brk from "./modules/brk-client/index.js" * @import { BrkClient, Index, SeriesData } from "./modules/brk-client/index.js" diff --git a/website/scripts/explorer/chain.js b/website/scripts/explorer/chain.js index 881220cda..0bf76d033 100644 --- a/website/scripts/explorer/chain.js +++ b/website/scripts/explorer/chain.js @@ -1,15 +1,19 @@ import { brk } from "../utils/client.js"; +import { onPlainClick } from "../utils/dom.js"; import { createCube } from "./cube.js"; import { createHeightElement, formatFeeRate } from "./render.js"; const LOOKAHEAD = 15; /** @type {HTMLDivElement} */ let chainEl; +/** @type {HTMLDivElement} */ let scrollEl; /** @type {HTMLDivElement} */ let blocksEl; /** @type {HTMLAnchorElement | null} */ let selectedCube = null; /** @type {IntersectionObserver} */ let olderObserver; /** @type {(block: BlockInfoV1) => void} */ let onSelect = () => {}; /** @type {(cube: HTMLAnchorElement) => void} */ let onCubeClick = () => {}; +/** @type {() => void} */ let onTip = () => {}; +/** @type {() => void} */ let onGenesis = () => {}; /** @type {Map} */ const blocksByHash = new Map(); @@ -22,32 +26,51 @@ let reachedTip = false; /** * @param {HTMLElement} parent - * @param {{ onSelect: (block: BlockInfoV1) => void, onCubeClick: (cube: HTMLAnchorElement) => void }} callbacks + * @param {{ + * onSelect: (block: BlockInfoV1) => void, + * onCubeClick: (cube: HTMLAnchorElement) => void, + * onTip: () => void, + * onGenesis: () => void, + * }} callbacks */ export function initChain(parent, callbacks) { onSelect = callbacks.onSelect; onCubeClick = callbacks.onCubeClick; + onTip = callbacks.onTip; + onGenesis = callbacks.onGenesis; chainEl = document.createElement("div"); chainEl.id = "chain"; parent.append(chainEl); + chainEl.append( + createControlLink("tip", "/block/tip", "Jump to chain tip", onTip), + ); + + chainEl.append( + createControlLink("gen", "/block/0", "Jump to genesis block", onGenesis), + ); + + scrollEl = document.createElement("div"); + scrollEl.classList.add("chain-scroll"); + chainEl.append(scrollEl); + blocksEl = document.createElement("div"); blocksEl.classList.add("blocks"); - chainEl.append(blocksEl); + scrollEl.append(blocksEl); olderObserver = new IntersectionObserver( (entries) => { if (entries[0].isIntersecting) loadOlder(); }, - { root: chainEl }, + { root: scrollEl }, ); - chainEl.addEventListener( + scrollEl.addEventListener( "scroll", () => { if (reachedTip || loadingNewer) return; - if (chainEl.scrollTop <= 50 && chainEl.scrollLeft <= 50) loadNewer(); + if (scrollEl.scrollTop <= 50 && scrollEl.scrollLeft <= 50) loadNewer(); }, { passive: true }, ); @@ -125,8 +148,8 @@ function appendNewerBlocks(blocks) { if (anchor && anchorRect) { const r = anchor.getBoundingClientRect(); - chainEl.scrollTop += r.top - anchorRect.top; - chainEl.scrollLeft += r.left - anchorRect.left; + scrollEl.scrollTop += r.top - anchorRect.top; + scrollEl.scrollLeft += r.left - anchorRect.left; } return true; } @@ -164,6 +187,7 @@ async function resolveHeight(hashOrHeight) { /** @param {BlockHash | Height | string | null} [hashOrHeight] @param {{ silent?: boolean }} [options] */ export async function goToCube(hashOrHeight, { silent } = {}) { + if (hashOrHeight === "tip") hashOrHeight = null; if (typeof hashOrHeight === "string" && /^\d+$/.test(hashOrHeight)) { hashOrHeight = Number(hashOrHeight); } @@ -263,14 +287,7 @@ function createBlockCube(block) { const fill = Math.min(1, virtualSize / 1_000_000); const { topFace, rightFace, leftFace } = createCube(cubeElement, fill); blocksByHash.set(block.id, block); - // Intercept plain left-clicks for SPA nav; let modified clicks - // (cmd/ctrl/shift/middle) and right-click fall through so the - // anchor's native open-in-new-tab / context-menu behavior works. - cubeElement.addEventListener("click", (e) => { - if (e.metaKey || e.ctrlKey || e.shiftKey || e.button !== 0) return; - e.preventDefault(); - onCubeClick(cubeElement); - }); + onPlainClick(cubeElement, () => onCubeClick(cubeElement)); const minerName = pool.name; @@ -341,3 +358,14 @@ function appendCube(cube) { setGap(cube); } +/** @param {"tip" | "gen"} label @param {string} href @param {string} title @param {() => void} handler */ +function createControlLink(label, href, title, handler) { + const a = document.createElement("a"); + a.classList.add("chain-edge", label); + a.href = href; + a.title = title; + a.textContent = label; + onPlainClick(a, handler); + return a; +} + diff --git a/website/scripts/explorer/index.js b/website/scripts/explorer/index.js index f5dc723d2..2666cb32a 100644 --- a/website/scripts/explorer/index.js +++ b/website/scripts/explorer/index.js @@ -77,6 +77,16 @@ export function init(selected) { navigate(); selectCube(cube); }, + onTip: () => { + history.pushState(null, "", "/block/tip"); + navigate(); + goToCube(null); + }, + onGenesis: () => { + history.pushState(null, "", "/block/0"); + navigate(); + goToCube(0); + }, }); initBlockDetails(explorerElement, handleLinkClick); diff --git a/website/scripts/utils/chart/index.js b/website/scripts/utils/chart/index.js index 7943c09f5..9ea8bfa9d 100644 --- a/website/scripts/utils/chart/index.js +++ b/website/scripts/utils/chart/index.js @@ -4,7 +4,7 @@ import { HistogramSeries, LineSeries, BaselineSeries, -} from "../../modules/lightweight-charts/5.1.0/dist/lightweight-charts.standalone.production.mjs"; +} from "../../modules/lightweight-charts/5.2.0/dist/lightweight-charts.standalone.production.mjs"; import { createLegend, createSeriesLegend } from "./legend.js"; import { capture } from "./capture.js"; import { colors } from "../colors.js"; diff --git a/website/scripts/utils/dom.js b/website/scripts/utils/dom.js index 663a7b2a1..be3429635 100644 --- a/website/scripts/utils/dom.js +++ b/website/scripts/utils/dom.js @@ -96,6 +96,18 @@ export function createAnchorElement({ return anchor; } +// Intercept plain left-clicks for SPA nav; let modified clicks +// (cmd/ctrl/shift/middle) and right-click fall through so the +// anchor's native open-in-new-tab / context-menu behavior works. +/** @param {HTMLElement} el @param {() => void} handler */ +export function onPlainClick(el, handler) { + el.addEventListener("click", (e) => { + if (e.metaKey || e.ctrlKey || e.shiftKey || e.button !== 0) return; + e.preventDefault(); + handler(); + }); +} + /** * @param {Object} arg * @param {string | HTMLElement} arg.inside diff --git a/website/styles/chart.css b/website/styles/chart.css index 959792b94..86bfd3023 100644 --- a/website/styles/chart.css +++ b/website/styles/chart.css @@ -58,7 +58,6 @@ display: flex; align-items: center; overflow-x: auto; - scrollbar-width: thin; padding: 0 var(--main-padding); padding-top: 0.375rem; diff --git a/website/styles/components.css b/website/styles/components.css index b726cdf7e..5028f829c 100644 --- a/website/styles/components.css +++ b/website/styles/components.css @@ -74,5 +74,4 @@ fieldset { min-width: 0; font-size: var(--font-size-sm); line-height: var(--line-height-sm); - scrollbar-width: thin; } diff --git a/website/styles/elements.css b/website/styles/elements.css index b47572124..6ddc9d567 100644 --- a/website/styles/elements.css +++ b/website/styles/elements.css @@ -125,7 +125,7 @@ h3 { html { background-color: var(--background-color); color: var(--color); - scrollbar-color: var(--off-color) var(--background-color); + scrollbar-color: var(--off-color) transparent; scrollbar-width: thin; overflow: hidden; } diff --git a/website/styles/main.css b/website/styles/main.css index ed36f55e7..b6bb4a713 100644 --- a/website/styles/main.css +++ b/website/styles/main.css @@ -145,7 +145,6 @@ footer { display: flex; gap: 1.125rem; overflow-x: auto; - scrollbar-width: thin; min-width: 0; margin: -0.5rem 0; padding: 0.5rem var(--main-padding); diff --git a/website/styles/panes/explorer.css b/website/styles/panes/explorer.css index 2177ec13b..020563ee7 100644 --- a/website/styles/panes/explorer.css +++ b/website/styles/panes/explorer.css @@ -89,16 +89,65 @@ #chain { flex-shrink: 0; - - @container aside (max-width: 767px) { - overflow-x: auto; - padding-bottom: 1rem; - } + position: relative; + padding: 0; @container aside (min-width: 768px) { height: 100%; - overflow-y: auto; - padding-right: calc(var(--main-padding) / 2); + } + + .chain-scroll { + padding: 0 var(--main-padding); + + @container aside (max-width: 767px) { + padding-bottom: 1rem; + overflow-x: auto; + } + + @container aside (min-width: 768px) { + padding: var(--main-padding); + padding-right: calc(var(--main-padding) / 2); + height: 100%; + overflow-y: auto; + } + } + + .chain-edge { + position: absolute; + font-size: var(--font-size-xs); + text-transform: uppercase; + letter-spacing: 0.1em; + font-weight: 500; + + @container aside (max-width: 767px) { + display: flex; + height: 100%; + width: var(--main-padding); + justify-content: center; + align-items: center; + writing-mode: vertical-lr; + text-orientation: upright; + text-decoration: none; + } + + @container aside (min-width: 768px) { + display: flex; + width: 100%; + height: var(--main-padding); + justify-content: center; + align-items: center; + padding-left: calc(var(--main-padding) / 2); + } + } + + .tip { + @container aside (min-width: 768px) { top: 0; } + @container aside (max-width: 767px) { left: 0; } + } + + .gen { + @container aside (min-width: 768px) { bottom: 0; } + @container aside (max-width: 767px) { top: 0; right: 0; } } .blocks {