From 8072c4670c4475c49497b52f4271bff2077d4470 Mon Sep 17 00:00:00 2001 From: nym21 Date: Tue, 21 Oct 2025 18:59:30 +0200 Subject: [PATCH] iterator: init + global: snapshot --- Cargo.lock | 37 +++--- crates/brk_bundler/Cargo.toml | 2 +- crates/brk_cli/Cargo.toml | 1 - crates/brk_computer/Cargo.toml | 1 - crates/brk_error/src/lib.rs | 8 ++ crates/brk_indexer/Cargo.toml | 1 - crates/brk_interface/Cargo.toml | 1 - crates/brk_iterator/Cargo.toml | 2 + crates/brk_iterator/examples/iterator.rs | 8 +- crates/brk_iterator/src/builder.rs | 69 +++++++++++ crates/brk_iterator/src/lib.rs | 70 ++++++++++- crates/brk_iterator/src/range.rs | 8 ++ crates/brk_iterator/src/source.rs | 44 +++++++ crates/brk_monitor/Cargo.toml | 2 +- crates/brk_monitor/src/lib.rs | 14 +-- crates/brk_reader/Cargo.toml | 3 +- crates/brk_reader/examples/reader.rs | 10 +- crates/brk_reader/src/any_block.rs | 8 +- crates/brk_reader/src/lib.rs | 51 ++++---- crates/brk_rpc/Cargo.toml | 3 +- crates/brk_rpc/src/inner.rs | 1 + crates/brk_rpc/src/lib.rs | 141 +++++++++++++++++++++-- crates/brk_server/Cargo.toml | 1 - crates/brk_structs/Cargo.toml | 3 +- crates/brk_structs/src/block.rs | 14 ++- crates/brk_structs/src/blockhash.rs | 21 ++-- crates/brk_structs/src/height.rs | 8 -- crates/brk_structs/src/tx.rs | 31 +---- crates/brk_structs/src/txid.rs | 13 +++ crates/brk_structs/src/txin.rs | 36 +----- crates/brk_structs/src/txout.rs | 9 +- 31 files changed, 456 insertions(+), 165 deletions(-) create mode 100644 crates/brk_iterator/src/builder.rs create mode 100644 crates/brk_iterator/src/range.rs create mode 100644 crates/brk_iterator/src/source.rs diff --git a/Cargo.lock b/Cargo.lock index e563d42ac..7cf837a67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,7 +606,6 @@ dependencies = [ name = "brk_cli" version = "0.0.111" dependencies = [ - "bitcoincore-rpc", "brk_binder", "brk_bundler", "brk_computer", @@ -634,7 +633,6 @@ version = "0.0.111" dependencies = [ "allocative", "bitcoin", - "bitcoincore-rpc", "brk_error", "brk_fetcher", "brk_grouper", @@ -712,7 +710,6 @@ name = "brk_indexer" version = "0.0.111" dependencies = [ "bitcoin", - "bitcoincore-rpc", "brk_error", "brk_fjall", "brk_grouper", @@ -732,7 +729,6 @@ name = "brk_interface" version = "0.0.111" dependencies = [ "bitcoin", - "bitcoincore-rpc", "brk_computer", "brk_error", "brk_indexer", @@ -752,8 +748,10 @@ name = "brk_iterator" version = "0.0.111" dependencies = [ "bitcoin", + "brk_error", "brk_reader", "brk_rpc", + "brk_structs", ] [[package]] @@ -784,7 +782,7 @@ name = "brk_monitor" version = "0.0.111" dependencies = [ "bitcoin", - "bitcoincore-rpc", + "brk_rpc", "brk_structs", "log", "parking_lot 0.12.5", @@ -796,11 +794,12 @@ name = "brk_reader" version = "0.0.111" dependencies = [ "bitcoin", - "bitcoincore-rpc", "brk_error", + "brk_rpc", "brk_structs", "crossbeam", "derive_deref", + "log", "parking_lot 0.12.5", "rayon", ] @@ -1205,6 +1204,7 @@ dependencies = [ "bitcoincore-rpc", "brk_error", "brk_logger", + "brk_structs", "log", "parking_lot 0.12.5", ] @@ -1215,7 +1215,6 @@ version = "0.0.111" dependencies = [ "axum", "bitcoin", - "bitcoincore-rpc", "brk-aide", "brk_computer", "brk_error", @@ -1272,7 +1271,6 @@ version = "0.0.111" dependencies = [ "allocative", "bitcoin", - "bitcoincore-rpc", "brk_error", "byteview 0.6.1", "derive_deref", @@ -2767,9 +2765,9 @@ checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" dependencies = [ "libc", ] @@ -2977,9 +2975,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "option-ext" @@ -3907,9 +3905,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "4.1.0" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164772177ee16e3b074e6019c63cd92cb3cecf38e8c40d097675958b86dd8084" +checksum = "d8e65c75143ce5d47c55b510297eeb1182f3c739b6043c537670e9fc18612dae" dependencies = [ "rustversion", ] @@ -4545,9 +4543,12 @@ dependencies = [ [[package]] name = "sugar_path" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8230d5b8a65a6d4d4a7e5ee8dbdd9312ba447a8b8329689a390a0945d69b57ce" +checksum = "48abcb2199ce37819c20dc7a72dc09e3263a00e598ff5089fe5fda92e0f63c37" +dependencies = [ + "smallvec", +] [[package]] name = "syn" @@ -5000,9 +5001,9 @@ checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" [[package]] name = "unicode-linebreak" diff --git a/crates/brk_bundler/Cargo.toml b/crates/brk_bundler/Cargo.toml index d71a2baad..4d8485097 100644 --- a/crates/brk_bundler/Cargo.toml +++ b/crates/brk_bundler/Cargo.toml @@ -16,5 +16,5 @@ rolldown = { version = "0.2.3", package = "brk_rolldown" } # rolldown = "0.1.0" # brk_rolldown = "0.2.3" # brk_rolldown = { path = "../../../rolldown/crates/rolldown"} -sugar_path = "1.2.0" +sugar_path = "1.2.1" tokio = { workspace = true } diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 9ce9d150c..07dabb60c 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -10,7 +10,6 @@ rust-version.workspace = true build = "build.rs" [dependencies] -bitcoincore-rpc = { workspace = true } brk_binder = { workspace = true } brk_bundler = { workspace = true } brk_computer = { workspace = true } diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 22a1ebbf5..c8e0ece6d 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -12,7 +12,6 @@ build = "build.rs" [dependencies] allocative = { workspace = true } bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } brk_error = { workspace = true } brk_fetcher = { workspace = true } brk_grouper = { workspace = true } diff --git a/crates/brk_error/src/lib.rs b/crates/brk_error/src/lib.rs index 1b49c7eb4..c8c3db891 100644 --- a/crates/brk_error/src/lib.rs +++ b/crates/brk_error/src/lib.rs @@ -20,6 +20,7 @@ pub enum Error { SystemTimeError(time::SystemTimeError), BitcoinConsensusEncode(bitcoin::consensus::encode::Error), BitcoinBip34Error(bitcoin::block::Bip34Error), + BitcoinHexError(bitcoin::consensus::encode::FromHexError), BitcoinFromScriptError(bitcoin::address::FromScriptError), SonicRS(sonic_rs::Error), ZeroCopyError, @@ -53,6 +54,12 @@ impl From for Error { } } +impl From for Error { + fn from(value: bitcoin::consensus::encode::FromHexError) -> Self { + Self::BitcoinHexError(value) + } +} + impl From for Error { fn from(value: bitcoin::address::FromScriptError) -> Self { Self::BitcoinFromScriptError(value) @@ -143,6 +150,7 @@ impl fmt::Display for Error { Error::BitcoinConsensusEncode(error) => Display::fmt(&error, f), Error::BitcoinBip34Error(error) => Display::fmt(&error, f), Error::BitcoinFromScriptError(error) => Display::fmt(&error, f), + Error::BitcoinHexError(error) => Display::fmt(&error, f), Error::BitcoinRPC(error) => Display::fmt(&error, f), Error::FjallV2(error) => Display::fmt(&error, f), // Error::FjallV3(error) => Display::fmt(&error, f), diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 4fb0ff9aa..03d9e8217 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -11,7 +11,6 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } brk_error = { workspace = true } brk_grouper = { workspace = true } brk_logger = { workspace = true } diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index 01d0d9fee..9b3425cab 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -11,7 +11,6 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } brk_computer = { workspace = true } brk_error = { workspace = true } brk_indexer = { workspace = true } diff --git a/crates/brk_iterator/Cargo.toml b/crates/brk_iterator/Cargo.toml index 9580aa2c8..69bde785e 100644 --- a/crates/brk_iterator/Cargo.toml +++ b/crates/brk_iterator/Cargo.toml @@ -11,5 +11,7 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } +brk_error = { workspace = true } brk_reader = { workspace = true } brk_rpc = { workspace = true } +brk_structs = { workspace = true } diff --git a/crates/brk_iterator/examples/iterator.rs b/crates/brk_iterator/examples/iterator.rs index f328e4d9d..d79b22606 100644 --- a/crates/brk_iterator/examples/iterator.rs +++ b/crates/brk_iterator/examples/iterator.rs @@ -1 +1,7 @@ -fn main() {} +use brk_iterator::BlockIterator; +use brk_reader::Reader; + +fn main() { + let reader = Reader::new(blocks_dir, rpc); + BlockIterator::last(10).reader(reader, client); +} diff --git a/crates/brk_iterator/src/builder.rs b/crates/brk_iterator/src/builder.rs new file mode 100644 index 000000000..b2be39ef9 --- /dev/null +++ b/crates/brk_iterator/src/builder.rs @@ -0,0 +1,69 @@ +use brk_error::Result; +use brk_reader::Reader; +use brk_rpc::Client; +use brk_structs::Height; + +use crate::{BlockIterator, BlockRange, Source}; + +pub struct BlockIteratorBuilder { + range: BlockRange, +} + +impl BlockIteratorBuilder { + pub fn new(range: BlockRange) -> Self { + Self { range } + } + + /// Build with automatic source selection (≤10 blocks = RPC, >10 = Reader) + pub fn smart(self, reader: &Reader, client: Client) -> Result { + let (start, end) = self.resolve_range(&client)?; + let count = end.saturating_sub(*start) + 1; + + let source = if count <= 10 { + Source::new_rpc(client, start, end) + } else { + Source::Reader { + receiver: reader.read(Some(start), Some(end)), + } + }; + + Ok(BlockIterator { source }) + } + + /// Build using RPC source + pub fn rpc(self, client: Client) -> Result { + let (start, end) = self.resolve_range(&client)?; + Ok(BlockIterator::from(Source::new_rpc(client, start, end))) + } + + /// Build using Reader source + pub fn reader(self, reader: &crate::Reader, client: Client) -> Result { + let (start, end) = self.resolve_range(&client)?; + Ok(BlockIterator::from(Source::Reader { + receiver: reader.read(Some(start), Some(end)), + })) + } + + /// Resolve the range to concrete start/end heights + fn resolve_range(&self, client: &Client) -> Result<(Height, Height)> { + match self.range { + BlockRange::Span { start, end } => Ok((start, end)), + BlockRange::Start { start } => { + let end = Height::new(client.get_block_count()? as u32); + Ok((start, end)) + } + BlockRange::End { end } => Ok((Height::ZERO, end)), + BlockRange::Last { n } => { + let end = Height::new(client.get_block_count()? as u32); + let start = Height::new((*end).saturating_sub(n - 1)); + Ok((start, end)) + } + } + } +} + +impl From for BlockIteratorBuilder { + fn from(range: BlockRange) -> Self { + Self { range } + } +} diff --git a/crates/brk_iterator/src/lib.rs b/crates/brk_iterator/src/lib.rs index c0733d805..17ac56526 100644 --- a/crates/brk_iterator/src/lib.rs +++ b/crates/brk_iterator/src/lib.rs @@ -1,13 +1,73 @@ -use brk_rpc::Client; +use brk_reader::Reader; +use brk_structs::{Block, Height}; +mod builder; +mod range; +mod source; + +use builder::*; +use range::*; +use source::*; + +/// Block iterator that can use either RPC or Reader pub struct BlockIterator { - client: Client, + source: Source, +} + +impl From for BlockIterator { + fn from(source: Source) -> Self { + Self { source } + } +} + +impl Iterator for BlockIterator { + type Item = Block; + + fn next(&mut self) -> Option { + match &mut self.source { + Source::Rpc { + client, + heights, + prev_hash, + } => { + let height = heights.next()?; + + let Ok(hash) = client.get_block_hash(height) else { + return None; + }; + + let Ok(block) = client.get_block(&hash) else { + return None; + }; + + if prev_hash + .as_ref() + .is_some_and(|prev_hash| block.header.prev_blockhash != prev_hash.into()) + { + return None; + } + + Some(Block::from((height, hash, block))) + } + Source::Reader { receiver } => receiver.recv().ok().map(|b| b.unwrap()), + } + } } impl BlockIterator { - pub fn new(client: Client) -> Self { - Self { client } + pub fn range(start: Height, end: Height) -> BlockIteratorBuilder { + BlockIteratorBuilder::from(BlockRange::Span { start, end }) } - pub fn iter() {} + pub fn start(start: Height) -> BlockIteratorBuilder { + BlockIteratorBuilder::from(BlockRange::Start { start }) + } + + pub fn end(end: Height) -> BlockIteratorBuilder { + BlockIteratorBuilder::from(BlockRange::End { end }) + } + + pub fn last(n: u32) -> BlockIteratorBuilder { + BlockIteratorBuilder::from(BlockRange::Last { n }) + } } diff --git a/crates/brk_iterator/src/range.rs b/crates/brk_iterator/src/range.rs new file mode 100644 index 000000000..c957e5348 --- /dev/null +++ b/crates/brk_iterator/src/range.rs @@ -0,0 +1,8 @@ +use brk_structs::Height; + +pub enum BlockRange { + Span { start: Height, end: Height }, + Start { start: Height }, + End { end: Height }, + Last { n: u32 }, +} diff --git a/crates/brk_iterator/src/source.rs b/crates/brk_iterator/src/source.rs new file mode 100644 index 000000000..ef0711bb7 --- /dev/null +++ b/crates/brk_iterator/src/source.rs @@ -0,0 +1,44 @@ +use std::vec; + +use brk_reader::Receiver; +use brk_rpc::Client; +use brk_structs::{BlockHash, Height, ReadBlock}; + +pub enum Source { + Rpc { + client: Client, + heights: vec::IntoIter, + prev_hash: Option, + }, + Reader { + receiver: Receiver, + }, +} + +impl Source { + pub fn new_rpc(client: Client, start: Height, end: Height) -> Self { + let heights = (*start..=*end) + .map(Height::new) + .collect::>() + .into_iter(); + + Self::Rpc { + client, + heights, + prev_hash: None, + } + } + + pub fn new_reader(client: Client, start: Height, end: Height) -> Self { + let heights = (*start..=*end) + .map(Height::new) + .collect::>() + .into_iter(); + + Self::Rpc { + client, + heights, + prev_hash: None, + } + } +} diff --git a/crates/brk_monitor/Cargo.toml b/crates/brk_monitor/Cargo.toml index 522b49213..81313f498 100644 --- a/crates/brk_monitor/Cargo.toml +++ b/crates/brk_monitor/Cargo.toml @@ -11,7 +11,7 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } +brk_rpc = { workspace = true } brk_structs = { workspace = true } log = { workspace = true } parking_lot = { workspace = true } diff --git a/crates/brk_monitor/src/lib.rs b/crates/brk_monitor/src/lib.rs index 992a734fb..a0436ed53 100644 --- a/crates/brk_monitor/src/lib.rs +++ b/crates/brk_monitor/src/lib.rs @@ -1,7 +1,7 @@ use std::{thread, time::Duration}; use bitcoin::consensus::encode; -use bitcoincore_rpc::{Client, RpcApi}; +use brk_rpc::Client; use brk_structs::{AddressBytes, AddressMempoolStats, Transaction, Txid}; use log::error; use parking_lot::{RwLock, RwLockReadGuard}; @@ -10,15 +10,15 @@ use rustc_hash::{FxHashMap, FxHashSet}; const MAX_FETCHES_PER_CYCLE: usize = 10_000; pub struct Mempool { - rpc: &'static Client, + client: Client, txs: RwLock>, addresses: RwLock)>>, } impl Mempool { - pub fn new(rpc: &'static Client) -> Self { + pub fn new(client: Client) -> Self { Self { - rpc, + client, txs: RwLock::new(FxHashMap::default()), addresses: RwLock::new(FxHashMap::default()), } @@ -45,7 +45,7 @@ impl Mempool { pub fn update(&self) -> Result<(), Box> { let txids = self - .rpc + .client .get_raw_mempool()? .into_iter() .map(Txid::from) @@ -62,11 +62,11 @@ impl Mempool { } .into_iter() .filter_map(|txid| { - self.rpc + self.client .get_raw_transaction_hex(&bitcoin::Txid::from(&txid), None) .ok() .and_then(|hex| encode::deserialize_hex::(&hex).ok()) - .map(|tx| Transaction::from_mempool(tx, self.rpc)) + .map(|tx| Transaction::from_mempool(tx, self.client)) .map(|tx| (txid, tx)) }) .collect::>(); diff --git a/crates/brk_reader/Cargo.toml b/crates/brk_reader/Cargo.toml index 0c58ac2a2..b9fc7ae2e 100644 --- a/crates/brk_reader/Cargo.toml +++ b/crates/brk_reader/Cargo.toml @@ -13,10 +13,11 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } brk_error = { workspace = true } +brk_rpc = { workspace = true } brk_structs = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_deref = { workspace = true } +log = { workspace = true } parking_lot = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_reader/examples/reader.rs b/crates/brk_reader/examples/reader.rs index 400bad4e9..441bfc4f2 100644 --- a/crates/brk_reader/examples/reader.rs +++ b/crates/brk_reader/examples/reader.rs @@ -1,8 +1,8 @@ use std::path::Path; -use bitcoincore_rpc::{Auth, Client}; use brk_error::Result; use brk_reader::Reader; +use brk_rpc::{Auth, Client}; #[allow(clippy::needless_doctest_main)] fn main() -> Result<()> { @@ -13,12 +13,14 @@ fn main() -> Result<()> { .join("Application Support") .join("Bitcoin"); - let rpc = Box::leak(Box::new(Client::new( + let client = Client::new( "http://localhost:8332", Auth::CookieFile(bitcoin_dir.join(".cookie")), - )?)); + )?; - let reader = Reader::new(bitcoin_dir.join("blocks"), rpc); + let blocks_dir = bitcoin_dir.join("blocks"); + + let reader = Reader::new(blocks_dir, client); let start = None; // let start = Some(916037_u32.into()); diff --git a/crates/brk_reader/src/any_block.rs b/crates/brk_reader/src/any_block.rs index 7fa0aae3c..989f1c561 100644 --- a/crates/brk_reader/src/any_block.rs +++ b/crates/brk_reader/src/any_block.rs @@ -1,6 +1,6 @@ use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor}; -use bitcoincore_rpc::RpcApi; use brk_error::Result; +use brk_rpc::Client; use brk_structs::{BlkMetadata, Block, Height, ReadBlock}; use crate::{XORBytes, XORIndex}; @@ -15,7 +15,7 @@ impl AnyBlock { pub fn decode( self, metadata: BlkMetadata, - rpc: &'static bitcoincore_rpc::Client, + client: &Client, mut xor_i: XORIndex, xor_bytes: XORBytes, start: Option, @@ -32,11 +32,11 @@ impl AnyBlock { let header = Header::consensus_decode(&mut cursor)?; - let hash = header.block_hash(); + let hash = header.block_hash().into(); let tx_count = VarInt::consensus_decode(&mut cursor)?.0; - let Ok(block_header_result) = rpc.get_block_header_info(&hash) else { + let Ok(block_header_result) = client.get_block_header_info(&hash) else { return Ok(Self::Skipped); }; diff --git a/crates/brk_reader/src/lib.rs b/crates/brk_reader/src/lib.rs index b3c6ab2c2..85a26a0a4 100644 --- a/crates/brk_reader/src/lib.rs +++ b/crates/brk_reader/src/lib.rs @@ -13,11 +13,13 @@ use std::{ }; use bitcoin::{block::Header, consensus::Decodable}; -use bitcoincore_rpc::RpcApi; use blk_index_to_blk_path::*; use brk_error::Result; -use brk_structs::{BlkMetadata, BlkPosition, Block, Height, ReadBlock}; -use crossbeam::channel::{Receiver, bounded}; +use brk_rpc::Client; +use brk_structs::{BlkMetadata, BlkPosition, BlockHash, Height, ReadBlock}; +pub use crossbeam::channel::Receiver; +use crossbeam::channel::bounded; +use log::error; use parking_lot::{RwLock, RwLockReadGuard}; use rayon::prelude::*; @@ -38,18 +40,18 @@ pub struct Reader { blk_index_to_blk_path: Arc>, xor_bytes: XORBytes, blocks_dir: PathBuf, - rpc: &'static bitcoincore_rpc::Client, + client: Client, } impl Reader { - pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self { + pub fn new(blocks_dir: PathBuf, client: Client) -> Self { Self { xor_bytes: XORBytes::from(blocks_dir.as_path()), blk_index_to_blk_path: Arc::new(RwLock::new(BlkIndexToBlkPath::scan( blocks_dir.as_path(), ))), blocks_dir, - rpc, + client, } } @@ -61,22 +63,13 @@ impl Reader { self.xor_bytes } - pub fn get(&self, height: Height) -> Result { - Ok(( - height, - self.rpc - .get_block(&self.rpc.get_block_hash(height.into())?)?, - ) - .into()) - } - /// /// Returns a crossbeam channel receiver that receives `Block` from an **inclusive** range (`start` and `end`) /// /// For an example checkout `./main.rs` /// pub fn read(&self, start: Option, end: Option) -> Receiver { - let rpc = self.rpc; + let client = self.client.clone(); let (send_bytes, recv_bytes) = bounded(BOUND_CAP / 2); let (send_block, recv_block) = bounded(BOUND_CAP); @@ -169,7 +162,7 @@ impl Reader { .into_par_iter() .try_for_each(|(metdata, any_block, xor_i)| { if let Ok(AnyBlock::Decoded(block)) = - any_block.decode(metdata, rpc, xor_i, xor_bytes, start, end) + any_block.decode(metdata, &client, xor_i, xor_bytes, start, end) && send_block.send(block).is_err() { return ControlFlow::Break(()); @@ -197,7 +190,7 @@ impl Reader { thread::spawn(move || { let mut current_height = start.unwrap_or_default(); - + let mut prev_hash: Option = None; let mut future_blocks = BTreeMap::default(); let _ = recv_block @@ -217,7 +210,21 @@ impl Reader { None } }) { - send_ordered.send(block).unwrap(); + if let Some(expected_prev) = prev_hash.as_ref() && block.header.prev_blockhash != expected_prev.into() { + error!( + "Chain discontinuity detected at height {}: expected prev_hash {}, got {}. Stopping iteration.", + *block.height(), + expected_prev, + block.hash() + ); + return ControlFlow::Break(()); + } + + prev_hash = Some(block.hash().clone()); + + if send_ordered.send(block).is_err() { + return ControlFlow::Break(()); + } current_height.increment(); } @@ -240,8 +247,8 @@ impl Reader { }; // If start is a very recent block we only look back X blk file before the last - if let Ok(count) = self.rpc.get_block_count() - && (count as u32).saturating_sub(*target_start) <= 3 + if let Ok(height) = self.client.get_last_height() + && (*height).saturating_sub(*target_start) <= 3 { return Ok(blk_index_to_blk_path .keys() @@ -332,7 +339,7 @@ impl Reader { let header = Header::consensus_decode(&mut std::io::Cursor::new(&header_bytes))?; - let height = self.rpc.get_block_info(&header.block_hash())?.height as u32; + let height = self.client.get_block_info(&header.block_hash())?.height as u32; Ok(Height::new(height)) } diff --git a/crates/brk_rpc/Cargo.toml b/crates/brk_rpc/Cargo.toml index 5e34b4286..fdb6afbeb 100644 --- a/crates/brk_rpc/Cargo.toml +++ b/crates/brk_rpc/Cargo.toml @@ -11,8 +11,9 @@ build = "build.rs" [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = "0.19.0" +bitcoincore-rpc = { workspace = true } brk_error = { workspace = true } brk_logger = { workspace = true } +brk_structs = { workspace = true } log = { workspace = true } parking_lot = { workspace = true } diff --git a/crates/brk_rpc/src/inner.rs b/crates/brk_rpc/src/inner.rs index 79c2e41f2..20d593524 100644 --- a/crates/brk_rpc/src/inner.rs +++ b/crates/brk_rpc/src/inner.rs @@ -6,6 +6,7 @@ use std::time::Duration; pub use bitcoincore_rpc::Auth; +#[derive(Debug)] pub struct ClientInner { url: String, auth: Auth, diff --git a/crates/brk_rpc/src/lib.rs b/crates/brk_rpc/src/lib.rs index f0bf0a523..1e925cd2a 100644 --- a/crates/brk_rpc/src/lib.rs +++ b/crates/brk_rpc/src/lib.rs @@ -1,9 +1,10 @@ -use bitcoin::BlockHash; -use bitcoincore_rpc::json::GetBlockResult; +use std::{mem, sync::Arc, time::Duration}; + +use bitcoin::consensus::encode; +use bitcoincore_rpc::json::{GetBlockHeaderResult, GetBlockResult, GetTxOutResult}; use bitcoincore_rpc::{Client as CoreClient, Error as RpcError, RpcApi}; use brk_error::Result; -use std::sync::Arc; -use std::time::Duration; +use brk_structs::{BlockHash, Height, Sats, Transaction, TxIn, TxOut, TxStatus, Txid, Vout}; pub use bitcoincore_rpc::Auth; @@ -16,7 +17,7 @@ use inner::ClientInner; /// /// Free to clone (Arc) /// -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct Client(Arc); impl Client { @@ -38,8 +39,126 @@ impl Client { )?))) } - pub fn get_block_info(&self, hash: &BlockHash) -> Result { - self.call(|c| c.get_block_info(hash)).map_err(Into::into) + pub fn get_block(&self, hash: &BlockHash) -> Result { + self.call(|c| c.get_block(hash.into())).map_err(Into::into) + } + + /// Returns the numbers of block in the longest chain. + pub fn get_last_height(&self) -> Result { + self.call(|c| c.get_block_count()) + .map(Height::from) + .map_err(Into::into) + } + + /// Get block hash at a given height + pub fn get_block_hash(&self, height: Height) -> Result { + self.call(|c| c.get_block_hash(height.into())) + .map(BlockHash::from) + .map_err(Into::into) + } + + pub fn get_block_info<'a, H>(&self, hash: &'a H) -> Result + where + &'a H: Into<&'a bitcoin::BlockHash>, + { + self.call(move |c| c.get_block_info(hash.into())) + .map_err(Into::into) + } + + pub fn get_block_header_info(&self, hash: &BlockHash) -> Result { + self.call(|c| c.get_block_header_info(hash.into())) + .map_err(Into::into) + } + + pub fn get_transaction(&self, txid: Txid) -> Result { + let mut tx = self.get_raw_transaction(&txid, None)?; + + let input = mem::take(&mut tx.input) + .into_iter() + .map(|txin| -> Result { + let txout_result = self.get_tx_out( + (&txin.previous_output.txid).into(), + txin.previous_output.vout.into(), + Some(true), + )?; + + let is_coinbase = txout_result.as_ref().is_none_or(|r| r.coinbase); + + let txout = if let Some(txout_result) = txout_result { + Some(TxOut::from(( + txout_result.script_pub_key.script()?, + Sats::from(txout_result.value.to_sat()), + ))) + } else { + None + }; + + Ok(TxIn { + is_coinbase, + prevout: txout, + txid: txin.previous_output.txid.into(), + vout: txin.previous_output.vout.into(), + script_sig: txin.script_sig, + script_sig_asm: (), + sequence: txin.sequence.into(), + inner_redeem_script_asm: (), + }) + }) + .collect::>>()?; + + let mut tx = Transaction { + index: None, + txid: tx.compute_txid().into(), + version: tx.version.into(), + total_sigop_cost: tx.total_sigop_cost(|_| None), + weight: tx.weight().into(), + lock_time: tx.lock_time.into(), + total_size: tx.total_size(), + fee: Sats::default(), + input, + output: tx.output.into_iter().map(TxOut::from).collect(), + status: TxStatus::UNCOMFIRMED, + }; + + tx.compute_fee(); + + Ok(tx) + } + + pub fn get_tx_out( + &self, + txid: &Txid, + vout: Vout, + include_mempool: Option, + ) -> Result> { + self.call(|c| c.get_tx_out(txid.into(), vout.into(), include_mempool)) + .map_err(Into::into) + } + + /// Get txids of all transactions in a memory pool + pub fn get_raw_mempool(&self) -> Result> { + self.call(|c| c.get_raw_mempool()) + .map(|v| unsafe { mem::transmute(v) }) + .map_err(Into::into) + } + + pub fn get_raw_transaction( + &self, + txid: &Txid, + block_hash: Option<&BlockHash>, + ) -> brk_error::Result { + let hex = self.get_raw_transaction_hex(txid, block_hash)?; + let tx = encode::deserialize_hex::(&hex)?; + Ok(tx) + } + + pub fn get_raw_transaction_hex( + &self, + txid: &Txid, + block_hash: Option<&BlockHash>, + ) -> Result { + self.call(|c| c.get_raw_transaction_hex(txid.into(), block_hash.map(|h| h.into()))) + .map_err(Into::into) } /// Checks if a block is in the main chain (has positive confirmations) @@ -63,19 +182,21 @@ impl Client { // Get the previous block hash and walk backwards let mut current_hash = block_info .previousblockhash + .map(BlockHash::from) .ok_or("Genesis block has no previous block")?; loop { if self.is_in_main_chain(¤t_hash)? { // Found a block in the main chain - let current_info = self.get_block_info(¤t_hash)?; + let current_info = self.get_block_header_info(¤t_hash)?; return Ok(current_info.height as u64); } // Continue walking backwards - let current_info = self.get_block_info(¤t_hash)?; + let current_info = self.get_block_header_info(¤t_hash)?; current_hash = current_info - .previousblockhash + .previous_block_hash + .map(BlockHash::from) .ok_or("Reached genesis without finding main chain")?; } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index d9de34cb2..9eedd5fa5 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -13,7 +13,6 @@ build = "build.rs" aide = { workspace = true } axum = { workspace = true } bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } brk_computer = { workspace = true } brk_error = { workspace = true } brk_fetcher = { workspace = true } diff --git a/crates/brk_structs/Cargo.toml b/crates/brk_structs/Cargo.toml index 28c057c21..643e852b3 100644 --- a/crates/brk_structs/Cargo.toml +++ b/crates/brk_structs/Cargo.toml @@ -12,14 +12,13 @@ build = "build.rs" [dependencies] allocative = { workspace = true } bitcoin = { workspace = true } -bitcoincore-rpc = { workspace = true } brk_error = { workspace = true } byteview = { workspace = true } derive_deref = { workspace = true } itoa = "1.0.15" jiff = { workspace = true } num_enum = "0.7.5" -rapidhash = "4.1.0" +rapidhash = "4.1.1" ryu = "1.0.20" schemars = { workspace = true } serde = { workspace = true } diff --git a/crates/brk_structs/src/block.rs b/crates/brk_structs/src/block.rs index b106712c8..37e5521c4 100644 --- a/crates/brk_structs/src/block.rs +++ b/crates/brk_structs/src/block.rs @@ -39,10 +39,18 @@ impl From<(Height, bitcoin::Block)> for Block { } impl From<(Height, bitcoin::BlockHash, bitcoin::Block)> for Block { + #[inline] fn from((height, hash, block): (Height, bitcoin::BlockHash, bitcoin::Block)) -> Self { + Self::from((height, BlockHash::from(hash), block)) + } +} + +impl From<(Height, BlockHash, bitcoin::Block)> for Block { + #[inline] + fn from((height, hash, block): (Height, BlockHash, bitcoin::Block)) -> Self { Self { height, - hash: hash.into(), + hash, block, } } @@ -80,6 +88,10 @@ impl ReadBlock { pub fn tx_metadata(&self) -> &Vec { &self.tx_metadata } + + pub fn unwrap(self) -> Block { + self.block + } } impl Deref for ReadBlock { diff --git a/crates/brk_structs/src/blockhash.rs b/crates/brk_structs/src/blockhash.rs index da0457151..800fa445e 100644 --- a/crates/brk_structs/src/blockhash.rs +++ b/crates/brk_structs/src/blockhash.rs @@ -1,18 +1,16 @@ use std::{fmt, mem}; use bitcoin::hashes::Hash; -use bitcoincore_rpc::{Client, RpcApi}; use derive_deref::Deref; use schemars::JsonSchema; use serde::{Serialize, Serializer}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::Height; - /// Block hash #[derive( Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, JsonSchema, )] +#[repr(C)] pub struct BlockHash([u8; 32]); impl From for BlockHash { @@ -21,22 +19,27 @@ impl From for BlockHash { } } +impl From<&bitcoin::BlockHash> for &BlockHash { + fn from(value: &bitcoin::BlockHash) -> Self { + unsafe { mem::transmute(value) } + } +} + impl From for bitcoin::BlockHash { fn from(value: BlockHash) -> Self { unsafe { mem::transmute(value) } } } -impl From<&BlockHash> for bitcoin::BlockHash { +impl From<&BlockHash> for &bitcoin::BlockHash { fn from(value: &BlockHash) -> Self { - bitcoin::BlockHash::from_slice(&value.0).unwrap() + unsafe { mem::transmute(value) } } } -impl TryFrom<(&Client, Height)> for BlockHash { - type Error = bitcoincore_rpc::Error; - fn try_from((rpc, height): (&Client, Height)) -> Result { - Ok(Self::from(rpc.get_block_hash(u64::from(height))?)) +impl From<&BlockHash> for bitcoin::BlockHash { + fn from(value: &BlockHash) -> Self { + bitcoin::BlockHash::from_slice(&value.0).unwrap() } } diff --git a/crates/brk_structs/src/height.rs b/crates/brk_structs/src/height.rs index 8838a5efc..c341dc8cd 100644 --- a/crates/brk_structs/src/height.rs +++ b/crates/brk_structs/src/height.rs @@ -4,7 +4,6 @@ use std::{ }; use allocative::Allocative; -use bitcoincore_rpc::{Client, RpcApi}; use byteview::ByteView; use derive_deref::Deref; use schemars::JsonSchema; @@ -213,13 +212,6 @@ impl From for u64 { } } -impl TryFrom<&Client> for Height { - type Error = bitcoincore_rpc::Error; - fn try_from(value: &Client) -> Result { - Ok((value.get_block_count()? as usize).into()) - } -} - impl From for Height { fn from(value: bitcoin::locktime::absolute::Height) -> Self { Self(value.to_consensus_u32()) diff --git a/crates/brk_structs/src/tx.rs b/crates/brk_structs/src/tx.rs index f7f3d375a..f5f6e76c2 100644 --- a/crates/brk_structs/src/tx.rs +++ b/crates/brk_structs/src/tx.rs @@ -1,5 +1,4 @@ use crate::{RawLockTime, Sats, TxIn, TxIndex, TxOut, TxStatus, TxVersion, Txid, Weight}; -use bitcoincore_rpc::Client; use schemars::JsonSchema; use serde::Serialize; use vecdb::CheckedSub; @@ -50,37 +49,17 @@ pub struct Transaction { } impl Transaction { - pub fn fee(&self) -> Option { - let in_ = self + pub fn fee(tx: &Transaction) -> Option { + let in_ = tx .input .iter() .map(|txin| txin.prevout.as_ref().map(|txout| txout.value)) .sum::>()?; - let out = self.output.iter().map(|txout| txout.value).sum::(); + let out = tx.output.iter().map(|txout| txout.value).sum::(); Some(in_.checked_sub(out).unwrap()) } -} -impl Transaction { - pub fn from_mempool(tx: bitcoin::Transaction, rpc: &Client) -> Self { - let mut this = Self { - index: None, - txid: tx.compute_txid().into(), - version: tx.version.into(), - total_sigop_cost: tx.total_sigop_cost(|_| None), - weight: tx.weight().into(), - lock_time: tx.lock_time.into(), - total_size: tx.total_size(), - fee: Sats::default(), - input: tx - .input - .into_iter() - .map(|txin| TxIn::from((txin, rpc))) - .collect(), - output: tx.output.into_iter().map(TxOut::from).collect(), - status: TxStatus::UNCOMFIRMED, - }; - this.fee = this.fee().unwrap_or_default(); - this + pub fn compute_fee(&mut self) { + self.fee = Self::fee(self).unwrap_or_default(); } } diff --git a/crates/brk_structs/src/txid.rs b/crates/brk_structs/src/txid.rs index 2d89c9cd2..019c2c625 100644 --- a/crates/brk_structs/src/txid.rs +++ b/crates/brk_structs/src/txid.rs @@ -25,6 +25,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; example = "2bb85f4b004be6da54f766c17c1e855187327112c231ef2ff35ebad0ea67c69e", example = "9a0b3b8305bb30cacf9e8443a90d53a76379fb3305047fdeaa4e4b0934a2a1ba" )] +#[repr(C)] pub struct Txid([u8; 32]); impl From for Txid { @@ -33,6 +34,12 @@ impl From for Txid { } } +impl From<&bitcoin::Txid> for &Txid { + fn from(value: &bitcoin::Txid) -> Self { + unsafe { mem::transmute(value) } + } +} + impl From for bitcoin::Txid { fn from(value: Txid) -> Self { unsafe { mem::transmute(value) } @@ -45,6 +52,12 @@ impl From<&Txid> for bitcoin::Txid { } } +impl From<&Txid> for &bitcoin::Txid { + fn from(value: &Txid) -> Self { + unsafe { mem::transmute(value) } + } +} + impl fmt::Display for Txid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&bitcoin::Txid::from(self).to_string()) diff --git a/crates/brk_structs/src/txin.rs b/crates/brk_structs/src/txin.rs index 6aed0a2c5..117c02210 100644 --- a/crates/brk_structs/src/txin.rs +++ b/crates/brk_structs/src/txin.rs @@ -1,6 +1,5 @@ use crate::{TxOut, Txid, Vout}; use bitcoin::{Script, ScriptBuf}; -use bitcoincore_rpc::{Client, RpcApi}; use schemars::JsonSchema; use serde::{Serialize, Serializer, ser::SerializeStruct}; @@ -27,13 +26,12 @@ pub struct TxIn { pub script_sig: ScriptBuf, /// Signature script in assembly format - #[allow(dead_code)] #[schemars( rename = "scriptsig_asm", with = "String", example = "OP_PUSHBYTES_4 ffff001d OP_PUSHBYTES_1 04 OP_PUSHBYTES_69 5468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73" )] - script_sig_asm: (), + pub script_sig_asm: (), // /// Witness data (for SegWit inputs) // #[schemars(example = vec!["3045022100d0c9936990bf00bdba15f425f0f360a223d5cbf81f4bf8477fe6c6d838fb5fae02207e42a8325a4dd41702bf065aa6e0a1b7b0b8ee92a5e6c182da018b0afc82c40601".to_string()])] @@ -54,7 +52,7 @@ pub struct TxIn { with = "Option", example = Some("OP_0 OP_PUSHBYTES_20 992a1f7420fc5285070d19c71ff2efb1e356ad2f".to_string()) )] - inner_redeem_script_asm: (), + pub inner_redeem_script_asm: (), } impl TxIn { @@ -67,36 +65,6 @@ impl TxIn { } } -impl From<(bitcoin::TxIn, &Client)> for TxIn { - fn from((txin, rpc): (bitcoin::TxIn, &Client)) -> Self { - let txout_result = rpc - .get_tx_out( - &txin.previous_output.txid, - txin.previous_output.vout, - Some(true), - ) - .unwrap(); - - let is_coinbase = txout_result.as_ref().is_none_or(|r| r.coinbase); - - // txin.witness - - // txin.script_sig.redeem_script() - - Self { - is_coinbase, - prevout: txout_result.map(TxOut::from), - txid: txin.previous_output.txid.into(), - vout: txin.previous_output.vout.into(), - script_sig: txin.script_sig, - script_sig_asm: (), - sequence: txin.sequence.into(), - // witness: - inner_redeem_script_asm: (), - } - } -} - impl Serialize for TxIn { fn serialize(&self, serializer: S) -> Result where diff --git a/crates/brk_structs/src/txout.rs b/crates/brk_structs/src/txout.rs index 483d63004..f45756742 100644 --- a/crates/brk_structs/src/txout.rs +++ b/crates/brk_structs/src/txout.rs @@ -1,6 +1,5 @@ use crate::{Address, AddressBytes, OutputType, Sats}; use bitcoin::ScriptBuf; -use bitcoincore_rpc::json::GetTxOutResult; use schemars::JsonSchema; use serde::{Serialize, Serializer, ser::SerializeStruct}; @@ -74,14 +73,14 @@ impl From for TxOut { } } -impl From for TxOut { - fn from(value: GetTxOutResult) -> Self { +impl From<(ScriptBuf, Sats)> for TxOut { + fn from((script, value): (ScriptBuf, Sats)) -> Self { Self { - script_pubkey: value.script_pub_key.script().unwrap(), + script_pubkey: script, script_pubkey_address: (), script_pubkey_asm: (), script_pubkey_type: (), - value: value.value.into(), + value, } } }