From 1c2afd14ddf240a6466693c1c682647115b05556 Mon Sep 17 00:00:00 2001 From: nym21 Date: Mon, 1 Sep 2025 20:34:27 +0200 Subject: [PATCH] global: fixes of Parser::new --- Cargo.lock | 6 ++++++ Cargo.toml | 4 ++-- crates/brk/README.md | 14 +++++++------- crates/brk_cli/src/lib.rs | 3 ++- crates/brk_computer/examples/computer.rs | 6 +++++- crates/brk_indexer/README.md | 16 ++++++++-------- crates/brk_indexer/examples/indexer.rs | 2 +- crates/brk_parser/README.md | 12 ++++++------ crates/brk_server/examples/main.rs | 2 +- 9 files changed, 38 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 367b0e24a..c704843d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3885,6 +3885,8 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "seqdb" version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54ab988c96efa9d275ca2b12bf2d3c6adec993b8e82ea31a88c984abdaa14fa" dependencies = [ "libc", "log", @@ -4711,6 +4713,8 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5c4ec34c376be3a41435eeb7672d0ea0e9c1d60c5d1d90218912588f91abea" dependencies = [ "ctrlc", "log", @@ -4729,6 +4733,8 @@ dependencies = [ [[package]] name = "vecdb_derive" version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778c4874c05822465e28cae6a7dead593a73124ec80afb85b85adae5ac883368" dependencies = [ "quote", "syn 2.0.106", diff --git a/Cargo.toml b/Cargo.toml index 5e32acd6f..39aa7d075 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,8 +52,8 @@ serde_bytes = "0.11.17" serde_derive = "1.0.219" serde_json = { version = "1.0.143", features = ["float_roundtrip"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] } -vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} -# vecdb = { version = "0.2.5", features = ["derive"]} +# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} +vecdb = { version = "0.2.5", features = ["derive"]} zerocopy = "0.8.26" zerocopy-derive = "0.8.26" diff --git a/crates/brk/README.md b/crates/brk/README.md index cb9313364..cbd2dcc2f 100644 --- a/crates/brk/README.md +++ b/crates/brk/README.md @@ -64,7 +64,7 @@ brk = { version = "0.0.88", features = ["parser", "indexer", "computer"] } use brk::{parser, indexer, computer}; // Core data pipeline only -let parser = parser::Parser::new(blocks_dir, output_dir, rpc); +let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc); let mut indexer = indexer::Indexer::forced_import(output_dir)?; let mut computer = computer::Computer::forced_import(output_dir, &indexer, None)?; ``` @@ -81,7 +81,7 @@ use brk::{structs, parser}; // Just parsing and types let height = structs::Height::new(800_000); -let parser = parser::Parser::new(blocks_dir, output_dir, rpc); +let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc); ``` ## Feature Flags @@ -113,7 +113,7 @@ use brk::*; // Full data pipeline setup let config = cli::Config::load()?; let rpc = /* Bitcoin Core RPC client */; -let parser = parser::Parser::new(config.blocks_dir, config.output_dir, rpc); +let parser = parser::Parser::new(config.blocks_dir, Some(config.output_dir), rpc); let mut indexer = indexer::Indexer::forced_import(&config.output_dir)?; let mut computer = computer::Computer::forced_import(&config.output_dir, &indexer, None)?; let interface = interface::Interface::build(&indexer, &computer); @@ -152,15 +152,15 @@ use brk::{structs, parser, error}; // Custom application with BRK components fn analyze_blocks() -> error::Result<()> { - let parser = parser::Parser::new(blocks_dir, output_dir, rpc); - + let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc); + parser.parse(None, None) .iter() .take(1000) // First 1000 blocks .for_each(|(height, block, hash)| { println!("Block {}: {} transactions", height, block.txdata.len()); }); - + Ok(()) } ``` @@ -194,4 +194,4 @@ For specific dependency information, see individual crate READMEs. --- -*This README was generated by Claude Code* \ No newline at end of file +*This README was generated by Claude Code* diff --git a/crates/brk_cli/src/lib.rs b/crates/brk_cli/src/lib.rs index 19f33e0ca..4c2e38b9f 100644 --- a/crates/brk_cli/src/lib.rs +++ b/crates/brk_cli/src/lib.rs @@ -13,6 +13,7 @@ use brk_bundler::bundle; use brk_computer::Computer; use brk_indexer::Indexer; use brk_interface::Interface; +use brk_parser::Parser; use brk_server::{Server, VERSION}; use log::info; use vecdb::Exit; @@ -46,7 +47,7 @@ pub fn run() -> color_eyre::Result<()> { let exit = Exit::new(); exit.set_ctrlc_handler(); - let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc); + let parser = Parser::new(config.blocksdir(), Some(config.brkdir()), rpc); let mut indexer = Indexer::forced_import(&config.brkdir())?; diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 56a3f3aa5..9f8fa901c 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -34,7 +34,11 @@ pub fn main() -> Result<()> { let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk"); // let outputs_dir = Path::new("../../_outputs"); - let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc); + let parser = Parser::new( + bitcoin_dir.join("blocks"), + Some(outputs_dir.to_path_buf()), + rpc, + ); let mut indexer = Indexer::forced_import(&outputs_dir)?; diff --git a/crates/brk_indexer/README.md b/crates/brk_indexer/README.md index 53999257b..8a28b6a89 100644 --- a/crates/brk_indexer/README.md +++ b/crates/brk_indexer/README.md @@ -25,7 +25,7 @@ **Key-Value Storage (lookups):** - Block hash prefixes → heights -- Transaction ID prefixes → transaction indices +- Transaction ID prefixes → transaction indices - Address byte hashes → type indices - Fast point queries by hash or address @@ -60,7 +60,7 @@ let rpc = Box::leak(Box::new(Client::new( // Create parser for Bitcoin Core block files let parser = Parser::new( Path::new("~/.bitcoin/blocks").to_path_buf(), - Path::new("./brk_data").to_path_buf(), + Some(Path::new("./brk_data").to_path_buf()), rpc ); @@ -85,19 +85,19 @@ use std::thread::sleep; // Continuous indexing loop for real-time updates loop { let start_time = Instant::now(); - + // Index new blocks let indexes = indexer.index(&parser, rpc, &exit, true)?; - - println!("Indexed to height {} in {:?}", + + println!("Indexed to height {} in {:?}", indexes.height, start_time.elapsed()); - + // Check for exit signal if exit.is_signaled() { println!("Graceful shutdown requested"); break; } - + // Wait before next update cycle sleep(Duration::from_secs(5 * 60)); } @@ -187,4 +187,4 @@ pub struct Indexes { --- -*This README was generated by Claude Code* \ No newline at end of file +*This README was generated by Claude Code* diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 9e247c54b..9b9b82a7a 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -33,7 +33,7 @@ fn main() -> Result<()> { let exit = Exit::new(); exit.set_ctrlc_handler(); - let parser = Parser::new(blocks_dir, outputs_dir.to_path_buf(), rpc); + let parser = Parser::new(blocks_dir, Some(outputs_dir.to_path_buf()), rpc); fs::create_dir_all(outputs_dir)?; diff --git a/crates/brk_parser/README.md b/crates/brk_parser/README.md index f65b499c0..2e49f7534 100644 --- a/crates/brk_parser/README.md +++ b/crates/brk_parser/README.md @@ -8,7 +8,7 @@ - **Sequential block access**: Blocks delivered in height order (0, 1, 2, ...) regardless of physical file storage - **Fork filtering**: Automatically excludes orphaned blocks using Bitcoin Core RPC verification -- **XOR encryption support**: Transparently handles XOR-encrypted block files +- **XOR encryption support**: Transparently handles XOR-encrypted block files - **High performance**: Multi-threaded parsing with ~500MB peak memory usage - **State persistence**: Caches parsing state for fast restarts @@ -47,7 +47,7 @@ let rpc = Box::leak(Box::new(Client::new( // Create parser let parser = Parser::new( Path::new("~/.bitcoin/blocks").to_path_buf(), - Path::new("./output").to_path_buf(), + Some(Path::new("./output").to_path_buf()), rpc, ); @@ -90,7 +90,7 @@ use bitcoin::Block; fn analyze_blockchain(parser: &Parser) { let mut total_transactions = 0; let mut total_outputs = 0; - + parser.parse(None, None) .iter() .for_each(|(height, block, _hash)| { @@ -98,12 +98,12 @@ fn analyze_blockchain(parser: &Parser) { total_outputs += block.txdata.iter() .map(|tx| tx.output.len()) .sum::(); - + if height.0 % 10000 == 0 { println!("Processed {} blocks", height); } }); - + println!("Total transactions: {}", total_transactions); println!("Total outputs: {}", total_outputs); } @@ -150,4 +150,4 @@ The parser saves parsing state in `{output_dir}/blk_index_to_blk_recap.json` con --- -*This README was generated by Claude Code* \ No newline at end of file +*This README was generated by Claude Code* diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index c178be87d..38213dc62 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -26,7 +26,7 @@ pub fn main() -> Result<()> { let exit = Exit::new(); exit.set_ctrlc_handler(); - let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir.to_path_buf(), rpc); + let parser = Parser::new(bitcoin_dir.join("blocks"), Some(brk_dir.to_path_buf()), rpc); let outputs_dir = Path::new("../../_outputs");