global: fixes of Parser::new

This commit is contained in:
nym21
2025-09-01 20:34:27 +02:00
parent fe5343c1d6
commit 1c2afd14dd
9 changed files with 38 additions and 27 deletions

6
Cargo.lock generated
View File

@@ -3885,6 +3885,8 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]] [[package]]
name = "seqdb" name = "seqdb"
version = "0.2.5" version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c54ab988c96efa9d275ca2b12bf2d3c6adec993b8e82ea31a88c984abdaa14fa"
dependencies = [ dependencies = [
"libc", "libc",
"log", "log",
@@ -4711,6 +4713,8 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]] [[package]]
name = "vecdb" name = "vecdb"
version = "0.2.5" version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e5c4ec34c376be3a41435eeb7672d0ea0e9c1d60c5d1d90218912588f91abea"
dependencies = [ dependencies = [
"ctrlc", "ctrlc",
"log", "log",
@@ -4729,6 +4733,8 @@ dependencies = [
[[package]] [[package]]
name = "vecdb_derive" name = "vecdb_derive"
version = "0.2.5" version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778c4874c05822465e28cae6a7dead593a73124ec80afb85b85adae5ac883368"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.106", "syn 2.0.106",

View File

@@ -52,8 +52,8 @@ serde_bytes = "0.11.17"
serde_derive = "1.0.219" serde_derive = "1.0.219"
serde_json = { version = "1.0.143", features = ["float_roundtrip"] } serde_json = { version = "1.0.143", features = ["float_roundtrip"] }
tokio = { version = "1.47.1", features = ["rt-multi-thread"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
# vecdb = { version = "0.2.5", features = ["derive"]} vecdb = { version = "0.2.5", features = ["derive"]}
zerocopy = "0.8.26" zerocopy = "0.8.26"
zerocopy-derive = "0.8.26" zerocopy-derive = "0.8.26"

View File

@@ -64,7 +64,7 @@ brk = { version = "0.0.88", features = ["parser", "indexer", "computer"] }
use brk::{parser, indexer, computer}; use brk::{parser, indexer, computer};
// Core data pipeline only // Core data pipeline only
let parser = parser::Parser::new(blocks_dir, output_dir, rpc); let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc);
let mut indexer = indexer::Indexer::forced_import(output_dir)?; let mut indexer = indexer::Indexer::forced_import(output_dir)?;
let mut computer = computer::Computer::forced_import(output_dir, &indexer, None)?; let mut computer = computer::Computer::forced_import(output_dir, &indexer, None)?;
``` ```
@@ -81,7 +81,7 @@ use brk::{structs, parser};
// Just parsing and types // Just parsing and types
let height = structs::Height::new(800_000); let height = structs::Height::new(800_000);
let parser = parser::Parser::new(blocks_dir, output_dir, rpc); let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc);
``` ```
## Feature Flags ## Feature Flags
@@ -113,7 +113,7 @@ use brk::*;
// Full data pipeline setup // Full data pipeline setup
let config = cli::Config::load()?; let config = cli::Config::load()?;
let rpc = /* Bitcoin Core RPC client */; let rpc = /* Bitcoin Core RPC client */;
let parser = parser::Parser::new(config.blocks_dir, config.output_dir, rpc); let parser = parser::Parser::new(config.blocks_dir, Some(config.output_dir), rpc);
let mut indexer = indexer::Indexer::forced_import(&config.output_dir)?; let mut indexer = indexer::Indexer::forced_import(&config.output_dir)?;
let mut computer = computer::Computer::forced_import(&config.output_dir, &indexer, None)?; let mut computer = computer::Computer::forced_import(&config.output_dir, &indexer, None)?;
let interface = interface::Interface::build(&indexer, &computer); let interface = interface::Interface::build(&indexer, &computer);
@@ -152,15 +152,15 @@ use brk::{structs, parser, error};
// Custom application with BRK components // Custom application with BRK components
fn analyze_blocks() -> error::Result<()> { fn analyze_blocks() -> error::Result<()> {
let parser = parser::Parser::new(blocks_dir, output_dir, rpc); let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc);
parser.parse(None, None) parser.parse(None, None)
.iter() .iter()
.take(1000) // First 1000 blocks .take(1000) // First 1000 blocks
.for_each(|(height, block, hash)| { .for_each(|(height, block, hash)| {
println!("Block {}: {} transactions", height, block.txdata.len()); println!("Block {}: {} transactions", height, block.txdata.len());
}); });
Ok(()) Ok(())
} }
``` ```
@@ -194,4 +194,4 @@ For specific dependency information, see individual crate READMEs.
--- ---
*This README was generated by Claude Code* *This README was generated by Claude Code*

View File

@@ -13,6 +13,7 @@ use brk_bundler::bundle;
use brk_computer::Computer; use brk_computer::Computer;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_interface::Interface; use brk_interface::Interface;
use brk_parser::Parser;
use brk_server::{Server, VERSION}; use brk_server::{Server, VERSION};
use log::info; use log::info;
use vecdb::Exit; use vecdb::Exit;
@@ -46,7 +47,7 @@ pub fn run() -> color_eyre::Result<()> {
let exit = Exit::new(); let exit = Exit::new();
exit.set_ctrlc_handler(); exit.set_ctrlc_handler();
let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc); let parser = Parser::new(config.blocksdir(), Some(config.brkdir()), rpc);
let mut indexer = Indexer::forced_import(&config.brkdir())?; let mut indexer = Indexer::forced_import(&config.brkdir())?;

View File

@@ -34,7 +34,11 @@ pub fn main() -> Result<()> {
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk"); let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs"); // let outputs_dir = Path::new("../../_outputs");
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc); let parser = Parser::new(
bitcoin_dir.join("blocks"),
Some(outputs_dir.to_path_buf()),
rpc,
);
let mut indexer = Indexer::forced_import(&outputs_dir)?; let mut indexer = Indexer::forced_import(&outputs_dir)?;

View File

@@ -25,7 +25,7 @@
**Key-Value Storage (lookups):** **Key-Value Storage (lookups):**
- Block hash prefixes → heights - Block hash prefixes → heights
- Transaction ID prefixes → transaction indices - Transaction ID prefixes → transaction indices
- Address byte hashes → type indices - Address byte hashes → type indices
- Fast point queries by hash or address - Fast point queries by hash or address
@@ -60,7 +60,7 @@ let rpc = Box::leak(Box::new(Client::new(
// Create parser for Bitcoin Core block files // Create parser for Bitcoin Core block files
let parser = Parser::new( let parser = Parser::new(
Path::new("~/.bitcoin/blocks").to_path_buf(), Path::new("~/.bitcoin/blocks").to_path_buf(),
Path::new("./brk_data").to_path_buf(), Some(Path::new("./brk_data").to_path_buf()),
rpc rpc
); );
@@ -85,19 +85,19 @@ use std::thread::sleep;
// Continuous indexing loop for real-time updates // Continuous indexing loop for real-time updates
loop { loop {
let start_time = Instant::now(); let start_time = Instant::now();
// Index new blocks // Index new blocks
let indexes = indexer.index(&parser, rpc, &exit, true)?; let indexes = indexer.index(&parser, rpc, &exit, true)?;
println!("Indexed to height {} in {:?}", println!("Indexed to height {} in {:?}",
indexes.height, start_time.elapsed()); indexes.height, start_time.elapsed());
// Check for exit signal // Check for exit signal
if exit.is_signaled() { if exit.is_signaled() {
println!("Graceful shutdown requested"); println!("Graceful shutdown requested");
break; break;
} }
// Wait before next update cycle // Wait before next update cycle
sleep(Duration::from_secs(5 * 60)); sleep(Duration::from_secs(5 * 60));
} }
@@ -187,4 +187,4 @@ pub struct Indexes {
--- ---
*This README was generated by Claude Code* *This README was generated by Claude Code*

View File

@@ -33,7 +33,7 @@ fn main() -> Result<()> {
let exit = Exit::new(); let exit = Exit::new();
exit.set_ctrlc_handler(); exit.set_ctrlc_handler();
let parser = Parser::new(blocks_dir, outputs_dir.to_path_buf(), rpc); let parser = Parser::new(blocks_dir, Some(outputs_dir.to_path_buf()), rpc);
fs::create_dir_all(outputs_dir)?; fs::create_dir_all(outputs_dir)?;

View File

@@ -8,7 +8,7 @@
- **Sequential block access**: Blocks delivered in height order (0, 1, 2, ...) regardless of physical file storage - **Sequential block access**: Blocks delivered in height order (0, 1, 2, ...) regardless of physical file storage
- **Fork filtering**: Automatically excludes orphaned blocks using Bitcoin Core RPC verification - **Fork filtering**: Automatically excludes orphaned blocks using Bitcoin Core RPC verification
- **XOR encryption support**: Transparently handles XOR-encrypted block files - **XOR encryption support**: Transparently handles XOR-encrypted block files
- **High performance**: Multi-threaded parsing with ~500MB peak memory usage - **High performance**: Multi-threaded parsing with ~500MB peak memory usage
- **State persistence**: Caches parsing state for fast restarts - **State persistence**: Caches parsing state for fast restarts
@@ -47,7 +47,7 @@ let rpc = Box::leak(Box::new(Client::new(
// Create parser // Create parser
let parser = Parser::new( let parser = Parser::new(
Path::new("~/.bitcoin/blocks").to_path_buf(), Path::new("~/.bitcoin/blocks").to_path_buf(),
Path::new("./output").to_path_buf(), Some(Path::new("./output").to_path_buf()),
rpc, rpc,
); );
@@ -90,7 +90,7 @@ use bitcoin::Block;
fn analyze_blockchain(parser: &Parser) { fn analyze_blockchain(parser: &Parser) {
let mut total_transactions = 0; let mut total_transactions = 0;
let mut total_outputs = 0; let mut total_outputs = 0;
parser.parse(None, None) parser.parse(None, None)
.iter() .iter()
.for_each(|(height, block, _hash)| { .for_each(|(height, block, _hash)| {
@@ -98,12 +98,12 @@ fn analyze_blockchain(parser: &Parser) {
total_outputs += block.txdata.iter() total_outputs += block.txdata.iter()
.map(|tx| tx.output.len()) .map(|tx| tx.output.len())
.sum::<usize>(); .sum::<usize>();
if height.0 % 10000 == 0 { if height.0 % 10000 == 0 {
println!("Processed {} blocks", height); println!("Processed {} blocks", height);
} }
}); });
println!("Total transactions: {}", total_transactions); println!("Total transactions: {}", total_transactions);
println!("Total outputs: {}", total_outputs); println!("Total outputs: {}", total_outputs);
} }
@@ -150,4 +150,4 @@ The parser saves parsing state in `{output_dir}/blk_index_to_blk_recap.json` con
--- ---
*This README was generated by Claude Code* *This README was generated by Claude Code*

View File

@@ -26,7 +26,7 @@ pub fn main() -> Result<()> {
let exit = Exit::new(); let exit = Exit::new();
exit.set_ctrlc_handler(); exit.set_ctrlc_handler();
let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir.to_path_buf(), rpc); let parser = Parser::new(bitcoin_dir.join("blocks"), Some(brk_dir.to_path_buf()), rpc);
let outputs_dir = Path::new("../../_outputs"); let outputs_dir = Path::new("../../_outputs");