mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-05-19 22:34:46 -07:00
global: fixes of Parser::new
This commit is contained in:
6
Cargo.lock
generated
6
Cargo.lock
generated
@@ -3885,6 +3885,8 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "seqdb"
|
name = "seqdb"
|
||||||
version = "0.2.5"
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c54ab988c96efa9d275ca2b12bf2d3c6adec993b8e82ea31a88c984abdaa14fa"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"log",
|
"log",
|
||||||
@@ -4711,6 +4713,8 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "vecdb"
|
name = "vecdb"
|
||||||
version = "0.2.5"
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3e5c4ec34c376be3a41435eeb7672d0ea0e9c1d60c5d1d90218912588f91abea"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ctrlc",
|
"ctrlc",
|
||||||
"log",
|
"log",
|
||||||
@@ -4729,6 +4733,8 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "vecdb_derive"
|
name = "vecdb_derive"
|
||||||
version = "0.2.5"
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "778c4874c05822465e28cae6a7dead593a73124ec80afb85b85adae5ac883368"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.106",
|
"syn 2.0.106",
|
||||||
|
|||||||
@@ -52,8 +52,8 @@ serde_bytes = "0.11.17"
|
|||||||
serde_derive = "1.0.219"
|
serde_derive = "1.0.219"
|
||||||
serde_json = { version = "1.0.143", features = ["float_roundtrip"] }
|
serde_json = { version = "1.0.143", features = ["float_roundtrip"] }
|
||||||
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
|
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
|
||||||
vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
|
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
|
||||||
# vecdb = { version = "0.2.5", features = ["derive"]}
|
vecdb = { version = "0.2.5", features = ["derive"]}
|
||||||
zerocopy = "0.8.26"
|
zerocopy = "0.8.26"
|
||||||
zerocopy-derive = "0.8.26"
|
zerocopy-derive = "0.8.26"
|
||||||
|
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ brk = { version = "0.0.88", features = ["parser", "indexer", "computer"] }
|
|||||||
use brk::{parser, indexer, computer};
|
use brk::{parser, indexer, computer};
|
||||||
|
|
||||||
// Core data pipeline only
|
// Core data pipeline only
|
||||||
let parser = parser::Parser::new(blocks_dir, output_dir, rpc);
|
let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc);
|
||||||
let mut indexer = indexer::Indexer::forced_import(output_dir)?;
|
let mut indexer = indexer::Indexer::forced_import(output_dir)?;
|
||||||
let mut computer = computer::Computer::forced_import(output_dir, &indexer, None)?;
|
let mut computer = computer::Computer::forced_import(output_dir, &indexer, None)?;
|
||||||
```
|
```
|
||||||
@@ -81,7 +81,7 @@ use brk::{structs, parser};
|
|||||||
|
|
||||||
// Just parsing and types
|
// Just parsing and types
|
||||||
let height = structs::Height::new(800_000);
|
let height = structs::Height::new(800_000);
|
||||||
let parser = parser::Parser::new(blocks_dir, output_dir, rpc);
|
let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc);
|
||||||
```
|
```
|
||||||
|
|
||||||
## Feature Flags
|
## Feature Flags
|
||||||
@@ -113,7 +113,7 @@ use brk::*;
|
|||||||
// Full data pipeline setup
|
// Full data pipeline setup
|
||||||
let config = cli::Config::load()?;
|
let config = cli::Config::load()?;
|
||||||
let rpc = /* Bitcoin Core RPC client */;
|
let rpc = /* Bitcoin Core RPC client */;
|
||||||
let parser = parser::Parser::new(config.blocks_dir, config.output_dir, rpc);
|
let parser = parser::Parser::new(config.blocks_dir, Some(config.output_dir), rpc);
|
||||||
let mut indexer = indexer::Indexer::forced_import(&config.output_dir)?;
|
let mut indexer = indexer::Indexer::forced_import(&config.output_dir)?;
|
||||||
let mut computer = computer::Computer::forced_import(&config.output_dir, &indexer, None)?;
|
let mut computer = computer::Computer::forced_import(&config.output_dir, &indexer, None)?;
|
||||||
let interface = interface::Interface::build(&indexer, &computer);
|
let interface = interface::Interface::build(&indexer, &computer);
|
||||||
@@ -152,15 +152,15 @@ use brk::{structs, parser, error};
|
|||||||
|
|
||||||
// Custom application with BRK components
|
// Custom application with BRK components
|
||||||
fn analyze_blocks() -> error::Result<()> {
|
fn analyze_blocks() -> error::Result<()> {
|
||||||
let parser = parser::Parser::new(blocks_dir, output_dir, rpc);
|
let parser = parser::Parser::new(blocks_dir, Some(output_dir), rpc);
|
||||||
|
|
||||||
parser.parse(None, None)
|
parser.parse(None, None)
|
||||||
.iter()
|
.iter()
|
||||||
.take(1000) // First 1000 blocks
|
.take(1000) // First 1000 blocks
|
||||||
.for_each(|(height, block, hash)| {
|
.for_each(|(height, block, hash)| {
|
||||||
println!("Block {}: {} transactions", height, block.txdata.len());
|
println!("Block {}: {} transactions", height, block.txdata.len());
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@@ -194,4 +194,4 @@ For specific dependency information, see individual crate READMEs.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
*This README was generated by Claude Code*
|
*This README was generated by Claude Code*
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ use brk_bundler::bundle;
|
|||||||
use brk_computer::Computer;
|
use brk_computer::Computer;
|
||||||
use brk_indexer::Indexer;
|
use brk_indexer::Indexer;
|
||||||
use brk_interface::Interface;
|
use brk_interface::Interface;
|
||||||
|
use brk_parser::Parser;
|
||||||
use brk_server::{Server, VERSION};
|
use brk_server::{Server, VERSION};
|
||||||
use log::info;
|
use log::info;
|
||||||
use vecdb::Exit;
|
use vecdb::Exit;
|
||||||
@@ -46,7 +47,7 @@ pub fn run() -> color_eyre::Result<()> {
|
|||||||
let exit = Exit::new();
|
let exit = Exit::new();
|
||||||
exit.set_ctrlc_handler();
|
exit.set_ctrlc_handler();
|
||||||
|
|
||||||
let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);
|
let parser = Parser::new(config.blocksdir(), Some(config.brkdir()), rpc);
|
||||||
|
|
||||||
let mut indexer = Indexer::forced_import(&config.brkdir())?;
|
let mut indexer = Indexer::forced_import(&config.brkdir())?;
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,11 @@ pub fn main() -> Result<()> {
|
|||||||
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
|
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
|
||||||
// let outputs_dir = Path::new("../../_outputs");
|
// let outputs_dir = Path::new("../../_outputs");
|
||||||
|
|
||||||
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);
|
let parser = Parser::new(
|
||||||
|
bitcoin_dir.join("blocks"),
|
||||||
|
Some(outputs_dir.to_path_buf()),
|
||||||
|
rpc,
|
||||||
|
);
|
||||||
|
|
||||||
let mut indexer = Indexer::forced_import(&outputs_dir)?;
|
let mut indexer = Indexer::forced_import(&outputs_dir)?;
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@
|
|||||||
|
|
||||||
**Key-Value Storage (lookups):**
|
**Key-Value Storage (lookups):**
|
||||||
- Block hash prefixes → heights
|
- Block hash prefixes → heights
|
||||||
- Transaction ID prefixes → transaction indices
|
- Transaction ID prefixes → transaction indices
|
||||||
- Address byte hashes → type indices
|
- Address byte hashes → type indices
|
||||||
- Fast point queries by hash or address
|
- Fast point queries by hash or address
|
||||||
|
|
||||||
@@ -60,7 +60,7 @@ let rpc = Box::leak(Box::new(Client::new(
|
|||||||
// Create parser for Bitcoin Core block files
|
// Create parser for Bitcoin Core block files
|
||||||
let parser = Parser::new(
|
let parser = Parser::new(
|
||||||
Path::new("~/.bitcoin/blocks").to_path_buf(),
|
Path::new("~/.bitcoin/blocks").to_path_buf(),
|
||||||
Path::new("./brk_data").to_path_buf(),
|
Some(Path::new("./brk_data").to_path_buf()),
|
||||||
rpc
|
rpc
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -85,19 +85,19 @@ use std::thread::sleep;
|
|||||||
// Continuous indexing loop for real-time updates
|
// Continuous indexing loop for real-time updates
|
||||||
loop {
|
loop {
|
||||||
let start_time = Instant::now();
|
let start_time = Instant::now();
|
||||||
|
|
||||||
// Index new blocks
|
// Index new blocks
|
||||||
let indexes = indexer.index(&parser, rpc, &exit, true)?;
|
let indexes = indexer.index(&parser, rpc, &exit, true)?;
|
||||||
|
|
||||||
println!("Indexed to height {} in {:?}",
|
println!("Indexed to height {} in {:?}",
|
||||||
indexes.height, start_time.elapsed());
|
indexes.height, start_time.elapsed());
|
||||||
|
|
||||||
// Check for exit signal
|
// Check for exit signal
|
||||||
if exit.is_signaled() {
|
if exit.is_signaled() {
|
||||||
println!("Graceful shutdown requested");
|
println!("Graceful shutdown requested");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait before next update cycle
|
// Wait before next update cycle
|
||||||
sleep(Duration::from_secs(5 * 60));
|
sleep(Duration::from_secs(5 * 60));
|
||||||
}
|
}
|
||||||
@@ -187,4 +187,4 @@ pub struct Indexes {
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
*This README was generated by Claude Code*
|
*This README was generated by Claude Code*
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ fn main() -> Result<()> {
|
|||||||
let exit = Exit::new();
|
let exit = Exit::new();
|
||||||
exit.set_ctrlc_handler();
|
exit.set_ctrlc_handler();
|
||||||
|
|
||||||
let parser = Parser::new(blocks_dir, outputs_dir.to_path_buf(), rpc);
|
let parser = Parser::new(blocks_dir, Some(outputs_dir.to_path_buf()), rpc);
|
||||||
|
|
||||||
fs::create_dir_all(outputs_dir)?;
|
fs::create_dir_all(outputs_dir)?;
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
- **Sequential block access**: Blocks delivered in height order (0, 1, 2, ...) regardless of physical file storage
|
- **Sequential block access**: Blocks delivered in height order (0, 1, 2, ...) regardless of physical file storage
|
||||||
- **Fork filtering**: Automatically excludes orphaned blocks using Bitcoin Core RPC verification
|
- **Fork filtering**: Automatically excludes orphaned blocks using Bitcoin Core RPC verification
|
||||||
- **XOR encryption support**: Transparently handles XOR-encrypted block files
|
- **XOR encryption support**: Transparently handles XOR-encrypted block files
|
||||||
- **High performance**: Multi-threaded parsing with ~500MB peak memory usage
|
- **High performance**: Multi-threaded parsing with ~500MB peak memory usage
|
||||||
- **State persistence**: Caches parsing state for fast restarts
|
- **State persistence**: Caches parsing state for fast restarts
|
||||||
|
|
||||||
@@ -47,7 +47,7 @@ let rpc = Box::leak(Box::new(Client::new(
|
|||||||
// Create parser
|
// Create parser
|
||||||
let parser = Parser::new(
|
let parser = Parser::new(
|
||||||
Path::new("~/.bitcoin/blocks").to_path_buf(),
|
Path::new("~/.bitcoin/blocks").to_path_buf(),
|
||||||
Path::new("./output").to_path_buf(),
|
Some(Path::new("./output").to_path_buf()),
|
||||||
rpc,
|
rpc,
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -90,7 +90,7 @@ use bitcoin::Block;
|
|||||||
fn analyze_blockchain(parser: &Parser) {
|
fn analyze_blockchain(parser: &Parser) {
|
||||||
let mut total_transactions = 0;
|
let mut total_transactions = 0;
|
||||||
let mut total_outputs = 0;
|
let mut total_outputs = 0;
|
||||||
|
|
||||||
parser.parse(None, None)
|
parser.parse(None, None)
|
||||||
.iter()
|
.iter()
|
||||||
.for_each(|(height, block, _hash)| {
|
.for_each(|(height, block, _hash)| {
|
||||||
@@ -98,12 +98,12 @@ fn analyze_blockchain(parser: &Parser) {
|
|||||||
total_outputs += block.txdata.iter()
|
total_outputs += block.txdata.iter()
|
||||||
.map(|tx| tx.output.len())
|
.map(|tx| tx.output.len())
|
||||||
.sum::<usize>();
|
.sum::<usize>();
|
||||||
|
|
||||||
if height.0 % 10000 == 0 {
|
if height.0 % 10000 == 0 {
|
||||||
println!("Processed {} blocks", height);
|
println!("Processed {} blocks", height);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
println!("Total transactions: {}", total_transactions);
|
println!("Total transactions: {}", total_transactions);
|
||||||
println!("Total outputs: {}", total_outputs);
|
println!("Total outputs: {}", total_outputs);
|
||||||
}
|
}
|
||||||
@@ -150,4 +150,4 @@ The parser saves parsing state in `{output_dir}/blk_index_to_blk_recap.json` con
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
*This README was generated by Claude Code*
|
*This README was generated by Claude Code*
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ pub fn main() -> Result<()> {
|
|||||||
let exit = Exit::new();
|
let exit = Exit::new();
|
||||||
exit.set_ctrlc_handler();
|
exit.set_ctrlc_handler();
|
||||||
|
|
||||||
let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir.to_path_buf(), rpc);
|
let parser = Parser::new(bitcoin_dir.join("blocks"), Some(brk_dir.to_path_buf()), rpc);
|
||||||
|
|
||||||
let outputs_dir = Path::new("../../_outputs");
|
let outputs_dir = Path::new("../../_outputs");
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user