Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-16 05:28:12 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: rework, made stateless

Browse Source
This commit is contained in:
nym21
2025-09-17 23:31:57 +02:00
parent 9524eafea1
commit cc5701ea62
22 changed files with 795 additions and 636 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_cli/src/lib.rs
+1 -1
View File
@@ -47,7 +47,7 @@ pub fn run() -> color_eyre::Result<()> {
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(config.blocksdir(), Some(config.brkdir()), rpc);
let parser = Parser::new(config.blocksdir(), rpc);
let mut indexer = Indexer::forced_import(&config.brkdir())?;
crates/brk_computer/examples/computer.rs
+1 -5
View File
@@ -34,11 +34,7 @@ pub fn main() -> Result<()> {
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs");
let parser = Parser::new(
bitcoin_dir.join("blocks"),
Some(outputs_dir.to_path_buf()),
rpc,
);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let mut indexer = Indexer::forced_import(&outputs_dir)?;
crates/brk_error/src/lib.rs
+14
View File
@@ -18,6 +18,7 @@ pub enum Error {
Minreq(minreq::Error),
SystemTimeError(time::SystemTimeError),
BitcoinConsensusEncode(bitcoin::consensus::encode::Error),
BitcoinBip34Error(bitcoin::block::Bip34Error),
SerdeJson(serde_json::Error),
ZeroCopyError,
Vecs(vecdb::Error),
@@ -30,6 +31,12 @@ pub enum Error {
String(String),
}
impl From<bitcoin::block::Bip34Error> for Error {
fn from(value: bitcoin::block::Bip34Error) -> Self {
Self::BitcoinBip34Error(value)
}
}
impl From<bitcoin::consensus::encode::Error> for Error {
fn from(value: bitcoin::consensus::encode::Error) -> Self {
Self::BitcoinConsensusEncode(value)
@@ -90,6 +97,12 @@ impl From<fjall::Error> for Error {
}
}
impl From<&'static str> for Error {
fn from(value: &'static str) -> Self {
Self::Str(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
@@ -106,6 +119,7 @@ impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::BitcoinConsensusEncode(error) => Display::fmt(&error, f),
Error::BitcoinBip34Error(error) => Display::fmt(&error, f),
Error::BitcoinRPC(error) => Display::fmt(&error, f),
Error::Fjall(error) => Display::fmt(&error, f),
Error::IO(error) => Display::fmt(&error, f),
crates/brk_indexer/examples/indexer.rs
+1 -1
View File
@@ -33,7 +33,7 @@ fn main() -> Result<()> {
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(blocks_dir, Some(outputs_dir.to_path_buf()), rpc);
let parser = Parser::new(blocks_dir, rpc);
fs::create_dir_all(&outputs_dir)?;
crates/brk_indexer/src/lib.rs
+9 -7
View File
@@ -8,9 +8,9 @@ use brk_error::{Error, Result};
use brk_parser::Parser;
use brk_store::AnyStore;
use brk_structs::{
AddressBytes, AddressBytesHash, BlockExtended, BlockHash, BlockHashPrefix, Height, InputIndex,
OutputIndex, OutputType, Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout,
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, InputIndex, OutputIndex, OutputType,
Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex,
Unit, Version, Vin, Vout,
};
use log::{error, info};
use rayon::prelude::*;
@@ -165,7 +165,10 @@ impl Indexer {
);
parser.parse(start, end).iter().try_for_each(
|(height, block, blockhash)| -> Result<()> {
|block| -> Result<()> {
let height = block.height();
let blockhash = block.hash();
info!("Indexing block {height}...");
idxs.height = height;
@@ -183,8 +186,7 @@ impl Indexer {
// Used to check rapidhash collisions
let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO ;
let blockhash = BlockHash::from(blockhash);
let blockhash_prefix = BlockHashPrefix::from(&blockhash);
let blockhash_prefix = BlockHashPrefix::from(blockhash);
if stores
.blockhashprefix_to_height
@@ -205,7 +207,7 @@ impl Indexer {
.height_to_coinbase_tag
.insert_if_needed( height, block.coinbase_tag().into(), height);
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
vecs.height_to_blockhash.push_if_needed(height, blockhash.clone())?;
vecs.height_to_difficulty
.push_if_needed(height, block.header.difficulty_float().into())?;
vecs.height_to_timestamp
crates/brk_parser/Cargo.toml
+1
View File
@@ -14,6 +14,7 @@ build = "build.rs"
[dependencies]
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_error = { workspace = true }
brk_structs = { workspace = true }
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
derive_deref = { workspace = true }
crates/brk_parser/examples/main.rs
+26 -20
View File
@@ -1,6 +1,7 @@
use std::path::Path;
use std::{collections::BTreeMap, path::Path};
use bitcoincore_rpc::{Auth, Client, Result};
use bitcoincore_rpc::{Auth, Client};
use brk_error::Result;
use brk_parser::Parser;
use brk_structs::Height;
@@ -12,35 +13,40 @@ fn main() -> Result<()> {
.join("Library")
.join("Application Support")
.join("Bitcoin");
let brk_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
let rpc = Box::leak(Box::new(Client::new(
"http://localhost:8332",
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let parser = Parser::new(bitcoin_dir.join("blocks"), Some(brk_dir), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
// let start = None;
// let end = None;
// parser
// .parse(start, end)
// .iter()
// .for_each(|(height, _block, hash)| {
// println!("{height}: {}", hash);
// });
let start = Some(915_155_u32.into());
let end = None;
let mut blk_index = 0;
let mut diff = BTreeMap::new();
parser.parse(start, end).iter().for_each(|block| {
println!("{}: {}", block.height(), block.hash());
let new_blk_index = block.position().blk_index;
if new_blk_index < blk_index {
diff.insert(blk_index - new_blk_index, block.height());
}
blk_index = new_blk_index;
});
let block_0 = parser.get(Height::new(0));
dbg!("{}", block_0.coinbase_tag());
// let v = diff.iter().rev().take(10).collect::<Vec<_>>();
let block_158251 = parser.get(Height::new(158251));
dbg!("{}", block_158251.coinbase_tag());
// let block_0 = parser.get(Height::new(0))?;
// dbg!("{}", block_0.coinbase_tag());
let block_173195 = parser.get(Height::new(173195));
dbg!("{}", block_173195.coinbase_tag());
// let block_158251 = parser.get(Height::new(158251))?;
// dbg!("{}", block_158251.coinbase_tag());
let block_840_000 = parser.get(Height::new(840_004));
dbg!("{}", block_840_000.coinbase_tag());
// let block_173195 = parser.get(Height::new(173195))?;
// dbg!("{}", block_173195.coinbase_tag());
// let block_840_000 = parser.get(Height::new(840_004))?;
// dbg!("{}", block_840_000.coinbase_tag());
dbg!(i.elapsed());
crates/brk_parser/examples/p2a.rs
+2 -3
View File
@@ -8,7 +8,6 @@ fn main() {
let i = std::time::Instant::now();
let bitcoin_dir = Path::new("").join("");
let brk_dir = Path::new("").join("");
let rpc = Box::leak(Box::new(
Client::new(
@@ -21,7 +20,7 @@ fn main() {
// let start = None;
// let end = None;
let parser = Parser::new(bitcoin_dir.join("blocks"), Some(brk_dir), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
// parser
// .parse(start, end)
@@ -43,7 +42,7 @@ fn main() {
// .script_pubkey
// );
let block_850_000 = parser.get(Height::new(850_000));
let block_850_000 = parser.get(Height::new(850_000)).unwrap();
let tx = block_850_000.txdata.iter().find(|tx| {
tx.compute_txid().to_string()
crates/brk_parser/src/any_block.rs
+49 -6
View File
@@ -1,16 +1,27 @@
use bitcoin::{Block, consensus::Decodable, io::Cursor};
use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor};
use bitcoincore_rpc::RpcApi;
use brk_error::Result;
use brk_structs::{Block, BlockPosition, Height, ParsedBlock};
use crate::{XORBytes, XORIndex};
pub enum AnyBlock {
Raw(Vec<u8>),
Decoded(Block),
Decoded(ParsedBlock),
Skipped,
}
impl AnyBlock {
pub fn decode(&mut self, xor_i: &mut XORIndex, xor_bytes: &XORBytes) {
let bytes = match self {
pub fn decode(
self,
position: BlockPosition,
rpc: &'static bitcoincore_rpc::Client,
mut xor_i: XORIndex,
xor_bytes: &XORBytes,
start: Option<Height>,
end: Option<Height>,
) -> Result<Self> {
let mut bytes = match self {
AnyBlock::Raw(bytes) => bytes,
_ => unreachable!(),
};
@@ -19,8 +30,40 @@ impl AnyBlock {
let mut cursor = Cursor::new(bytes);
let block = Block::consensus_decode(&mut cursor).unwrap();
let header = Header::consensus_decode(&mut cursor)?;
*self = AnyBlock::Decoded(block);
let hash = header.block_hash();
let tx_count = VarInt::consensus_decode(&mut cursor)?.0;
let Ok(block_header_result) = rpc.get_block_header_info(&hash) else {
return Ok(Self::Skipped);
};
let height = Height::from(block_header_result.height);
if let Some(start) = start
&& start > height
{
return Ok(Self::Skipped);
} else if let Some(end) = end
&& end < height
{
return Ok(Self::Skipped);
} else if block_header_result.confirmations <= 0 {
return Ok(Self::Skipped);
}
let mut txdata = Vec::with_capacity(tx_count as usize);
for _ in 0..tx_count {
let tx = Transaction::consensus_decode(&mut cursor)?;
txdata.push(tx);
}
let block = bitcoin::Block { header, txdata };
let block = Block::from((height, hash, block));
let block = ParsedBlock::from((block, position));
Ok(Self::Decoded(block))
}
}
crates/brk_parser/src/blk_index_to_blk_path.rs
+3 -3
View File
@@ -7,7 +7,7 @@ use std::{
use derive_deref::{Deref, DerefMut};
const BLK: &str = "blk";
const DAT: &str = ".dat";
const DOT_DAT: &str = ".dat";
#[derive(Debug, Deref, DerefMut)]
pub struct BlkIndexToBlkPath(BTreeMap<u16, PathBuf>);
@@ -24,7 +24,7 @@ impl BlkIndexToBlkPath {
if is_file {
let file_name = path.file_name().unwrap().to_str().unwrap();
file_name.starts_with(BLK) && file_name.ends_with(DAT)
file_name.starts_with(BLK) && file_name.ends_with(DOT_DAT)
} else {
false
}
@@ -32,7 +32,7 @@ impl BlkIndexToBlkPath {
.map(|path| {
let file_name = path.file_name().unwrap().to_str().unwrap();
let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
let blk_index = file_name[BLK.len()..(file_name.len() - DOT_DAT.len())]
.parse::<u16>()
.unwrap();
crates/brk_parser/src/blk_index_to_blk_recap.rs
-107
View File
@@ -1,107 +0,0 @@
use std::{
collections::{BTreeMap, BTreeSet},
fs::File,
io::{BufReader, BufWriter},
path::{Path, PathBuf},
};
use crate::{BlkIndexToBlkPath, Height, blk_recap::BlkRecap};
#[derive(Debug)]
pub struct BlkIndexToBlkRecap {
pub path: PathBuf,
pub tree: BTreeMap<u16, BlkRecap>,
}
impl BlkIndexToBlkRecap {
pub fn import(
outputs_dir: &Path,
blk_index_to_blk_path: &BlkIndexToBlkPath,
start: Option<Height>,
) -> (Self, u16) {
let path = outputs_dir.join("blk_index_to_blk_recap.json");
let tree = {
if let Ok(file) = File::open(&path) {
let reader = BufReader::new(file);
serde_json::from_reader(reader).unwrap_or_default()
} else {
BTreeMap::default()
}
};
let mut slf = Self { path, tree };
let min_removed = slf.clean_outdated(blk_index_to_blk_path);
let blk_index = slf.get_start_recap(min_removed, start);
(slf, blk_index)
}
fn clean_outdated(&mut self, blk_index_to_blk_path: &BlkIndexToBlkPath) -> Option<u16> {
let mut min_removed_blk_index: Option<u16> = None;
let mut unprocessed_keys = self.tree.keys().copied().collect::<BTreeSet<_>>();
blk_index_to_blk_path
.iter()
.for_each(|(blk_index, blk_path)| {
unprocessed_keys.remove(blk_index);
if let Some(blk_recap) = self.tree.get(blk_index)
&& blk_recap.has_different_modified_time(blk_path)
{
self.tree.remove(blk_index).unwrap();
if min_removed_blk_index.is_none_or(|_blk_index| *blk_index < _blk_index) {
min_removed_blk_index.replace(*blk_index);
}
}
});
unprocessed_keys.into_iter().for_each(|blk_index| {
self.tree.remove(&blk_index).unwrap();
if min_removed_blk_index.is_none_or(|_blk_index| blk_index < _blk_index) {
min_removed_blk_index.replace(blk_index);
}
});
min_removed_blk_index
}
pub fn get_start_recap(&mut self, min_removed: Option<u16>, start: Option<Height>) -> u16 {
if start.is_none() {
return 0;
}
let height = start.unwrap();
let mut start = None;
if let Some(found) = self
.tree
.iter()
.find(|(_, recap)| recap.max_height >= height)
{
start = Some(*found.0);
}
if let Some(min_removed) = min_removed
&& start.is_none_or(|start| start > min_removed)
{
start = Some(min_removed);
}
// Should only be none if asking for a too high start
start.unwrap_or_else(|| self.tree.last_key_value().map_or(0, |(i, _)| *i))
}
pub fn export(&self) {
let file = File::create(&self.path).unwrap_or_else(|e| {
dbg!(e);
dbg!(&self.path);
panic!("Cannot write file");
});
serde_json::to_writer(&mut BufWriter::new(file), &self.tree).unwrap();
}
}
crates/brk_parser/src/blk_metadata.rs
-18
View File
@@ -1,18 +0,0 @@
use std::path::Path;
use crate::path_to_modified_time;
#[derive(Debug, Clone, Copy)]
pub struct BlkMetadata {
pub index: u16,
pub modified_time: u64,
}
impl BlkMetadata {
pub fn new(index: u16, path: &Path) -> Self {
Self {
index,
modified_time: path_to_modified_time(path),
}
}
}
crates/brk_parser/src/blk_recap.rs
-19
View File
@@ -1,19 +0,0 @@
use std::path::Path;
use brk_structs::Height;
use serde::{Deserialize, Serialize};
use crate::path_to_modified_time;
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[repr(C)]
pub struct BlkRecap {
pub max_height: Height,
pub modified_time: u64,
}
impl BlkRecap {
pub fn has_different_modified_time(&self, blk_path: &Path) -> bool {
self.modified_time != path_to_modified_time(blk_path)
}
}
crates/brk_parser/src/lib.rs
+167 -124
View File
@@ -1,28 +1,29 @@
#![doc = include_str!("../README.md")]
use std::{cmp::Ordering, collections::BTreeMap, fs, ops::ControlFlow, path::PathBuf, thread};
use std::{
collections::BTreeMap,
fs::{self, File},
io::Read,
mem,
ops::ControlFlow,
path::PathBuf,
thread,
};
use bitcoin::BlockHash;
use bitcoin::{block::Header, consensus::Decodable};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use blk_recap::BlkRecap;
use brk_structs::{Block, Height};
use brk_error::Result;
use brk_structs::{Block, BlockPosition, Height, ParsedBlock};
use crossbeam::channel::{Receiver, bounded};
use rayon::prelude::*;
mod any_block;
mod blk_index_to_blk_path;
mod blk_index_to_blk_recap;
mod blk_metadata;
mod blk_recap;
mod utils;
mod xor_bytes;
mod xor_index;
use any_block::*;
use blk_index_to_blk_recap::*;
use blk_metadata::*;
use utils::*;
use xor_bytes::*;
use xor_index::*;
@@ -33,41 +34,29 @@ const BOUND_CAP: usize = 50;
pub struct Parser {
blocks_dir: PathBuf,
outputs_dir: Option<PathBuf>,
rpc: &'static bitcoincore_rpc::Client,
}
impl Parser {
pub fn new(
blocks_dir: PathBuf,
outputs_dir: Option<PathBuf>,
rpc: &'static bitcoincore_rpc::Client,
) -> Self {
Self {
blocks_dir,
outputs_dir,
rpc,
}
pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { blocks_dir, rpc }
}
// pub fn get(&self, height: Height) -> Block {
// self.parse(Some(height), Some(height))
// .iter()
// .next()
// .unwrap()
// .1
// }
pub fn get(&self, height: Height) -> Result<Block> {
Ok((
height,
self.rpc
.get_block(&self.rpc.get_block_hash(height.into())?)?,
)
.into())
}
///
/// Returns a crossbeam channel receiver that receives `(Height, Block, BlockHash)` tuples from an **inclusive** range (`start` and `end`)
/// Returns a crossbeam channel receiver that receives `Block` from an **inclusive** range (`start` and `end`)
///
/// For an example checkout `./main.rs`
///
pub fn parse(
&self,
start: Option<Height>,
end: Option<Height>,
) -> Receiver<(Height, bitcoin::Block, BlockHash)> {
pub fn parse(&self, start: Option<Height>, end: Option<Height>) -> Receiver<ParsedBlock> {
let blocks_dir = self.blocks_dir.as_path();
let rpc = self.rpc;
@@ -77,25 +66,19 @@ impl Parser {
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);
let (mut blk_index_to_blk_recap, blk_index) = BlkIndexToBlkRecap::import(
self.outputs_dir.as_ref().unwrap(),
&blk_index_to_blk_path,
start,
);
let xor_bytes = XORBytes::from(blocks_dir);
thread::spawn(move || {
let xor_bytes = xor_bytes;
let first_blk_index = self
.find_start_blk_index(start, &blk_index_to_blk_path, &xor_bytes)
.unwrap_or_default();
let _ = blk_index_to_blk_path.range(blk_index..).try_for_each(
thread::spawn(move || {
let _ = blk_index_to_blk_path.range(first_blk_index..).try_for_each(
move |(blk_index, blk_path)| {
let mut xor_i = XORIndex::default();
let blk_index = *blk_index;
let blk_metadata = BlkMetadata::new(blk_index, blk_path.as_path());
let mut blk_bytes_ = fs::read(blk_path).unwrap();
let blk_bytes = blk_bytes_.as_mut_slice();
let blk_bytes_len = blk_bytes.len();
@@ -120,18 +103,27 @@ impl Parser {
}
}
let len = u32::from_le_bytes(
let deser_len = u32::from_le_bytes(
xor_i
.bytes(&mut blk_bytes[i..(i + 4)], &xor_bytes)
.try_into()
.unwrap(),
) as usize;
);
let len = deser_len as usize;
i += 4;
let block_bytes = (blk_bytes[i..(i + len)]).to_vec();
if send_bytes
.send((blk_metadata, AnyBlock::Raw(block_bytes), xor_i))
.send((
BlockPosition {
blk_index,
offset: i,
len: deser_len,
},
AnyBlock::Raw(block_bytes),
xor_i,
))
.is_err()
{
return ControlFlow::Break(());
@@ -151,24 +143,20 @@ impl Parser {
let mut bulk = vec![];
let drain_and_send = |bulk: &mut Vec<_>| {
let drain_and_send = |bulk: &mut Vec<(BlockPosition, AnyBlock, XORIndex)>| {
// Using a vec and sending after to not end up with stuck threads in par iter
bulk.par_iter_mut().for_each(|(_, any_block, xor_i)| {
AnyBlock::decode(any_block, xor_i, &xor_bytes);
});
mem::take(bulk)
.into_par_iter()
.try_for_each(|(position, any_block, xor_i)| {
if let Ok(AnyBlock::Decoded(block)) =
any_block.decode(position, rpc, xor_i, &xor_bytes, start, end)
&& send_block.send(block).is_err()
{
return ControlFlow::Break(());
}
bulk.drain(..).try_for_each(|(blk_metadata, any_block, _)| {
let block = match any_block {
AnyBlock::Decoded(block) => block,
_ => unreachable!(),
};
if send_block.send((blk_metadata, block)).is_err() {
return ControlFlow::Break(());
}
ControlFlow::Continue(())
})
ControlFlow::Continue(())
})
};
recv_bytes.iter().try_for_each(|tuple| {
@@ -192,83 +180,138 @@ impl Parser {
let _ = recv_block
.iter()
.try_for_each(|(blk_metadata, block)| -> ControlFlow<(), _> {
let hash = block.block_hash();
let header = rpc.get_block_header_info(&hash);
if header.is_err() {
return ControlFlow::Continue(());
}
let header = header.unwrap();
if header.confirmations <= 0 {
return ControlFlow::Continue(());
}
let height = Height::from(header.height);
let len = blk_index_to_blk_recap.tree.len();
if blk_metadata.index == len as u16 || blk_metadata.index + 1 == len as u16 {
match (len as u16).cmp(&blk_metadata.index) {
Ordering::Equal => {
if len % 21 == 0 {
blk_index_to_blk_recap.export();
}
}
Ordering::Less => panic!(),
Ordering::Greater => {}
}
blk_index_to_blk_recap
.tree
.entry(blk_metadata.index)
.and_modify(|recap| {
if recap.max_height < height {
recap.max_height = height;
}
})
.or_insert(BlkRecap {
max_height: height,
modified_time: blk_metadata.modified_time,
});
}
let mut opt = if current_height == height {
Some((block, hash))
.try_for_each(|block| -> ControlFlow<(), _> {
let mut opt = if current_height == block.height() {
Some(block)
} else {
if start.is_none_or(|start| start <= height)
&& end.is_none_or(|end| end >= height)
{
future_blocks.insert(height, (block, hash));
}
future_blocks.insert(block.height(), block);
None
};
while let Some((block, hash)) = opt.take().or_else(|| {
while let Some(block) = opt.take().or_else(|| {
if !future_blocks.is_empty() {
future_blocks.remove(&current_height)
} else {
None
}
}) {
if end.is_some_and(|end| end < current_height) {
return ControlFlow::Break(());
}
send_ordered.send((current_height, block, hash)).unwrap();
if end.is_some_and(|end| end == current_height) {
return ControlFlow::Break(());
}
send_ordered.send(block).unwrap();
current_height.increment();
}
ControlFlow::Continue(())
});
blk_index_to_blk_recap.export();
});
recv_ordered
}
fn find_start_blk_index(
&self,
target_start: Option<Height>,
blk_index_to_blk_path: &BlkIndexToBlkPath,
xor_bytes: &XORBytes,
) -> Result<u16> {
let Some(target_start) = target_start else {
return Ok(0);
};
// If start is a very recent block we only look back X blk file before the last
if let Ok(count) = self.rpc.get_block_count()
&& (count as u32).saturating_sub(*target_start) <= 3
{
return Ok(blk_index_to_blk_path
.keys()
.rev()
.nth(2)
.cloned()
.unwrap_or_default());
}
let blk_indices: Vec<u16> = blk_index_to_blk_path
.range(0..)
.map(|(idx, _)| *idx)
.collect();
if blk_indices.is_empty() {
return Ok(0);
}
let mut left = 0;
let mut right = blk_indices.len() - 1;
let mut best_start_idx = 0;
while left <= right {
let mid = (left + right) / 2;
let blk_index = blk_indices[mid];
if let Some(blk_path) = blk_index_to_blk_path.get(&blk_index) {
match self.get_first_block_height(blk_path, xor_bytes) {
Ok(height) => {
if height <= target_start {
best_start_idx = mid;
if mid == usize::MAX {
break;
}
left = mid + 1;
} else {
if mid == 0 {
break;
}
right = mid - 1;
}
}
Err(_) => {
if mid == usize::MAX {
break;
}
left = mid + 1;
}
}
} else {
break;
}
}
// buffer for worst-case scenarios when a block as far behind
let final_idx = best_start_idx.saturating_sub(21);
Ok(blk_indices.get(final_idx).copied().unwrap_or(0))
}
fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: &XORBytes) -> Result<Height> {
let mut file = File::open(blk_path)?;
let mut xor_i = XORIndex::default();
let mut current_4bytes = [0; 4];
let mut byte_buffer = [0u8; 1];
loop {
if file.read_exact(&mut byte_buffer).is_err() {
return Err("No magic bytes found".into());
}
current_4bytes.rotate_left(1);
current_4bytes[3] = xor_i.byte(byte_buffer[0], xor_bytes);
if current_4bytes == MAGIC_BYTES {
break;
}
}
let mut size_bytes = [0u8; 4];
file.read_exact(&mut size_bytes)?;
let _block_size =
u32::from_le_bytes(xor_i.bytes(&mut size_bytes, xor_bytes).try_into().unwrap());
let mut header_bytes = [0u8; 80];
file.read_exact(&mut header_bytes)?;
xor_i.bytes(&mut header_bytes, xor_bytes);
let header = Header::consensus_decode(&mut std::io::Cursor::new(&header_bytes))?;
let height = self.rpc.get_block_info(&header.block_hash())?.height as u32;
Ok(Height::new(height))
}
}
crates/brk_parser/src/utils.rs
-11
View File
@@ -1,11 +0,0 @@
use std::{fs, path::Path, time::UNIX_EPOCH};
pub fn path_to_modified_time(path: &Path) -> u64 {
fs::metadata(path)
.unwrap()
.modified()
.unwrap()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs()
}
crates/brk_server/examples/main.rs
+1 -2
View File
@@ -17,7 +17,6 @@ pub fn main() -> Result<()> {
let process = true;
let bitcoin_dir = Path::new("");
let brk_dir = Path::new("");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
@@ -26,7 +25,7 @@ pub fn main() -> Result<()> {
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(bitcoin_dir.join("blocks"), Some(brk_dir.to_path_buf()), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let outputs_dir = Path::new("../../_outputs");
crates/brk_structs/src/structs/block.rs
+109 -29
View File
@@ -1,22 +1,35 @@
use std::{borrow::Cow, collections::HashMap, ops::Deref};
use std::{
borrow::Cow,
collections::{BTreeMap, HashMap},
io::Cursor,
ops::Deref,
};
use bitcoin::{Txid, block::Header, consensus::Decodable};
use bitcoin::{block::Header, consensus::Decodable};
use brk_error::Result;
use crate::BlockPosition;
use super::{BlockHash, Height};
#[derive(Debug)]
pub struct Block {
pub block: bitcoin::Block,
pub hash: BlockHash,
pub height: Height,
pub tx_positions: HashMap<Txid, usize>, // txid -> offset
pub block_start_offset: usize,
height: Height,
hash: BlockHash,
block: bitcoin::Block,
}
impl Block {
pub fn parse(block_data: &[u8], file_offset: usize) -> Result<Self> {
let mut cursor = std::io::Cursor::new(block_data);
pub fn height(&self) -> Height {
self.height
}
pub fn hash(&self) -> &BlockHash {
&self.hash
}
pub fn parse(block_data: &[u8]) -> Result<Self> {
let mut cursor = Cursor::new(block_data);
let header = Header::consensus_decode(&mut cursor)?;
@@ -46,17 +59,9 @@ impl Block {
block,
hash: hash.into(),
height: Height::ZERO,
tx_positions,
block_start_offset: file_offset,
})
}
pub fn get_absolute_position(&self, txid: &Txid) -> Option<usize> {
self.tx_positions
.get(txid)
.map(|offset| self.block_start_offset + offset)
}
pub fn coinbase_tag(&self) -> Cow<'_, str> {
String::from_utf8_lossy(
self.txdata
@@ -69,6 +74,22 @@ impl Block {
}
}
impl From<(Height, bitcoin::Block)> for Block {
fn from((height, block): (Height, bitcoin::Block)) -> Self {
Self::from((height, block.block_hash(), block))
}
}
impl From<(Height, bitcoin::BlockHash, bitcoin::Block)> for Block {
fn from((height, hash, block): (Height, bitcoin::BlockHash, bitcoin::Block)) -> Self {
Self {
height,
hash: hash.into(),
block,
}
}
}
impl Deref for Block {
type Target = bitcoin::Block;
fn deref(&self) -> &Self::Target {
@@ -76,19 +97,78 @@ impl Deref for Block {
}
}
pub trait BlockExtended {
fn coinbase_tag(&self) -> Cow<'_, str>;
#[derive(Debug)]
pub struct ParsedBlock {
block: Block,
position: BlockPosition,
tx_positions: BTreeMap<usize, usize>, // txid -> offset
}
impl BlockExtended for bitcoin::Block {
fn coinbase_tag(&self) -> Cow<'_, str> {
String::from_utf8_lossy(
self.txdata
.first()
.and_then(|tx| tx.input.first())
.unwrap()
.script_sig
.as_bytes(),
)
impl From<(Block, BlockPosition)> for ParsedBlock {
fn from((block, position): (Block, BlockPosition)) -> Self {
Self {
block,
position,
tx_positions: BTreeMap::default(),
}
}
}
impl ParsedBlock {
pub fn position(&self) -> &BlockPosition {
&self.position
}
pub fn tx_positions(&self) -> &BTreeMap<usize, usize> {
&self.tx_positions
}
// pub fn parse(block_data: &[u8], file_offset: usize) -> Result<Self> {
// let mut cursor = std::io::Cursor::new(block_data);
// let header = Header::consensus_decode(&mut cursor)?;
// // Parse transactions with positions
// let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize;
// let mut transactions = Vec::with_capacity(tx_count);
// let mut tx_positions = HashMap::with_capacity(tx_count);
// for _ in 0..tx_count {
// let start = cursor.position() as usize;
// let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?;
// tx_positions.insert(tx.compute_txid(), start);
// transactions.push(tx);
// }
// let block = bitcoin::Block {
// header,
// txdata: transactions,
// };
// // block.bip34_block_height()
// let hash = block.block_hash();
// Ok(Block {
// block,
// hash: hash.into(),
// height: Height::ZERO,
// tx_positions,
// block_start_offset: file_offset,
// })
// }
// pub fn get_absolute_position(&self, txid: &Txid) -> Option<usize> {
// self.tx_positions
// .get(txid)
// .map(|offset| self.block_start_offset + offset)
// }
}
impl Deref for ParsedBlock {
type Target = Block;
fn deref(&self) -> &Self::Target {
&self.block
}
}
crates/brk_structs/src/structs/blockmeta.rs
+6
View File
@@ -0,0 +1,6 @@
#[derive(Debug, Clone, Copy)]
pub struct BlockPosition {
pub blk_index: u16,
pub offset: usize,
pub len: u32,
}
crates/brk_structs/src/structs/mod.rs
+2
View File
@@ -7,6 +7,7 @@ mod bitcoin;
mod block;
mod blockhash;
mod blockhashprefix;
mod blockmeta;
mod cents;
mod date;
mod dateindex;
@@ -71,6 +72,7 @@ pub use bitcoin::*;
pub use block::*;
pub use blockhash::*;
pub use blockhashprefix::*;
pub use blockmeta::*;
pub use cents::*;
pub use date::*;
pub use dateindex::*;