Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-25 17:58:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: add positions

Browse Source
This commit is contained in:
nym21
2025-09-18 19:45:16 +02:00
parent cc5701ea62
commit 43117825d7
27 changed files with 458 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_cli/Cargo.toml
+1 -1
View File
@@ -27,7 +27,7 @@ log = { workspace = true }
minreq = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.5"
toml = "0.9.6"
zip = { version = "5.1.1", default-features = false, features = ["deflate"] }
[[bin]]
crates/brk_cli/src/lib.rs
+2 -4
View File
@@ -73,7 +73,7 @@ pub fn run() -> color_eyre::Result<()> {
.enable_all()
.build()?
.block_on(async {
let interface = Interface::build(&indexer, &computer);
let interface = Interface::build(&parser, &indexer, &computer);
let website = config.website();
@@ -135,9 +135,7 @@ pub fn run() -> color_eyre::Result<()> {
let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap();
// dbg!(&starting_indexes);
computer.compute(&indexer, starting_indexes, &exit).unwrap();
computer.compute(&indexer, starting_indexes, &parser, &exit).unwrap();
info!("Waiting for new blocks...");
crates/brk_computer/examples/computer.rs
+1 -1
View File
@@ -45,7 +45,7 @@ pub fn main() -> Result<()> {
loop {
let i = Instant::now();
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &parser, &exit)?;
dbg!(i.elapsed());
sleep(Duration::from_secs(10));
}
crates/brk_computer/src/constants.rs
+22 -18
View File
@@ -12,8 +12,6 @@ use super::{
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
@@ -35,15 +33,21 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
let db = Database::open(&parent.join("constants"))?;
pub fn forced_import(
parent_path: &Path,
parent_version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let db = Database::open(&parent_path.join("constants"))?;
let version = parent_version + Version::ZERO;
let this = Self {
constant_0: ComputedVecsFromHeight::forced_import(
&db,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -51,7 +55,7 @@ impl Vecs {
&db,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -59,7 +63,7 @@ impl Vecs {
&db,
"constant_2",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -67,7 +71,7 @@ impl Vecs {
&db,
"constant_3",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -75,7 +79,7 @@ impl Vecs {
&db,
"constant_4",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -83,7 +87,7 @@ impl Vecs {
&db,
"constant_38_2",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -91,7 +95,7 @@ impl Vecs {
&db,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -99,7 +103,7 @@ impl Vecs {
&db,
"constant_61_8",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -107,7 +111,7 @@ impl Vecs {
&db,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -115,7 +119,7 @@ impl Vecs {
&db,
"constant_600",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -123,7 +127,7 @@ impl Vecs {
&db,
"constant_minus_1",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -131,7 +135,7 @@ impl Vecs {
&db,
"constant_minus_2",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -139,7 +143,7 @@ impl Vecs {
&db,
"constant_minus_3",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -147,7 +151,7 @@ impl Vecs {
&db,
"constant_minus_4",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
crates/brk_computer/src/lib.rs
+34 -14
View File
@@ -5,6 +5,7 @@ use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::Version;
use log::info;
use vecdb::{AnyCollectableVec, Exit, Format};
@@ -17,6 +18,7 @@ mod grouped;
mod indexes;
mod market;
mod pools;
mod positions;
mod price;
mod stateful;
mod states;
@@ -31,15 +33,16 @@ use states::*;
#[derive(Clone)]
pub struct Computer {
pub indexes: indexes::Vecs,
pub chain: chain::Vecs,
pub cointime: cointime::Vecs,
pub constants: constants::Vecs,
pub fetched: Option<fetched::Vecs>,
pub indexes: indexes::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub positions: positions::Vecs,
pub price: Option<price::Vecs>,
pub chain: chain::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
pub cointime: cointime::Vecs,
}
const VERSION: Version = Version::new(4);
@@ -87,6 +90,7 @@ impl Computer {
&indexes,
price.as_ref(),
)?,
positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
@@ -109,15 +113,12 @@ impl Computer {
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
parser: &Parser,
exit: &Exit,
) -> Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(&self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
@@ -131,7 +132,25 @@ impl Computer {
)?;
}
info!("Computing positions...");
self.positions
.compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
let constants = scope.spawn(|| -> Result<()> {
info!("Computing constants...");
self.constants
.compute(&self.indexes, &starting_indexes, exit)?;
Ok(())
});
// let positions = scope.spawn(|| -> Result<()> {
// info!("Computing positions...");
// self.positions
// .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
// Ok(())
// });
let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
self.chain.compute(
@@ -149,6 +168,8 @@ impl Computer {
self.market.compute(price, &starting_indexes, exit)?;
}
constants.join().unwrap()?;
// positions.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;
@@ -162,8 +183,6 @@ impl Computer {
exit,
)?;
// return Ok(());
info!("Computing stateful...");
self.stateful.compute(
indexer,
@@ -191,14 +210,15 @@ impl Computer {
let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.fetched.iter().flat_map(|v| v.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.chain.iter_any_collectable()));
iter = Box::new(iter.chain(self.cointime.iter_any_collectable()));
iter = Box::new(iter.chain(self.constants.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes.iter_any_collectable()));
iter = Box::new(iter.chain(self.market.iter_any_collectable()));
iter = Box::new(iter.chain(self.chain.iter_any_collectable()));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.positions.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.stateful.iter_any_collectable()));
iter = Box::new(iter.chain(self.cointime.iter_any_collectable()));
iter
}
crates/brk_computer/src/positions.rs
+133
View File
@@ -0,0 +1,133 @@
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::{BlkPosition, Height, TxIndex, Version};
use log::info;
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
GenericStoredVec, PAGE_SIZE, VecIterator,
};
use super::{Indexes, indexes};
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub height_to_position: CompressedVec<Height, BlkPosition>,
pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("positions"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let version = parent_version + Version::ZERO;
let this = Self {
height_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
)?,
txindex_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
)?,
db,
};
this.db.retain_regions(
this.iter_any_collectable()
.flat_map(|v| v.region_names())
.collect(),
)?;
Ok(this)
}
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
parser: &Parser,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, parser, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
parser: &Parser,
exit: &Exit,
) -> Result<()> {
let min_txindex =
TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
let min_height = indexes
.txindex_to_height
.iter()
.unwrap_get_inner(min_txindex)
.min(starting_indexes.height);
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
parser
.parse(Some(min_height), None)
.iter()
.try_for_each(|block| -> Result<()> {
let height = block.height();
info!("{height}");
self.height_to_position
.forced_push_at(height, *block.position(), exit)?;
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
block.tx_positions().iter().enumerate().try_for_each(
|(index, position)| -> Result<()> {
let txindex = txindex + index;
self.txindex_to_position
.forced_push_at(txindex, *position, exit)?;
Ok(())
},
)?;
// Stuck, why ??
if *height % 1_000 == 0 {
let _lock = exit.lock();
self.height_to_position.flush()?;
self.txindex_to_position.flush()?;
}
Ok(())
})?;
let _lock = exit.lock();
self.height_to_position.flush()?;
self.txindex_to_position.flush()?;
Ok(())
}
pub fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
Box::new(
[
&self.height_to_position as &dyn AnyCollectableVec,
&self.txindex_to_position,
]
.into_iter(),
)
}
}
crates/brk_indexer/src/vecs.rs
+17 -19
View File
@@ -20,7 +20,6 @@ use crate::Indexes;
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub emptyoutputindex_to_txindex: CompressedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: RawVec<Height, BlockHash>,
pub height_to_difficulty: CompressedVec<Height, StoredF64>,
@@ -39,7 +38,7 @@ pub struct Vecs {
pub height_to_first_p2wshaddressindex: CompressedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: CompressedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: CompressedVec<Height, UnknownOutputIndex>,
/// Doesn't guarantee continuity due to possible reorgs
/// Doesn't guarantee continuity due to possible reorgs and more generally the nature of mining
pub height_to_timestamp: CompressedVec<Height, Timestamp>,
pub height_to_total_size: CompressedVec<Height, StoredU64>,
pub height_to_weight: CompressedVec<Height, Weight>,
@@ -72,9 +71,11 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(parent: &Path, version: Version) -> Result<Self> {
let db = Database::open(&parent.join("vecs"))?;
db.set_min_len(PAGE_SIZE * 50_000_000)?;
let db_positions = Database::open(&parent.join("vecs/positions"))?;
db_positions.set_min_len(PAGE_SIZE * 1_000_000)?;
let this = Self {
emptyoutputindex_to_txindex: CompressedVec::forced_import(&db, "txindex", version)?,
height_to_blockhash: RawVec::forced_import(&db, "blockhash", version)?,
@@ -192,12 +193,8 @@ impl Vecs {
db,
};
// self.db.retain_regions(
// this.vecs()
// .into_iter()
// .flat_map(|v| v.region_names())
// .collect(),
// )?;
this.db
.retain_regions(this.iter().flat_map(|v| v.region_names()).collect())?;
Ok(this)
}
@@ -349,16 +346,15 @@ impl Vecs {
}
pub fn flush(&mut self, height: Height) -> Result<()> {
self.mut_vecs()
.into_par_iter()
self.iter_mut()
.par_bridge()
.try_for_each(|vec| vec.stamped_flush(Stamp::from(height)))?;
self.db.flush()?;
Ok(())
}
pub fn starting_height(&mut self) -> Height {
self.mut_vecs()
.into_iter()
self.iter_mut()
.map(|vec| {
let h = Height::from(vec.stamp());
if h > Height::ZERO { h.incremented() } else { h }
@@ -372,9 +368,9 @@ impl Vecs {
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
vec![
&self.emptyoutputindex_to_txindex,
pub fn iter(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
[
&self.emptyoutputindex_to_txindex as &dyn AnyCollectableVec,
&self.height_to_blockhash,
&self.height_to_difficulty,
&self.height_to_first_emptyoutputindex,
@@ -419,11 +415,12 @@ impl Vecs {
&self.txindex_to_txversion,
&self.unknownoutputindex_to_txindex,
]
.into_iter()
}
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStoredVec> {
vec![
&mut self.emptyoutputindex_to_txindex,
fn iter_mut(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.emptyoutputindex_to_txindex as &mut dyn AnyStoredVec,
&mut self.height_to_blockhash,
&mut self.height_to_difficulty,
&mut self.height_to_first_emptyoutputindex,
@@ -468,5 +465,6 @@ impl Vecs {
&mut self.txindex_to_txversion,
&mut self.unknownoutputindex_to_txindex,
]
.into_iter()
}
}
crates/brk_interface/Cargo.toml
+2
View File
@@ -10,9 +10,11 @@ rust-version.workspace = true
build = "build.rs"
[dependencies]
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_error = { workspace = true }
brk_indexer = { workspace = true }
brk_parser = { workspace = true }
brk_structs = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
crates/brk_interface/examples/main.rs
+27 -5
View File
@@ -1,18 +1,40 @@
use std::path::Path;
use std::{fs, path::Path};
use brk_computer::Computer;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_interface::{Index, Interface, Params, ParamsOpt};
use brk_parser::Parser;
use vecdb::Exit;
pub fn main() -> Result<()> {
let outputs_dir = Path::new("../../_outputs");
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let indexer = Indexer::forced_import(outputs_dir)?;
let blocks_dir = bitcoin_dir.join("blocks");
let computer = Computer::forced_import(outputs_dir, &indexer, None)?;
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
fs::create_dir_all(&outputs_dir)?;
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let interface = Interface::build(&indexer, &computer);
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(blocks_dir, rpc);
let indexer = Indexer::forced_import(&outputs_dir)?;
let computer = Computer::forced_import(&outputs_dir, &indexer, None)?;
let interface = Interface::build(&parser, &indexer, &computer);
dbg!(interface.search_and_format(Params {
index: Index::Height,
crates/brk_interface/src/lib.rs
+9 -1
View File
@@ -5,6 +5,7 @@ use std::{collections::BTreeMap, sync::OnceLock};
use brk_computer::Computer;
use brk_error::{Error, Result};
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::Height;
use nucleo_matcher::{
Config, Matcher,
@@ -42,18 +43,21 @@ pub fn cached_errors() -> &'static Cache<String, String> {
#[allow(dead_code)]
pub struct Interface<'a> {
vecs: Vecs<'a>,
parser: &'a Parser,
indexer: &'a Indexer,
computer: &'a Computer,
}
impl<'a> Interface<'a> {
pub fn build(indexer: &Indexer, computer: &Computer) -> Self {
pub fn build(parser: &Parser, indexer: &Indexer, computer: &Computer) -> Self {
let parser = parser.static_clone();
let indexer = indexer.static_clone();
let computer = computer.static_clone();
let vecs = Vecs::build(indexer, computer);
Self {
vecs,
parser,
indexer,
computer,
}
@@ -262,6 +266,10 @@ impl<'a> Interface<'a> {
self.vecs.id_to_indexes(id)
}
pub fn parser(&self) -> &Parser {
self.parser
}
pub fn indexer(&self) -> &Indexer {
self.indexer
}
crates/brk_interface/src/vecs.rs
+1 -5
View File
@@ -27,11 +27,7 @@ impl<'a> Vecs<'a> {
pub fn build(indexer: &'a Indexer, computer: &'a Computer) -> Self {
let mut this = Vecs::default();
indexer
.vecs
.vecs()
.into_iter()
.for_each(|vec| this.insert(vec));
indexer.vecs.iter().for_each(|vec| this.insert(vec));
computer
.iter_any_collectable()
crates/brk_parser/Cargo.toml
+1
View File
@@ -18,6 +18,7 @@ brk_error = { workspace = true }
brk_structs = { workspace = true }
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
derive_deref = { workspace = true }
parking_lot = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
crates/brk_parser/examples/main.rs
+2 -3
View File
@@ -3,7 +3,6 @@ use std::{collections::BTreeMap, path::Path};
use bitcoincore_rpc::{Auth, Client};
use brk_error::Result;
use brk_parser::Parser;
use brk_structs::Height;
#[allow(clippy::needless_doctest_main)]
fn main() -> Result<()> {
@@ -21,13 +20,13 @@ fn main() -> Result<()> {
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let start = Some(915_155_u32.into());
let start = Some(915_138_u32.into());
let end = None;
let mut blk_index = 0;
let mut diff = BTreeMap::new();
parser.parse(start, end).iter().for_each(|block| {
println!("{}: {}", block.height(), block.hash());
let new_blk_index = block.position().blk_index;
let new_blk_index = block.position().blk_index();
if new_blk_index < blk_index {
diff.insert(blk_index - new_blk_index, block.height());
}
crates/brk_parser/src/any_block.rs
+7 -4
View File
@@ -1,7 +1,7 @@
use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor};
use bitcoincore_rpc::RpcApi;
use brk_error::Result;
use brk_structs::{Block, BlockPosition, Height, ParsedBlock};
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use crate::{XORBytes, XORIndex};
@@ -14,10 +14,10 @@ pub enum AnyBlock {
impl AnyBlock {
pub fn decode(
self,
position: BlockPosition,
position: BlkPosition,
rpc: &'static bitcoincore_rpc::Client,
mut xor_i: XORIndex,
xor_bytes: &XORBytes,
xor_bytes: XORBytes,
start: Option<Height>,
end: Option<Height>,
) -> Result<Self> {
@@ -55,14 +55,17 @@ impl AnyBlock {
}
let mut txdata = Vec::with_capacity(tx_count as usize);
let mut tx_positions = Vec::with_capacity(tx_count as usize);
for _ in 0..tx_count {
let tx_position = BlkPosition::new(position.blk_index(), cursor.position() as u32);
tx_positions.push(tx_position);
let tx = Transaction::consensus_decode(&mut cursor)?;
txdata.push(tx);
}
let block = bitcoin::Block { header, txdata };
let block = Block::from((height, hash, block));
let block = ParsedBlock::from((block, position));
let block = ParsedBlock::from((block, position, tx_positions));
Ok(Self::Decoded(block))
}
crates/brk_parser/src/blk_index_to_blk_path.rs
+1 -1
View File
@@ -9,7 +9,7 @@ use derive_deref::{Deref, DerefMut};
const BLK: &str = "blk";
const DOT_DAT: &str = ".dat";
#[derive(Debug, Deref, DerefMut)]
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct BlkIndexToBlkPath(BTreeMap<u16, PathBuf>);
impl BlkIndexToBlkPath {
crates/brk_parser/src/lib.rs
+43 -26
View File
@@ -7,6 +7,7 @@ use std::{
mem,
ops::ControlFlow,
path::PathBuf,
sync::Arc,
thread,
};
@@ -14,8 +15,9 @@ use bitcoin::{block::Header, consensus::Decodable};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use brk_error::Result;
use brk_structs::{Block, BlockPosition, Height, ParsedBlock};
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use crossbeam::channel::{Receiver, bounded};
use parking_lot::{RwLock, RwLockReadGuard};
use rayon::prelude::*;
mod any_block;
@@ -24,22 +26,40 @@ mod xor_bytes;
mod xor_index;
use any_block::*;
use xor_bytes::*;
use xor_index::*;
pub use xor_bytes::*;
pub use xor_index::*;
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 50;
#[derive(Debug, Clone)]
pub struct Parser {
blk_index_to_blk_path: Arc<RwLock<BlkIndexToBlkPath>>,
xor_bytes: XORBytes,
blocks_dir: PathBuf,
rpc: &'static bitcoincore_rpc::Client,
}
impl Parser {
pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { blocks_dir, rpc }
Self {
xor_bytes: XORBytes::from(blocks_dir.as_path()),
blk_index_to_blk_path: Arc::new(RwLock::new(BlkIndexToBlkPath::scan(
blocks_dir.as_path(),
))),
blocks_dir,
rpc,
}
}
pub fn blk_index_to_blk_path(&self) -> RwLockReadGuard<'_, BlkIndexToBlkPath> {
self.blk_index_to_blk_path.read()
}
pub fn xor_bytes(&self) -> XORBytes {
self.xor_bytes
}
pub fn get(&self, height: Height) -> Result<Block> {
@@ -57,19 +77,19 @@ impl Parser {
/// For an example checkout `./main.rs`
///
pub fn parse(&self, start: Option<Height>, end: Option<Height>) -> Receiver<ParsedBlock> {
let blocks_dir = self.blocks_dir.as_path();
let rpc = self.rpc;
let (send_bytes, recv_bytes) = bounded(BOUND_CAP);
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_ordered, recv_ordered) = bounded(BOUND_CAP);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(&self.blocks_dir);
*self.blk_index_to_blk_path.write() = blk_index_to_blk_path.clone();
let xor_bytes = XORBytes::from(blocks_dir);
let xor_bytes = self.xor_bytes;
let first_blk_index = self
.find_start_blk_index(start, &blk_index_to_blk_path, &xor_bytes)
.find_start_blk_index(start, &blk_index_to_blk_path, xor_bytes)
.unwrap_or_default();
thread::spawn(move || {
@@ -95,7 +115,7 @@ impl Parser {
current_4bytes.rotate_left(1);
current_4bytes[3] = xor_i.byte(blk_bytes[i], &xor_bytes);
current_4bytes[3] = xor_i.byte(blk_bytes[i], xor_bytes);
i += 1;
if current_4bytes == MAGIC_BYTES {
@@ -103,27 +123,20 @@ impl Parser {
}
}
let deser_len = u32::from_le_bytes(
let position = BlkPosition::new(blk_index, i as u32);
let len = u32::from_le_bytes(
xor_i
.bytes(&mut blk_bytes[i..(i + 4)], &xor_bytes)
.bytes(&mut blk_bytes[i..(i + 4)], xor_bytes)
.try_into()
.unwrap(),
);
let len = deser_len as usize;
) as usize;
i += 4;
let block_bytes = (blk_bytes[i..(i + len)]).to_vec();
if send_bytes
.send((
BlockPosition {
blk_index,
offset: i,
len: deser_len,
},
AnyBlock::Raw(block_bytes),
xor_i,
))
.send((position, AnyBlock::Raw(block_bytes), xor_i))
.is_err()
{
return ControlFlow::Break(());
@@ -143,13 +156,13 @@ impl Parser {
let mut bulk = vec![];
let drain_and_send = |bulk: &mut Vec<(BlockPosition, AnyBlock, XORIndex)>| {
let drain_and_send = |bulk: &mut Vec<(BlkPosition, AnyBlock, XORIndex)>| {
// Using a vec and sending after to not end up with stuck threads in par iter
mem::take(bulk)
.into_par_iter()
.try_for_each(|(position, any_block, xor_i)| {
if let Ok(AnyBlock::Decoded(block)) =
any_block.decode(position, rpc, xor_i, &xor_bytes, start, end)
any_block.decode(position, rpc, xor_i, xor_bytes, start, end)
&& send_block.send(block).is_err()
{
return ControlFlow::Break(());
@@ -211,7 +224,7 @@ impl Parser {
&self,
target_start: Option<Height>,
blk_index_to_blk_path: &BlkIndexToBlkPath,
xor_bytes: &XORBytes,
xor_bytes: XORBytes,
) -> Result<u16> {
let Some(target_start) = target_start else {
return Ok(0);
@@ -280,7 +293,7 @@ impl Parser {
Ok(blk_indices.get(final_idx).copied().unwrap_or(0))
}
fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: &XORBytes) -> Result<Height> {
fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: XORBytes) -> Result<Height> {
let mut file = File::open(blk_path)?;
let mut xor_i = XORIndex::default();
let mut current_4bytes = [0; 4];
@@ -314,4 +327,8 @@ impl Parser {
Ok(Height::new(height))
}
pub fn static_clone(&self) -> &'static Self {
Box::leak(Box::new(self.clone()))
}
}
crates/brk_parser/src/xor_index.rs
+2 -2
View File
@@ -4,7 +4,7 @@ use crate::xor_bytes::{XOR_LEN, XORBytes};
pub struct XORIndex(usize);
impl XORIndex {
pub fn bytes<'a>(&mut self, bytes: &'a mut [u8], xor_bytes: &XORBytes) -> &'a mut [u8] {
pub fn bytes<'a>(&mut self, bytes: &'a mut [u8], xor_bytes: XORBytes) -> &'a mut [u8] {
let len = bytes.len();
let mut bytes_index = 0;
@@ -18,7 +18,7 @@ impl XORIndex {
}
#[inline]
pub fn byte(&mut self, mut byte: u8, xor_bytes: &XORBytes) -> u8 {
pub fn byte(&mut self, mut byte: u8, xor_bytes: XORBytes) -> u8 {
byte ^= xor_bytes[self.0];
self.increment();
byte
crates/brk_server/examples/main.rs
+2 -2
View File
@@ -39,7 +39,7 @@ pub fn main() -> Result<()> {
.enable_all()
.build()?
.block_on(async {
let interface = Interface::build(&indexer, &computer);
let interface = Interface::build(&parser, &indexer, &computer);
let server = Server::new(interface, None);
@@ -53,7 +53,7 @@ pub fn main() -> Result<()> {
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &parser, &exit)?;
while block_count == rpc.get_block_count()? {
sleep(Duration::from_secs(1))
crates/brk_server/src/api/mod.rs
+35 -12
View File
@@ -1,4 +1,8 @@
use std::str::FromStr;
use std::{
fs::File,
io::{BufReader, Seek, SeekFrom},
str::FromStr,
};
use axum::{
Json, Router,
@@ -7,9 +11,10 @@ use axum::{
response::{IntoResponse, Redirect, Response},
routing::get,
};
use bitcoin::{Address, Network, absolute::LockTime};
use bitcoin::{Address, Network, Transaction, absolute::LockTime, consensus::Decodable};
use bitcoincore_rpc::bitcoin;
use brk_interface::{IdParam, Index, PaginatedIndexParam, PaginationParam, Params, ParamsOpt};
use brk_parser::XORIndex;
use brk_structs::{
AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, Txid, TxidPrefix,
};
@@ -45,13 +50,6 @@ impl ApiRoutes for Router<AppState> {
let computer = interface.computer();
let stores = &indexer.stores;
let hash = AddressBytesHash::from(&address);
dbg!(&hash);
dbg!(
&address,
address.address_type(),
address.script_pubkey(),
OutputType::from(&address)
);
let Ok(Some(addri)) = stores
.addressbyteshash_to_typeindex
@@ -60,8 +58,6 @@ impl ApiRoutes for Router<AppState> {
return "Unknown address".into_response();
};
println!("Script pubkey: {}", address.script_pubkey());
println!("Address type: {:?}", address.address_type());
let output_type = OutputType::from(&address);
let stateful = &computer.stateful;
let price = computer.price.as_ref().map(|v| {
@@ -178,11 +174,38 @@ impl ApiRoutes for Router<AppState> {
.unwrap_get_inner(txindex);
let locktime = LockTime::from(rawlocktime);
let parser = interface.parser();
let computer = interface.computer();
let position = computer.positions.txindex_to_position.iter().unwrap_get_inner(txindex);
let blk_index_to_blk_path = parser.blk_index_to_blk_path();
let Some(blk_path) = blk_index_to_blk_path.get(&position.blk_index()) else {
return "Unknown blk index".into_response();
};
let mut xori = XORIndex::default();
xori.add_assign(position.offset() as usize);
let Ok(file) = File::open(blk_path) else {
return "Error opening blk file".into_response();
};
let mut reader = BufReader::new(file);
if reader.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
return "Error seeking position in blk file".into_response();
}
let Ok(tx) = Transaction::consensus_decode(&mut reader) else {
return "Error decoding transaction".into_response();
};
Json(serde_json::json!({
"txid": txid,
"index": txindex,
"version": version,
"locktime": locktime
"locktime": locktime,
"tx": tx,
}))
.into_response()
},
crates/brk_structs/src/structs/blkposition.rs
+22
View File
@@ -0,0 +1,22 @@
use serde::Serialize;
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout, StoredCompressed,
)]
pub struct BlkPosition(u64);
impl BlkPosition {
pub fn new(blk_index: u16, offset: u32) -> Self {
Self(((blk_index as u64) << 32) | offset as u64)
}
pub fn blk_index(&self) -> u16 {
(self.0 >> 31) as u16
}
pub fn offset(&self) -> u32 {
self.0 as u32
}
}
crates/brk_structs/src/structs/block.rs
+9 -93
View File
@@ -1,14 +1,6 @@
use std::{
borrow::Cow,
collections::{BTreeMap, HashMap},
io::Cursor,
ops::Deref,
};
use std::{borrow::Cow, ops::Deref};
use bitcoin::{block::Header, consensus::Decodable};
use brk_error::Result;
use crate::BlockPosition;
use crate::BlkPosition;
use super::{BlockHash, Height};
@@ -28,40 +20,6 @@ impl Block {
&self.hash
}
pub fn parse(block_data: &[u8]) -> Result<Self> {
let mut cursor = Cursor::new(block_data);
let header = Header::consensus_decode(&mut cursor)?;
// Parse transactions with positions
let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize;
let mut transactions = Vec::with_capacity(tx_count);
let mut tx_positions = HashMap::with_capacity(tx_count);
for _ in 0..tx_count {
let start = cursor.position() as usize;
let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?;
tx_positions.insert(tx.compute_txid(), start);
transactions.push(tx);
}
let block = bitcoin::Block {
header,
txdata: transactions,
};
// block.bip34_block_height()
let hash = block.block_hash();
Ok(Block {
block,
hash: hash.into(),
height: Height::ZERO,
})
}
pub fn coinbase_tag(&self) -> Cow<'_, str> {
String::from_utf8_lossy(
self.txdata
@@ -100,70 +58,28 @@ impl Deref for Block {
#[derive(Debug)]
pub struct ParsedBlock {
block: Block,
position: BlockPosition,
tx_positions: BTreeMap<usize, usize>, // txid -> offset
position: BlkPosition,
tx_positions: Vec<BlkPosition>,
}
impl From<(Block, BlockPosition)> for ParsedBlock {
fn from((block, position): (Block, BlockPosition)) -> Self {
impl From<(Block, BlkPosition, Vec<BlkPosition>)> for ParsedBlock {
fn from((block, position, tx_positions): (Block, BlkPosition, Vec<BlkPosition>)) -> Self {
Self {
block,
position,
tx_positions: BTreeMap::default(),
tx_positions,
}
}
}
impl ParsedBlock {
pub fn position(&self) -> &BlockPosition {
pub fn position(&self) -> &BlkPosition {
&self.position
}
pub fn tx_positions(&self) -> &BTreeMap<usize, usize> {
pub fn tx_positions(&self) -> &Vec<BlkPosition> {
&self.tx_positions
}
// pub fn parse(block_data: &[u8], file_offset: usize) -> Result<Self> {
// let mut cursor = std::io::Cursor::new(block_data);
// let header = Header::consensus_decode(&mut cursor)?;
// // Parse transactions with positions
// let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize;
// let mut transactions = Vec::with_capacity(tx_count);
// let mut tx_positions = HashMap::with_capacity(tx_count);
// for _ in 0..tx_count {
// let start = cursor.position() as usize;
// let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?;
// tx_positions.insert(tx.compute_txid(), start);
// transactions.push(tx);
// }
// let block = bitcoin::Block {
// header,
// txdata: transactions,
// };
// // block.bip34_block_height()
// let hash = block.block_hash();
// Ok(Block {
// block,
// hash: hash.into(),
// height: Height::ZERO,
// tx_positions,
// block_start_offset: file_offset,
// })
// }
// pub fn get_absolute_position(&self, txid: &Txid) -> Option<usize> {
// self.tx_positions
// .get(txid)
// .map(|offset| self.block_start_offset + offset)
// }
}
impl Deref for ParsedBlock {
crates/brk_structs/src/structs/blockmeta.rs
-6
View File
@@ -1,6 +0,0 @@
#[derive(Debug, Clone, Copy)]
pub struct BlockPosition {
pub blk_index: u16,
pub offset: usize,
pub len: u32,
}
crates/brk_structs/src/structs/mod.rs
+2 -2
View File
@@ -4,10 +4,10 @@ mod addressbytes;
mod addressbyteshash;
mod anyaddressindex;
mod bitcoin;
mod blkposition;
mod block;
mod blockhash;
mod blockhashprefix;
mod blockmeta;
mod cents;
mod date;
mod dateindex;
@@ -69,10 +69,10 @@ pub use addressbytes::*;
pub use addressbyteshash::*;
pub use anyaddressindex::*;
pub use bitcoin::*;
pub use blkposition::*;
pub use block::*;
pub use blockhash::*;
pub use blockhashprefix::*;
pub use blockmeta::*;
pub use cents::*;
pub use date::*;
pub use dateindex::*;