Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 14:11:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

general: fixed builds

Browse Source
This commit is contained in:
nym21
2025-03-03 19:36:17 +01:00
parent d24096374f
commit fc6f12fb22
16 changed files with 229 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+54
View File
@@ -352,7 +352,9 @@ dependencies = [
"clap",
"color-eyre",
"log",
"serde",
"tabled",
"toml",
]
[[package]]
@@ -2251,6 +2253,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -2493,6 +2504,40 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]]
name = "toml_datetime"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
"winnow",
]
[[package]]
name = "tower"
version = "0.5.2"
@@ -2762,6 +2807,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1"
dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen-rt"
version = "0.33.0"
Cargo.toml
+1 -1
View File
@@ -21,7 +21,7 @@ brk_query = { version = "0", path = "crates/brk_query" }
brk_server = { version = "0", path = "crates/brk_server" }
brk_vec = { version = "0", path = "crates/brk_vec" }
byteview = "0.5.4"
clap = { version = "4.5.31", features = ["derive"] }
clap = { version = "4.5.31", features = ["derive", "string"] }
color-eyre = "0.6.3"
derive_deref = "1.1.1"
fjall = "2.6.7"
README.md
+1 -1
View File
@@ -1,6 +1,6 @@
# Bitcoin Research Kit
The Bitcoin Research Kit is a suite of tools designed to extract, compute, serve and display data stored on a Bitcoin Core node.
The Bitcoin Research Kit is a suite of tools designed to parse, index, compute, serve and display data stored on a Bitcoin Core node.
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) and [electrs](https://github.com/romanz/electrs) all in one package with a particular focus on the self-hosting experience.
crates/brk_computer/src/lib.rs
+50 -28
View File
@@ -18,21 +18,27 @@ use storage::{Stores, Vecs};
#[derive(Clone)]
pub struct Computer {
path: PathBuf,
pub vecs: Vecs,
pub stores: Stores,
vecs: Option<Vecs>,
stores: Option<Stores>,
}
impl Computer {
pub fn import(computed_dir: &Path) -> color_eyre::Result<Self> {
let vecs = Vecs::import(&computed_dir.join("vecs"))?;
let stores = Stores::import(&computed_dir.join("stores"))?;
Ok(Self {
pub fn new(computed_dir: &Path) -> Self {
Self {
path: computed_dir.to_owned(),
vecs,
stores,
})
vecs: None,
stores: None,
}
}
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(&self.path.join("vecs"))?);
Ok(())
}
pub fn import_stores(&mut self) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(&self.path.join("stores"))?);
Ok(())
}
}
@@ -40,65 +46,65 @@ impl Computer {
pub fn compute(&mut self, indexer: &mut Indexer, starting_indexes: Indexes, exit: &Exit) -> color_eyre::Result<()> {
info!("Computing...");
let height_count = indexer.vecs.height_to_size.len();
let txindexes_count = indexer.vecs.txindex_to_txid.len();
let txinindexes_count = indexer.vecs.txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs.txoutindex_to_addressindex.len();
let height_count = indexer.vecs().height_to_size.len();
let txindexes_count = indexer.vecs().txindex_to_txid.len();
let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
// TODO: Remove all outdated
// self.vecs.txindex_to_last_txinindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs.txindex_to_first_txinindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// txinindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs.txindex_to_first_txinindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// &mut self.vecs.txindex_to_last_txinindex,
// exit,
// )?;
// self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs.txindex_to_first_txoutindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// txoutindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs.txindex_to_first_txoutindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// &mut self.vecs.txindex_to_last_txoutindex,
// exit,
// )?;
self.vecs.height_to_height.compute_transform(
self.mut_vecs().height_to_height.compute_transform(
starting_indexes.height,
&mut indexer.vecs.height_to_timestamp,
&mut indexer.mut_vecs().height_to_timestamp,
|_, height| height,
exit,
)?;
self.vecs.height_to_date.compute_transform(
self.mut_vecs().height_to_date.compute_transform(
starting_indexes.height,
&mut indexer.vecs.height_to_timestamp,
&mut indexer.mut_vecs().height_to_timestamp,
|timestamp, _| Date::from(*timestamp),
exit,
)?;
// self.vecs.height_to_last_txindex.compute_last_index_from_first(
// starting_indexes.height,
// &mut indexer.vecs.height_to_first_txindex,
// &mut indexer.vecs().height_to_first_txindex,
// height_count,
// exit,
// )?;
// self.vecs.txindex_to_height.compute_inverse_less_to_more(
// starting_indexes.height,
// &mut indexer.vecs.height_to_first_txindex,
// &mut indexer.vecs().height_to_first_txindex,
// &mut self.vecs.height_to_last_txindex,
// exit,
// )?;
@@ -106,16 +112,16 @@ impl Computer {
// self.vecs.txindex_to_is_coinbase.compute_is_first_ordered(
// starting_indexes.txindex,
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs.height_to_first_txindex,
// &mut indexer.vecs().height_to_first_txindex,
// exit,
// )?;
// self.vecs.txindex_to_fee.compute_transform(
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs.height_to_first_txindex,
// &mut indexer.vecs().height_to_first_txindex,
// )?;
let date_count = self.vecs.height_to_date.len();
let date_count = self.vecs().height_to_date.len();
// self.vecs.height_to_dateindex.compute(...)
@@ -144,4 +150,20 @@ impl Computer {
pub fn path(&self) -> &Path {
&self.path
}
pub fn vecs(&self) -> &Vecs {
self.vecs.as_ref().unwrap()
}
pub fn mut_vecs(&mut self) -> &mut Vecs {
self.vecs.as_mut().unwrap()
}
pub fn stores(&self) -> &Stores {
self.stores.as_ref().unwrap()
}
pub fn mut_stores(&mut self) -> &mut Stores {
self.stores.as_mut().unwrap()
}
}
crates/brk_computer/src/main.rs
+9 -5
View File
@@ -14,20 +14,24 @@ pub fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let data_dir = Path::new("../../../bitcoin");
let bitcoin_dir = Path::new("../../../bitcoin");
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
rpc::Auth::CookieFile(Path::new(bitcoin_dir).join(".cookie")),
)?));
let exit = Exit::new();
let parser = Parser::new(data_dir, rpc);
let parser = Parser::new(bitcoin_dir, rpc);
let outputs_dir = Path::new("../../_outputs");
let mut indexer = Indexer::import(&outputs_dir.join("indexed"))?;
let mut indexer = Indexer::new(&outputs_dir.join("indexed"))?;
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::import(&outputs_dir.join("computed"))?;
let mut computer = Computer::new(&outputs_dir.join("computed"));
computer.import_stores()?;
computer.import_vecs()?;
loop {
let block_count = rpc.get_block_count()?;
crates/brk_indexer/src/lib.rs
+64 -32
View File
@@ -5,7 +5,7 @@
use std::{
collections::BTreeMap,
path::Path,
path::{Path, PathBuf},
str::FromStr,
thread::{self},
};
@@ -19,7 +19,7 @@ pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use color_eyre::eyre::{ContextCompat, eyre};
use log::{debug, info};
use log::info;
use rayon::prelude::*;
mod indexes;
mod stores;
@@ -33,37 +33,63 @@ const SNAPSHOT_BLOCK_RANGE: usize = 1000;
#[derive(Clone)]
pub struct Indexer {
pub vecs: Vecs,
pub stores: Stores,
path: PathBuf,
vecs: Option<Vecs>,
stores: Option<Stores>,
}
impl Indexer {
pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
pub fn new(indexes_dir: &Path) -> color_eyre::Result<Self> {
setrlimit()?;
Ok(Self {
path: indexes_dir.to_owned(),
vecs: None,
stores: None,
})
}
debug!("Importing indexes...");
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(&self.path.join("vecs"))?);
Ok(())
}
let vecs = Vecs::import(&indexes_dir.join("vecs"))?;
let stores = Stores::import(&indexes_dir.join("stores"))?;
Ok(Self { vecs, stores })
pub fn import_stores(&mut self) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(&self.path.join("stores"))?);
Ok(())
}
pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<Indexes> {
let check_collisions = true;
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| {
let indexes = Indexes::default();
indexes.push_if_needed(&mut self.vecs).unwrap();
indexes
});
let starting_indexes = Indexes::try_from((self.vecs.as_mut().unwrap(), self.stores.as_ref().unwrap(), rpc))
.unwrap_or_else(|_| {
let indexes = Indexes::default();
indexes.push_if_needed(self.vecs.as_mut().unwrap()).unwrap();
indexes
});
exit.block();
self.stores.rollback_if_needed(&self.vecs, &starting_indexes)?;
self.vecs.rollback_if_needed(&starting_indexes)?;
self.stores
.as_mut()
.unwrap()
.rollback_if_needed(self.vecs.as_ref().unwrap(), &starting_indexes)?;
self.vecs.as_mut().unwrap().rollback_if_needed(&starting_indexes)?;
exit.release();
let vecs = self.vecs.as_mut().unwrap();
let stores = self.stores.as_mut().unwrap();
let mut idxs = starting_indexes.clone();
let start = Some(idxs.height);
let end = None; //Some(Height::new(400_000));
if starting_indexes.height > Height::try_from(rpc)? || end.is_some_and(|end| starting_indexes.height > end) {
return Ok(starting_indexes);
}
info!("Started indexing...");
let export_if_needed =
|stores: &mut Stores, vecs: &mut Vecs, height: Height, rem: bool, exit: &Exit| -> color_eyre::Result<()> {
if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() {
@@ -78,20 +104,6 @@ impl Indexer {
Ok(())
};
let vecs = &mut self.vecs;
let stores = &mut self.stores;
let mut idxs = starting_indexes.clone();
let start = Some(idxs.height);
let end = None; //Some(Height::new(400_000));
if starting_indexes.height > Height::try_from(rpc)? || end.is_some_and(|end| starting_indexes.height > end) {
return Ok(starting_indexes);
}
info!("Started indexing...");
parser.parse(start, None).iter().try_for_each(
|(height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {height}...");
@@ -639,6 +651,26 @@ impl Indexer {
Ok(starting_indexes)
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn vecs(&self) -> &Vecs {
self.vecs.as_ref().unwrap()
}
pub fn mut_vecs(&mut self) -> &mut Vecs {
self.vecs.as_mut().unwrap()
}
pub fn stores(&self) -> &Stores {
self.stores.as_ref().unwrap()
}
pub fn mut_stores(&mut self) -> &mut Stores {
self.stores.as_mut().unwrap()
}
}
#[derive(Debug)]
crates/brk_indexer/src/main.rs
+7 -5
View File
@@ -13,22 +13,24 @@ fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let data_dir = Path::new("../../../bitcoin");
let bitcoin_dir = Path::new("../../../bitcoin");
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
rpc::Auth::CookieFile(Path::new(bitcoin_dir).join(".cookie")),
)?));
let exit = Exit::new();
let parser = Parser::new(data_dir, rpc);
let parser = Parser::new(bitcoin_dir, rpc);
let mut indexer = Indexer::new(Path::new("../../_outputs/indexed"))?;
indexer.import_stores()?;
indexer.import_vecs()?;
loop {
let block_count = rpc.get_block_count()?;
info!("{block_count} blocks found.");
let mut indexer = Indexer::import(Path::new("../../_outputs/indexed"))?;
indexer.index(&parser, rpc, &exit)?;
info!("Waiting for new blocks...");
crates/brk_indexer/src/stores/meta.rs
+3 -3
View File
@@ -27,16 +27,16 @@ impl StoreMeta {
Self::reset_(path)?;
}
let this = Self {
let slf = Self {
pathbuf: path.to_owned(),
version,
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
len: Self::read_length_(path)?,
};
this.version.write(&this.path_version())?;
slf.version.write(&slf.path_version())?;
Ok(this)
Ok(slf)
}
pub fn len(&self) -> usize {
crates/brk_parser/src/blk_index_to_blk_path.rs
+2 -2
View File
@@ -13,8 +13,8 @@ const DAT: &str = ".dat";
pub struct BlkIndexToBlkPath(BTreeMap<u16, PathBuf>);
impl BlkIndexToBlkPath {
pub fn scan(data_dir: &Path) -> Self {
let blocks_dir = data_dir.join("blocks");
pub fn scan(bitcoin_dir: &Path) -> Self {
let blocks_dir = bitcoin_dir.join("blocks");
Self(
fs::read_dir(blocks_dir)
crates/brk_parser/src/blk_index_to_blk_recap.rs
+2 -2
View File
@@ -14,8 +14,8 @@ pub struct BlkIndexToBlkRecap {
}
impl BlkIndexToBlkRecap {
pub fn import(data_dir: &Path, blk_index_to_blk_path: &BlkIndexToBlkPath, start: Option<Height>) -> (Self, u16) {
let path = data_dir.join("blk_index_to_blk_recap.json");
pub fn import(bitcoin_dir: &Path, blk_index_to_blk_path: &BlkIndexToBlkPath, start: Option<Height>) -> (Self, u16) {
let path = bitcoin_dir.join("blk_index_to_blk_recap.json");
let tree = {
if let Ok(file) = File::open(&path) {
crates/brk_parser/src/lib.rs
+7 -7
View File
@@ -47,14 +47,14 @@ const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 50;
pub struct Parser {
data_dir: PathBuf,
bitcoin_dir: PathBuf,
rpc: &'static bitcoincore_rpc::Client,
}
impl Parser {
pub fn new(data_dir: &Path, rpc: &'static bitcoincore_rpc::Client) -> Self {
pub fn new(bitcoin_dir: &Path, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self {
data_dir: data_dir.to_owned(),
bitcoin_dir: bitcoin_dir.to_owned(),
rpc,
}
}
@@ -69,19 +69,19 @@ impl Parser {
/// For an example checkout `./main.rs`
///
pub fn parse(&self, start: Option<Height>, end: Option<Height>) -> Receiver<(Height, Block, BlockHash)> {
let data_dir = self.data_dir.as_path();
let bitcoin_dir = self.bitcoin_dir.as_path();
let rpc = self.rpc;
let (send_bytes, recv_bytes) = bounded(BOUND_CAP);
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(data_dir);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(bitcoin_dir);
let (mut blk_index_to_blk_recap, blk_index) =
BlkIndexToBlkRecap::import(data_dir, &blk_index_to_blk_path, start);
BlkIndexToBlkRecap::import(bitcoin_dir, &blk_index_to_blk_path, start);
let xor_bytes = XORBytes::from(data_dir);
let xor_bytes = XORBytes::from(bitcoin_dir);
thread::spawn(move || {
let xor_bytes = xor_bytes;
crates/brk_parser/src/main.rs
+3 -3
View File
@@ -7,11 +7,11 @@ use brk_parser::Parser;
fn main() {
let i = std::time::Instant::now();
let data_dir = Path::new("../../../bitcoin");
let bitcoin_dir = Path::new("../../../bitcoin");
let rpc = Box::leak(Box::new(
Client::new(
"http://localhost:8332",
Auth::CookieFile(Path::new(data_dir).join(".cookie")),
Auth::CookieFile(Path::new(bitcoin_dir).join(".cookie")),
)
.unwrap(),
));
@@ -19,7 +19,7 @@ fn main() {
let start = None;
let end = None;
let parser = Parser::new(data_dir, rpc);
let parser = Parser::new(bitcoin_dir, rpc);
parser.parse(start, end).iter().for_each(|(height, _block, hash)| {
println!("{height}: {hash}");
crates/brk_query/src/lib.rs
+11 -2
View File
@@ -31,8 +31,17 @@ impl<'a> Query<'a> {
pub fn build(indexer: &'a Indexer, computer: &'a Computer) -> Self {
let mut vecs = VecIdToIndexToVec::default();
indexer.vecs.as_any_vecs().into_iter().for_each(|vec| vecs.insert(vec));
computer.vecs.as_any_vecs().into_iter().for_each(|vec| vecs.insert(vec));
indexer
.vecs()
.as_any_vecs()
.into_iter()
.for_each(|vec| vecs.insert(vec));
computer
.vecs()
.as_any_vecs()
.into_iter()
.for_each(|vec| vecs.insert(vec));
Self {
vecid_to_index_to_vec: vecs,
crates/brk_query/src/main.rs
+4 -2
View File
@@ -9,9 +9,11 @@ pub fn main() -> color_eyre::Result<()> {
let outputs_dir = Path::new("../../_outputs");
let indexer = Indexer::import(&outputs_dir.join("indexed"))?;
let mut indexer = Indexer::new(&outputs_dir.join("indexed"))?;
indexer.import_vecs()?;
let computer = Computer::import(&outputs_dir.join("computed"))?;
let mut computer = Computer::new(&outputs_dir.join("computed"));
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);
crates/brk_server/src/main.rs
+9 -5
View File
@@ -14,20 +14,24 @@ pub fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let data_dir = Path::new("../../../bitcoin");
let bitcoin_dir = Path::new("../../../bitcoin");
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
rpc::Auth::CookieFile(Path::new(bitcoin_dir).join(".cookie")),
)?));
let exit = Exit::new();
let parser = Parser::new(data_dir, rpc);
let parser = Parser::new(bitcoin_dir, rpc);
let outputs_dir = Path::new("../../_outputs");
let mut indexer = Indexer::import(&outputs_dir.join("indexed"))?;
let mut indexer = Indexer::new(&outputs_dir.join("indexed"))?;
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::import(&outputs_dir.join("computed"))?;
let mut computer = Computer::new(&outputs_dir.join("computed"));
computer.import_stores()?;
computer.import_vecs()?;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
publish.sh
+2
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
cargo check
cd crates/brk
cd ../brk_core