Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-13 07:18:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

vec: compression part 2 and done

Browse Source
This commit is contained in:
nym21
2025-03-14 16:00:47 +01:00
parent c459a3033d
commit a995eb2929
22 changed files with 799 additions and 524 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
crates/brk_indexer/examples/main.rs
View File

@@ -2,12 +2,11 @@ use std::path::Path;
use brk_core::{default_bitcoin_path, dot_brk_path};
use brk_exit::Exit;
use brk_indexer::{Indexer, rpc::RpcApi};
use brk_indexer::Indexer;
use brk_parser::{
Parser,
rpc::{self},
};
use log::info;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -26,24 +25,12 @@ fn main() -> color_eyre::Result<()> {
let outputs = dot_brk_path().join("outputs");
let mut indexer = Indexer::new(outputs.join("indexed").to_owned(), true)?;
let mut indexer = Indexer::new(outputs.join("indexed").to_owned(), true, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
// loop {
let block_count = rpc.get_block_count()?;
info!("{block_count} blocks found.");
indexer.index(&parser, rpc, &exit)?;
info!("Waiting for new blocks...");
// while block_count == rpc.get_block_count()? {
// sleep(Duration::from_secs(1))
// }
// }
#[allow(unreachable_code)]
Ok(())
}

12
crates/brk_indexer/src/lib.rs
View File

@@ -38,21 +38,27 @@ pub struct Indexer {
vecs: Option<Vecs>,
stores: Option<Stores>,
check_collisions: bool,
compressed: Compressed,
}
impl Indexer {
pub fn new(indexes_dir: PathBuf, check_collisions: bool) -> color_eyre::Result<Self> {
pub fn new(
indexes_dir: PathBuf,
compressed: bool,
check_collisions: bool,
) -> color_eyre::Result<Self> {
setrlimit()?;
Ok(Self {
path: indexes_dir,
vecs: None,
stores: None,
compressed: Compressed::from(compressed),
check_collisions,
})
}
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(&self.path.join("vecs"))?);
self.vecs = Some(Vecs::import(&self.path.join("vecs"), self.compressed)?);
Ok(())
}
@@ -131,7 +137,7 @@ impl Indexer {
idxs.height = height;
let check_collisions = self.check_collisions && height > Height::new(886_000);
let check_collisions = self.check_collisions && height > Height::new(200_000);
let blockhash = BlockHash::from(blockhash);
let blockhash_prefix = BlockHashPrefix::from(&blockhash);

5
crates/brk_indexer/src/vecs/base.rs
View File

@@ -23,7 +23,7 @@ where
pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result<Self> {
let mut vec = brk_vec::StorableVec::forced_import(path, version, compressed)?;
vec.init_big_cache()?;
vec.enable_large_cache();
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
@@ -51,8 +51,7 @@ where
pub fn flush(&mut self, height: Height) -> io::Result<()> {
height.write(&self.path_height())?;
self.vec.flush()?;
self.vec.init_big_cache()
self.vec.flush()
}
}

68
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -64,24 +64,24 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
addressindex_to_addresstype: StorableVec::import(
&path.join("addressindex_to_addresstype"),
Version::from(1),
Compressed::YES,
compressed,
)?,
addressindex_to_addresstypeindex: StorableVec::import(
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
addressindex_to_height: StorableVec::import(
&path.join("addressindex_to_height"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_blockhash: StorableVec::import(
&path.join("height_to_blockhash"),
@@ -91,102 +91,102 @@ impl Vecs {
height_to_difficulty: StorableVec::import(
&path.join("height_to_difficulty"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_addressindex: StorableVec::import(
&path.join("height_to_first_addressindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_emptyindex: StorableVec::import(
&path.join("height_to_first_emptyindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_multisigindex: StorableVec::import(
&path.join("height_to_first_multisigindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_opreturnindex: StorableVec::import(
&path.join("height_to_first_opreturnindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_pushonlyindex: StorableVec::import(
&path.join("height_to_first_pushonlyindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_txindex: StorableVec::import(
&path.join("height_to_first_txindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_txinindex: StorableVec::import(
&path.join("height_to_first_txinindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_txoutindex: StorableVec::import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_unknownindex: StorableVec::import(
&path.join("height_to_first_unkownindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2pk33index: StorableVec::import(
&path.join("height_to_first_p2pk33index"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2pk65index: StorableVec::import(
&path.join("height_to_first_p2pk65index"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2pkhindex: StorableVec::import(
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2shindex: StorableVec::import(
&path.join("height_to_first_p2shindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2trindex: StorableVec::import(
&path.join("height_to_first_p2trindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2wpkhindex: StorableVec::import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_first_p2wshindex: StorableVec::import(
&path.join("height_to_first_p2wshindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_size: StorableVec::import(
&path.join("height_to_size"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_timestamp: StorableVec::import(
&path.join("height_to_timestamp"),
Version::from(1),
Compressed::YES,
compressed,
)?,
height_to_weight: StorableVec::import(
&path.join("height_to_weight"),
Version::from(1),
Compressed::YES,
compressed,
)?,
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
@@ -226,7 +226,7 @@ impl Vecs {
txindex_to_first_txinindex: StorableVec::import(
&path.join("txindex_to_first_txinindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txindex_to_first_txoutindex: StorableVec::import(
&path.join("txindex_to_first_txoutindex"),
@@ -236,12 +236,12 @@ impl Vecs {
txindex_to_height: StorableVec::import(
&path.join("txindex_to_height"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txindex_to_locktime: StorableVec::import(
&path.join("txindex_to_locktime"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txindex_to_txid: StorableVec::import(
&path.join("txindex_to_txid"),
@@ -251,37 +251,37 @@ impl Vecs {
txindex_to_base_size: StorableVec::import(
&path.join("txindex_to_base_size"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txindex_to_total_size: StorableVec::import(
&path.join("txindex_to_total_size"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txindex_to_is_explicitly_rbf: StorableVec::import(
&path.join("txindex_to_is_explicitly_rbf"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txindex_to_txversion: StorableVec::import(
&path.join("txindex_to_txversion"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txinindex_to_txoutindex: StorableVec::import(
&path.join("txinindex_to_txoutindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txoutindex_to_addressindex: StorableVec::import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
Compressed::YES,
compressed,
)?,
txoutindex_to_value: StorableVec::import(
&path.join("txoutindex_to_value"),
Version::from(1),
Compressed::YES,
compressed,
)?,
})
}