Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-10 06:09:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

compression: added everywhere

Browse Source
This commit is contained in:
nym21
2025-03-14 18:10:03 +01:00
parent a995eb2929
commit 36d97ad5ca
12 changed files with 94 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
crates/brk_cli/src/query.rs
View File

@@ -8,14 +8,12 @@ use crate::run::RunConfig;
pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let config = RunConfig::import(None)?;
let mut indexer = Indexer::new(
config.indexeddir(),
config.compressed(),
config.check_collisions(),
)?;
let compressed = config.compressed();
let mut indexer = Indexer::new(config.indexeddir(), compressed, config.check_collisions())?;
indexer.import_vecs()?;
let mut computer = Computer::new(config.computeddir(), None);
let mut computer = Computer::new(config.computeddir(), None, compressed);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

10
crates/brk_cli/src/run.rs
View File

@@ -26,11 +26,9 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
let mut indexer = Indexer::new(
config.indexeddir(),
config.compressed(),
config.check_collisions(),
)?;
let compressed = config.compressed();
let mut indexer = Indexer::new(config.indexeddir(), compressed, config.check_collisions())?;
indexer.import_stores()?;
indexer.import_vecs()?;
@@ -38,7 +36,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
.fetch()
.then(|| Fetcher::import(Some(config.harsdir().as_path())).unwrap());
let mut computer = Computer::new(config.computeddir(), fetcher);
let mut computer = Computer::new(config.computeddir(), fetcher, compressed);
computer.import_stores()?;
computer.import_vecs()?;

6
crates/brk_computer/examples/main.rs
View File

@@ -28,13 +28,15 @@ pub fn main() -> color_eyre::Result<()> {
let outputs_dir = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs_dir.join("indexed"), true, true)?;
let compressed = true;
let mut indexer = Indexer::new(outputs_dir.join("indexed"), compressed, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
let fetcher = Fetcher::import(None)?;
let mut computer = Computer::new(outputs_dir.join("computed"), Some(fetcher));
let mut computer = Computer::new(outputs_dir.join("computed"), Some(fetcher), compressed);
computer.import_stores()?;
computer.import_vecs()?;

6
crates/brk_computer/src/lib.rs
View File

@@ -12,6 +12,7 @@ pub use brk_parser::rpc;
mod storage;
use brk_vec::Compressed;
use log::info;
use storage::{Stores, Vecs};
@@ -21,15 +22,17 @@ pub struct Computer {
fetcher: Option<Fetcher>,
vecs: Option<Vecs>,
stores: Option<Stores>,
compressed: Compressed,
}
impl Computer {
pub fn new(computed_dir: PathBuf, fetcher: Option<Fetcher>) -> Self {
pub fn new(computed_dir: PathBuf, fetcher: Option<Fetcher>, compressed: bool) -> Self {
Self {
path: computed_dir,
fetcher,
vecs: None,
stores: None,
compressed: Compressed::from(compressed),
}
}
@@ -37,6 +40,7 @@ impl Computer {
self.vecs = Some(Vecs::import(
&self.path.join("vecs"),
self.fetcher.is_some(),
self.compressed,
)?);
Ok(())
}

4
crates/brk_computer/src/storage/vecs/base.rs
View File

@@ -24,8 +24,8 @@ where
I: StoredIndex,
T: StoredType,
{
pub fn import(path: &Path, version: Version) -> brk_vec::Result<Self> {
let vec = brk_vec::StorableVec::forced_import(path, version, Compressed::YES)?;
pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result<Self> {
let vec = brk_vec::StorableVec::forced_import(path, version, compressed)?;
Ok(Self {
computed_version: None,

15
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, ops::Deref, path::Path};
use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Value, Version};
use brk_vec::{AnyStorableVec, Compressed, Value, Version};
use super::StorableVec;
@@ -25,54 +25,65 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_date: StorableVec::import(
&path.join("dateindex_to_date"),
Version::from(1),
compressed,
)?,
dateindex_to_dateindex: StorableVec::import(
&path.join("dateindex_to_dateindex"),
Version::from(1),
compressed,
)?,
dateindex_to_first_height: StorableVec::import(
&path.join("dateindex_to_first_height"),
Version::from(1),
compressed,
)?,
dateindex_to_last_height: StorableVec::import(
&path.join("dateindex_to_last_height"),
Version::from(1),
compressed,
)?,
height_to_real_date: StorableVec::import(
&path.join("height_to_real_date"),
Version::from(1),
compressed,
)?,
height_to_fixed_date: StorableVec::import(
&path.join("height_to_fixed_date"),
Version::from(1),
compressed,
)?,
height_to_dateindex: StorableVec::import(
&path.join("height_to_dateindex"),
Version::from(1),
compressed,
)?,
height_to_height: StorableVec::import(
&path.join("height_to_height"),
Version::from(1),
compressed,
)?,
height_to_last_txindex: StorableVec::import(
&path.join("height_to_last_txindex"),
Version::from(1),
compressed,
)?,
txindex_to_last_txinindex: StorableVec::import(
&path.join("txindex_to_last_txinindex"),
Version::from(1),
compressed,
)?,
txindex_to_last_txoutindex: StorableVec::import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
compressed,
)?,
})
}

51
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -6,7 +6,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Value, Version};
use brk_vec::{AnyStorableVec, Compressed, Value, Version};
use super::{Indexes, StorableVec, indexes};
@@ -37,82 +37,119 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_ohlc_in_cents: StorableVec::import(
&path.join("dateindex_to_ohlc_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_ohlc: StorableVec::import(
&path.join("dateindex_to_ohlc"),
Version::from(1),
compressed,
)?,
dateindex_to_close_in_cents: StorableVec::import(
&path.join("dateindex_to_close_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_close: StorableVec::import(
&path.join("dateindex_to_close"),
Version::from(1),
compressed,
)?,
dateindex_to_high_in_cents: StorableVec::import(
&path.join("dateindex_to_high_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_high: StorableVec::import(
&path.join("dateindex_to_high"),
Version::from(1),
compressed,
)?,
dateindex_to_low_in_cents: StorableVec::import(
&path.join("dateindex_to_low_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_low: StorableVec::import(
&path.join("dateindex_to_low"),
Version::from(1),
compressed,
)?,
dateindex_to_open_in_cents: StorableVec::import(
&path.join("dateindex_to_open_in_cents"),
Version::from(1),
compressed,
)?,
dateindex_to_open: StorableVec::import(
&path.join("dateindex_to_open"),
Version::from(1),
compressed,
)?,
dateindex_to_sats_per_dollar: StorableVec::import(
&path.join("dateindex_to_sats_per_dollar"),
Version::from(1),
compressed,
)?,
height_to_ohlc_in_cents: StorableVec::import(
&path.join("height_to_ohlc_in_cents"),
Version::from(1),
compressed,
)?,
height_to_ohlc: StorableVec::import(
&path.join("height_to_ohlc"),
Version::from(1),
compressed,
)?,
height_to_ohlc: StorableVec::import(&path.join("height_to_ohlc"), Version::from(1))?,
height_to_close_in_cents: StorableVec::import(
&path.join("height_to_close_in_cents"),
Version::from(1),
compressed,
)?,
height_to_close: StorableVec::import(
&path.join("height_to_close"),
Version::from(1),
compressed,
)?,
height_to_close: StorableVec::import(&path.join("height_to_close"), Version::from(1))?,
height_to_high_in_cents: StorableVec::import(
&path.join("height_to_high_in_cents"),
Version::from(1),
compressed,
)?,
height_to_high: StorableVec::import(
&path.join("height_to_high"),
Version::from(1),
compressed,
)?,
height_to_high: StorableVec::import(&path.join("height_to_high"), Version::from(1))?,
height_to_low_in_cents: StorableVec::import(
&path.join("height_to_low_in_cents"),
Version::from(1),
compressed,
)?,
height_to_low: StorableVec::import(
&path.join("height_to_low"),
Version::from(1),
compressed,
)?,
height_to_low: StorableVec::import(&path.join("height_to_low"), Version::from(1))?,
height_to_open_in_cents: StorableVec::import(
&path.join("height_to_open_in_cents"),
Version::from(1),
compressed,
)?,
height_to_open: StorableVec::import(
&path.join("height_to_open"),
Version::from(1),
compressed,
)?,
height_to_open: StorableVec::import(&path.join("height_to_open"), Version::from(1))?,
height_to_sats_per_dollar: StorableVec::import(
&path.join("height_to_sats_per_dollar"),
Version::from(1),
compressed,
)?,
})
}

10
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, path::Path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::AnyStorableVec;
use brk_vec::{AnyStorableVec, Compressed};
mod base;
mod indexes;
@@ -21,13 +21,13 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path, fetch: bool) -> color_eyre::Result<Self> {
pub fn import(path: &Path, fetch: bool, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
indexes: indexes::Vecs::import(path)?,
transactions: transactions::Vecs::import(path)?,
marketprice: fetch.then(|| marketprice::Vecs::import(path).unwrap()),
indexes: indexes::Vecs::import(path, compressed)?,
transactions: transactions::Vecs::import(path, compressed)?,
marketprice: fetch.then(|| marketprice::Vecs::import(path, compressed).unwrap()),
})
}

5
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, path::Path};
use brk_core::Txindex;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Version};
use brk_vec::{AnyStorableVec, Compressed, Version};
use super::{Indexes, StorableVec, indexes};
@@ -29,7 +29,7 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -50,6 +50,7 @@ impl Vecs {
txindex_to_is_coinbase: StorableVec::import(
&path.join("txindex_to_is_coinbase"),
Version::from(1),
compressed,
)?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
// txindex_to_inputs_count: StorableVec::forced_import(

3
crates/brk_indexer/src/lib.rs
View File

@@ -137,7 +137,8 @@ impl Indexer {
idxs.height = height;
let check_collisions = self.check_collisions && height > Height::new(200_000);
// Used to check rapidhash collisions
let check_collisions = self.check_collisions && height > Height::new(500_000);
let blockhash = BlockHash::from(blockhash);
let blockhash_prefix = BlockHashPrefix::from(&blockhash);

6
crates/brk_query/examples/main.rs
View File

@@ -9,10 +9,12 @@ pub fn main() -> color_eyre::Result<()> {
let outputs_dir = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs_dir.join("indexed"), true, true)?;
let compressed = true;
let mut indexer = Indexer::new(outputs_dir.join("indexed"), compressed, true)?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"), None);
let mut computer = Computer::new(outputs_dir.join("computed"), None, compressed);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

6
crates/brk_server/examples/main.rs
View File

@@ -31,13 +31,15 @@ pub fn main() -> color_eyre::Result<()> {
let outputs_dir = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs_dir.join("indexed"), true, true)?;
let compressed = true;
let mut indexer = Indexer::new(outputs_dir.join("indexed"), compressed, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
let fetcher = Some(Fetcher::import(None)?);
let mut computer = Computer::new(outputs_dir.join("computed"), fetcher);
let mut computer = Computer::new(outputs_dir.join("computed"), fetcher, compressed);
computer.import_stores()?;
computer.import_vecs()?;