diff --git a/crates/brk_cli/src/query.rs b/crates/brk_cli/src/query.rs index 326c992c2..68e0ca09f 100644 --- a/crates/brk_cli/src/query.rs +++ b/crates/brk_cli/src/query.rs @@ -8,14 +8,12 @@ use crate::run::RunConfig; pub fn query(params: QueryParams) -> color_eyre::Result<()> { let config = RunConfig::import(None)?; - let mut indexer = Indexer::new( - config.indexeddir(), - config.compressed(), - config.check_collisions(), - )?; + let compressed = config.compressed(); + + let mut indexer = Indexer::new(config.indexeddir(), compressed, config.check_collisions())?; indexer.import_vecs()?; - let mut computer = Computer::new(config.computeddir(), None); + let mut computer = Computer::new(config.computeddir(), None, compressed); computer.import_vecs()?; let query = Query::build(&indexer, &computer); diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index a850caa7c..599fe86ae 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -26,11 +26,9 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { let parser = brk_parser::Parser::new(config.blocksdir(), rpc); - let mut indexer = Indexer::new( - config.indexeddir(), - config.compressed(), - config.check_collisions(), - )?; + let compressed = config.compressed(); + + let mut indexer = Indexer::new(config.indexeddir(), compressed, config.check_collisions())?; indexer.import_stores()?; indexer.import_vecs()?; @@ -38,7 +36,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { .fetch() .then(|| Fetcher::import(Some(config.harsdir().as_path())).unwrap()); - let mut computer = Computer::new(config.computeddir(), fetcher); + let mut computer = Computer::new(config.computeddir(), fetcher, compressed); computer.import_stores()?; computer.import_vecs()?; diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index b37b9241c..d267298d0 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -28,13 +28,15 @@ pub fn main() -> color_eyre::Result<()> { let outputs_dir = Path::new("../../_outputs"); - let mut indexer = Indexer::new(outputs_dir.join("indexed"), true, true)?; + let compressed = true; + + let mut indexer = Indexer::new(outputs_dir.join("indexed"), compressed, true)?; indexer.import_stores()?; indexer.import_vecs()?; let fetcher = Fetcher::import(None)?; - let mut computer = Computer::new(outputs_dir.join("computed"), Some(fetcher)); + let mut computer = Computer::new(outputs_dir.join("computed"), Some(fetcher), compressed); computer.import_stores()?; computer.import_vecs()?; diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index f4ef3dea3..cb01b03b5 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -12,6 +12,7 @@ pub use brk_parser::rpc; mod storage; +use brk_vec::Compressed; use log::info; use storage::{Stores, Vecs}; @@ -21,15 +22,17 @@ pub struct Computer { fetcher: Option, vecs: Option, stores: Option, + compressed: Compressed, } impl Computer { - pub fn new(computed_dir: PathBuf, fetcher: Option) -> Self { + pub fn new(computed_dir: PathBuf, fetcher: Option, compressed: bool) -> Self { Self { path: computed_dir, fetcher, vecs: None, stores: None, + compressed: Compressed::from(compressed), } } @@ -37,6 +40,7 @@ impl Computer { self.vecs = Some(Vecs::import( &self.path.join("vecs"), self.fetcher.is_some(), + self.compressed, )?); Ok(()) } diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index 46e407576..d95b7118e 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -24,8 +24,8 @@ where I: StoredIndex, T: StoredType, { - pub fn import(path: &Path, version: Version) -> brk_vec::Result { - let vec = brk_vec::StorableVec::forced_import(path, version, Compressed::YES)?; + pub fn import(path: &Path, version: Version, compressed: Compressed) -> brk_vec::Result { + let vec = brk_vec::StorableVec::forced_import(path, version, compressed)?; Ok(Self { computed_version: None, diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index c201bd48b..964422640 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -3,7 +3,7 @@ use std::{fs, ops::Deref, path::Path}; use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Value, Version}; +use brk_vec::{AnyStorableVec, Compressed, Value, Version}; use super::StorableVec; @@ -25,54 +25,65 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path) -> color_eyre::Result { + pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { dateindex_to_date: StorableVec::import( &path.join("dateindex_to_date"), Version::from(1), + compressed, )?, dateindex_to_dateindex: StorableVec::import( &path.join("dateindex_to_dateindex"), Version::from(1), + compressed, )?, dateindex_to_first_height: StorableVec::import( &path.join("dateindex_to_first_height"), Version::from(1), + compressed, )?, dateindex_to_last_height: StorableVec::import( &path.join("dateindex_to_last_height"), Version::from(1), + compressed, )?, height_to_real_date: StorableVec::import( &path.join("height_to_real_date"), Version::from(1), + compressed, )?, height_to_fixed_date: StorableVec::import( &path.join("height_to_fixed_date"), Version::from(1), + compressed, )?, height_to_dateindex: StorableVec::import( &path.join("height_to_dateindex"), Version::from(1), + compressed, )?, height_to_height: StorableVec::import( &path.join("height_to_height"), Version::from(1), + compressed, )?, height_to_last_txindex: StorableVec::import( &path.join("height_to_last_txindex"), Version::from(1), + compressed, )?, txindex_to_last_txinindex: StorableVec::import( &path.join("txindex_to_last_txinindex"), Version::from(1), + compressed, )?, txindex_to_last_txoutindex: StorableVec::import( &path.join("txindex_to_last_txoutindex"), Version::from(1), + compressed, )?, }) } diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index f24e0c082..9dc7bc4f4 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -6,7 +6,7 @@ use brk_core::{ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Value, Version}; +use brk_vec::{AnyStorableVec, Compressed, Value, Version}; use super::{Indexes, StorableVec, indexes}; @@ -37,82 +37,119 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path) -> color_eyre::Result { + pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { dateindex_to_ohlc_in_cents: StorableVec::import( &path.join("dateindex_to_ohlc_in_cents"), Version::from(1), + compressed, )?, dateindex_to_ohlc: StorableVec::import( &path.join("dateindex_to_ohlc"), Version::from(1), + compressed, )?, dateindex_to_close_in_cents: StorableVec::import( &path.join("dateindex_to_close_in_cents"), Version::from(1), + compressed, )?, dateindex_to_close: StorableVec::import( &path.join("dateindex_to_close"), Version::from(1), + compressed, )?, dateindex_to_high_in_cents: StorableVec::import( &path.join("dateindex_to_high_in_cents"), Version::from(1), + compressed, )?, dateindex_to_high: StorableVec::import( &path.join("dateindex_to_high"), Version::from(1), + compressed, )?, dateindex_to_low_in_cents: StorableVec::import( &path.join("dateindex_to_low_in_cents"), Version::from(1), + compressed, )?, dateindex_to_low: StorableVec::import( &path.join("dateindex_to_low"), Version::from(1), + compressed, )?, dateindex_to_open_in_cents: StorableVec::import( &path.join("dateindex_to_open_in_cents"), Version::from(1), + compressed, )?, dateindex_to_open: StorableVec::import( &path.join("dateindex_to_open"), Version::from(1), + compressed, )?, dateindex_to_sats_per_dollar: StorableVec::import( &path.join("dateindex_to_sats_per_dollar"), Version::from(1), + compressed, )?, height_to_ohlc_in_cents: StorableVec::import( &path.join("height_to_ohlc_in_cents"), Version::from(1), + compressed, + )?, + height_to_ohlc: StorableVec::import( + &path.join("height_to_ohlc"), + Version::from(1), + compressed, )?, - height_to_ohlc: StorableVec::import(&path.join("height_to_ohlc"), Version::from(1))?, height_to_close_in_cents: StorableVec::import( &path.join("height_to_close_in_cents"), Version::from(1), + compressed, + )?, + height_to_close: StorableVec::import( + &path.join("height_to_close"), + Version::from(1), + compressed, )?, - height_to_close: StorableVec::import(&path.join("height_to_close"), Version::from(1))?, height_to_high_in_cents: StorableVec::import( &path.join("height_to_high_in_cents"), Version::from(1), + compressed, + )?, + height_to_high: StorableVec::import( + &path.join("height_to_high"), + Version::from(1), + compressed, )?, - height_to_high: StorableVec::import(&path.join("height_to_high"), Version::from(1))?, height_to_low_in_cents: StorableVec::import( &path.join("height_to_low_in_cents"), Version::from(1), + compressed, + )?, + height_to_low: StorableVec::import( + &path.join("height_to_low"), + Version::from(1), + compressed, )?, - height_to_low: StorableVec::import(&path.join("height_to_low"), Version::from(1))?, height_to_open_in_cents: StorableVec::import( &path.join("height_to_open_in_cents"), Version::from(1), + compressed, + )?, + height_to_open: StorableVec::import( + &path.join("height_to_open"), + Version::from(1), + compressed, )?, - height_to_open: StorableVec::import(&path.join("height_to_open"), Version::from(1))?, height_to_sats_per_dollar: StorableVec::import( &path.join("height_to_sats_per_dollar"), Version::from(1), + compressed, )?, }) } diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index d777eaaa8..393f33bdc 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -3,7 +3,7 @@ use std::{fs, path::Path}; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::AnyStorableVec; +use brk_vec::{AnyStorableVec, Compressed}; mod base; mod indexes; @@ -21,13 +21,13 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path, fetch: bool) -> color_eyre::Result { + pub fn import(path: &Path, fetch: bool, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { - indexes: indexes::Vecs::import(path)?, - transactions: transactions::Vecs::import(path)?, - marketprice: fetch.then(|| marketprice::Vecs::import(path).unwrap()), + indexes: indexes::Vecs::import(path, compressed)?, + transactions: transactions::Vecs::import(path, compressed)?, + marketprice: fetch.then(|| marketprice::Vecs::import(path, compressed).unwrap()), }) } diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 84177c347..eab1b847c 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -3,7 +3,7 @@ use std::{fs, path::Path}; use brk_core::Txindex; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Version}; +use brk_vec::{AnyStorableVec, Compressed, Version}; use super::{Indexes, StorableVec, indexes}; @@ -29,7 +29,7 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path) -> color_eyre::Result { + pub fn import(path: &Path, compressed: Compressed) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { @@ -50,6 +50,7 @@ impl Vecs { txindex_to_is_coinbase: StorableVec::import( &path.join("txindex_to_is_coinbase"), Version::from(1), + compressed, )?, // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?, // txindex_to_inputs_count: StorableVec::forced_import( diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index df4bc143d..e89c887bb 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -137,7 +137,8 @@ impl Indexer { idxs.height = height; - let check_collisions = self.check_collisions && height > Height::new(200_000); + // Used to check rapidhash collisions + let check_collisions = self.check_collisions && height > Height::new(500_000); let blockhash = BlockHash::from(blockhash); let blockhash_prefix = BlockHashPrefix::from(&blockhash); diff --git a/crates/brk_query/examples/main.rs b/crates/brk_query/examples/main.rs index 81f22a45b..7f8c5b42c 100644 --- a/crates/brk_query/examples/main.rs +++ b/crates/brk_query/examples/main.rs @@ -9,10 +9,12 @@ pub fn main() -> color_eyre::Result<()> { let outputs_dir = Path::new("../../_outputs"); - let mut indexer = Indexer::new(outputs_dir.join("indexed"), true, true)?; + let compressed = true; + + let mut indexer = Indexer::new(outputs_dir.join("indexed"), compressed, true)?; indexer.import_vecs()?; - let mut computer = Computer::new(outputs_dir.join("computed"), None); + let mut computer = Computer::new(outputs_dir.join("computed"), None, compressed); computer.import_vecs()?; let query = Query::build(&indexer, &computer); diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 842f53865..3a72ec3e8 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -31,13 +31,15 @@ pub fn main() -> color_eyre::Result<()> { let outputs_dir = Path::new("../../_outputs"); - let mut indexer = Indexer::new(outputs_dir.join("indexed"), true, true)?; + let compressed = true; + + let mut indexer = Indexer::new(outputs_dir.join("indexed"), compressed, true)?; indexer.import_stores()?; indexer.import_vecs()?; let fetcher = Some(Fetcher::import(None)?); - let mut computer = Computer::new(outputs_dir.join("computed"), fetcher); + let mut computer = Computer::new(outputs_dir.join("computed"), fetcher, compressed); computer.import_stores()?; computer.import_vecs()?;