diff --git a/Cargo.lock b/Cargo.lock index 379ff27b0..4fabac1e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -519,7 +519,7 @@ dependencies = [ "brk_logger", "brk_parser", "brk_server", - "brk_vec", + "brk_vecs", "clap", "clap_derive", "color-eyre", @@ -543,7 +543,7 @@ dependencies = [ "brk_logger", "brk_parser", "brk_store", - "brk_vec", + "brk_vecs", "color-eyre", "derive_deref", "fjall", @@ -624,7 +624,7 @@ dependencies = [ "brk_computer", "brk_core", "brk_indexer", - "brk_vec", + "brk_vecs", "color-eyre", "derive_deref", "rmcp", @@ -997,7 +997,7 @@ dependencies = [ "brk_logger", "brk_mcp", "brk_parser", - "brk_vec", + "brk_vecs", "clap", "clap_derive", "color-eyre", @@ -1057,9 +1057,10 @@ dependencies = [ name = "brk_vecs" version = "0.0.81" dependencies = [ - "bincode", "brk_core", "brk_exit", + "clap", + "clap_derive", "libc", "log", "memmap2", diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index c3b74a84e..3f2b959e6 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -17,7 +17,7 @@ brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_server = { workspace = true } -brk_vec = { workspace = true } +brk_vecs = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } color-eyre = { workspace = true } diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index d6190d2e9..7569a3557 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -7,13 +7,12 @@ use bitcoincore_rpc::{self, Auth, Client}; use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path}; use brk_fetcher::Fetcher; use brk_server::Website; -use brk_vec::{Computation, Format}; +use brk_vecs::{Computation, Format}; use clap::Parser; use clap_derive::Parser; use color_eyre::eyre::eyre; use serde::{Deserialize, Serialize}; - #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] #[command(version, about)] pub struct Config { @@ -32,7 +31,6 @@ pub struct Config { #[arg(long, value_name = "PATH")] brkdir: Option, - /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved #[serde(default, deserialize_with = "default_on_error")] #[arg(short, long)] @@ -124,7 +122,6 @@ impl Config { config_saved.brkdir = Some(brkdir); } - if let Some(computation) = config_args.computation.take() { config_saved.computation = Some(computation); } @@ -298,7 +295,6 @@ impl Config { self.outputsdir().join("hars") } - fn path_cookiefile(&self) -> PathBuf { self.rpccookiefile.as_ref().map_or_else( || self.bitcoindir().join(".cookie"), diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 6dbc6de4f..1b91586c0 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -18,7 +18,7 @@ brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_store = { workspace = true } -brk_vec = { workspace = true } +brk_vecs = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } fjall = { workspace = true } diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index e4551a86a..f0140d70a 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_vec::{Computation, Format}; +use brk_vecs::{Computation, Format}; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_computer/src/all.rs b/crates/brk_computer/src/all.rs index 902614870..d6e003cd7 100644 --- a/crates/brk_computer/src/all.rs +++ b/crates/brk_computer/src/all.rs @@ -1,10 +1,10 @@ -use std::path::Path; +use std::{path::Path, sync::Arc}; use brk_core::Version; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, Computation, Format}; +use brk_vecs::{AnyCollectableVec, Computation, File, Format}; use log::info; use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions}; @@ -27,16 +27,19 @@ pub struct Vecs { } impl Vecs { + #[allow(clippy::too_many_arguments)] pub fn import( - path: &Path, + file: &Arc, version: Version, indexer: &Indexer, fetch: bool, computation: Computation, format: Format, + fetched_file: &Arc, + states_path: &Path, ) -> color_eyre::Result { let indexes = indexes::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, indexer, computation, @@ -45,7 +48,8 @@ impl Vecs { let fetched = fetch.then(|| { fetched::Vecs::forced_import( - path, + file, + fetched_file, version + VERSION + Version::ZERO, computation, format, @@ -56,43 +60,44 @@ impl Vecs { Ok(Self { blocks: blocks::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, computation, format, &indexes, )?, mining: mining::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, computation, format, &indexes, )?, constants: constants::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, computation, format, &indexes, )?, market: market::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, computation, format, &indexes, )?, stateful: stateful::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, computation, format, &indexes, fetched.as_ref(), + states_path, )?, transactions: transactions::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, indexer, &indexes, @@ -101,7 +106,7 @@ impl Vecs { fetched.as_ref(), )?, cointime: cointime::Vecs::forced_import( - path, + file, version + VERSION + Version::ZERO, computation, format, diff --git a/crates/brk_computer/src/blocks.rs b/crates/brk_computer/src/blocks.rs index 26eb86b6d..f2c8b1074 100644 --- a/crates/brk_computer/src/blocks.rs +++ b/crates/brk_computer/src/blocks.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize, @@ -6,7 +6,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format}; +use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, File, Format}; use crate::grouped::Source; @@ -34,7 +34,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, computation: Computation, format: Format, @@ -42,13 +42,13 @@ impl Vecs { ) -> color_eyre::Result { Ok(Self { height_to_interval: EagerVec::forced_import( - path, + file, "interval", version + VERSION + Version::ZERO, format, )?, timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( - path, + file, "timestamp", Source::Compute, version + VERSION + Version::ZERO, @@ -58,7 +58,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, indexes_to_block_interval: ComputedVecsFromHeight::forced_import( - path, + file, "block_interval", Source::None, version + VERSION + Version::ZERO, @@ -71,7 +71,7 @@ impl Vecs { .add_average(), )?, indexes_to_block_count: ComputedVecsFromHeight::forced_import( - path, + file, "block_count", Source::Compute, version + VERSION + Version::ZERO, @@ -81,7 +81,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_weight: ComputedVecsFromHeight::forced_import( - path, + file, "block_weight", Source::None, version + VERSION + Version::ZERO, @@ -91,7 +91,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_size: ComputedVecsFromHeight::forced_import( - path, + file, "block_size", Source::None, version + VERSION + Version::ZERO, @@ -101,13 +101,13 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, height_to_vbytes: EagerVec::forced_import( - path, + file, "vbytes", version + VERSION + Version::ZERO, format, )?, indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( - path, + file, "block_vbytes", Source::None, version + VERSION + Version::ZERO, @@ -117,13 +117,13 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, difficultyepoch_to_timestamp: EagerVec::forced_import( - path, + file, "timestamp", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_timestamp: EagerVec::forced_import( - path, + file, "timestamp", version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index c0eb5d8e8..bf8326322 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; +use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator}; use super::{ Indexes, fetched, @@ -46,7 +46,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, computation: Computation, format: Format, @@ -57,7 +57,7 @@ impl Vecs { Ok(Self { indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import( - path, + file, "coinblocks_created", Source::Compute, version + VERSION + Version::ZERO, @@ -67,7 +67,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import( - path, + file, "coinblocks_stored", Source::Compute, version + VERSION + Version::ZERO, @@ -77,7 +77,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_liveliness: ComputedVecsFromHeight::forced_import( - path, + file, "liveliness", Source::Compute, version + VERSION + Version::ZERO, @@ -87,7 +87,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaultedness: ComputedVecsFromHeight::forced_import( - path, + file, "vaultedness", Source::Compute, version + VERSION + Version::ZERO, @@ -97,7 +97,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import( - path, + file, "activity_to_vaultedness_ratio", Source::Compute, version + VERSION + Version::ZERO, @@ -107,7 +107,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import( - path, + file, "vaulted_supply", Source::Compute, version + VERSION + Version::ONE, @@ -118,7 +118,7 @@ impl Vecs { indexes, )?, indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import( - path, + file, "active_supply", Source::Compute, version + VERSION + Version::ONE, @@ -129,7 +129,7 @@ impl Vecs { indexes, )?, indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import( - path, + file, "thermo_cap", Source::Compute, version + VERSION + Version::ONE, @@ -139,7 +139,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_investor_cap: ComputedVecsFromHeight::forced_import( - path, + file, "investor_cap", Source::Compute, version + VERSION + Version::ONE, @@ -149,7 +149,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import( - path, + file, "vaulted_cap", Source::Compute, version + VERSION + Version::ONE, @@ -159,7 +159,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_active_cap: ComputedVecsFromHeight::forced_import( - path, + file, "active_cap", Source::Compute, version + VERSION + Version::ONE, @@ -169,7 +169,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import( - path, + file, "vaulted_price", Source::Compute, version + VERSION + Version::ZERO, @@ -179,7 +179,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "vaulted_price", Source::None, version + VERSION + Version::ZERO, @@ -189,7 +189,7 @@ impl Vecs { true, )?, indexes_to_active_price: ComputedVecsFromHeight::forced_import( - path, + file, "active_price", Source::Compute, version + VERSION + Version::ZERO, @@ -199,7 +199,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "active_price", Source::None, version + VERSION + Version::ZERO, @@ -209,7 +209,7 @@ impl Vecs { true, )?, indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import( - path, + file, "true_market_mean", Source::Compute, version + VERSION + Version::ZERO, @@ -219,7 +219,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "true_market_mean", Source::None, version + VERSION + Version::ZERO, @@ -229,7 +229,7 @@ impl Vecs { true, )?, indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import( - path, + file, "cointime_value_destroyed", Source::Compute, version + VERSION + Version::ZERO, @@ -239,7 +239,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import( - path, + file, "cointime_value_created", Source::Compute, version + VERSION + Version::ZERO, @@ -249,7 +249,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import( - path, + file, "cointime_value_stored", Source::Compute, version + VERSION + Version::ZERO, @@ -259,7 +259,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_price: ComputedVecsFromHeight::forced_import( - path, + file, "cointime_price", Source::Compute, version + VERSION + Version::ZERO, @@ -269,7 +269,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import( - path, + file, "cointime_cap", Source::Compute, version + VERSION + Version::ZERO, @@ -279,7 +279,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "cointime_price", Source::None, version + VERSION + Version::ZERO, diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index 2b6c0b8fb..769e4da8f 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{StoredU8, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format}; +use brk_vecs::{AnyCollectableVec, AnyVec, Computation, File, Format}; use crate::grouped::Source; @@ -25,7 +25,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, computation: Computation, format: Format, @@ -33,7 +33,7 @@ impl Vecs { ) -> color_eyre::Result { Ok(Self { constant_0: ComputedVecsFromHeight::forced_import( - path, + file, "constant_0", Source::Compute, version + VERSION + Version::ZERO, @@ -43,7 +43,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_1: ComputedVecsFromHeight::forced_import( - path, + file, "constant_1", Source::Compute, version + VERSION + Version::ZERO, @@ -53,7 +53,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_50: ComputedVecsFromHeight::forced_import( - path, + file, "constant_50", Source::Compute, version + VERSION + Version::ZERO, @@ -63,7 +63,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_100: ComputedVecsFromHeight::forced_import( - path, + file, "constant_100", Source::Compute, version + VERSION + Version::ZERO, diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index 8b7ea7b15..068349c97 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, @@ -8,8 +8,8 @@ use brk_core::{ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex, +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, StoredIndex, VecIterator, }; @@ -76,103 +76,100 @@ const VERSION_IN_SATS: Version = Version::ZERO; impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, + fetched_file: &Arc, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, ) -> color_eyre::Result { - let mut fetched_path = path.to_owned(); - fetched_path.pop(); - fetched_path = fetched_path.join("fetched"); - Ok(Self { dateindex_to_ohlc_in_cents: EagerVec::forced_import( - &fetched_path, + fetched_file, "ohlc_in_cents", version + Version::ZERO, format, )?, dateindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, dateindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, dateindex_to_close_in_cents: EagerVec::forced_import( - path, + file, "close_in_cents", version + VERSION + Version::ZERO, format, )?, dateindex_to_high_in_cents: EagerVec::forced_import( - path, + file, "high_in_cents", version + VERSION + Version::ZERO, format, )?, dateindex_to_low_in_cents: EagerVec::forced_import( - path, + file, "low_in_cents", version + VERSION + Version::ZERO, format, )?, dateindex_to_open_in_cents: EagerVec::forced_import( - path, + file, "open_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_ohlc_in_cents: EagerVec::forced_import( - &fetched_path, + fetched_file, "ohlc_in_cents", version + Version::ZERO, format, )?, height_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, height_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, height_to_close_in_cents: EagerVec::forced_import( - path, + file, "close_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_high_in_cents: EagerVec::forced_import( - path, + file, "high_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_low_in_cents: EagerVec::forced_import( - path, + file, "low_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_open_in_cents: EagerVec::forced_import( - path, + file, "open_in_cents", version + VERSION + Version::ZERO, format, )?, timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( - path, + file, "open", Source::Compute, version + VERSION + Version::ZERO, @@ -182,7 +179,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, timeindexes_to_high: ComputedVecsFromDateIndex::forced_import( - path, + file, "high", Source::Compute, version + VERSION + Version::ZERO, @@ -192,7 +189,7 @@ impl Vecs { VecBuilderOptions::default().add_max(), )?, timeindexes_to_low: ComputedVecsFromDateIndex::forced_import( - path, + file, "low", Source::Compute, version + VERSION + Version::ZERO, @@ -202,7 +199,7 @@ impl Vecs { VecBuilderOptions::default().add_min(), )?, timeindexes_to_close: ComputedVecsFromDateIndex::forced_import( - path, + file, "close", Source::Compute, version + VERSION + Version::ZERO, @@ -212,7 +209,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import( - path, + file, "open_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -222,7 +219,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import( - path, + file, "high_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -232,7 +229,7 @@ impl Vecs { VecBuilderOptions::default().add_max(), )?, timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import( - path, + file, "low_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -242,7 +239,7 @@ impl Vecs { VecBuilderOptions::default().add_min(), )?, timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import( - path, + file, "close_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -252,143 +249,143 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( - path, + file, "open", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_first(), )?, chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( - path, + file, "high", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_max(), )?, chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( - path, + file, "low", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_min(), )?, chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( - path, + file, "close", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_last(), )?, chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import( - path, + file, "open_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_first(), )?, chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import( - path, + file, "high_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_max(), )?, chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import( - path, + file, "low_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_min(), )?, chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import( - path, + file, "close_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_last(), )?, weekindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, weekindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, difficultyepoch_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, monthindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, monthindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, quarterindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, quarterindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, semesterindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, semesterindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, yearindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, yearindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, )?, - // halvingepoch_to_ohlc: StorableVec::forced_import(path, + // halvingepoch_to_ohlc: StorableVec::forced_import(file, // "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?, decadeindex_to_ohlc: EagerVec::forced_import( - path, + file, "ohlc", version + VERSION + Version::ZERO, format, )?, decadeindex_to_ohlc_in_sats: EagerVec::forced_import( - path, + file, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/builder_computed.rs b/crates/brk_computer/src/grouped/builder_computed.rs index 25b828a06..627fa177a 100644 --- a/crates/brk_computer/src/grouped/builder_computed.rs +++ b/crates/brk_computer/src/grouped/builder_computed.rs @@ -1,10 +1,10 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{FromCoarserIndex, Result, Version}; use brk_exit::Exit; -use brk_vec::{ +use brk_vecs::{ AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation, - ComputedVec, ComputedVecFrom2, Format, StoredIndex, + ComputedVec, ComputedVecFrom2, File, Format, StoredIndex, }; use crate::grouped::{EagerVecBuilder, VecBuilderOptions}; @@ -39,7 +39,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, version: Version, format: Format, @@ -66,7 +66,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, &maybe_suffix("first"), version + VERSION + Version::ZERO, format, @@ -91,7 +91,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, name, version + VERSION + Version::ZERO, format, @@ -100,7 +100,7 @@ where source .as_ref() .unwrap_or_else(|| { - dbg!(path, name, I::to_string()); + dbg!(file, name, I::to_string()); panic!() }) .clone() @@ -124,7 +124,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, &maybe_suffix("min"), version + VERSION + Version::ZERO, format, @@ -149,7 +149,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, &maybe_suffix("max"), version + VERSION + Version::ZERO, format, @@ -174,7 +174,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, &maybe_suffix("average"), version + VERSION + Version::ZERO, format, @@ -206,7 +206,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, &(if !options.last && !options.average && !options.min && !options.max { name.to_string() } else { @@ -241,7 +241,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - path, + file, &suffix("cumulative"), version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index 3a7d6e536..57001e54c 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -1,8 +1,10 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{CheckedSub, Result, StoredUsize, Version}; use brk_exit::Exit; -use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType}; +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, AnyVec, EagerVec, File, Format, StoredIndex, StoredType, +}; use color_eyre::eyre::ContextCompat; use crate::utils::get_percentile; @@ -37,7 +39,7 @@ where T: ComputedType, { pub fn forced_import( - path: &Path, + file: &Arc, name: &str, version: Version, format: Format, @@ -59,7 +61,7 @@ where first: options.first.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("first"), version + VERSION + Version::ZERO, format, @@ -69,13 +71,13 @@ where }), last: options.last.then(|| { Box::new( - EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(), + EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(), ) }), min: options.min.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("min"), version + VERSION + Version::ZERO, format, @@ -86,7 +88,7 @@ where max: options.max.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("max"), version + VERSION + Version::ZERO, format, @@ -97,7 +99,7 @@ where median: options.median.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("median"), version + VERSION + Version::ZERO, format, @@ -108,7 +110,7 @@ where average: options.average.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("average"), version + VERSION + Version::ZERO, format, @@ -119,7 +121,7 @@ where sum: options.sum.then(|| { Box::new( EagerVec::forced_import( - path, + file, &(if !options.last && !options.average && !options.min && !options.max { name.to_string() } else { @@ -134,7 +136,7 @@ where cumulative: options.cumulative.then(|| { Box::new( EagerVec::forced_import( - path, + file, &suffix("cumulative"), version + VERSION + Version::ZERO, format, @@ -145,7 +147,7 @@ where _90p: options._90p.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("90p"), version + VERSION + Version::ZERO, format, @@ -156,7 +158,7 @@ where _75p: options._75p.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("75p"), version + VERSION + Version::ZERO, format, @@ -167,7 +169,7 @@ where _25p: options._25p.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("25p"), version + VERSION + Version::ZERO, format, @@ -178,7 +180,7 @@ where _10p: options._10p.then(|| { Box::new( EagerVec::forced_import( - path, + file, &maybe_suffix("10p"), version + VERSION + Version::ZERO, format, @@ -309,7 +311,7 @@ where .inspect_err(|_| { dbg!( &values, - max.path(), + max.name(), first_indexes.name(), first_index, count_indexes.name(), diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index a49c68a2c..f4ca684dc 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex, @@ -6,8 +6,8 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format, }; use crate::{Indexes, grouped::ComputedVecBuilder, indexes}; @@ -37,7 +37,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -47,11 +47,11 @@ where options: VecBuilderOptions, ) -> color_eyre::Result { let dateindex = source.is_compute().then(|| { - EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap() + EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap() }); let dateindex_extra = EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -64,7 +64,7 @@ where Ok(Self { weekindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -75,7 +75,7 @@ where options.into(), )?, monthindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -86,7 +86,7 @@ where options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -97,7 +97,7 @@ where options.into(), )?, semesterindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -108,7 +108,7 @@ where options.into(), )?, yearindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -119,7 +119,7 @@ where options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 15630aa05..3bbe1e7d3 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, @@ -6,8 +6,8 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format, }; use crate::{ @@ -45,7 +45,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -55,11 +55,11 @@ where options: VecBuilderOptions, ) -> color_eyre::Result { let height = source.is_compute().then(|| { - EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap() + EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap() }); let height_extra = EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -67,7 +67,7 @@ where )?; let dateindex = EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -78,7 +78,7 @@ where Ok(Self { weekindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -89,7 +89,7 @@ where options.into(), )?, monthindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -100,7 +100,7 @@ where options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -111,7 +111,7 @@ where options.into(), )?, semesterindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -122,7 +122,7 @@ where options.into(), )?, yearindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -133,7 +133,7 @@ where options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -143,12 +143,12 @@ where indexes.decadeindex_to_decadeindex.boxed_clone(), options.into(), )?, - // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, + // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?, height, height_extra, dateindex, difficultyepoch: EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index cefd2df77..d59ca609c 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{DifficultyEpoch, Height, Result, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, EagerVec, Format}; +use brk_vecs::{AnyCollectableVec, EagerVec, File, Format}; use crate::{Indexes, indexes}; @@ -28,17 +28,17 @@ where f64: From, { pub fn forced_import( - path: &Path, + file: &Arc, name: &str, version: Version, format: Format, options: VecBuilderOptions, ) -> color_eyre::Result { let height = - EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?; + EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?; let height_extra = EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -51,13 +51,13 @@ where height, height_extra, difficultyepoch: EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, options, )?, - // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, + // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?, }) } diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index a20e0dbef..c346d101d 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, @@ -6,9 +6,9 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ +use brk_vecs::{ AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec, - Format, StoredIndex, VecIterator, + File, Format, StoredIndex, VecIterator, }; use crate::{ @@ -46,7 +46,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -57,13 +57,13 @@ where ) -> color_eyre::Result { let txindex = source.is_compute().then(|| { Box::new( - EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format) + EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format) .unwrap(), ) }); let height = EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -73,7 +73,7 @@ where let options = options.remove_percentiles(); let dateindex = EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -82,7 +82,7 @@ where Ok(Self { weekindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -93,7 +93,7 @@ where options.into(), )?, monthindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -104,7 +104,7 @@ where options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -115,7 +115,7 @@ where options.into(), )?, semesterindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -126,7 +126,7 @@ where options.into(), )?, yearindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -137,7 +137,7 @@ where options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, @@ -152,13 +152,13 @@ where height, dateindex, difficultyepoch: EagerVecBuilder::forced_import( - path, + file, name, version + VERSION + Version::ZERO, format, options, )?, - // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?, + // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?, }) } diff --git a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs index 5d144d157..53eee2292 100644 --- a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -1,10 +1,10 @@ -use std::{f32, path::Path}; +use std::{f32, sync::Arc}; use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format, StoredIndex, VecIterator, }; @@ -60,7 +60,7 @@ const VERSION: Version = Version::ZERO; impl ComputedRatioVecsFromDateIndex { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -74,7 +74,7 @@ impl ComputedRatioVecsFromDateIndex { Ok(Self { price: source.is_compute().then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, name, Source::Compute, version + VERSION, @@ -86,7 +86,7 @@ impl ComputedRatioVecsFromDateIndex { .unwrap() }), ratio: ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio"), Source::Compute, version + VERSION + Version::ZERO, @@ -97,7 +97,7 @@ impl ComputedRatioVecsFromDateIndex { )?, ratio_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -110,7 +110,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1w_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_1w_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -123,7 +123,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1m_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_1m_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -136,7 +136,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_1y_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -149,7 +149,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_4y_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_4y_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -162,7 +162,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_sma_momentum_oscillator: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_1y_sma_momentum_oscillator"), Source::Compute, version + VERSION + Version::ZERO, @@ -175,7 +175,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -188,7 +188,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_4y_sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_4y_sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -201,7 +201,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_1y_sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -214,7 +214,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_9: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p99_9"), Source::Compute, version + VERSION + Version::ZERO, @@ -227,7 +227,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_5: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p99_5"), Source::Compute, version + VERSION + Version::ZERO, @@ -240,7 +240,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p99"), Source::Compute, version + VERSION + Version::ZERO, @@ -253,7 +253,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p1"), Source::Compute, version + VERSION + Version::ZERO, @@ -266,7 +266,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_5: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p0_5"), Source::Compute, version + VERSION + Version::ZERO, @@ -279,7 +279,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_1: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p0_1"), Source::Compute, version + VERSION + Version::ZERO, @@ -292,7 +292,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p1sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -305,7 +305,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p2sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p2sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -318,7 +318,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p3sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p3sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -331,7 +331,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m1sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_m1sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -344,7 +344,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m2sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_m2sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -357,7 +357,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m3sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_m3sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -370,7 +370,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_9_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p99_9_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -383,7 +383,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_5_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p99_5_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -396,7 +396,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p99_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -409,7 +409,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p1_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -422,7 +422,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_5_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p0_5_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -435,7 +435,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_1_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p0_1_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -448,7 +448,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p1sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -461,7 +461,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p2sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p2sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -474,7 +474,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p3sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_p3sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -487,7 +487,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m1sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_m1sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -500,7 +500,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m2sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_m2sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -513,7 +513,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m3sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_m3sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -526,7 +526,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_zscore: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_zscore"), Source::Compute, version + VERSION + Version::ZERO, @@ -539,7 +539,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_4y_zscore: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_4y_zscore"), Source::Compute, version + VERSION + Version::ZERO, @@ -552,7 +552,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_zscore: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_ratio_1y_zscore"), Source::Compute, version + VERSION + Version::ZERO, @@ -1069,6 +1069,10 @@ impl ComputedRatioVecsFromDateIndex { Ok(()) })?; + drop(sma_iter); + drop(_4y_sma_iter); + drop(_1y_sma_iter); + self.mut_ratio_vecs() .into_iter() .try_for_each(|v| v.safe_flush(exit))?; diff --git a/crates/brk_computer/src/grouped/source.rs b/crates/brk_computer/src/grouped/source.rs index d8886e4f5..aade698fc 100644 --- a/crates/brk_computer/src/grouped/source.rs +++ b/crates/brk_computer/src/grouped/source.rs @@ -1,4 +1,4 @@ -use brk_vec::BoxedAnyIterableVec; +use brk_vecs::BoxedAnyIterableVec; #[derive(Clone)] pub enum Source { diff --git a/crates/brk_computer/src/grouped/type.rs b/crates/brk_computer/src/grouped/type.rs index 357e6d6ba..df737cc71 100644 --- a/crates/brk_computer/src/grouped/type.rs +++ b/crates/brk_computer/src/grouped/type.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vec::StoredType; +use brk_vecs::StoredType; pub trait ComputedType where diff --git a/crates/brk_computer/src/grouped/value_from_dateindex.rs b/crates/brk_computer/src/grouped/value_from_dateindex.rs index e2ec90ba1..08ba4ff50 100644 --- a/crates/brk_computer/src/grouped/value_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/value_from_dateindex.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; +use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec}; use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes}; @@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromDateIndex { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -33,7 +33,7 @@ impl ComputedValueVecsFromDateIndex { ) -> color_eyre::Result { Ok(Self { sats: ComputedVecsFromDateIndex::forced_import( - path, + file, name, source, version + VERSION, @@ -43,7 +43,7 @@ impl ComputedValueVecsFromDateIndex { options, )?, bitcoin: ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_in_btc"), Source::Compute, version + VERSION, @@ -54,7 +54,7 @@ impl ComputedValueVecsFromDateIndex { )?, dollars: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &format!("{name}_in_usd"), Source::Compute, version + VERSION, diff --git a/crates/brk_computer/src/grouped/value_from_height.rs b/crates/brk_computer/src/grouped/value_from_height.rs index 033463328..3eb6b92a2 100644 --- a/crates/brk_computer/src/grouped/value_from_height.rs +++ b/crates/brk_computer/src/grouped/value_from_height.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; +use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec}; use crate::{Indexes, fetched, grouped::Source, indexes}; @@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromHeight { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -33,7 +33,7 @@ impl ComputedValueVecsFromHeight { ) -> color_eyre::Result { Ok(Self { sats: ComputedVecsFromHeight::forced_import( - path, + file, name, source, version + VERSION, @@ -43,7 +43,7 @@ impl ComputedValueVecsFromHeight { options, )?, bitcoin: ComputedVecsFromHeight::forced_import( - path, + file, &format!("{name}_in_btc"), Source::Compute, version + VERSION, @@ -54,7 +54,7 @@ impl ComputedValueVecsFromHeight { )?, dollars: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &format!("{name}_in_usd"), Source::Compute, version + VERSION, diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index 4b529bdb2..a651506c8 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -1,11 +1,11 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ +use brk_vecs::{ AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3, - Format, LazyVecFrom1, StoredIndex, StoredVec, + File, Format, LazyVecFrom1, StoredIndex, StoredVec, }; use crate::{Indexes, fetched, grouped::Source, indexes}; @@ -38,7 +38,7 @@ const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromTxindex { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, name: &str, indexes: &indexes::Vecs, source: Source, @@ -54,7 +54,7 @@ impl ComputedValueVecsFromTxindex { let name_in_usd = format!("{name}_in_usd"); let sats = ComputedVecsFromTxindex::forced_import( - path, + file, name, source.clone(), version + VERSION, @@ -79,7 +79,7 @@ impl ComputedValueVecsFromTxindex { ); let bitcoin = ComputedVecsFromTxindex::forced_import( - path, + file, &name_in_btc, Source::None, version + VERSION, @@ -92,7 +92,7 @@ impl ComputedValueVecsFromTxindex { let dollars_txindex = fetched.map(|fetched| { ComputedVecFrom3::forced_import_or_init_from_3( computation, - path, + file, &name_in_usd, version + VERSION, format, @@ -127,7 +127,7 @@ impl ComputedValueVecsFromTxindex { dollars_txindex, dollars: compute_dollars.then(|| { ComputedVecsFromTxindex::forced_import( - path, + file, &name_in_usd, Source::None, version + VERSION, diff --git a/crates/brk_computer/src/grouped/value_height.rs b/crates/brk_computer/src/grouped/value_height.rs index 24848351d..77c7e6378 100644 --- a/crates/brk_computer/src/grouped/value_height.rs +++ b/crates/brk_computer/src/grouped/value_height.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec}; +use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, File, Format, StoredVec}; use crate::{Indexes, fetched, grouped::Source, indexes}; @@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO; impl ComputedHeightValueVecs { pub fn forced_import( - path: &Path, + file: &Arc, name: &str, source: Source, version: Version, @@ -27,18 +27,18 @@ impl ComputedHeightValueVecs { ) -> color_eyre::Result { Ok(Self { sats: source.is_compute().then(|| { - EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format) + EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format) .unwrap() }), bitcoin: EagerVec::forced_import( - path, + file, &format!("{name}_in_btc"), version + VERSION + Version::ZERO, format, )?, dollars: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &format!("{name}_in_usd"), version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index 9cac4dbe8..853905d2c 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -1,4 +1,4 @@ -use std::{ops::Deref, path::Path}; +use std::{ops::Deref, sync::Arc}; use brk_core::{ Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height, @@ -10,9 +10,9 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ +use brk_vecs::{ AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1, - ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator, + ComputedVecFrom2, EagerVec, File, Format, StoredIndex, VecIterator, }; const VERSION: Version = Version::ZERO; @@ -97,7 +97,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, indexer: &Indexer, computation: Computation, @@ -105,7 +105,7 @@ impl Vecs { ) -> color_eyre::Result { let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "outputindex", version + VERSION + Version::ZERO, format, @@ -115,7 +115,7 @@ impl Vecs { let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "inputindex", version + VERSION + Version::ZERO, format, @@ -125,7 +125,7 @@ impl Vecs { let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "txindex", version + VERSION + Version::ZERO, format, @@ -135,7 +135,7 @@ impl Vecs { let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2( computation, - path, + file, "input_count", version + VERSION + Version::ZERO, format, @@ -158,7 +158,7 @@ impl Vecs { let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2( computation, - path, + file, "output_count", version + VERSION + Version::ZERO, format, @@ -181,7 +181,7 @@ impl Vecs { let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2pk33addressindex", version + VERSION + Version::ZERO, format, @@ -190,7 +190,7 @@ impl Vecs { )?; let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2pk65addressindex", version + VERSION + Version::ZERO, format, @@ -199,7 +199,7 @@ impl Vecs { )?; let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2pkhaddressindex", version + VERSION + Version::ZERO, format, @@ -208,7 +208,7 @@ impl Vecs { )?; let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2shaddressindex", version + VERSION + Version::ZERO, format, @@ -217,7 +217,7 @@ impl Vecs { )?; let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2traddressindex", version + VERSION + Version::ZERO, format, @@ -226,7 +226,7 @@ impl Vecs { )?; let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2wpkhaddressindex", version + VERSION + Version::ZERO, format, @@ -235,7 +235,7 @@ impl Vecs { )?; let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2wshaddressindex", version + VERSION + Version::ZERO, format, @@ -244,7 +244,7 @@ impl Vecs { )?; let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2aaddressindex", version + VERSION + Version::ZERO, format, @@ -253,7 +253,7 @@ impl Vecs { )?; let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "p2msoutputindex", version + VERSION + Version::ZERO, format, @@ -262,7 +262,7 @@ impl Vecs { )?; let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "emptyoutputindex", version + VERSION + Version::ZERO, format, @@ -271,7 +271,7 @@ impl Vecs { )?; let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "unknownoutputindex", version + VERSION + Version::ZERO, format, @@ -280,7 +280,7 @@ impl Vecs { )?; let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "opreturnindex", version + VERSION + Version::ZERO, format, @@ -308,259 +308,259 @@ impl Vecs { unknownoutputindex_to_unknownoutputindex, dateindex_to_date: EagerVec::forced_import( - path, + file, "date", version + VERSION + Version::ZERO, format, )?, dateindex_to_dateindex: EagerVec::forced_import( - path, + file, "dateindex", version + VERSION + Version::ZERO, format, )?, dateindex_to_first_height: EagerVec::forced_import( - path, + file, "first_height", version + VERSION + Version::ZERO, format, )?, dateindex_to_monthindex: EagerVec::forced_import( - path, + file, "monthindex", version + VERSION + Version::ZERO, format, )?, dateindex_to_weekindex: EagerVec::forced_import( - path, + file, "weekindex", version + VERSION + Version::ZERO, format, )?, decadeindex_to_decadeindex: EagerVec::forced_import( - path, + file, "decadeindex", version + VERSION + Version::ZERO, format, )?, decadeindex_to_first_yearindex: EagerVec::forced_import( - path, + file, "first_yearindex", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_difficultyepoch: EagerVec::forced_import( - path, + file, "difficultyepoch", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_first_height: EagerVec::forced_import( - path, + file, "first_height", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_first_height: EagerVec::forced_import( - path, + file, "first_height", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_halvingepoch: EagerVec::forced_import( - path, + file, "halvingepoch", version + VERSION + Version::ZERO, format, )?, height_to_date: EagerVec::forced_import( - path, + file, "date", version + VERSION + Version::ZERO, format, )?, height_to_difficultyepoch: EagerVec::forced_import( - path, + file, "difficultyepoch", version + VERSION + Version::ZERO, format, )?, height_to_halvingepoch: EagerVec::forced_import( - path, + file, "halvingepoch", version + VERSION + Version::ZERO, format, )?, height_to_height: EagerVec::forced_import( - path, + file, "height", version + VERSION + Version::ZERO, format, )?, monthindex_to_first_dateindex: EagerVec::forced_import( - path, + file, "first_dateindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_monthindex: EagerVec::forced_import( - path, + file, "monthindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_quarterindex: EagerVec::forced_import( - path, + file, "quarterindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_semesterindex: EagerVec::forced_import( - path, + file, "semesterindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_yearindex: EagerVec::forced_import( - path, + file, "yearindex", version + VERSION + Version::ZERO, format, )?, quarterindex_to_first_monthindex: EagerVec::forced_import( - path, + file, "first_monthindex", version + VERSION + Version::ZERO, format, )?, semesterindex_to_first_monthindex: EagerVec::forced_import( - path, + file, "first_monthindex", version + VERSION + Version::ZERO, format, )?, weekindex_to_first_dateindex: EagerVec::forced_import( - path, + file, "first_dateindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_first_monthindex: EagerVec::forced_import( - path, + file, "first_monthindex", version + VERSION + Version::ZERO, format, )?, quarterindex_to_quarterindex: EagerVec::forced_import( - path, + file, "quarterindex", version + VERSION + Version::ZERO, format, )?, semesterindex_to_semesterindex: EagerVec::forced_import( - path, + file, "semesterindex", version + VERSION + Version::ZERO, format, )?, weekindex_to_weekindex: EagerVec::forced_import( - path, + file, "weekindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_decadeindex: EagerVec::forced_import( - path, + file, "decadeindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_yearindex: EagerVec::forced_import( - path, + file, "yearindex", version + VERSION + Version::ZERO, format, )?, height_to_date_fixed: EagerVec::forced_import( - path, + file, "date_fixed", version + VERSION + Version::ZERO, format, )?, height_to_dateindex: EagerVec::forced_import( - path, + file, "dateindex", version + VERSION + Version::ZERO, format, )?, txindex_to_height: EagerVec::forced_import( - path, + file, "height", version + VERSION + Version::ZERO, format, )?, height_to_timestamp_fixed: EagerVec::forced_import( - path, + file, "timestamp_fixed", version + VERSION + Version::ZERO, format, )?, height_to_txindex_count: EagerVec::forced_import( - path, + file, "txindex_count", version + VERSION + Version::ZERO, format, )?, dateindex_to_height_count: EagerVec::forced_import( - path, + file, "height_count", version + VERSION + Version::ZERO, format, )?, weekindex_to_dateindex_count: EagerVec::forced_import( - path, + file, "dateindex_count", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_height_count: EagerVec::forced_import( - path, + file, "height_count", version + VERSION + Version::ZERO, format, )?, monthindex_to_dateindex_count: EagerVec::forced_import( - path, + file, "dateindex_count", version + VERSION + Version::ZERO, format, )?, quarterindex_to_monthindex_count: EagerVec::forced_import( - path, + file, "monthindex_count", version + VERSION + Version::ZERO, format, )?, semesterindex_to_monthindex_count: EagerVec::forced_import( - path, + file, "monthindex_count", version + VERSION + Version::ZERO, format, )?, yearindex_to_monthindex_count: EagerVec::forced_import( - path, + file, "monthindex_count", version + VERSION + Version::ZERO, format, )?, decadeindex_to_yearindex_count: EagerVec::forced_import( - path, + file, "yearindex_count", version + VERSION + Version::ZERO, format, )?, outputindex_to_txindex: EagerVec::forced_import( - path, + file, "txindex", version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index d92820b5b..7b342ba32 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -3,13 +3,13 @@ #![doc = include_str!("../examples/main.rs")] #![doc = "```"] -use std::path::Path; +use std::{path::Path, sync::Arc}; use brk_core::Version; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{Computation, Format}; +use brk_vecs::{Computation, File, Format}; use log::info; mod all; @@ -47,15 +47,22 @@ impl Computer { fetcher: Option, format: Format, ) -> color_eyre::Result { + let computed_path = outputs_dir.join("computed"); + let states_path = computed_path.join("states"); + + let file = Arc::new(File::open(&computed_path.join("vecs"))?); + let file_fetched = Arc::new(File::open(&outputs_dir.join("fetched/vecs"))?); + Ok(Self { vecs: all::Vecs::import( - // TODO: Give self.path, join inside import - &outputs_dir.join("vecs/computed"), + &file, VERSION + Version::ZERO, indexer, fetcher.is_some(), computation, format, + &file_fetched, + &states_path, )?, fetcher, }) diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index 7560aa285..a96e1435a 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -1,9 +1,9 @@ -use std::{path::Path, thread}; +use std::{sync::Arc, thread}; use brk_core::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredUsize, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, Computation, EagerVec, Format, StoredIndex, VecIterator}; +use brk_vecs::{AnyCollectableVec, Computation, EagerVec, File, Format, StoredIndex, VecIterator}; use crate::grouped::Source; @@ -163,7 +163,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, computation: Computation, format: Format, @@ -171,25 +171,25 @@ impl Vecs { ) -> color_eyre::Result { Ok(Self { height_to_marketcap: EagerVec::forced_import( - path, + file, "marketcap", version + VERSION + Version::ZERO, format, )?, height_to_ath: EagerVec::forced_import( - path, + file, "ath", version + VERSION + Version::ZERO, format, )?, height_to_drawdown: EagerVec::forced_import( - path, + file, "drawdown", version + VERSION + Version::ZERO, format, )?, indexes_to_marketcap: ComputedVecsFromDateIndex::forced_import( - path, + file, "marketcap", Source::Compute, version + VERSION + Version::ZERO, @@ -199,7 +199,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_ath: ComputedVecsFromDateIndex::forced_import( - path, + file, "ath", Source::Compute, version + VERSION + Version::ZERO, @@ -209,7 +209,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_drawdown: ComputedVecsFromDateIndex::forced_import( - path, + file, "drawdown", Source::Compute, version + VERSION + Version::ZERO, @@ -219,7 +219,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_days_since_ath: ComputedVecsFromDateIndex::forced_import( - path, + file, "days_since_ath", Source::Compute, version + VERSION + Version::ZERO, @@ -229,7 +229,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_max_days_between_aths: ComputedVecsFromDateIndex::forced_import( - path, + file, "max_days_between_aths", Source::Compute, version + VERSION + Version::ZERO, @@ -239,7 +239,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_max_years_between_aths: ComputedVecsFromDateIndex::forced_import( - path, + file, "max_years_between_aths", Source::Compute, version + VERSION + Version::ZERO, @@ -250,7 +250,7 @@ impl Vecs { )?, indexes_to_1w_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "1w_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -260,7 +260,7 @@ impl Vecs { true, )?, indexes_to_8d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "8d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -270,7 +270,7 @@ impl Vecs { true, )?, indexes_to_13d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "13d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -280,7 +280,7 @@ impl Vecs { true, )?, indexes_to_21d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "21d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -290,7 +290,7 @@ impl Vecs { true, )?, indexes_to_1m_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "1m_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -300,7 +300,7 @@ impl Vecs { true, )?, indexes_to_34d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "34d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -310,7 +310,7 @@ impl Vecs { true, )?, indexes_to_55d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "55d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -320,7 +320,7 @@ impl Vecs { true, )?, indexes_to_89d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "89d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -330,7 +330,7 @@ impl Vecs { true, )?, indexes_to_144d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "144d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -340,7 +340,7 @@ impl Vecs { true, )?, indexes_to_200d_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "200d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -350,7 +350,7 @@ impl Vecs { true, )?, indexes_to_1y_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "1y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -360,7 +360,7 @@ impl Vecs { true, )?, indexes_to_2y_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "2y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -370,7 +370,7 @@ impl Vecs { true, )?, indexes_to_200w_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "200w_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -380,7 +380,7 @@ impl Vecs { true, )?, indexes_to_4y_sma: ComputedRatioVecsFromDateIndex::forced_import( - path, + file, "4y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -391,7 +391,7 @@ impl Vecs { )?, _1d_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1d_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -401,7 +401,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1w_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -411,7 +411,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -421,7 +421,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "3m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -431,7 +431,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "6m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -441,7 +441,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -451,7 +451,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "2y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -461,7 +461,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "3y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -471,7 +471,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "4y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -481,7 +481,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "5y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -491,7 +491,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "6y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -501,7 +501,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "8y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -511,7 +511,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "10y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -521,7 +521,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "2y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -531,7 +531,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "3y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -541,7 +541,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "4y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -551,7 +551,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "5y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -561,7 +561,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "6y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -571,7 +571,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "8y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -581,7 +581,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "10y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -592,7 +592,7 @@ impl Vecs { )?, _1w_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1w_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -602,7 +602,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -612,7 +612,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "3m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -622,7 +622,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "6m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -632,7 +632,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "1y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -642,7 +642,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "2y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -652,7 +652,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "3y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -662,7 +662,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "4y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -672,7 +672,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "5y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -682,7 +682,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "6y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -692,7 +692,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "8y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -702,7 +702,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "10y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -712,7 +712,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "2y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -722,7 +722,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "3y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -732,7 +732,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "4y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -742,7 +742,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "5y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -752,7 +752,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "6y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -762,7 +762,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "8y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -772,7 +772,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - path, + file, "10y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -782,7 +782,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "1w_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -792,7 +792,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "1m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -802,7 +802,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "3m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -812,7 +812,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "6m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -822,7 +822,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "1y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -832,7 +832,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "2y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -842,7 +842,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "3y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -852,7 +852,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "4y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -862,7 +862,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "5y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -872,7 +872,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "6y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -882,7 +882,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "8y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -892,7 +892,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "10y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -902,7 +902,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1d_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_1d_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -912,7 +912,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1w_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_1w_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -922,7 +922,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1m_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_1m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -932,7 +932,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_3m_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_3m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -942,7 +942,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_6m_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_6m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -952,7 +952,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_1y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -962,7 +962,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_2y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_2y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -972,7 +972,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_3y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_3y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -982,7 +982,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_4y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_4y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -992,7 +992,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_5y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_5y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1002,7 +1002,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_6y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_6y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1012,7 +1012,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_8y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_8y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1022,7 +1022,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_10y_ago: ComputedVecsFromDateIndex::forced_import( - path, + file, "price_10y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1032,7 +1032,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "1w_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1042,7 +1042,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "1m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1052,7 +1052,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "3m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1062,7 +1062,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "6m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1072,7 +1072,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "1y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1082,7 +1082,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "2y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1092,7 +1092,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "3y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1102,7 +1102,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "4y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1112,7 +1112,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "5y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1122,7 +1122,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "6y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1132,7 +1132,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "8y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1142,7 +1142,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "10y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1153,7 +1153,7 @@ impl Vecs { )?, dca_class_2025_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2025_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1163,7 +1163,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2024_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1173,7 +1173,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2023_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1183,7 +1183,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2022_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1193,7 +1193,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2021_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1203,7 +1203,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2020_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1213,7 +1213,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2019_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1223,7 +1223,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2018_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1233,7 +1233,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2017_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1243,7 +1243,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2016_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1253,7 +1253,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_stack: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2015_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1264,7 +1264,7 @@ impl Vecs { )?, dca_class_2025_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2025_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1274,7 +1274,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2024_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1284,7 +1284,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2023_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1294,7 +1294,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2022_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1304,7 +1304,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2021_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1314,7 +1314,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2020_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1324,7 +1324,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2019_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1334,7 +1334,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2018_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1344,7 +1344,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2017_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1354,7 +1354,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2016_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1364,7 +1364,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_avg_price: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2015_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1375,7 +1375,7 @@ impl Vecs { )?, dca_class_2025_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2025_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1385,7 +1385,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2024_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1395,7 +1395,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2023_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1405,7 +1405,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2022_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1415,7 +1415,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2021_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1425,7 +1425,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2020_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1435,7 +1435,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2019_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1445,7 +1445,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2018_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1455,7 +1455,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2017_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1465,7 +1465,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2016_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1475,7 +1475,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_returns: ComputedVecsFromDateIndex::forced_import( - path, + file, "dca_class_2015_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1486,7 +1486,7 @@ impl Vecs { )?, indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex::forced_import( - path, + file, "200d_sma_x2_4", Source::Compute, version + VERSION + Version::ZERO, @@ -1496,7 +1496,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex::forced_import( - path, + file, "200d_sma_x0_8", Source::Compute, version + VERSION + Version::ZERO, diff --git a/crates/brk_computer/src/mining.rs b/crates/brk_computer/src/mining.rs index 09cea1805..52f7ddbfc 100644 --- a/crates/brk_computer/src/mining.rs +++ b/crates/brk_computer/src/mining.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; +use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator}; use crate::grouped::Source; @@ -24,7 +24,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, computation: Computation, format: Format, @@ -32,7 +32,7 @@ impl Vecs { ) -> color_eyre::Result { Ok(Self { indexes_to_difficulty: ComputedVecsFromHeight::forced_import( - path, + file, "difficulty", Source::None, version + VERSION + Version::ZERO, @@ -42,7 +42,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import( - path, + file, "difficultyepoch", Source::Compute, version + VERSION + Version::ZERO, @@ -52,7 +52,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import( - path, + file, "halvingepoch", Source::Compute, version + VERSION + Version::ZERO, diff --git a/crates/brk_computer/src/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs index ff5ea45fd..e8e12e72f 100644 --- a/crates/brk_computer/src/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -1,10 +1,10 @@ -use std::{ops::Deref, path::Path}; +use std::{ops::Deref, path::Path, sync::Arc}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator, +use brk_vecs::{ + AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, VecIterator, }; use crate::{ @@ -35,7 +35,7 @@ pub struct Vecs { impl Vecs { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, cohort_name: Option<&str>, computation: Computation, format: Format, @@ -57,13 +57,13 @@ impl Vecs { compute_dollars, )?, height_to_address_count: EagerVec::forced_import( - path, + file, &suffix("address_count"), version + VERSION + Version::ZERO, format, )?, indexes_to_address_count: ComputedVecsFromHeight::forced_import( - path, + file, &suffix("address_count"), Source::None, version + VERSION + Version::ZERO, @@ -73,7 +73,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, inner: common::Vecs::forced_import( - path, + file, cohort_name, computation, format, diff --git a/crates/brk_computer/src/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs index 83b984f09..a1ee7ad5b 100644 --- a/crates/brk_computer/src/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{path::Path, sync::Arc}; use brk_core::{ AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex, @@ -6,7 +6,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyIterableVec, Computation, Format}; +use brk_vecs::{AnyIterableVec, Computation, File, Format}; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; @@ -25,7 +25,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>); impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, _computation: Computation, format: Format, @@ -37,7 +37,7 @@ impl Vecs { AddressGroups { amount_range: ByAmountRange { _0sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_with_0sats"), _computation, format, @@ -48,7 +48,7 @@ impl Vecs { true, )?, _1sat_to_10sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1sat_under_10sats"), _computation, format, @@ -59,7 +59,7 @@ impl Vecs { true, )?, _10sats_to_100sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10sats_under_100sats"), _computation, format, @@ -70,7 +70,7 @@ impl Vecs { true, )?, _100sats_to_1k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100sats_under_1k_sats"), _computation, format, @@ -81,7 +81,7 @@ impl Vecs { true, )?, _1k_sats_to_10k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1k_sats_under_10k_sats"), _computation, format, @@ -92,7 +92,7 @@ impl Vecs { true, )?, _10k_sats_to_100k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10k_sats_under_100k_sats"), _computation, format, @@ -103,7 +103,7 @@ impl Vecs { true, )?, _100k_sats_to_1m_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100k_sats_under_1m_sats"), _computation, format, @@ -114,7 +114,7 @@ impl Vecs { true, )?, _1m_sats_to_10m_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1m_sats_under_10m_sats"), _computation, format, @@ -125,7 +125,7 @@ impl Vecs { true, )?, _10m_sats_to_1btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10m_sats_under_1btc"), _computation, format, @@ -136,7 +136,7 @@ impl Vecs { true, )?, _1btc_to_10btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1btc_under_10btc"), _computation, format, @@ -147,7 +147,7 @@ impl Vecs { true, )?, _10btc_to_100btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10btc_under_100btc"), _computation, format, @@ -158,7 +158,7 @@ impl Vecs { true, )?, _100btc_to_1k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100btc_under_1k_btc"), _computation, format, @@ -169,7 +169,7 @@ impl Vecs { true, )?, _1k_btc_to_10k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1k_btc_under_10k_btc"), _computation, format, @@ -180,7 +180,7 @@ impl Vecs { true, )?, _10k_btc_to_100k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10k_btc_under_100k_btc"), _computation, format, @@ -191,7 +191,7 @@ impl Vecs { true, )?, _100k_btc_or_more: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100k_btc"), _computation, format, @@ -204,7 +204,7 @@ impl Vecs { }, lt_amount: ByLowerThanAmount { _10sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_10sats"), _computation, format, @@ -215,7 +215,7 @@ impl Vecs { true, )?, _100sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_100sats"), _computation, format, @@ -226,7 +226,7 @@ impl Vecs { true, )?, _1k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_1k_sats"), _computation, format, @@ -237,7 +237,7 @@ impl Vecs { true, )?, _10k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_10k_sats"), _computation, format, @@ -248,7 +248,7 @@ impl Vecs { true, )?, _100k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_100k_sats"), _computation, format, @@ -259,7 +259,7 @@ impl Vecs { true, )?, _1m_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_1m_sats"), _computation, format, @@ -270,7 +270,7 @@ impl Vecs { true, )?, _10m_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_10m_sats"), _computation, format, @@ -281,7 +281,7 @@ impl Vecs { true, )?, _1btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_1btc"), _computation, format, @@ -292,7 +292,7 @@ impl Vecs { true, )?, _10btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_10btc"), _computation, format, @@ -303,7 +303,7 @@ impl Vecs { true, )?, _100btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_100btc"), _computation, format, @@ -314,7 +314,7 @@ impl Vecs { true, )?, _1k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_1k_btc"), _computation, format, @@ -325,7 +325,7 @@ impl Vecs { true, )?, _10k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_10k_btc"), _computation, format, @@ -336,7 +336,7 @@ impl Vecs { true, )?, _100k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_under_100k_btc"), _computation, format, @@ -349,7 +349,7 @@ impl Vecs { }, ge_amount: ByGreatEqualAmount { _1sat: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1sat"), _computation, format, @@ -360,7 +360,7 @@ impl Vecs { true, )?, _10sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10sats"), _computation, format, @@ -371,7 +371,7 @@ impl Vecs { true, )?, _100sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100sats"), _computation, format, @@ -382,7 +382,7 @@ impl Vecs { true, )?, _1k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1k_sats"), _computation, format, @@ -393,7 +393,7 @@ impl Vecs { true, )?, _10k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10k_sats"), _computation, format, @@ -404,7 +404,7 @@ impl Vecs { true, )?, _100k_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100k_sats"), _computation, format, @@ -415,7 +415,7 @@ impl Vecs { true, )?, _1m_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1m_sats"), _computation, format, @@ -426,7 +426,7 @@ impl Vecs { true, )?, _10m_sats: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10m_sats"), _computation, format, @@ -437,7 +437,7 @@ impl Vecs { true, )?, _1btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1btc"), _computation, format, @@ -448,7 +448,7 @@ impl Vecs { true, )?, _10btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10btc"), _computation, format, @@ -459,7 +459,7 @@ impl Vecs { true, )?, _100btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_100btc"), _computation, format, @@ -470,7 +470,7 @@ impl Vecs { true, )?, _1k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_1k_btc"), _computation, format, @@ -481,7 +481,7 @@ impl Vecs { true, )?, _10k_btc: address_cohort::Vecs::forced_import( - path, + file, Some("addrs_above_10k_btc"), _computation, format, diff --git a/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs index 334c7207e..1424c9cc6 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs @@ -1,5 +1,5 @@ use brk_core::{ByAddressType, Height}; -use brk_vec::VecIterator; +use brk_vecs::VecIterator; use derive_deref::{Deref, DerefMut}; use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; diff --git a/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs index 48c009a5c..8ec38ba83 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs @@ -1,6 +1,6 @@ use brk_core::{ByAddressType, Height, Result, StoredUsize}; use brk_exit::Exit; -use brk_vec::EagerVec; +use brk_vecs::EagerVec; use derive_deref::{Deref, DerefMut}; use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; diff --git a/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs index 73aa42124..89746fab2 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs @@ -1,6 +1,6 @@ use brk_core::{ByAddressType, StoredUsize}; use brk_exit::Exit; -use brk_vec::AnyCollectableVec; +use brk_vecs::AnyCollectableVec; use derive_deref::{Deref, DerefMut}; use crate::{ diff --git a/crates/brk_computer/src/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs index 65b043ac7..c8bba9681 100644 --- a/crates/brk_computer/src/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -1,13 +1,13 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version, }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ +use brk_vecs::{ AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec, - Format, VecIterator, + File, Format, VecIterator, }; use crate::{ @@ -128,7 +128,7 @@ pub struct Vecs { impl Vecs { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, cohort_name: Option<&str>, computation: Computation, format: Format, @@ -146,7 +146,7 @@ impl Vecs { let dateindex_to_supply_in_profit = compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_profit"), version + VERSION + Version::ZERO, format, @@ -156,7 +156,7 @@ impl Vecs { let dateindex_to_supply_even = compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_even"), version + VERSION + Version::ZERO, format, @@ -166,7 +166,7 @@ impl Vecs { let dateindex_to_supply_in_loss = compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_loss"), version + VERSION + Version::ZERO, format, @@ -176,7 +176,7 @@ impl Vecs { let dateindex_to_unrealized_profit = compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("unrealized_profit"), version + VERSION + Version::ZERO, format, @@ -186,7 +186,7 @@ impl Vecs { let dateindex_to_unrealized_loss = compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("unrealized_loss"), version + VERSION + Version::ZERO, format, @@ -197,7 +197,7 @@ impl Vecs { Ok(Self { height_to_supply_in_profit: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_profit"), version + VERSION + Version::ZERO, format, @@ -206,7 +206,7 @@ impl Vecs { }), indexes_to_supply_in_profit: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_in_profit"), dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -221,7 +221,7 @@ impl Vecs { dateindex_to_supply_in_profit, height_to_supply_even: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_even"), version + VERSION + Version::ZERO, format, @@ -230,7 +230,7 @@ impl Vecs { }), indexes_to_supply_even: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_even"), dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -245,7 +245,7 @@ impl Vecs { dateindex_to_supply_even, height_to_supply_in_loss: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_loss"), version + VERSION + Version::ZERO, format, @@ -254,7 +254,7 @@ impl Vecs { }), indexes_to_supply_in_loss: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_in_loss"), dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -269,7 +269,7 @@ impl Vecs { dateindex_to_supply_in_loss, height_to_unrealized_profit: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("unrealized_profit"), version + VERSION + Version::ZERO, format, @@ -278,7 +278,7 @@ impl Vecs { }), indexes_to_unrealized_profit: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("unrealized_profit"), dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -292,7 +292,7 @@ impl Vecs { dateindex_to_unrealized_profit, height_to_unrealized_loss: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("unrealized_loss"), version + VERSION + Version::ZERO, format, @@ -301,7 +301,7 @@ impl Vecs { }), height_to_min_price_paid: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("min_price_paid"), version + VERSION + Version::ZERO, format, @@ -310,7 +310,7 @@ impl Vecs { }), height_to_max_price_paid: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("max_price_paid"), version + VERSION + Version::ZERO, format, @@ -319,7 +319,7 @@ impl Vecs { }), indexes_to_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("unrealized_loss"), dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -333,7 +333,7 @@ impl Vecs { dateindex_to_unrealized_loss, height_to_realized_cap: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("realized_cap"), version + VERSION + Version::ZERO, format, @@ -342,7 +342,7 @@ impl Vecs { }), indexes_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_cap"), Source::None, version + VERSION + Version::ZERO, @@ -355,7 +355,7 @@ impl Vecs { }), indexes_to_min_price_paid: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("min_price_paid"), Source::None, version + VERSION + Version::ZERO, @@ -368,7 +368,7 @@ impl Vecs { }), indexes_to_max_price_paid: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("max_price_paid"), Source::None, version + VERSION + Version::ZERO, @@ -380,13 +380,13 @@ impl Vecs { .unwrap() }), height_to_supply: EagerVec::forced_import( - path, + file, &suffix("supply"), version + VERSION + Version::ZERO, format, )?, height_to_supply_value: ComputedHeightValueVecs::forced_import( - path, + file, &suffix("supply"), Source::None, version + VERSION + Version::ZERO, @@ -394,7 +394,7 @@ impl Vecs { compute_dollars, )?, indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( - path, + file, &suffix("supply"), Source::Compute, version + VERSION + Version::ONE, @@ -405,13 +405,13 @@ impl Vecs { indexes, )?, height_to_utxo_count: EagerVec::forced_import( - path, + file, &suffix("utxo_count"), version + VERSION + Version::ZERO, format, )?, indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( - path, + file, &suffix("utxo_count"), Source::None, version + VERSION + Version::ZERO, @@ -422,7 +422,7 @@ impl Vecs { )?, indexes_to_realized_price: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -435,7 +435,7 @@ impl Vecs { }), indexes_to_realized_price_extra: compute_dollars.then(|| { ComputedRatioVecsFromDateIndex::forced_import( - path, + file, &suffix("realized_price"), Source::None, version + VERSION + Version::ZERO, @@ -448,7 +448,7 @@ impl Vecs { }), height_to_realized_profit: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("realized_profit"), version + VERSION + Version::ZERO, format, @@ -457,7 +457,7 @@ impl Vecs { }), indexes_to_realized_profit: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_profit"), Source::None, version + VERSION + Version::ZERO, @@ -472,7 +472,7 @@ impl Vecs { }), height_to_realized_loss: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("realized_loss"), version + VERSION + Version::ZERO, format, @@ -481,7 +481,7 @@ impl Vecs { }), indexes_to_realized_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_loss"), Source::None, version + VERSION + Version::ZERO, @@ -496,7 +496,7 @@ impl Vecs { }), indexes_to_negative_realized_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("negative_realized_loss"), Source::Compute, version + VERSION + Version::ONE, @@ -509,7 +509,7 @@ impl Vecs { }), height_to_value_created: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("value_created"), version + VERSION + Version::ZERO, format, @@ -518,7 +518,7 @@ impl Vecs { }), indexes_to_value_created: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("value_created"), Source::None, version + VERSION + Version::ZERO, @@ -531,7 +531,7 @@ impl Vecs { }), indexes_to_realized_value: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_value"), Source::Compute, version + VERSION + Version::ZERO, @@ -544,7 +544,7 @@ impl Vecs { }), height_to_adjusted_value_created: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("adjusted_value_created"), version + VERSION + Version::ZERO, format, @@ -553,7 +553,7 @@ impl Vecs { }), indexes_to_adjusted_value_created: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("adjusted_value_created"), Source::None, version + VERSION + Version::ZERO, @@ -566,7 +566,7 @@ impl Vecs { }), height_to_value_destroyed: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("value_destroyed"), version + VERSION + Version::ZERO, format, @@ -575,7 +575,7 @@ impl Vecs { }), indexes_to_value_destroyed: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("value_destroyed"), Source::None, version + VERSION + Version::ZERO, @@ -588,7 +588,7 @@ impl Vecs { }), height_to_adjusted_value_destroyed: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("adjusted_value_destroyed"), version + VERSION + Version::ZERO, format, @@ -597,7 +597,7 @@ impl Vecs { }), indexes_to_adjusted_value_destroyed: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("adjusted_value_destroyed"), Source::None, version + VERSION + Version::ZERO, @@ -610,7 +610,7 @@ impl Vecs { }), indexes_to_realized_cap_30d_change: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("realized_cap_30d_change"), Source::Compute, version + VERSION + Version::ZERO, @@ -623,7 +623,7 @@ impl Vecs { }), indexes_to_net_realized_profit_and_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("net_realized_profit_and_loss"), Source::Compute, version + VERSION + Version::ZERO, @@ -638,7 +638,7 @@ impl Vecs { }), dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("sell_side_risk_ratio"), version + VERSION + Version::ONE, format, @@ -647,7 +647,7 @@ impl Vecs { }), dateindex_to_spent_output_profit_ratio: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("spent_output_profit_ratio"), version + VERSION + Version::ZERO, format, @@ -656,7 +656,7 @@ impl Vecs { }), dateindex_to_adjusted_spent_output_profit_ratio: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("adjusted_spent_output_profit_ratio"), version + VERSION + Version::ZERO, format, @@ -664,7 +664,7 @@ impl Vecs { .unwrap() }), height_to_halved_supply_value: ComputedHeightValueVecs::forced_import( - path, + file, &suffix("halved_supply"), Source::Compute, version + VERSION + Version::ZERO, @@ -672,7 +672,7 @@ impl Vecs { compute_dollars, )?, indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import( - path, + file, &suffix("halved_supply"), Source::Compute, version + VERSION + Version::ZERO, @@ -684,7 +684,7 @@ impl Vecs { )?, height_to_negative_unrealized_loss: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("negative_unrealized_loss"), version + VERSION + Version::ZERO, format, @@ -693,7 +693,7 @@ impl Vecs { }), indexes_to_negative_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("negative_unrealized_loss"), Source::Compute, version + VERSION + Version::ZERO, @@ -706,7 +706,7 @@ impl Vecs { }), height_to_net_unrealized_profit_and_loss: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("net_unrealized_profit_and_loss"), version + VERSION + Version::ZERO, format, @@ -715,7 +715,7 @@ impl Vecs { }), indexes_to_net_unrealized_profit_and_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("net_unrealized_profit_and_loss"), Source::Compute, version + VERSION + Version::ZERO, @@ -729,7 +729,7 @@ impl Vecs { height_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then( || { EagerVec::forced_import( - path, + file, &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), version + VERSION + Version::ONE, format, @@ -740,7 +740,7 @@ impl Vecs { indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then( || { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), Source::Compute, version + VERSION + Version::ONE, @@ -754,7 +754,7 @@ impl Vecs { ), indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_profit_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::ZERO, @@ -767,7 +767,7 @@ impl Vecs { }), indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("realized_loss_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::ZERO, @@ -781,7 +781,7 @@ impl Vecs { indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then( || { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("net_realized_profit_and_loss_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::ONE, @@ -795,7 +795,7 @@ impl Vecs { ), height_to_supply_even_value: compute_dollars.then(|| { ComputedHeightValueVecs::forced_import( - path, + file, &suffix("supply_even"), Source::None, version + VERSION + Version::ZERO, @@ -806,7 +806,7 @@ impl Vecs { }), height_to_supply_in_loss_value: compute_dollars.then(|| { ComputedHeightValueVecs::forced_import( - path, + file, &suffix("supply_in_loss"), Source::None, version + VERSION + Version::ZERO, @@ -817,7 +817,7 @@ impl Vecs { }), height_to_supply_in_profit_value: compute_dollars.then(|| { ComputedHeightValueVecs::forced_import( - path, + file, &suffix("supply_in_profit"), Source::None, version + VERSION + Version::ZERO, @@ -828,7 +828,7 @@ impl Vecs { }), height_to_supply_even_relative_to_own_supply: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_even_relative_to_own_supply"), version + VERSION + Version::ONE, format, @@ -837,7 +837,7 @@ impl Vecs { }), height_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_loss_relative_to_own_supply"), version + VERSION + Version::ONE, format, @@ -846,7 +846,7 @@ impl Vecs { }), height_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_profit_relative_to_own_supply"), version + VERSION + Version::ONE, format, @@ -855,7 +855,7 @@ impl Vecs { }), indexes_to_supply_even_relative_to_own_supply: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_even_relative_to_own_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -868,7 +868,7 @@ impl Vecs { }), indexes_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_in_loss_relative_to_own_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -881,7 +881,7 @@ impl Vecs { }), indexes_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_in_profit_relative_to_own_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -894,7 +894,7 @@ impl Vecs { }), indexes_to_supply_relative_to_circulating_supply: compute_relative_to_all.then(|| { ComputedVecsFromHeight::forced_import( - path, + file, &suffix("supply_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -909,7 +909,7 @@ impl Vecs { && compute_dollars) .then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_even_relative_to_circulating_supply"), version + VERSION + Version::ONE, format, @@ -920,7 +920,7 @@ impl Vecs { && compute_dollars) .then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_loss_relative_to_circulating_supply"), version + VERSION + Version::ONE, format, @@ -931,7 +931,7 @@ impl Vecs { && compute_dollars) .then(|| { EagerVec::forced_import( - path, + file, &suffix("supply_in_profit_relative_to_circulating_supply"), version + VERSION + Version::ONE, format, @@ -942,7 +942,7 @@ impl Vecs { && compute_dollars) .then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_even_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -957,7 +957,7 @@ impl Vecs { && compute_dollars) .then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_in_loss_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -972,7 +972,7 @@ impl Vecs { && compute_dollars) .then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("supply_in_profit_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -984,19 +984,19 @@ impl Vecs { .unwrap() }), height_to_satblocks_destroyed: EagerVec::forced_import( - path, + file, &suffix("satblocks_destroyed"), version + VERSION + Version::ZERO, format, )?, height_to_satdays_destroyed: EagerVec::forced_import( - path, + file, &suffix("satdays_destroyed"), version + VERSION + Version::ZERO, format, )?, indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import( - path, + file, &suffix("coinblocks_destroyed"), Source::Compute, version + VERSION + Version::TWO, @@ -1006,7 +1006,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import( - path, + file, &suffix("coindays_destroyed"), Source::Compute, version + VERSION + Version::TWO, @@ -1017,7 +1017,7 @@ impl Vecs { )?, indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("net_realized_profit_and_loss_cumulative_30d_change"), Source::Compute, version + VERSION + Version::new(3), @@ -1030,7 +1030,7 @@ impl Vecs { }), indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::new(3), @@ -1043,7 +1043,7 @@ impl Vecs { }), indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - path, + file, &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"), Source::Compute, version + VERSION + Version::new(3), diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index f94a3f449..7596af042 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1,4 +1,4 @@ -use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; +use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread}; use brk_core::{ AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex, @@ -9,9 +9,9 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyIndexedVec, AnyVec, CollectableVec, Computation, EagerVec, Format, - GenericStoredVec, IndexedVec, Mmap, StoredIndex, StoredVec, VecIterator, +use brk_vecs::{ + AnyCollectableVec, AnyStampedVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format, + GenericStoredVec, Reader, Stamp, StampedVec, StoredIndex, StoredVec, VecIterator, }; use log::info; use rayon::prelude::*; @@ -71,16 +71,16 @@ pub struct Vecs { pub utxo_cohorts: utxo_cohorts::Vecs, pub address_cohorts: address_cohorts::Vecs, - pub p2pk33addressindex_to_anyaddressindex: IndexedVec, - pub p2pk65addressindex_to_anyaddressindex: IndexedVec, - pub p2pkhaddressindex_to_anyaddressindex: IndexedVec, - pub p2shaddressindex_to_anyaddressindex: IndexedVec, - pub p2traddressindex_to_anyaddressindex: IndexedVec, - pub p2wpkhaddressindex_to_anyaddressindex: IndexedVec, - pub p2wshaddressindex_to_anyaddressindex: IndexedVec, - pub p2aaddressindex_to_anyaddressindex: IndexedVec, - pub loadedaddressindex_to_loadedaddressdata: IndexedVec, - pub emptyaddressindex_to_emptyaddressdata: IndexedVec, + pub p2pk33addressindex_to_anyaddressindex: StampedVec, + pub p2pk65addressindex_to_anyaddressindex: StampedVec, + pub p2pkhaddressindex_to_anyaddressindex: StampedVec, + pub p2shaddressindex_to_anyaddressindex: StampedVec, + pub p2traddressindex_to_anyaddressindex: StampedVec, + pub p2wpkhaddressindex_to_anyaddressindex: StampedVec, + pub p2wshaddressindex_to_anyaddressindex: StampedVec, + pub p2aaddressindex_to_anyaddressindex: StampedVec, + pub loadedaddressindex_to_loadedaddressdata: StampedVec, + pub emptyaddressindex_to_emptyaddressdata: StampedVec, pub indexes_to_address_count: ComputedVecsFromHeight, pub indexes_to_empty_address_count: ComputedVecsFromHeight, @@ -88,36 +88,34 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, computation: Computation, format: Format, indexes: &indexes::Vecs, fetched: Option<&fetched::Vecs>, + states_path: &Path, ) -> color_eyre::Result { let compute_dollars = fetched.is_some(); - let mut root_path = path.to_owned(); - root_path.pop(); - root_path.pop(); - let states_path = root_path.join("states"); + let chain_file = Arc::new(File::open(&states_path.join("chain"))?); Ok(Self { chain_state: StoredVec::forced_import( - &states_path, + &chain_file, "chain", version + VERSION + Version::ZERO, Format::Raw, )?, height_to_unspendable_supply: EagerVec::forced_import( - path, + file, "unspendable_supply", version + VERSION + Version::ZERO, format, )?, indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( - path, + file, "unspendable_supply", Source::None, version + VERSION + Version::ZERO, @@ -128,13 +126,13 @@ impl Vecs { indexes, )?, height_to_opreturn_supply: EagerVec::forced_import( - path, + file, "opreturn_supply", version + VERSION + Version::ZERO, format, )?, indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( - path, + file, "opreturn_supply", Source::None, version + VERSION + Version::ZERO, @@ -145,7 +143,7 @@ impl Vecs { indexes, )?, indexes_to_address_count: ComputedVecsFromHeight::forced_import( - path, + file, "address_count", Source::Compute, version + VERSION + Version::ZERO, @@ -155,7 +153,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import( - path, + file, "empty_address_count", Source::Compute, version + VERSION + Version::ZERO, @@ -167,49 +165,49 @@ impl Vecs { addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { p2pk65: EagerVec::forced_import( - path, + file, "p2pk65_address_count", version + VERSION + Version::ZERO, format, )?, p2pk33: EagerVec::forced_import( - path, + file, "p2pk33_address_count", version + VERSION + Version::ZERO, format, )?, p2pkh: EagerVec::forced_import( - path, + file, "p2pkh_address_count", version + VERSION + Version::ZERO, format, )?, p2sh: EagerVec::forced_import( - path, + file, "p2sh_address_count", version + VERSION + Version::ZERO, format, )?, p2wpkh: EagerVec::forced_import( - path, + file, "p2wpkh_address_count", version + VERSION + Version::ZERO, format, )?, p2wsh: EagerVec::forced_import( - path, + file, "p2wsh_address_count", version + VERSION + Version::ZERO, format, )?, p2tr: EagerVec::forced_import( - path, + file, "p2tr_address_count", version + VERSION + Version::ZERO, format, )?, p2a: EagerVec::forced_import( - path, + file, "p2a_address_count", version + VERSION + Version::ZERO, format, @@ -219,49 +217,49 @@ impl Vecs { addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { p2pk65: EagerVec::forced_import( - path, + file, "p2pk65_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2pk33: EagerVec::forced_import( - path, + file, "p2pk33_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2pkh: EagerVec::forced_import( - path, + file, "p2pkh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2sh: EagerVec::forced_import( - path, + file, "p2sh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2wpkh: EagerVec::forced_import( - path, + file, "p2wpkh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2wsh: EagerVec::forced_import( - path, + file, "p2wsh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2tr: EagerVec::forced_import( - path, + file, "p2tr_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2a: EagerVec::forced_import( - path, + file, "p2a_empty_address_count", version + VERSION + Version::ZERO, format, @@ -271,7 +269,7 @@ impl Vecs { addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from( ByAddressType { p2pk65: ComputedVecsFromHeight::forced_import( - path, + file, "p2pk65_address_count", Source::None, version + VERSION + Version::ZERO, @@ -281,7 +279,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pk33: ComputedVecsFromHeight::forced_import( - path, + file, "p2pk33_address_count", Source::None, version + VERSION + Version::ZERO, @@ -291,7 +289,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pkh: ComputedVecsFromHeight::forced_import( - path, + file, "p2pkh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -301,7 +299,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2sh: ComputedVecsFromHeight::forced_import( - path, + file, "p2sh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -311,7 +309,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wpkh: ComputedVecsFromHeight::forced_import( - path, + file, "p2wpkh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -321,7 +319,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wsh: ComputedVecsFromHeight::forced_import( - path, + file, "p2wsh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -331,7 +329,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2tr: ComputedVecsFromHeight::forced_import( - path, + file, "p2tr_address_count", Source::None, version + VERSION + Version::ZERO, @@ -341,7 +339,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2a: ComputedVecsFromHeight::forced_import( - path, + file, "p2a_address_count", Source::None, version + VERSION + Version::ZERO, @@ -355,7 +353,7 @@ impl Vecs { addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from( ByAddressType { p2pk65: ComputedVecsFromHeight::forced_import( - path, + file, "p2pk65_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -365,7 +363,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pk33: ComputedVecsFromHeight::forced_import( - path, + file, "p2pk33_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -375,7 +373,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pkh: ComputedVecsFromHeight::forced_import( - path, + file, "p2pkh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -385,7 +383,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2sh: ComputedVecsFromHeight::forced_import( - path, + file, "p2sh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -395,7 +393,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wpkh: ComputedVecsFromHeight::forced_import( - path, + file, "p2wpkh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -405,7 +403,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wsh: ComputedVecsFromHeight::forced_import( - path, + file, "p2wsh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -415,7 +413,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2tr: ComputedVecsFromHeight::forced_import( - path, + file, "p2tr_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -425,7 +423,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2a: ComputedVecsFromHeight::forced_import( - path, + file, "p2a_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -437,81 +435,81 @@ impl Vecs { }, ), utxo_cohorts: utxo_cohorts::Vecs::forced_import( - path, + file, version, computation, format, indexes, fetched, - &states_path, + states_path, )?, address_cohorts: address_cohorts::Vecs::forced_import( - path, + file, version, computation, format, indexes, fetched, - &states_path, + states_path, )?, - p2aaddressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2aaddressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2pk33addressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2pk33addressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2pk65addressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2pk65addressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2pkhaddressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2pkhaddressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2shaddressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2shaddressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2traddressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2traddressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2wpkhaddressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2wpkhaddressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2wshaddressindex_to_anyaddressindex: IndexedVec::forced_import( - path, + p2wshaddressindex_to_anyaddressindex: StampedVec::forced_import( + file, "anyaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - loadedaddressindex_to_loadedaddressdata: IndexedVec::forced_import( - path, + loadedaddressindex_to_loadedaddressdata: StampedVec::forced_import( + file, "loadedaddressdata", version + VERSION + Version::ZERO, Format::Raw, )?, - emptyaddressindex_to_emptyaddressdata: IndexedVec::forced_import( - path, + emptyaddressindex_to_emptyaddressdata: StampedVec::forced_import( + file, "emptyaddressdata", version + VERSION + Version::ZERO, Format::Raw, @@ -637,54 +635,54 @@ impl Vecs { ) .min(chain_state_starting_height) .min( - self.p2pk33addressindex_to_anyaddressindex - .height() + Height::from(u64::from( + self.p2pk33addressindex_to_anyaddressindex.stamp(), + )) + .incremented(), + ) + .min( + Height::from(u64::from( + self.p2pk65addressindex_to_anyaddressindex.stamp(), + )) + .incremented(), + ) + .min( + Height::from(u64::from(self.p2pkhaddressindex_to_anyaddressindex.stamp())) .incremented(), ) .min( - self.p2pk65addressindex_to_anyaddressindex - .height() + Height::from(u64::from(self.p2shaddressindex_to_anyaddressindex.stamp())) .incremented(), ) .min( - self.p2pkhaddressindex_to_anyaddressindex - .height() + Height::from(u64::from(self.p2traddressindex_to_anyaddressindex.stamp())) .incremented(), ) .min( - self.p2shaddressindex_to_anyaddressindex - .height() + Height::from(u64::from( + self.p2wpkhaddressindex_to_anyaddressindex.stamp(), + )) + .incremented(), + ) + .min( + Height::from(u64::from(self.p2wshaddressindex_to_anyaddressindex.stamp())) .incremented(), ) .min( - self.p2traddressindex_to_anyaddressindex - .height() + Height::from(u64::from(self.p2aaddressindex_to_anyaddressindex.stamp())) .incremented(), ) .min( - self.p2wpkhaddressindex_to_anyaddressindex - .height() - .incremented(), + Height::from(u64::from( + self.loadedaddressindex_to_loadedaddressdata.stamp(), + )) + .incremented(), ) .min( - self.p2wshaddressindex_to_anyaddressindex - .height() - .incremented(), - ) - .min( - self.p2aaddressindex_to_anyaddressindex - .height() - .incremented(), - ) - .min( - self.loadedaddressindex_to_loadedaddressdata - .height() - .incremented(), - ) - .min( - self.emptyaddressindex_to_emptyaddressdata - .height() - .incremented(), + Height::from(u64::from( + self.emptyaddressindex_to_emptyaddressdata.stamp(), + )) + .incremented(), ) .min(Height::from(self.height_to_unspendable_supply.len())) .min(Height::from(self.height_to_opreturn_supply.len())) @@ -749,13 +747,13 @@ impl Vecs { .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; }; - let last_height = indexer.vecs.height_to_blockhash.height(); + let last_height = Height::from(u64::from(indexer.vecs.height_to_blockhash.stamp())); if starting_height <= last_height { - let inputindex_to_outputindex_mmap = inputindex_to_outputindex.create_mmap()?; - let outputindex_to_value_mmap = outputindex_to_value.create_mmap()?; - let outputindex_to_outputtype_mmap = outputindex_to_outputtype.create_mmap()?; - let outputindex_to_typeindex_mmap = outputindex_to_typeindex.create_mmap()?; + let inputindex_to_outputindex_reader = inputindex_to_outputindex.create_reader(); + let outputindex_to_value_reader = outputindex_to_value.create_reader(); + let outputindex_to_outputtype_reader = outputindex_to_outputtype.create_reader(); + let outputindex_to_typeindex_reader = outputindex_to_typeindex.create_reader(); let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); @@ -828,14 +826,14 @@ impl Vecs { AddressTypeToTypeIndexTree::>::default(); let mut addresstype_to_typeindex_to_emptyaddressdata = AddressTypeToTypeIndexTree::>::default(); - let mut addresstypeindex_to_anyaddressindex_mmap_opt = - ByAddressType::>::default(); - let mut anyaddressindex_to_anyaddressdata_mmap_opt = - ByAnyAddress::>::default(); + let mut addresstypeindex_to_anyaddressindex_reader_opt = + ByAddressType::>::default(); + let mut anyaddressindex_to_anyaddressdata_reader_opt = + ByAnyAddress::>::default(); - self.reset_mmaps_options( - &mut addresstypeindex_to_anyaddressindex_mmap_opt, - &mut anyaddressindex_to_anyaddressdata_mmap_opt, + self.reset_readers_options( + &mut addresstypeindex_to_anyaddressindex_reader_opt, + &mut anyaddressindex_to_anyaddressdata_reader_opt, ); (height.unwrap_to_usize()..height_to_date_fixed.len()) @@ -912,17 +910,17 @@ impl Vecs { .map(OutputIndex::from) .map(|outputindex| { let value = outputindex_to_value - .unwrap_read(outputindex, &outputindex_to_value_mmap); + .unwrap_read(outputindex, &outputindex_to_value_reader); let output_type = outputindex_to_outputtype - .unwrap_read(outputindex, &outputindex_to_outputtype_mmap); + .unwrap_read(outputindex, &outputindex_to_outputtype_reader); if output_type.is_not_address() { return (value, output_type, None); } let typeindex = outputindex_to_typeindex - .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); + .unwrap_read(outputindex, &outputindex_to_typeindex_reader); let addressdata_opt = Self::get_addressdatawithsource( output_type, @@ -930,8 +928,8 @@ impl Vecs { &first_addressindexes, &addresstype_to_typeindex_to_loadedaddressdata, &addresstype_to_typeindex_to_emptyaddressdata, - &addresstypeindex_to_anyaddressindex_mmap_opt, - &anyaddressindex_to_anyaddressdata_mmap_opt, + &addresstypeindex_to_anyaddressindex_reader_opt, + &anyaddressindex_to_anyaddressdata_reader_opt, &self.p2pk33addressindex_to_anyaddressindex, &self.p2pk65addressindex_to_anyaddressindex, &self.p2pkhaddressindex_to_anyaddressindex, @@ -996,13 +994,13 @@ impl Vecs { .map(InputIndex::from) .map(|inputindex| { let outputindex = - inputindex_to_outputindex.unwrap_read(inputindex, &inputindex_to_outputindex_mmap); + inputindex_to_outputindex.unwrap_read(inputindex, &inputindex_to_outputindex_reader); let value = outputindex_to_value - .unwrap_read(outputindex, &outputindex_to_value_mmap); + .unwrap_read(outputindex, &outputindex_to_value_reader); let input_type = outputindex_to_outputtype - .unwrap_read(outputindex, &outputindex_to_outputtype_mmap); + .unwrap_read(outputindex, &outputindex_to_outputtype_reader); let prev_height = *outputindex_range_to_height.get(outputindex).unwrap(); @@ -1012,7 +1010,7 @@ impl Vecs { } let typeindex = outputindex_to_typeindex - .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); + .unwrap_read(outputindex, &outputindex_to_typeindex_reader); let addressdata_opt = Self::get_addressdatawithsource( input_type, @@ -1020,8 +1018,8 @@ impl Vecs { &first_addressindexes, &addresstype_to_typeindex_to_loadedaddressdata, &addresstype_to_typeindex_to_emptyaddressdata, - &addresstypeindex_to_anyaddressindex_mmap_opt, - &anyaddressindex_to_anyaddressdata_mmap_opt, + &addresstypeindex_to_anyaddressindex_reader_opt, + &anyaddressindex_to_anyaddressdata_reader_opt, &self.p2pk33addressindex_to_anyaddressindex, &self.p2pk65addressindex_to_anyaddressindex, &self.p2pkhaddressindex_to_anyaddressindex, @@ -1240,11 +1238,14 @@ impl Vecs { let _lock = exit.lock(); + addresstypeindex_to_anyaddressindex_reader_opt.take(); + anyaddressindex_to_anyaddressdata_reader_opt.take(); + self.flush_states(height, &chain_state, mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata), mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), exit)?; - self.reset_mmaps_options( - &mut addresstypeindex_to_anyaddressindex_mmap_opt, - &mut anyaddressindex_to_anyaddressdata_mmap_opt, + self.reset_readers_options( + &mut addresstypeindex_to_anyaddressindex_reader_opt, + &mut anyaddressindex_to_anyaddressdata_reader_opt, ); } @@ -1431,18 +1432,18 @@ impl Vecs { addresstype_to_typeindex_to_emptyaddressdata: &AddressTypeToTypeIndexTree< WithAddressDataSource, >, - addresstypeindex_to_anyaddressindex_mmap_opt: &ByAddressType>, - anyaddressindex_to_anyaddressdata_mmap_opt: &ByAnyAddress>, - p2pk33addressindex_to_anyaddressindex: &IndexedVec, - p2pk65addressindex_to_anyaddressindex: &IndexedVec, - p2pkhaddressindex_to_anyaddressindex: &IndexedVec, - p2shaddressindex_to_anyaddressindex: &IndexedVec, - p2traddressindex_to_anyaddressindex: &IndexedVec, - p2wpkhaddressindex_to_anyaddressindex: &IndexedVec, - p2wshaddressindex_to_anyaddressindex: &IndexedVec, - p2aaddressindex_to_anyaddressindex: &IndexedVec, - loadedaddressindex_to_loadedaddressdata: &IndexedVec, - emptyaddressindex_to_emptyaddressdata: &IndexedVec, + addresstypeindex_to_anyaddressindex_reader_opt: &ByAddressType>, + anyaddressindex_to_anyaddressdata_reader_opt: &ByAnyAddress>, + p2pk33addressindex_to_anyaddressindex: &StampedVec, + p2pk65addressindex_to_anyaddressindex: &StampedVec, + p2pkhaddressindex_to_anyaddressindex: &StampedVec, + p2shaddressindex_to_anyaddressindex: &StampedVec, + p2traddressindex_to_anyaddressindex: &StampedVec, + p2wpkhaddressindex_to_anyaddressindex: &StampedVec, + p2wshaddressindex_to_anyaddressindex: &StampedVec, + p2aaddressindex_to_anyaddressindex: &StampedVec, + loadedaddressindex_to_loadedaddressdata: &StampedVec, + emptyaddressindex_to_emptyaddressdata: &StampedVec, ) -> Option> { if *first_addressindexes.get(address_type).unwrap() <= typeindex { return Some(WithAddressDataSource::New(LoadedAddressData::default())); @@ -1460,7 +1461,7 @@ impl Vecs { return None; } - let mmap = addresstypeindex_to_anyaddressindex_mmap_opt + let mmap = addresstypeindex_to_anyaddressindex_reader_opt .get_unwrap(address_type) .as_ref() .unwrap(); @@ -1498,7 +1499,7 @@ impl Vecs { Some(match anyaddressindex.to_enum() { AnyAddressDataIndexEnum::Loaded(loadedaddressindex) => { - let mmap = anyaddressindex_to_anyaddressdata_mmap_opt + let mmap = anyaddressindex_to_anyaddressdata_reader_opt .loaded .as_ref() .unwrap(); @@ -1515,7 +1516,7 @@ impl Vecs { )) } AnyAddressDataIndexEnum::Empty(emtpyaddressindex) => { - let mmap = anyaddressindex_to_anyaddressdata_mmap_opt + let mmap = anyaddressindex_to_anyaddressdata_reader_opt .empty .as_ref() .unwrap(); @@ -1534,60 +1535,60 @@ impl Vecs { }) } - fn reset_mmaps_options( + fn reset_readers_options( &self, - addresstypeindex_to_anyaddressindex_mmap_opt: &mut ByAddressType>, - anyaddressindex_to_anyaddressdata_mmap_opt: &mut ByAnyAddress>, + addresstypeindex_to_anyaddressindex_reader_opt: &mut ByAddressType>, + anyaddressindex_to_anyaddressdata_reader_opt: &mut ByAnyAddress>, ) { - addresstypeindex_to_anyaddressindex_mmap_opt.p2pk65.replace( - self.p2pk65addressindex_to_anyaddressindex - .create_mmap() - .unwrap(), - ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2pk33.replace( - self.p2pk33addressindex_to_anyaddressindex - .create_mmap() - .unwrap(), - ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2pkh.replace( - self.p2pkhaddressindex_to_anyaddressindex - .create_mmap() - .unwrap(), - ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2sh.replace( + addresstypeindex_to_anyaddressindex_reader_opt + .p2pk65 + .replace( + self.p2pk65addressindex_to_anyaddressindex + .create_static_reader(), + ); + addresstypeindex_to_anyaddressindex_reader_opt + .p2pk33 + .replace( + self.p2pk33addressindex_to_anyaddressindex + .create_static_reader(), + ); + addresstypeindex_to_anyaddressindex_reader_opt + .p2pkh + .replace( + self.p2pkhaddressindex_to_anyaddressindex + .create_static_reader(), + ); + addresstypeindex_to_anyaddressindex_reader_opt.p2sh.replace( self.p2shaddressindex_to_anyaddressindex - .create_mmap() - .unwrap(), + .create_static_reader(), ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2wpkh.replace( - self.p2wpkhaddressindex_to_anyaddressindex - .create_mmap() - .unwrap(), - ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2wsh.replace( - self.p2wshaddressindex_to_anyaddressindex - .create_mmap() - .unwrap(), - ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2tr.replace( + addresstypeindex_to_anyaddressindex_reader_opt + .p2wpkh + .replace( + self.p2wpkhaddressindex_to_anyaddressindex + .create_static_reader(), + ); + addresstypeindex_to_anyaddressindex_reader_opt + .p2wsh + .replace( + self.p2wshaddressindex_to_anyaddressindex + .create_static_reader(), + ); + addresstypeindex_to_anyaddressindex_reader_opt.p2tr.replace( self.p2traddressindex_to_anyaddressindex - .create_mmap() - .unwrap(), + .create_static_reader(), ); - addresstypeindex_to_anyaddressindex_mmap_opt.p2a.replace( + addresstypeindex_to_anyaddressindex_reader_opt.p2a.replace( self.p2aaddressindex_to_anyaddressindex - .create_mmap() - .unwrap(), + .create_static_reader(), ); - anyaddressindex_to_anyaddressdata_mmap_opt.loaded.replace( + anyaddressindex_to_anyaddressdata_reader_opt.loaded.replace( self.loadedaddressindex_to_loadedaddressdata - .create_mmap() - .unwrap(), + .create_static_reader(), ); - anyaddressindex_to_anyaddressdata_mmap_opt.empty.replace( + anyaddressindex_to_anyaddressdata_reader_opt.empty.replace( self.emptyaddressindex_to_emptyaddressdata - .create_mmap() - .unwrap(), + .create_static_reader(), ); } @@ -1765,16 +1766,26 @@ impl Vecs { }) })?; - self.p2pk33addressindex_to_anyaddressindex.flush(height)?; - self.p2pk65addressindex_to_anyaddressindex.flush(height)?; - self.p2pkhaddressindex_to_anyaddressindex.flush(height)?; - self.p2shaddressindex_to_anyaddressindex.flush(height)?; - self.p2traddressindex_to_anyaddressindex.flush(height)?; - self.p2wpkhaddressindex_to_anyaddressindex.flush(height)?; - self.p2wshaddressindex_to_anyaddressindex.flush(height)?; - self.p2aaddressindex_to_anyaddressindex.flush(height)?; - self.loadedaddressindex_to_loadedaddressdata.flush(height)?; - self.emptyaddressindex_to_emptyaddressdata.flush(height)?; + self.p2pk33addressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2pk65addressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2pkhaddressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2shaddressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2traddressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2wpkhaddressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2wshaddressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.p2aaddressindex_to_anyaddressindex + .flush(Stamp::from(u64::from(height)))?; + self.loadedaddressindex_to_loadedaddressdata + .flush(Stamp::from(u64::from(height)))?; + self.emptyaddressindex_to_emptyaddressdata + .flush(Stamp::from(u64::from(height)))?; self.chain_state.truncate_if_needed(Height::ZERO)?; chain_state.iter().for_each(|block_state| { diff --git a/crates/brk_computer/src/stateful/range_map.rs b/crates/brk_computer/src/stateful/range_map.rs index ee52d4da1..ef4d20db4 100644 --- a/crates/brk_computer/src/stateful/range_map.rs +++ b/crates/brk_computer/src/stateful/range_map.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use brk_vec::{IndexedVec, StoredIndex, StoredType}; +use brk_vecs::{StampedVec, StoredIndex, StoredType}; #[derive(Debug)] pub struct RangeMap(BTreeMap); @@ -20,12 +20,12 @@ where } } -impl From<&IndexedVec> for RangeMap +impl From<&StampedVec> for RangeMap where I: StoredIndex, T: StoredIndex + StoredType, { - fn from(vec: &IndexedVec) -> Self { + fn from(vec: &StampedVec) -> Self { Self( vec.into_iter() .map(|(i, v)| (v.into_owned(), i)) diff --git a/crates/brk_computer/src/stateful/trait.rs b/crates/brk_computer/src/stateful/trait.rs index 8b059d52f..fca7c6cce 100644 --- a/crates/brk_computer/src/stateful/trait.rs +++ b/crates/brk_computer/src/stateful/trait.rs @@ -1,7 +1,7 @@ use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIterableVec}; +use brk_vecs::{AnyCollectableVec, AnyIterableVec}; use crate::{Indexes, fetched, indexes, market}; diff --git a/crates/brk_computer/src/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs index 959edf4b0..5bb562dd9 100644 --- a/crates/brk_computer/src/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -1,9 +1,9 @@ -use std::{ops::Deref, path::Path}; +use std::{ops::Deref, path::Path, sync::Arc}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format}; +use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, File, Format}; use crate::{ Indexes, UTXOCohortState, fetched, indexes, market, @@ -25,7 +25,7 @@ pub struct Vecs { impl Vecs { #[allow(clippy::too_many_arguments)] pub fn forced_import( - path: &Path, + file: &Arc, cohort_name: Option<&str>, computation: Computation, format: Format, @@ -48,7 +48,7 @@ impl Vecs { )?, inner: common::Vecs::forced_import( - path, + file, cohort_name, computation, format, diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index cadbdd428..3dc33305a 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, ops::ControlFlow, path::Path}; +use std::{collections::BTreeMap, ops::ControlFlow, path::Path, sync::Arc}; use brk_core::{ Bitcoin, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge, @@ -7,7 +7,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyIterableVec, Computation, Format, StoredIndex}; +use brk_vecs::{AnyIterableVec, Computation, File, Format, StoredIndex}; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; @@ -26,7 +26,7 @@ pub struct Vecs(UTXOGroups<(GroupFilter, utxo_cohort::Vecs)>); impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, _computation: Computation, format: Format, @@ -37,7 +37,7 @@ impl Vecs { Ok(Self( UTXOGroups { all: utxo_cohort::Vecs::forced_import( - path, + file, None, _computation, format, @@ -50,7 +50,7 @@ impl Vecs { )?, term: ByTerm { short: utxo_cohort::Vecs::forced_import( - path, + file, Some("short_term_holders"), _computation, format, @@ -62,7 +62,7 @@ impl Vecs { true, )?, long: utxo_cohort::Vecs::forced_import( - path, + file, Some("long_term_holders"), _computation, format, @@ -76,7 +76,7 @@ impl Vecs { }, epoch: ByEpoch { _0: utxo_cohort::Vecs::forced_import( - path, + file, Some("epoch_0"), _computation, format, @@ -88,7 +88,7 @@ impl Vecs { true, )?, _1: utxo_cohort::Vecs::forced_import( - path, + file, Some("epoch_1"), _computation, format, @@ -100,7 +100,7 @@ impl Vecs { true, )?, _2: utxo_cohort::Vecs::forced_import( - path, + file, Some("epoch_2"), _computation, format, @@ -112,7 +112,7 @@ impl Vecs { true, )?, _3: utxo_cohort::Vecs::forced_import( - path, + file, Some("epoch_3"), _computation, format, @@ -124,7 +124,7 @@ impl Vecs { true, )?, _4: utxo_cohort::Vecs::forced_import( - path, + file, Some("epoch_4"), _computation, format, @@ -138,7 +138,7 @@ impl Vecs { }, _type: BySpendableType { p2pk65: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2pk65"), _computation, format, @@ -150,7 +150,7 @@ impl Vecs { false, )?, p2pk33: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2pk33"), _computation, format, @@ -162,7 +162,7 @@ impl Vecs { false, )?, p2pkh: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2pkh"), _computation, format, @@ -174,7 +174,7 @@ impl Vecs { false, )?, p2sh: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2sh"), _computation, format, @@ -186,7 +186,7 @@ impl Vecs { false, )?, p2wpkh: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2wpkh"), _computation, format, @@ -198,7 +198,7 @@ impl Vecs { false, )?, p2wsh: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2wsh"), _computation, format, @@ -210,7 +210,7 @@ impl Vecs { false, )?, p2tr: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2tr"), _computation, format, @@ -222,7 +222,7 @@ impl Vecs { false, )?, p2a: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2a"), _computation, format, @@ -234,7 +234,7 @@ impl Vecs { false, )?, p2ms: utxo_cohort::Vecs::forced_import( - path, + file, Some("p2ms_outputs"), _computation, format, @@ -246,7 +246,7 @@ impl Vecs { false, )?, empty: utxo_cohort::Vecs::forced_import( - path, + file, Some("empty_outputs"), _computation, format, @@ -258,7 +258,7 @@ impl Vecs { false, )?, unknown: utxo_cohort::Vecs::forced_import( - path, + file, Some("unknown_outputs"), _computation, format, @@ -272,7 +272,7 @@ impl Vecs { }, max_age: ByMaxAge { _1w: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_1w_old"), _computation, format, @@ -284,7 +284,7 @@ impl Vecs { true, )?, _1m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_1m_old"), _computation, format, @@ -296,7 +296,7 @@ impl Vecs { true, )?, _2m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_2m_old"), _computation, format, @@ -308,7 +308,7 @@ impl Vecs { true, )?, _3m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_3m_old"), _computation, format, @@ -320,7 +320,7 @@ impl Vecs { true, )?, _4m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_4m_old"), _computation, format, @@ -332,7 +332,7 @@ impl Vecs { true, )?, _5m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_5m_old"), _computation, format, @@ -344,7 +344,7 @@ impl Vecs { true, )?, _6m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_6m_old"), _computation, format, @@ -356,7 +356,7 @@ impl Vecs { true, )?, _1y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_1y_old"), _computation, format, @@ -368,7 +368,7 @@ impl Vecs { true, )?, _2y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_2y_old"), _computation, format, @@ -380,7 +380,7 @@ impl Vecs { true, )?, _3y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_3y_old"), _computation, format, @@ -392,7 +392,7 @@ impl Vecs { true, )?, _4y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_4y_old"), _computation, format, @@ -404,7 +404,7 @@ impl Vecs { true, )?, _5y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_5y_old"), _computation, format, @@ -416,7 +416,7 @@ impl Vecs { true, )?, _6y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_6y_old"), _computation, format, @@ -428,7 +428,7 @@ impl Vecs { true, )?, _7y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_7y_old"), _computation, format, @@ -440,7 +440,7 @@ impl Vecs { true, )?, _8y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_8y_old"), _computation, format, @@ -452,7 +452,7 @@ impl Vecs { true, )?, _10y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_10y_old"), _computation, format, @@ -464,7 +464,7 @@ impl Vecs { true, )?, _12y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_12y_old"), _computation, format, @@ -476,7 +476,7 @@ impl Vecs { true, )?, _15y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_15y_old"), _computation, format, @@ -490,7 +490,7 @@ impl Vecs { }, min_age: ByMinAge { _1d: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1d_old"), _computation, format, @@ -502,7 +502,7 @@ impl Vecs { true, )?, _1w: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1w_old"), _computation, format, @@ -514,7 +514,7 @@ impl Vecs { true, )?, _1m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1m_old"), _computation, format, @@ -526,7 +526,7 @@ impl Vecs { true, )?, _2m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_2m_old"), _computation, format, @@ -538,7 +538,7 @@ impl Vecs { true, )?, _3m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_3m_old"), _computation, format, @@ -550,7 +550,7 @@ impl Vecs { true, )?, _4m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_4m_old"), _computation, format, @@ -562,7 +562,7 @@ impl Vecs { true, )?, _5m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_5m_old"), _computation, format, @@ -574,7 +574,7 @@ impl Vecs { true, )?, _6m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_6m_old"), _computation, format, @@ -586,7 +586,7 @@ impl Vecs { true, )?, _1y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1y_old"), _computation, format, @@ -598,7 +598,7 @@ impl Vecs { true, )?, _2y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_2y_old"), _computation, format, @@ -610,7 +610,7 @@ impl Vecs { true, )?, _3y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_3y_old"), _computation, format, @@ -622,7 +622,7 @@ impl Vecs { true, )?, _4y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_4y_old"), _computation, format, @@ -634,7 +634,7 @@ impl Vecs { true, )?, _5y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_5y_old"), _computation, format, @@ -646,7 +646,7 @@ impl Vecs { true, )?, _6y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_6y_old"), _computation, format, @@ -658,7 +658,7 @@ impl Vecs { true, )?, _7y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_7y_old"), _computation, format, @@ -670,7 +670,7 @@ impl Vecs { true, )?, _8y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_8y_old"), _computation, format, @@ -682,7 +682,7 @@ impl Vecs { true, )?, _10y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_10y_old"), _computation, format, @@ -694,7 +694,7 @@ impl Vecs { true, )?, _12y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_12y_old"), _computation, format, @@ -708,7 +708,7 @@ impl Vecs { }, age_range: ByAgeRange { up_to_1d: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_up_to_1d_old"), _computation, format, @@ -720,7 +720,7 @@ impl Vecs { true, )?, _1d_to_1w: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1d_up_to_1w_old"), _computation, format, @@ -732,7 +732,7 @@ impl Vecs { true, )?, _1w_to_1m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1w_up_to_1m_old"), _computation, format, @@ -744,7 +744,7 @@ impl Vecs { true, )?, _1m_to_2m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1m_up_to_2m_old"), _computation, format, @@ -756,7 +756,7 @@ impl Vecs { true, )?, _2m_to_3m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_2m_up_to_3m_old"), _computation, format, @@ -768,7 +768,7 @@ impl Vecs { true, )?, _3m_to_4m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_3m_up_to_4m_old"), _computation, format, @@ -780,7 +780,7 @@ impl Vecs { true, )?, _4m_to_5m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_4m_up_to_5m_old"), _computation, format, @@ -792,7 +792,7 @@ impl Vecs { true, )?, _5m_to_6m: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_5m_up_to_6m_old"), _computation, format, @@ -804,7 +804,7 @@ impl Vecs { true, )?, _6m_to_1y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_6m_up_to_1y_old"), _computation, format, @@ -816,7 +816,7 @@ impl Vecs { true, )?, _1y_to_2y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_1y_up_to_2y_old"), _computation, format, @@ -828,7 +828,7 @@ impl Vecs { true, )?, _2y_to_3y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_2y_up_to_3y_old"), _computation, format, @@ -840,7 +840,7 @@ impl Vecs { true, )?, _3y_to_4y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_3y_up_to_4y_old"), _computation, format, @@ -852,7 +852,7 @@ impl Vecs { true, )?, _4y_to_5y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_4y_up_to_5y_old"), _computation, format, @@ -864,7 +864,7 @@ impl Vecs { true, )?, _5y_to_6y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_5y_up_to_6y_old"), _computation, format, @@ -876,7 +876,7 @@ impl Vecs { true, )?, _6y_to_7y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_6y_up_to_7y_old"), _computation, format, @@ -888,7 +888,7 @@ impl Vecs { true, )?, _7y_to_8y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_7y_up_to_8y_old"), _computation, format, @@ -900,7 +900,7 @@ impl Vecs { true, )?, _8y_to_10y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_8y_up_to_10y_old"), _computation, format, @@ -912,7 +912,7 @@ impl Vecs { true, )?, _10y_to_12y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_10y_up_to_12y_old"), _computation, format, @@ -924,7 +924,7 @@ impl Vecs { true, )?, _12y_to_15y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_12y_up_to_15y_old"), _computation, format, @@ -936,7 +936,7 @@ impl Vecs { true, )?, from_15y: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_at_least_15y_old"), _computation, format, @@ -950,7 +950,7 @@ impl Vecs { }, amount_range: ByAmountRange { _0sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_with_0sats"), _computation, format, @@ -962,7 +962,7 @@ impl Vecs { false, )?, _1sat_to_10sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1sat_under_10sats"), _computation, format, @@ -974,7 +974,7 @@ impl Vecs { false, )?, _10sats_to_100sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10sats_under_100sats"), _computation, format, @@ -986,7 +986,7 @@ impl Vecs { false, )?, _100sats_to_1k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100sats_under_1k_sats"), _computation, format, @@ -998,7 +998,7 @@ impl Vecs { false, )?, _1k_sats_to_10k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1k_sats_under_10k_sats"), _computation, format, @@ -1010,7 +1010,7 @@ impl Vecs { false, )?, _10k_sats_to_100k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10k_sats_under_100k_sats"), _computation, format, @@ -1022,7 +1022,7 @@ impl Vecs { false, )?, _100k_sats_to_1m_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100k_sats_under_1m_sats"), _computation, format, @@ -1034,7 +1034,7 @@ impl Vecs { false, )?, _1m_sats_to_10m_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1m_sats_under_10m_sats"), _computation, format, @@ -1046,7 +1046,7 @@ impl Vecs { false, )?, _10m_sats_to_1btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10m_sats_under_1btc"), _computation, format, @@ -1058,7 +1058,7 @@ impl Vecs { false, )?, _1btc_to_10btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1btc_under_10btc"), _computation, format, @@ -1070,7 +1070,7 @@ impl Vecs { false, )?, _10btc_to_100btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10btc_under_100btc"), _computation, format, @@ -1082,7 +1082,7 @@ impl Vecs { false, )?, _100btc_to_1k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100btc_under_1k_btc"), _computation, format, @@ -1094,7 +1094,7 @@ impl Vecs { false, )?, _1k_btc_to_10k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1k_btc_under_10k_btc"), _computation, format, @@ -1106,7 +1106,7 @@ impl Vecs { false, )?, _10k_btc_to_100k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10k_btc_under_100k_btc"), _computation, format, @@ -1118,7 +1118,7 @@ impl Vecs { false, )?, _100k_btc_or_more: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100k_btc"), _computation, format, @@ -1132,7 +1132,7 @@ impl Vecs { }, lt_amount: ByLowerThanAmount { _10sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_10sats"), _computation, format, @@ -1144,7 +1144,7 @@ impl Vecs { false, )?, _100sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_100sats"), _computation, format, @@ -1156,7 +1156,7 @@ impl Vecs { false, )?, _1k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_1k_sats"), _computation, format, @@ -1168,7 +1168,7 @@ impl Vecs { false, )?, _10k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_10k_sats"), _computation, format, @@ -1180,7 +1180,7 @@ impl Vecs { false, )?, _100k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_100k_sats"), _computation, format, @@ -1192,7 +1192,7 @@ impl Vecs { false, )?, _1m_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_1m_sats"), _computation, format, @@ -1204,7 +1204,7 @@ impl Vecs { false, )?, _10m_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_10m_sats"), _computation, format, @@ -1216,7 +1216,7 @@ impl Vecs { false, )?, _1btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_1btc"), _computation, format, @@ -1228,7 +1228,7 @@ impl Vecs { false, )?, _10btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_10btc"), _computation, format, @@ -1240,7 +1240,7 @@ impl Vecs { false, )?, _100btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_100btc"), _computation, format, @@ -1252,7 +1252,7 @@ impl Vecs { false, )?, _1k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_1k_btc"), _computation, format, @@ -1264,7 +1264,7 @@ impl Vecs { false, )?, _10k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_10k_btc"), _computation, format, @@ -1276,7 +1276,7 @@ impl Vecs { false, )?, _100k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_under_100k_btc"), _computation, format, @@ -1290,7 +1290,7 @@ impl Vecs { }, ge_amount: ByGreatEqualAmount { _1sat: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1sat"), _computation, format, @@ -1302,7 +1302,7 @@ impl Vecs { false, )?, _10sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10sats"), _computation, format, @@ -1314,7 +1314,7 @@ impl Vecs { false, )?, _100sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100sats"), _computation, format, @@ -1326,7 +1326,7 @@ impl Vecs { false, )?, _1k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1k_sats"), _computation, format, @@ -1338,7 +1338,7 @@ impl Vecs { false, )?, _10k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10k_sats"), _computation, format, @@ -1350,7 +1350,7 @@ impl Vecs { false, )?, _100k_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100k_sats"), _computation, format, @@ -1362,7 +1362,7 @@ impl Vecs { false, )?, _1m_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1m_sats"), _computation, format, @@ -1374,7 +1374,7 @@ impl Vecs { false, )?, _10m_sats: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10m_sats"), _computation, format, @@ -1386,7 +1386,7 @@ impl Vecs { false, )?, _1btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1btc"), _computation, format, @@ -1398,7 +1398,7 @@ impl Vecs { false, )?, _10btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10btc"), _computation, format, @@ -1410,7 +1410,7 @@ impl Vecs { false, )?, _100btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_100btc"), _computation, format, @@ -1422,7 +1422,7 @@ impl Vecs { false, )?, _1k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_1k_btc"), _computation, format, @@ -1434,7 +1434,7 @@ impl Vecs { false, )?, _10k_btc: utxo_cohort::Vecs::forced_import( - path, + file, Some("utxos_above_10k_btc"), _computation, format, diff --git a/crates/brk_computer/src/transactions.rs b/crates/brk_computer/src/transactions.rs index 220e544ea..1c54e2dbe 100644 --- a/crates/brk_computer/src/transactions.rs +++ b/crates/brk_computer/src/transactions.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32, @@ -6,9 +6,9 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{ +use brk_vecs::{ AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec, - ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator, + ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, File, Format, StoredIndex, VecIterator, }; use crate::grouped::{ @@ -86,7 +86,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( - path: &Path, + file: &Arc, version: Version, indexer: &Indexer, indexes: &indexes::Vecs, @@ -98,7 +98,7 @@ impl Vecs { let inputindex_to_value = ComputedVec::forced_import_or_init_from_2( computation, - path, + file, "value", version + VERSION + Version::ZERO, format, @@ -125,7 +125,7 @@ impl Vecs { let txindex_to_weight = ComputedVec::forced_import_or_init_from_2( computation, - path, + file, "weight", version + VERSION + Version::ZERO, format, @@ -153,7 +153,7 @@ impl Vecs { let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1( computation, - path, + file, "vsize", version + VERSION + Version::ZERO, format, @@ -170,7 +170,7 @@ impl Vecs { let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2( computation, - path, + file, "is_coinbase", version + VERSION + Version::ZERO, format, @@ -194,7 +194,7 @@ impl Vecs { let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3( computation, - path, + file, "input_value", version + VERSION + Version::ZERO, format, @@ -230,7 +230,7 @@ impl Vecs { // let indexes_to_input_value: ComputedVecsFromTxindex = // ComputedVecsFromTxindex::forced_import( - // path, + // file, // "input_value", // true, // version + VERSION + Version::ZERO, @@ -244,7 +244,7 @@ impl Vecs { let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3( computation, - path, + file, "output_value", version + VERSION + Version::ZERO, format, @@ -280,7 +280,7 @@ impl Vecs { // let indexes_to_output_value: ComputedVecsFromTxindex = // ComputedVecsFromTxindex::forced_import( - // path, + // file, // "output_value", // true, // version + VERSION + Version::ZERO, @@ -294,7 +294,7 @@ impl Vecs { let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2( computation, - path, + file, "fee", version + VERSION + Version::ZERO, format, @@ -317,7 +317,7 @@ impl Vecs { let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2( computation, - path, + file, "feerate", version + VERSION + Version::ZERO, format, @@ -337,7 +337,7 @@ impl Vecs { Ok(Self { indexes_to_tx_count: ComputedVecsFromHeight::forced_import( - path, + file, "tx_count", Source::Compute, version + VERSION + Version::ZERO, @@ -352,7 +352,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_input_count: ComputedVecsFromTxindex::forced_import( - path, + file, "input_count", Source::None, version + VERSION + Version::ZERO, @@ -367,7 +367,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_output_count: ComputedVecsFromTxindex::forced_import( - path, + file, "output_count", Source::None, version + VERSION + Version::ZERO, @@ -382,7 +382,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( - path, + file, "tx_v1", Source::Compute, version + VERSION + Version::ZERO, @@ -392,7 +392,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v2: ComputedVecsFromHeight::forced_import( - path, + file, "tx_v2", Source::Compute, version + VERSION + Version::ZERO, @@ -402,7 +402,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v3: ComputedVecsFromHeight::forced_import( - path, + file, "tx_v3", Source::Compute, version + VERSION + Version::ZERO, @@ -412,7 +412,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_fee: ComputedValueVecsFromTxindex::forced_import( - path, + file, "fee", indexes, Source::Vec(txindex_to_fee.boxed_clone()), @@ -428,7 +428,7 @@ impl Vecs { .add_average(), )?, indexes_to_feerate: ComputedVecsFromTxindex::forced_import( - path, + file, "feerate", Source::None, version + VERSION + Version::ZERO, @@ -441,7 +441,7 @@ impl Vecs { .add_average(), )?, indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import( - path, + file, "tx_vsize", Source::None, version + VERSION + Version::ZERO, @@ -454,7 +454,7 @@ impl Vecs { .add_average(), )?, indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import( - path, + file, "tx_weight", Source::None, version + VERSION + Version::ZERO, @@ -467,7 +467,7 @@ impl Vecs { .add_average(), )?, indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( - path, + file, "subsidy", Source::Compute, version + VERSION + Version::ZERO, @@ -483,7 +483,7 @@ impl Vecs { indexes, )?, indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import( - path, + file, "coinbase", Source::Compute, version + VERSION + Version::ZERO, @@ -499,7 +499,7 @@ impl Vecs { indexes, )?, indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import( - path, + file, "unclaimed_rewards", Source::Compute, version + VERSION + Version::ZERO, @@ -510,7 +510,7 @@ impl Vecs { indexes, )?, indexes_to_p2a_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2a_count", Source::Compute, version + VERSION + Version::ZERO, @@ -525,7 +525,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2ms_count", Source::Compute, version + VERSION + Version::ZERO, @@ -540,7 +540,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2pk33_count", Source::Compute, version + VERSION + Version::ZERO, @@ -555,7 +555,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2pk65_count", Source::Compute, version + VERSION + Version::ZERO, @@ -570,7 +570,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2pkh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -585,7 +585,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2sh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -600,7 +600,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2tr_count", Source::Compute, version + VERSION + Version::ZERO, @@ -615,7 +615,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2wpkh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -630,7 +630,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import( - path, + file, "p2wsh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -645,7 +645,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import( - path, + file, "opreturn_count", Source::Compute, version + VERSION + Version::ZERO, @@ -660,7 +660,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import( - path, + file, "unknownoutput_count", Source::Compute, version + VERSION + Version::ZERO, @@ -675,7 +675,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import( - path, + file, "emptyoutput_count", Source::Compute, version + VERSION + Version::ZERO, @@ -690,7 +690,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import( - path, + file, "exact_utxo_count", Source::Compute, version + VERSION + Version::ZERO, diff --git a/crates/brk_core/src/groups/by_address_type.rs b/crates/brk_core/src/groups/by_address_type.rs index 17289bcfb..c157c1290 100644 --- a/crates/brk_core/src/groups/by_address_type.rs +++ b/crates/brk_core/src/groups/by_address_type.rs @@ -171,3 +171,11 @@ where self.p2a += rhs.p2a; } } + +impl ByAddressType> { + pub fn take(&mut self) { + self.as_mut_vec().into_iter().for_each(|opt| { + opt.take(); + }); + } +} diff --git a/crates/brk_core/src/groups/by_any_address.rs b/crates/brk_core/src/groups/by_any_address.rs index 13591e9c4..6ac5ca291 100644 --- a/crates/brk_core/src/groups/by_any_address.rs +++ b/crates/brk_core/src/groups/by_any_address.rs @@ -3,3 +3,10 @@ pub struct ByAnyAddress { pub loaded: T, pub empty: T, } + +impl ByAnyAddress> { + pub fn take(&mut self) { + self.loaded.take(); + self.empty.take(); + } +} diff --git a/crates/brk_core/src/structs/addressbyteshash.rs b/crates/brk_core/src/structs/addressbyteshash.rs index 538be5977..c6b453fb1 100644 --- a/crates/brk_core/src/structs/addressbyteshash.rs +++ b/crates/brk_core/src/structs/addressbyteshash.rs @@ -1,5 +1,3 @@ -use std::hash::Hasher; - use byteview::ByteView; use derive_deref::Deref; use zerocopy::{FromBytes, IntoBytes}; diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index 0cf7bcdce..0e9f40581 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -35,7 +35,7 @@ impl Height { pub const ZERO: Self = Self(0); pub const MAX: Self = Self(u32::MAX); - pub fn new(height: u32) -> Self { + pub const fn new(height: u32) -> Self { Self(height) } diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 4669d49b3..9e1fcdf29 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -15,9 +15,9 @@ fn main() -> color_eyre::Result<()> { brk_logger::init(Some(Path::new(".log"))); // let bitcoin_dir = brk_core::default_bitcoin_path(); - let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin"); + let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); // let outputs_dir = brk_core::default_brk_path().join("outputs"); - let outputs_dir = Path::new("/Volumes/WD_BLACK/brk/outputs"); + let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( "http://localhost:8332", @@ -33,7 +33,7 @@ fn main() -> color_eyre::Result<()> { loop { let i = Instant::now(); - indexer.index(&parser, rpc, &exit, false)?; + indexer.index(&parser, rpc, &exit, true)?; dbg!(i.elapsed()); sleep(Duration::from_secs(5 * 60)); diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index 8b44290ac..b28620b04 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -89,8 +89,10 @@ impl Indexes { impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { type Error = color_eyre::Report; fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result { - // Height at which we wanna start: min last saved + 1 or 0 - let starting_height = vecs.starting_height().min(stores.starting_height()); + // Height at which we want to start: min last saved + 1 or 0 + let vecs_starting_height = vecs.starting_height(); + let stores_starting_height = stores.starting_height(); + let starting_height = vecs_starting_height.min(stores_starting_height); let range = u32::from( starting_height diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 88b8bd8d9..704f0a3ba 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -3,18 +3,18 @@ #![doc = include_str!("../examples/indexer.rs")] #![doc = "```"] -use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread}; +use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant}; use bitcoin::{Transaction, TxIn, TxOut}; use brk_core::{ AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex, - OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, - Unit, Version, Vin, Vout, setrlimit, + OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, + TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit, }; use brk_exit::Exit; use brk_parser::Parser; use brk_store::AnyStore; -use brk_vecs::{AnyVec, File, Reader, VecIterator}; +use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator}; use color_eyre::eyre::{ContextCompat, eyre}; use log::{error, info}; use rayon::prelude::*; @@ -27,7 +27,7 @@ pub use stores::*; pub use vecs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1000; -const COLLISIONS_CHECKED_UP_TO: u32 = 893_000; +const COLLISIONS_CHECKED_UP_TO: Height = Height::new(415_000); const VERSION: Version = Version::ONE; #[derive(Clone)] @@ -41,11 +41,18 @@ impl Indexer { pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result { setrlimit()?; - let file = Arc::new(File::open(&outputs_dir.join("vecs"))?); + let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?); + + let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?; + + file.set_min_len(PAGE_SIZE * 50_000_000)?; Ok(Self { - vecs: Vecs::forced_import(&file, VERSION + Version::ZERO)?, - stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?, + vecs, + stores: Stores::forced_import( + &outputs_dir.join("indexed/stores"), + VERSION + Version::ZERO, + )?, file, }) } @@ -57,6 +64,8 @@ impl Indexer { exit: &Exit, check_collisions: bool, ) -> color_eyre::Result { + let file = self.file.clone(); + let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)) .unwrap_or_else(|_report| Indexes::default()); @@ -87,23 +96,30 @@ impl Indexer { info!("Started indexing..."); - let export_if_needed = |stores: &mut Stores, - vecs: &mut Vecs, - height: Height, - rem: bool, - exit: &Exit| - -> color_eyre::Result { - if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem { - return Ok(false); - } - - info!("Exporting..."); - let _lock = exit.lock(); - stores.commit(height)?; - vecs.flush(height)?; - Ok(true) + let should_export = |height: Height, rem: bool| -> bool { + height != 0 && (height % SNAPSHOT_BLOCK_RANGE == 0) != rem }; + let export = + |stores: &mut Stores, vecs: &mut Vecs, height: Height, exit: &Exit| -> Result<()> { + info!("Exporting..."); + let _lock = exit.lock(); + thread::scope(|scope| -> Result<()> { + scope.spawn(|| { + let i = Instant::now(); + stores.commit(height).unwrap(); + info!("Commited stores in {}s", i.elapsed().as_secs()); + }); + let i = Instant::now(); + vecs.flush(height)?; + info!("Flushed vecs in {}s", i.elapsed().as_secs()); + let i = Instant::now(); + file.flush()?; + info!("Flushed file in {}s", i.elapsed().as_secs()); + Ok(()) + }) + }; + let mut txindex_to_first_outputindex_reader_opt = None; let mut p2pk65addressindex_to_p2pk65bytes_reader_opt = None; let mut p2pk33addressindex_to_p2pk33bytes_reader_opt = None; @@ -181,7 +197,7 @@ impl Indexer { let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_reader_opt.as_ref().unwrap(); // Used to check rapidhash collisions - let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO); + let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO ; let blockhash = BlockHash::from(blockhash); let blockhash_prefix = BlockHashPrefix::from(&blockhash); @@ -739,19 +755,18 @@ impl Indexer { idxs.inputindex += InputIndex::from(inputs_len); idxs.outputindex += OutputIndex::from(outputs_len); - txindex_to_first_outputindex_reader_opt.take(); - p2pk65addressindex_to_p2pk65bytes_reader_opt.take(); - p2pk33addressindex_to_p2pk33bytes_reader_opt.take(); - p2pkhaddressindex_to_p2pkhbytes_reader_opt.take(); - p2shaddressindex_to_p2shbytes_reader_opt.take(); - p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take(); - p2wshaddressindex_to_p2wshbytes_reader_opt.take(); - p2traddressindex_to_p2trbytes_reader_opt.take(); - p2aaddressindex_to_p2abytes_reader_opt.take(); - let exported = export_if_needed(stores, vecs, height, false, exit)?; - - if exported { + if should_export(height, false) { + txindex_to_first_outputindex_reader_opt.take(); + p2pk65addressindex_to_p2pk65bytes_reader_opt.take(); + p2pk33addressindex_to_p2pk33bytes_reader_opt.take(); + p2pkhaddressindex_to_p2pkhbytes_reader_opt.take(); + p2shaddressindex_to_p2shbytes_reader_opt.take(); + p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take(); + p2wshaddressindex_to_p2wshbytes_reader_opt.take(); + p2traddressindex_to_p2trbytes_reader_opt.take(); + p2aaddressindex_to_p2abytes_reader_opt.take(); + export(stores, vecs, height, exit)?; reset_mmaps_options( vecs, &mut txindex_to_first_outputindex_reader_opt, @@ -770,9 +785,13 @@ impl Indexer { }, )?; - export_if_needed(stores, vecs, idxs.height, true, exit)?; + if should_export(idxs.height, true) { + export(stores, vecs, idxs.height, exit)?; + } - unsafe { libc::sync() } + let i = Instant::now(); + file.punch_holes()?; + info!("Punched holes in file in {}s", i.elapsed().as_secs()); Ok(starting_indexes) } diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index db3479110..acdf1927c 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -11,7 +11,7 @@ repository.workspace = true brk_core = { workspace = true } brk_computer = { workspace = true } brk_indexer = { workspace = true } -brk_vec = { workspace = true } +brk_vecs = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } rmcp = { workspace = true } diff --git a/crates/brk_interface/examples/main.rs b/crates/brk_interface/examples/main.rs index 3bf1d70d6..23538a588 100644 --- a/crates/brk_interface/examples/main.rs +++ b/crates/brk_interface/examples/main.rs @@ -3,7 +3,7 @@ use std::path::Path; use brk_computer::Computer; use brk_indexer::Indexer; use brk_interface::{Index, Interface, Params, ParamsOpt}; -use brk_vec::{Computation, Format}; +use brk_vecs::{Computation, Format}; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index e9bf3df43..7383fd2c9 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -8,7 +8,7 @@ use std::collections::BTreeMap; use brk_computer::Computer; use brk_core::{Height, Result}; use brk_indexer::Indexer; -use brk_vec::{AnyCollectableVec, AnyIndexedVec}; +use brk_vecs::{AnyCollectableVec, AnyStampedVec}; use tabled::settings::Style; mod deser; @@ -47,7 +47,7 @@ impl<'a> Interface<'a> { } pub fn get_height(&self) -> Height { - self._indexer.vecs.height_to_blockhash.height() + Height::from(u64::from(self._indexer.vecs.height_to_blockhash.stamp())) } pub fn search(&self, params: &Params) -> Vec<(String, &&dyn AnyCollectableVec)> { diff --git a/crates/brk_interface/src/vecs.rs b/crates/brk_interface/src/vecs.rs index 8f081b960..7353d1ffa 100644 --- a/crates/brk_interface/src/vecs.rs +++ b/crates/brk_interface/src/vecs.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use brk_computer::Computer; use brk_indexer::Indexer; -use brk_vec::AnyCollectableVec; +use brk_vecs::AnyCollectableVec; use derive_deref::{Deref, DerefMut}; use crate::pagination::{PaginatedIndexParam, PaginationParam}; diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 50907bd44..57e1b6a22 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -20,7 +20,7 @@ brk_interface = { workspace = true } brk_logger = { workspace = true } brk_mcp = { workspace = true } brk_parser = { workspace = true } -brk_vec = { workspace = true } +brk_vecs = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } color-eyre = { workspace = true } diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index fd7c6e710..266bcd79b 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -8,7 +8,7 @@ use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::Parser; use brk_server::{Server, Website}; -use brk_vec::{Computation, Format}; +use brk_vecs::{Computation, Format}; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_vecs/Cargo.toml b/crates/brk_vecs/Cargo.toml index 33bd0cbc1..040dbed6b 100644 --- a/crates/brk_vecs/Cargo.toml +++ b/crates/brk_vecs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brk_vecs" -description = "A storeable vec" +description = "A simple index/value store" keywords = ["vec", "disk", "data"] categories = ["database"] version.workspace = true @@ -10,13 +10,14 @@ homepage.workspace = true repository.workspace = true [dependencies] -bincode = { workspace = true } brk_core = { workspace = true } brk_exit = { workspace = true } +clap = { workspace = true } +clap_derive = { workspace = true } libc = "0.2.174" log = { workspace = true } memmap2 = "0.9.7" -parking_lot = {workspace = true} +parking_lot = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/brk_vecs/examples/file.rs b/crates/brk_vecs/examples/file.rs index cef7d2b92..05f073ba2 100644 --- a/crates/brk_vecs/examples/file.rs +++ b/crates/brk_vecs/examples/file.rs @@ -8,13 +8,13 @@ fn main() -> Result<()> { let file = File::open(Path::new("vecs"))?; - let file_min_len = PAGE_SIZE * 1_000_000; - let min_regions = 20_000; + // let file_min_len = PAGE_SIZE * 1_000_000; + // let min_regions = 20_000; - file.set_min_len(file_min_len)?; - file.set_min_regions(min_regions)?; + // file.set_min_len(file_min_len)?; + // file.set_min_regions(min_regions)?; - let (region1_i, region1) = file.create_region_if_needed("region1")?; + let (region1_i, _) = file.create_region_if_needed("region1")?; { let layout = file.layout(); @@ -150,9 +150,11 @@ fn main() -> Result<()> { assert!(layout.start_to_hole().is_empty()); } - let (region1_i, region1) = file.create_region_if_needed("region1")?; - let (region2_i, region2) = file.create_region_if_needed("region2")?; - let (region3_i, region3) = file.create_region_if_needed("region3")?; + let (region1_i, _) = file.create_region_if_needed("region1")?; + let (region2_i, _) = file.create_region_if_needed("region2")?; + let (region3_i, _) = file.create_region_if_needed("region3")?; + + // dbg!(file.layout()); { let regions = file.regions(); @@ -228,7 +230,7 @@ fn main() -> Result<()> { ); } - let (region2_i, region2) = file.create_region_if_needed("region2")?; + let (region2_i, _) = file.create_region_if_needed("region2")?; { assert!(region2_i == 1) @@ -312,7 +314,7 @@ fn main() -> Result<()> { assert!(start_to_hole.is_empty()); } - let (region2_i, region2) = file.create_region_if_needed("region2")?; + let (region2_i, _) = file.create_region_if_needed("region2")?; { let regions = file.regions(); @@ -408,12 +410,14 @@ fn main() -> Result<()> { file.write_all_to_region(region2_i.into(), &[1; 6000])?; - let (region4_i, region4) = file.create_region_if_needed("region4")?; - file.remove_region(region2_i.into()); - file.remove_region(region4_i.into()); + let (region4_i, _) = file.create_region_if_needed("region4")?; + file.remove_region(region2_i.into())?; + file.remove_region(region4_i.into())?; - dbg!(file.regions()); - dbg!(file.layout()); + let regions = file.regions(); + dbg!(®ions); + let layout = file.layout(); + dbg!(&layout); Ok(()) } diff --git a/crates/brk_vecs/src/file/layout.rs b/crates/brk_vecs/src/file/layout.rs index 848697907..7da1a2e16 100644 --- a/crates/brk_vecs/src/file/layout.rs +++ b/crates/brk_vecs/src/file/layout.rs @@ -14,9 +14,8 @@ pub struct Layout { impl From<&Regions> for Layout { fn from(value: &Regions) -> Self { let mut start_to_index = BTreeMap::new(); - let mut start_to_hole = BTreeMap::new(); - let mut prev_end = 0; + let index_to_region = value.index_to_region(); value .index_to_region() @@ -27,13 +26,20 @@ impl From<&Regions> for Layout { let region = region.read(); let start = region.start(); start_to_index.insert(start, index); - if prev_end != start { - start_to_hole.insert(prev_end, start - prev_end); - } - let reserved = region.reserved(); - prev_end = start + reserved; }); + let mut start_to_hole = BTreeMap::new(); + + let mut prev_end = 0; + + start_to_index.iter().for_each(|(&start, &index)| { + if prev_end != start { + start_to_hole.insert(prev_end, start - prev_end); + } + let reserved = index_to_region[index].as_ref().unwrap().read().reserved(); + prev_end = start + reserved; + }); + Self { start_to_index, start_to_hole, @@ -64,13 +70,13 @@ impl Layout { let last = self.get_last_region(); let is_last = last.is_some_and(|(_, other_index)| index == other_index); if is_last { - debug_assert!(self.start_to_hole.range(last.unwrap().0..).next().is_none()); + assert!(self.start_to_hole.range(last.unwrap().0..).next().is_none()); } is_last } pub fn insert_region(&mut self, start: u64, index: usize) { - debug_assert!(self.start_to_index.insert(start, index).is_none()) + assert!(self.start_to_index.insert(start, index).is_none()) // TODO: Other checks related to holes ? } diff --git a/crates/brk_vecs/src/file/mod.rs b/crates/brk_vecs/src/file/mod.rs index 5f0792bdd..ba250e857 100644 --- a/crates/brk_vecs/src/file/mod.rs +++ b/crates/brk_vecs/src/file/mod.rs @@ -18,6 +18,7 @@ mod regions; pub use identifier::*; use layout::*; +use rayon::prelude::*; pub use reader::*; pub use region::*; use regions::*; @@ -39,6 +40,7 @@ impl File { fs::create_dir_all(path)?; let regions = Regions::open(path)?; + let layout = Layout::from(®ions); let file = OpenOptions::new() @@ -159,91 +161,113 @@ impl File { at: Option, ) -> Result<()> { let regions = self.regions.read(); - let Some(region) = regions.get_region(identifier.clone()) else { + let Some(region_lock) = regions.get_region(identifier.clone()) else { return Err(Error::Str("Unknown region")); }; + let region_index = regions.identifier_to_index(identifier).unwrap(); - drop(regions); - let region_lock = region.read(); - let start = region_lock.start(); - let reserved = region_lock.reserved(); - let len = region_lock.len(); + let region = region_lock.read(); + let start = region.start(); + let reserved = region.reserved(); + let len = region.len(); + drop(region); + let data_len = data.len() as u64; - drop(region_lock); let new_len = at.map_or(len + data_len, |at| (at + data_len).max(len)); - // dbg!(new_len); - let write_start = at.unwrap_or(start + len); - if at.is_some_and(|at| at < start || at >= start + reserved) { + if at.is_some_and(|at| at >= start + reserved) { return Err(Error::Str("Invalid at parameter")); } // Write to reserved space if possible if new_len <= reserved { - // dbh!("Write to reserved space"); - // dbg!(write_start); + // println!( + // "Write to {region_index} reserved space at {}", + // start + at.unwrap_or(len) + // ); - self.write(write_start, data); - - let regions = self.regions.read(); - let mut region_lock = region.write(); - if len != new_len { - region_lock.set_len(new_len); + if at.is_none() { + self.write(start + len, data); } - regions.write_to_mmap(®ion_lock, region_index); + + let mut region = region_lock.write(); + if let Some(at) = at { + self.write(start + at, data); + } + if len != new_len { + region.set_len(new_len); + } + regions.write_to_mmap(®ion, region_index); + return Ok(()); } - let mut layout_lock = self.layout.write(); + // let layouat_lock = self.layout.read(); - debug_assert!(new_len > reserved); + assert!(new_len > reserved); let mut new_reserved = reserved; while new_len > new_reserved { new_reserved *= 2; } - debug_assert!(new_len <= new_reserved); + assert!(new_len <= new_reserved); let added_reserve = new_reserved - reserved; + let mut layout = self.layout.write(); + // If is last continue writing - if layout_lock.is_last_region(region_index) { - // dbg!("Append to file"); - // dbg!(start, new_reserved, start + new_reserved); + if layout.is_last_region(region_index) { + // println!( + // "{region_index} Append to file at {}", + // start + at.unwrap_or(len) + // ); self.set_min_len(start + new_reserved)?; - self.write(write_start, data); + if at.is_none() { + self.write(start + len, data); + } + let mut region = region_lock.write(); + region.set_reserved(new_reserved); + if let Some(at) = at { + self.write(start + at, data); + } + region.set_len(new_len); + regions.write_to_mmap(®ion, region_index); - let regions = self.regions.read(); - let mut region_lock = region.write(); - region_lock.set_reserved(new_reserved); - region_lock.set_len(new_len); - regions.write_to_mmap(®ion_lock, region_index); return Ok(()); } // Expand region to the right if gap is wide enough let hole_start = start + reserved; - let gap = layout_lock.get_hole(hole_start); - if gap.is_some_and(|gap| gap >= added_reserve) { - // dbg!("Expand to hole"); + if layout + .get_hole(hole_start) + .is_some_and(|gap| gap >= added_reserve) + { + // println!("Expand {region_index} to hole"); - self.write(write_start, data); + layout.remove_or_compress_hole_to_right(hole_start, added_reserve); + drop(layout); - layout_lock.remove_or_compress_hole_to_right(hole_start, added_reserve); - drop(layout_lock); - - let regions = self.regions.read(); - let mut region_lock = region.write(); - region_lock.set_reserved(new_reserved); - region_lock.set_len(new_len); - regions.write_to_mmap(®ion_lock, region_index); + if at.is_none() { + self.write(start + len, data); + } + let mut region = region_lock.write(); + region.set_reserved(new_reserved); + if let Some(at) = at { + self.write(start + at, data); + } + region.set_len(new_len); + regions.write_to_mmap(®ion, region_index); return Ok(()); } // Find hole big enough to move the region - if let Some(hole_start) = layout_lock.find_smallest_adequate_hole(new_reserved) { - // dbg!("Move to hole"); + if let Some(hole_start) = layout.find_smallest_adequate_hole(new_reserved) { + // println!("Move {region_index} to hole at {hole_start}"); + + layout.remove_or_compress_hole_to_right(hole_start, new_reserved); + // drop(layout); self.write( hole_start, @@ -251,37 +275,40 @@ impl File { ); self.write(hole_start + len, data); - let regions = self.regions.read(); - let mut region_lock = region.write(); + // let mut layout = self.layout.write(); + let mut region = region_lock.write(); + layout.move_region(hole_start, region_index, ®ion)?; + region.set_start(hole_start); + region.set_reserved(new_reserved); + region.set_len(new_len); - layout_lock.remove_or_compress_hole_to_right(hole_start, new_reserved); - layout_lock.move_region(hole_start, region_index, ®ion_lock)?; + drop(layout); - region_lock.set_start(hole_start); - region_lock.set_reserved(new_reserved); - region_lock.set_len(new_len); - regions.write_to_mmap(®ion_lock, region_index); - - drop(layout_lock); - - self.punch_hole(start, reserved)?; + regions.write_to_mmap(®ion, region_index); return Ok(()); } // Write at the end - // dbg!("Move and write at the end"); - let regions = self.regions.read(); - let mut region_lock = region.write(); - let (last_region_start, last_region_index) = layout_lock.get_last_region().unwrap(); + // let mut region_lock = region.write(); + let (last_region_start, last_region_index) = layout.get_last_region().unwrap(); let new_start = last_region_start + regions .get_region_from_index(last_region_index) .unwrap() .read() .reserved(); + // println!( + // "Move {region_index} to the end, from {start}..{} to {new_start}..{}", + // start + reserved, + // new_start + new_reserved + // ); + self.set_min_len(new_start + new_reserved)?; + let mut region = region_lock.write(); + layout.move_region(new_start, region_index, ®ion)?; + self.write( new_start, &self.mmap.read()[start as usize..(start + len) as usize], @@ -290,23 +317,22 @@ impl File { // dbg!(new_start, region_index, ®ion_lock, new_reserved, new_len); - layout_lock.move_region(new_start, region_index, ®ion_lock)?; + region.set_start(new_start); + region.set_reserved(new_reserved); + region.set_len(new_len); - region_lock.set_start(new_start); - region_lock.set_reserved(new_reserved); - region_lock.set_len(new_len); - regions.write_to_mmap(®ion_lock, region_index); + drop(layout); - self.punch_hole(start, reserved)?; + regions.write_to_mmap(®ion, region_index); Ok(()) } - fn write(&self, start: u64, data: &[u8]) { + fn write(&self, at: u64, data: &[u8]) { let mmap = self.mmap.read(); let data_len = data.len(); - let start = start as usize; + let start = at as usize; let end = start + data_len; if end > mmap.len() { @@ -376,11 +402,85 @@ impl File { Ok(unsafe { MmapOptions::new().map_mut(file)? }) } + pub fn regions(&self) -> RwLockReadGuard<'_, Regions> { + self.regions.read() + } + + pub fn layout(&self) -> RwLockReadGuard<'_, Layout> { + self.layout.read() + } + + pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> { + self.mmap.read() + } + + fn ceil_number_to_page_size_multiple(num: u64) -> u64 { + (num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1 + } + + fn data_path(&self) -> PathBuf { + Self::data_path_(&self.path) + } + fn data_path_(path: &Path) -> PathBuf { + path.join("data") + } + + pub fn disk_usage(&self) -> String { + let path = self.data_path(); + + let output = std::process::Command::new("du") + .arg("-h") + .arg(&path) + .output() + .expect("Failed to run du"); + + String::from_utf8_lossy(&output.stdout) + .replace(path.to_str().unwrap(), " ") + .trim() + .to_string() + } + + pub fn flush(&self) -> Result<()> { + let mmap = self.mmap.read(); + let regions = self.regions.read(); + mmap.flush()?; + regions.flush() + } + + pub fn punch_holes(&self) -> Result<()> { + let file = self.file.write(); + let mmap = self.mmap.read(); + let layout = self.layout.read(); + Self::punch_holes_(&file, &mmap, &layout) + } + + fn punch_holes_(file: &fs::File, mmap: &MmapMut, layout: &Layout) -> Result<()> { + layout + .start_to_hole() + .par_iter() + .try_for_each(|(&start, &hole)| -> Result<()> { + assert!(start % PAGE_SIZE == 0); + assert!(hole % PAGE_SIZE == 0); + let has_old_data = (((start / PAGE_SIZE) as usize) + ..((start + hole) / PAGE_SIZE) as usize) + .any(|i| mmap[i * PAGE_SIZE as usize] != 0); + if has_old_data { + Self::punch_hole_(file, start, hole) + } else { + Ok(()) + } + }) + } + fn punch_hole(&self, start: u64, length: u64) -> Result<()> { let file = self.file.write(); Self::punch_hole_macos(&file, start, length) } + fn punch_hole_(file: &fs::File, start: u64, length: u64) -> Result<()> { + Self::punch_hole_macos(file, start, length) + } + #[cfg(target_os = "macos")] fn punch_hole_macos(file: &fs::File, start: u64, length: u64) -> Result<()> { let fpunchhole = FPunchhole { @@ -405,56 +505,6 @@ impl File { Ok(()) } - - pub fn regions(&self) -> RwLockReadGuard<'_, Regions> { - self.regions.read() - } - - pub fn layout(&self) -> RwLockReadGuard<'_, Layout> { - self.layout.read() - } - - pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> { - self.mmap.read() - } - - fn ceil_number_to_page_size_multiple(num: u64) -> u64 { - (num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1 - } - - fn data_path(&self) -> PathBuf { - Self::data_path_(&self.path) - } - fn data_path_(path: &Path) -> PathBuf { - path.join("data") - } - - pub fn flush(&self) -> Result<()> { - self.mmap.write().flush().map_err(|e| e.into()) - } - - pub fn sync_data(&self) -> Result<()> { - self.file.read().sync_data().map_err(|e| e.into()) - } - - pub fn sync_all(&self) -> Result<()> { - self.file.read().sync_all().map_err(|e| e.into()) - } - - pub fn disk_usage(&self) -> String { - let path = self.data_path(); - - let output = std::process::Command::new("du") - .arg("-h") - .arg(&path) - .output() - .expect("Failed to run du"); - - String::from_utf8_lossy(&output.stdout) - .replace(path.to_str().unwrap(), " ") - .trim() - .to_string() - } } #[repr(C)] diff --git a/crates/brk_vecs/src/file/reader.rs b/crates/brk_vecs/src/file/reader.rs index 88c151a7f..ad0df263d 100644 --- a/crates/brk_vecs/src/file/reader.rs +++ b/crates/brk_vecs/src/file/reader.rs @@ -18,7 +18,7 @@ impl<'a> Reader<'a> { } pub fn read(&self, offset: u64, len: u64) -> &[u8] { - debug_assert!(offset + len <= self.region.len()); + assert!(offset + len <= self.region.len()); let start = self.region.start() + offset; let end = start + len; &self.mmap[start as usize..end as usize] diff --git a/crates/brk_vecs/src/file/region.rs b/crates/brk_vecs/src/file/region.rs index 3803074be..ec74f5deb 100644 --- a/crates/brk_vecs/src/file/region.rs +++ b/crates/brk_vecs/src/file/region.rs @@ -18,10 +18,10 @@ pub const SIZE_OF_REGION: usize = size_of::(); impl Region { pub fn new(start: u64, len: u64, reserved: u64) -> Self { - debug_assert!(start % PAGE_SIZE == 0); - debug_assert!(reserved >= PAGE_SIZE); - debug_assert!(reserved % PAGE_SIZE == 0); - debug_assert!(len <= reserved); + assert!(start % PAGE_SIZE == 0); + assert!(reserved >= PAGE_SIZE); + assert!(reserved % PAGE_SIZE == 0); + assert!(len <= reserved); Self { start, @@ -35,7 +35,7 @@ impl Region { } pub fn set_start(&mut self, start: u64) { - debug_assert!(start % PAGE_SIZE == 0); + assert!(start % PAGE_SIZE == 0); self.start = start } @@ -44,7 +44,7 @@ impl Region { } pub fn set_len(&mut self, len: u64) { - debug_assert!(len <= self.reserved()); + assert!(len <= self.reserved()); self.len = len } @@ -53,9 +53,9 @@ impl Region { } pub fn set_reserved(&mut self, reserved: u64) { - debug_assert!(self.len() <= reserved); - debug_assert!(reserved >= PAGE_SIZE); - debug_assert!(reserved % PAGE_SIZE == 0); + assert!(self.len() <= reserved); + assert!(reserved >= PAGE_SIZE); + assert!(reserved % PAGE_SIZE == 0); self.reserved = reserved; } @@ -66,7 +66,7 @@ impl Region { } pub trait RegionReader { - fn create_reader(self, file: &File) -> Reader<'_>; + fn create_reader(self, file: &File) -> Reader; } impl<'a> RegionReader for RwLockReadGuard<'a, Region> { diff --git a/crates/brk_vecs/src/file/regions.rs b/crates/brk_vecs/src/file/regions.rs index 0a3a5cfd2..c7773e86c 100644 --- a/crates/brk_vecs/src/file/regions.rs +++ b/crates/brk_vecs/src/file/regions.rs @@ -1,15 +1,14 @@ use std::{ collections::HashMap, fs::{self, OpenOptions}, - io::{BufReader, BufWriter}, - path::Path, + io::{Cursor, Read}, + path::{Path, PathBuf}, sync::Arc, }; -use bincode::{decode_from_std_read, encode_into_std_write}; use brk_core::{Error, Result}; use memmap2::MmapMut; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockWriteGuard}; use zerocopy::{FromBytes, IntoBytes}; use super::{ @@ -20,7 +19,7 @@ use super::{ #[derive(Debug)] pub struct Regions { id_to_index: HashMap, - id_to_index_file: fs::File, + id_to_index_path: PathBuf, index_to_region: Vec>>>, index_to_region_file: fs::File, index_to_region_mmap: MmapMut, @@ -32,19 +31,11 @@ impl Regions { fs::create_dir_all(&path)?; - let id_to_index_file = OpenOptions::new() - .read(true) - .create(true) - .write(true) - .truncate(false) - .open(path.join("id_to_index"))?; + let id_to_index_path = path.join("id_to_index"); - let mut id_to_index: HashMap = HashMap::new(); - - if id_to_index_file.metadata()?.len() > 0 { - let mut reader = BufReader::new(&id_to_index_file); - id_to_index = decode_from_std_read(&mut reader, bincode::config::standard())?; - } + let id_to_index: HashMap = + deserialize_hashmap_binary(&fs::read(&id_to_index_path).unwrap_or_default()) + .unwrap_or_default(); let index_to_region_file = OpenOptions::new() .read(true) @@ -77,7 +68,7 @@ impl Regions { Ok(Self { id_to_index, - id_to_index_file, + id_to_index_path, index_to_region, index_to_region_file, index_to_region_mmap, @@ -109,15 +100,18 @@ impl Regions { self.set_min_len(((index + 1) * SIZE_OF_REGION) as u64)?; - self.write_to_mmap(®ion, index); + let region_lock = RwLock::new(region); - let region_arc = Arc::new(RwLock::new(region)); + self.write_to_mmap(®ion_lock.write(), index); + + let region_arc = Arc::new(region_lock); let region_opt = Some(region_arc.clone()); if index < self.index_to_region.len() { self.index_to_region[index] = region_opt } else { self.index_to_region.push(region_opt); + self.index_to_region_mmap.flush()?; } if self.id_to_index.insert(id, index).is_some() { @@ -129,8 +123,10 @@ impl Regions { } fn flush_id_to_index(&mut self) -> Result<()> { - let mut writer = BufWriter::new(&mut self.id_to_index_file); - encode_into_std_write(&self.id_to_index, &mut writer, bincode::config::standard())?; + fs::write( + &self.id_to_index_path, + serialize_hashmap_binary(&self.id_to_index), + )?; Ok(()) } @@ -208,6 +204,8 @@ impl Regions { return Ok(None); }; + self.index_to_region_mmap.flush()?; + self.id_to_index .remove(&self.find_id_from_index(index).unwrap().to_owned()); @@ -216,10 +214,10 @@ impl Regions { Ok(Some(region)) } - pub fn write_to_mmap(&self, region: &Region, index: usize) { + pub fn write_to_mmap(&self, region: &RwLockWriteGuard, index: usize) { + let mmap = &self.index_to_region_mmap; let start = index * SIZE_OF_REGION; let end = start + SIZE_OF_REGION; - let mmap = &self.index_to_region_mmap; if end > mmap.len() { unreachable!("Trying to write beyond mmap") @@ -229,4 +227,56 @@ impl Regions { slice[start..end].copy_from_slice(region.as_bytes()); } + + pub fn flush(&self) -> Result<()> { + self.index_to_region_mmap.flush().map_err(|e| e.into()) + } +} + +fn serialize_hashmap_binary(map: &HashMap) -> Vec { + let mut buffer = Vec::new(); + + buffer.extend_from_slice(&map.len().to_ne_bytes()); + + for (key, value) in map { + buffer.extend_from_slice(&key.len().to_ne_bytes()); + buffer.extend_from_slice(key.as_bytes()); + buffer.extend_from_slice(&value.to_ne_bytes()); + } + + buffer +} + +fn deserialize_hashmap_binary(data: &[u8]) -> Result> { + let mut cursor = Cursor::new(data); + let mut buffer = [0u8; 8]; + + cursor + .read_exact(&mut buffer) + .map_err(|_| Error::Str("Failed to read entry count"))?; + let entry_count = usize::from_ne_bytes(buffer); + + let mut map = HashMap::with_capacity(entry_count); + + for _ in 0..entry_count { + cursor + .read_exact(&mut buffer) + .map_err(|_| Error::Str("Failed to read key length"))?; + let key_len = usize::from_ne_bytes(buffer); + + let mut key_bytes = vec![0u8; key_len]; + cursor + .read_exact(&mut key_bytes) + .map_err(|_| Error::Str("Failed to read key"))?; + let key = String::from_utf8(key_bytes).map_err(|_| Error::Str("Invalid UTF-8 in key"))?; + + cursor + .read_exact(&mut buffer) + .map_err(|_| Error::Str("Failed to read value"))?; + let value = usize::from_ne_bytes(buffer); + + map.insert(key, value); + } + + Ok(map) } diff --git a/crates/brk_vecs/src/lib.rs b/crates/brk_vecs/src/lib.rs index a9ba1a2cd..27363e3de 100644 --- a/crates/brk_vecs/src/lib.rs +++ b/crates/brk_vecs/src/lib.rs @@ -6,4 +6,8 @@ use variants::*; pub use file::{File, PAGE_SIZE, Reader}; pub use traits::*; -pub use variants::{AnyStampedVec, Format, RawVec, Stamp, StampedVec}; +pub use variants::{ + AnyStampedVec, CompressedVec, Computation, ComputedVec, ComputedVecFrom1, ComputedVecFrom2, + ComputedVecFrom3, EagerVec, Format, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, Stamp, + StampedVec, StoredVec, +}; diff --git a/crates/brk_vecs/src/traits/any.rs b/crates/brk_vecs/src/traits/any.rs index 572c06ead..84c516788 100644 --- a/crates/brk_vecs/src/traits/any.rs +++ b/crates/brk_vecs/src/traits/any.rs @@ -1,7 +1,5 @@ use brk_core::{Height, Version}; -use crate::{File, Reader}; - use super::{BoxedVecIterator, StoredIndex, StoredType}; pub fn i64_to_usize(i: i64, len: usize) -> usize { @@ -39,30 +37,6 @@ pub trait AnyVec: Send + Sync { ) } - fn file(&self) -> &File; - - fn region_index(&self) -> usize; - - /// Be careful with deadlocks - /// - /// You'll want to drop the reader before mutable ops - fn create_reader(&self) -> Reader<'_> { - self.create_static_reader() - } - - /// Be careful with deadlocks - /// - /// You'll want to drop the reader before mutable ops - fn create_static_reader(&self) -> Reader<'static> { - unsafe { - std::mem::transmute( - self.file() - .create_region_reader(self.region_index().into()) - .unwrap(), - ) - } - } - #[inline] fn i64_to_usize(&self, i: i64) -> usize { let len = self.len(); diff --git a/crates/brk_vecs/src/traits/generic.rs b/crates/brk_vecs/src/traits/generic.rs index e7b4eab84..a3bd75748 100644 --- a/crates/brk_vecs/src/traits/generic.rs +++ b/crates/brk_vecs/src/traits/generic.rs @@ -6,7 +6,7 @@ use std::{ use brk_core::{Error, Result}; -use crate::{AnyVec, HEADER_OFFSET, Header, file::Reader}; +use crate::{AnyVec, File, HEADER_OFFSET, Header, file::Reader}; use super::{StoredIndex, StoredType}; @@ -18,22 +18,46 @@ where { const SIZE_OF_T: usize = size_of::(); + fn file(&self) -> &File; + + fn region_index(&self) -> usize; + + /// Be careful with deadlocks + /// + /// You'll want to drop the reader before mutable ops + fn create_reader(&self) -> Reader { + self.create_static_reader() + } + + /// Be careful with deadlocks + /// + /// You'll want to drop the reader before mutable ops + fn create_static_reader(&self) -> Reader<'static> { + unsafe { + std::mem::transmute( + self.file() + .create_region_reader(self.region_index().into()) + .unwrap(), + ) + } + } + #[inline] - fn unwrap_read(&self, index: I, reader: &Reader<'_>) -> T { + fn unwrap_read(&self, index: I, reader: &Reader) -> T { self.read(index, reader).unwrap().unwrap() } #[inline] - fn read(&self, index: I, reader: &Reader<'_>) -> Result> { + fn read(&self, index: I, reader: &Reader) -> Result> { self.read_(index.to_usize()?, reader) } - fn read_(&self, index: usize, reader: &Reader<'_>) -> Result>; + fn read_(&self, index: usize, reader: &Reader) -> Result>; #[inline] - fn get_or_read(&self, index: I, reader: &Reader<'_>) -> Result>> { + fn get_or_read(&self, index: I, reader: &Reader) -> Result>> { self.get_or_read_(index.to_usize()?, reader) } #[inline] - fn get_or_read_(&self, index: usize, reader: &Reader<'_>) -> Result>> { + fn get_or_read_(&self, index: usize, reader: &Reader) -> Result>> { let stored_len = self.stored_len(); if index >= stored_len { @@ -117,7 +141,7 @@ where fn holes(&self) -> &BTreeSet; fn mut_holes(&mut self) -> &mut BTreeSet; - fn take(&mut self, index: I, reader: &Reader<'_>) -> Result> { + fn take(&mut self, index: I, reader: &Reader) -> Result> { let opt = self.get_or_read(index, reader)?.map(|v| v.into_owned()); if opt.is_some() { self.unchecked_delete(index); @@ -205,7 +229,7 @@ where Self::vec_region_name_(self.name()) } fn vec_region_name_(name: &str) -> String { - format!("{name}_{}", I::to_string()) + format!("{}_to_{name}", I::to_string()) } fn holes_region_name(&self) -> String { diff --git a/crates/brk_vecs/src/variants/compressed/mod.rs b/crates/brk_vecs/src/variants/compressed/mod.rs index 2804d686f..5e0e590c1 100644 --- a/crates/brk_vecs/src/variants/compressed/mod.rs +++ b/crates/brk_vecs/src/variants/compressed/mod.rs @@ -92,7 +92,7 @@ where // }) } - fn decode_page(&self, page_index: usize, reader: &Reader<'_>) -> Result> { + fn decode_page(&self, page_index: usize, reader: &Reader) -> Result> { Self::decode_page_( self.stored_len(), page_index, @@ -104,7 +104,7 @@ where fn decode_page_( stored_len: usize, page_index: usize, - reader: &Reader<'_>, + reader: &Reader, compressed_pages_meta: &CompressedPagesMetadata, ) -> Result> { if Self::page_index_to_index(page_index) >= stored_len { @@ -178,8 +178,16 @@ where I: StoredIndex, T: StoredType, { + fn file(&self) -> &File { + self.inner.file() + } + + fn region_index(&self) -> usize { + self.inner.region_index() + } + #[inline] - fn read_(&self, index: usize, reader: &Reader<'_>) -> Result> { + fn read_(&self, index: usize, reader: &Reader) -> Result> { let page_index = Self::index_to_page_index(index); let decoded_index = index % Self::PER_PAGE; @@ -398,14 +406,6 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } - - fn file(&self) -> &File { - self.inner.file() - } - - fn region_index(&self) -> usize { - self.inner.region_index() - } } impl Clone for CompressedVec { diff --git a/crates/brk_vecs/src/variants/mod.rs b/crates/brk_vecs/src/variants/mod.rs index 95f3f0e74..5e38d6fa6 100644 --- a/crates/brk_vecs/src/variants/mod.rs +++ b/crates/brk_vecs/src/variants/mod.rs @@ -1,9 +1,15 @@ mod compressed; +mod computed; +mod eager; +mod lazy; mod raw; mod stamped; mod stored; pub use compressed::*; +pub use computed::*; +pub use eager::*; +pub use lazy::*; pub use raw::*; pub use stamped::*; pub use stored::*; diff --git a/crates/brk_vecs/src/variants/raw/mod.rs b/crates/brk_vecs/src/variants/raw/mod.rs index 5c438a9f0..1ef9613b2 100644 --- a/crates/brk_vecs/src/variants/raw/mod.rs +++ b/crates/brk_vecs/src/variants/raw/mod.rs @@ -7,12 +7,13 @@ use std::{ }; use brk_core::{Error, Result, Version}; +use parking_lot::RwLock; use rayon::prelude::*; use crate::{ AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, - File, GenericStoredVec, StoredIndex, StoredType, - file::{Reader, RegionReader}, + File, GenericStoredVec, Reader, StoredIndex, StoredType, + file::{Region, RegionReader}, }; use super::Format; @@ -28,6 +29,7 @@ const VERSION: Version = Version::ONE; #[derive(Debug)] pub struct RawVec { file: Arc, + region: Arc>, region_index: usize, header: Header, @@ -68,7 +70,6 @@ where if region_len > 0 && (region_len < HEADER_OFFSET || (region_len - HEADER_OFFSET) % Self::SIZE_OF_T != 0) { - dbg!(region_len); return Err(Error::Str("Region was saved incorrectly")); } @@ -101,6 +102,7 @@ where Ok(Self { file: file.clone(), + region: region.clone(), region_index, header, name: Box::leak(Box::new(name.to_string())), @@ -143,7 +145,7 @@ where T: StoredType, { #[inline] - fn read_(&self, index: usize, reader: &Reader<'_>) -> Result> { + fn read_(&self, index: usize, reader: &Reader) -> Result> { let slice = reader.read( (index * Self::SIZE_OF_T + HEADER_OFFSET) as u64, (Self::SIZE_OF_T) as u64, @@ -289,6 +291,14 @@ where fn reset(&mut self) -> Result<()> { self.reset_() } + + fn file(&self) -> &File { + &self.file + } + + fn region_index(&self) -> usize { + self.region_index + } } impl AnyVec for RawVec @@ -320,20 +330,13 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } - - fn file(&self) -> &File { - &self.file - } - - fn region_index(&self) -> usize { - self.region_index - } } impl Clone for RawVec { fn clone(&self) -> Self { Self { file: self.file.clone(), + region: self.region.clone(), region_index: self.region_index, header: self.header.clone(), name: self.name, diff --git a/crates/brk_vecs/src/variants/stamped/mod.rs b/crates/brk_vecs/src/variants/stamped/mod.rs index ff9ab898b..acb7663bf 100644 --- a/crates/brk_vecs/src/variants/stamped/mod.rs +++ b/crates/brk_vecs/src/variants/stamped/mod.rs @@ -27,18 +27,16 @@ where version: Version, format: Format, ) -> Result { - Ok(Self( - StoredVec::forced_import(file, name, version, format).unwrap(), - )) + Ok(Self(StoredVec::forced_import(file, name, version, format)?)) } #[inline] - pub fn unwrap_read(&self, index: I, reader: &Reader<'_>) -> T { + pub fn unwrap_read(&self, index: I, reader: &Reader) -> T { self.0.unwrap_read(index, reader) } #[inline] - pub fn get_or_read(&self, index: I, reader: &Reader<'_>) -> Result>> { + pub fn get_or_read(&self, index: I, reader: &Reader) -> Result>> { self.0.get_or_read(index, reader) } @@ -95,7 +93,7 @@ where self.0.update(index, value) } - pub fn take(&mut self, index: I, reader: &Reader<'_>) -> Result> { + pub fn take(&mut self, index: I, reader: &Reader) -> Result> { self.0.take(index, reader) } @@ -131,6 +129,14 @@ where pub fn hasnt(&self, index: I) -> Result { self.0.has(index).map(|b| !b) } + + pub fn create_reader(&self) -> Reader { + self.0.create_reader() + } + + pub fn create_static_reader(&self) -> Reader<'static> { + self.0.create_static_reader() + } } impl AnyVec for StampedVec @@ -162,14 +168,6 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } - - fn file(&self) -> &File { - self.0.file() - } - - fn region_index(&self) -> usize { - self.0.region_index() - } } pub trait AnyStampedVec: AnyVec { diff --git a/crates/brk_vecs/src/variants/stamped/stamp.rs b/crates/brk_vecs/src/variants/stamped/stamp.rs index c598f423e..e31fa6a5c 100644 --- a/crates/brk_vecs/src/variants/stamped/stamp.rs +++ b/crates/brk_vecs/src/variants/stamped/stamp.rs @@ -1,6 +1,8 @@ use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Default, Clone, Copy, FromBytes, IntoBytes, Immutable, KnownLayout)] +#[derive( + Debug, Default, Clone, Copy, PartialEq, Eq, FromBytes, IntoBytes, Immutable, KnownLayout, +)] pub struct Stamp(u64); impl Stamp { diff --git a/crates/brk_vecs/src/variants/stored/format.rs b/crates/brk_vecs/src/variants/stored/format.rs index b3f7bc47a..ab5e5452c 100644 --- a/crates/brk_vecs/src/variants/stored/format.rs +++ b/crates/brk_vecs/src/variants/stored/format.rs @@ -1,9 +1,12 @@ use std::{fs, io, path::Path}; use brk_core::{Error, Result}; +use clap_derive::ValueEnum; use serde::{Deserialize, Serialize}; -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive( + Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum, +)] pub enum Format { Compressed, #[default] diff --git a/crates/brk_vecs/src/variants/stored/mod.rs b/crates/brk_vecs/src/variants/stored/mod.rs index 48d65a4e8..4708f0ad5 100644 --- a/crates/brk_vecs/src/variants/stored/mod.rs +++ b/crates/brk_vecs/src/variants/stored/mod.rs @@ -40,9 +40,10 @@ where } if format.is_compressed() { - Ok(Self::Compressed(CompressedVec::forced_import( - file, name, version, - )?)) + todo!(); + // Ok(Self::Compressed(CompressedVec::forced_import( + // file, name, version, + // )?)) } else { Ok(Self::Raw(RawVec::forced_import(file, name, version)?)) } @@ -55,7 +56,23 @@ where T: StoredType, { #[inline] - fn read_(&self, index: usize, reader: &Reader<'_>) -> Result> { + fn file(&self) -> &File { + match self { + StoredVec::Raw(v) => v.file(), + StoredVec::Compressed(v) => v.file(), + } + } + + #[inline] + fn region_index(&self) -> usize { + match self { + StoredVec::Raw(v) => v.region_index(), + StoredVec::Compressed(v) => v.region_index(), + } + } + + #[inline] + fn read_(&self, index: usize, reader: &Reader) -> Result> { match self { StoredVec::Raw(v) => v.read_(index, reader), StoredVec::Compressed(v) => v.read_(index, reader), @@ -187,22 +204,6 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } - - #[inline] - fn file(&self) -> &File { - match self { - StoredVec::Raw(v) => v.file(), - StoredVec::Compressed(v) => v.file(), - } - } - - #[inline] - fn region_index(&self) -> usize { - match self { - StoredVec::Raw(v) => v.region_index(), - StoredVec::Compressed(v) => v.region_index(), - } - } } #[derive(Debug)]