diff --git a/Cargo.lock b/Cargo.lock index 62d9c67a2..6b8877b3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1263,6 +1263,7 @@ version = "0.0.111" dependencies = [ "brk_traversable_derive", "brk_types", + "serde", "vecdb", ] diff --git a/crates/brk_computer/src/blks.rs b/crates/brk_computer/src/blks.rs index 92629c0b1..333afa75c 100644 --- a/crates/brk_computer/src/blks.rs +++ b/crates/brk_computer/src/blks.rs @@ -6,7 +6,8 @@ use brk_reader::Reader; use brk_traversable::Traversable; use brk_types::{BlkPosition, Height, TxIndex, Version}; use vecdb::{ - AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, PAGE_SIZE, PcoVec, TypedVecIterator, + AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, Importable, PAGE_SIZE, PcoVec, + TypedVecIterator, }; use super::Indexes; @@ -34,7 +35,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/chain.rs b/crates/brk_computer/src/chain.rs index 52837b142..f2cd07507 100644 --- a/crates/brk_computer/src/chain.rs +++ b/crates/brk_computer/src/chain.rs @@ -10,8 +10,8 @@ use brk_types::{ TxVersion, Version, WeekIndex, Weight, YearIndex, }; use vecdb::{ - Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, IterableVec, LazyVecFrom1, - LazyVecFrom2, PAGE_SIZE, TypedVecIterator, VecIndex, unlikely, + Database, EagerVec, Exit, GenericStoredVec, Importable, IterableCloneableVec, IterableVec, + LazyVecFrom1, LazyVecFrom2, PAGE_SIZE, PcoVec, TypedVecIterator, VecIndex, unlikely, }; use crate::grouped::{ @@ -47,13 +47,13 @@ pub struct Vecs { pub yearindex_to_block_count_target: LazyVecFrom1, pub decadeindex_to_block_count_target: LazyVecFrom1, - pub height_to_interval: EagerVec, - pub height_to_24h_block_count: EagerVec, - pub height_to_24h_coinbase_sum: EagerVec, - pub height_to_24h_coinbase_usd_sum: EagerVec, - pub height_to_vbytes: EagerVec, - pub difficultyepoch_to_timestamp: EagerVec, - pub halvingepoch_to_timestamp: EagerVec, + pub height_to_interval: EagerVec>, + pub height_to_24h_block_count: EagerVec>, + pub height_to_24h_coinbase_sum: EagerVec>, + pub height_to_24h_coinbase_usd_sum: EagerVec>, + pub height_to_vbytes: EagerVec>, + pub difficultyepoch_to_timestamp: EagerVec>, + pub halvingepoch_to_timestamp: EagerVec>, pub timeindexes_to_timestamp: ComputedVecsFromDateIndex, pub indexes_to_block_count: ComputedVecsFromHeight, pub indexes_to_1w_block_count: ComputedVecsFromDateIndex, @@ -71,11 +71,11 @@ pub struct Vecs { pub indexes_to_fee: ComputedValueVecsFromTxindex, pub indexes_to_fee_rate: ComputedVecsFromTxindex, /// Value == 0 when Coinbase - pub txindex_to_input_value: EagerVec, + pub txindex_to_input_value: EagerVec>, pub indexes_to_sent: ComputedValueVecsFromHeight, // pub indexes_to_input_value: ComputedVecsFromTxindex, pub indexes_to_opreturn_count: ComputedVecsFromHeight, - pub txindex_to_output_value: EagerVec, + pub txindex_to_output_value: EagerVec>, // pub indexes_to_output_value: ComputedVecsFromTxindex, pub indexes_to_p2a_count: ComputedVecsFromHeight, pub indexes_to_p2ms_count: ComputedVecsFromHeight, @@ -95,17 +95,17 @@ pub struct Vecs { pub indexes_to_tx_vsize: ComputedVecsFromTxindex, pub indexes_to_tx_weight: ComputedVecsFromTxindex, pub indexes_to_unknownoutput_count: ComputedVecsFromHeight, - pub txinindex_to_value: EagerVec, + pub txinindex_to_value: EagerVec>, pub indexes_to_input_count: ComputedVecsFromTxindex, pub txindex_to_is_coinbase: LazyVecFrom2, pub indexes_to_output_count: ComputedVecsFromTxindex, pub txindex_to_vsize: LazyVecFrom1, pub txindex_to_weight: LazyVecFrom2, - pub txindex_to_fee: EagerVec, - pub txindex_to_fee_rate: EagerVec, + pub txindex_to_fee: EagerVec>, + pub txindex_to_fee_rate: EagerVec>, pub indexes_to_exact_utxo_count: ComputedVecsFromHeight, - pub dateindex_to_fee_dominance: EagerVec, - pub dateindex_to_subsidy_dominance: EagerVec, + pub dateindex_to_fee_dominance: EagerVec>, + pub dateindex_to_subsidy_dominance: EagerVec>, pub indexes_to_subsidy_usd_1y_sma: Option>, pub indexes_to_puell_multiple: Option>, pub indexes_to_hash_rate: ComputedVecsFromHeight, @@ -155,8 +155,8 @@ impl Vecs { let compute_dollars = price.is_some(); - let txinindex_to_value: EagerVec = - EagerVec::forced_import_compressed(&db, "value", version + Version::ZERO)?; + let txinindex_to_value: EagerVec> = + EagerVec::forced_import(&db, "value", version + Version::ZERO)?; let txindex_to_weight = LazyVecFrom2::init( "weight", @@ -201,16 +201,15 @@ impl Vecs { ); let txindex_to_input_value = - EagerVec::forced_import_compressed(&db, "input_value", version + Version::ZERO)?; + EagerVec::forced_import(&db, "input_value", version + Version::ZERO)?; let txindex_to_output_value = - EagerVec::forced_import_compressed(&db, "output_value", version + Version::ZERO)?; + EagerVec::forced_import(&db, "output_value", version + Version::ZERO)?; - let txindex_to_fee = - EagerVec::forced_import_compressed(&db, "fee", version + Version::ZERO)?; + let txindex_to_fee = EagerVec::forced_import(&db, "fee", version + Version::ZERO)?; let txindex_to_fee_rate = - EagerVec::forced_import_compressed(&db, "fee_rate", version + Version::ZERO)?; + EagerVec::forced_import(&db, "fee_rate", version + Version::ZERO)?; let dateindex_to_block_count_target = LazyVecFrom1::init( "block_count_target", @@ -263,11 +262,7 @@ impl Vecs { semesterindex_to_block_count_target, yearindex_to_block_count_target, decadeindex_to_block_count_target, - height_to_interval: EagerVec::forced_import_compressed( - &db, - "interval", - version + Version::ZERO, - )?, + height_to_interval: EagerVec::forced_import(&db, "interval", version + Version::ZERO)?, timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( &db, "timestamp", @@ -345,22 +340,18 @@ impl Vecs { .add_percentiles() .add_cumulative(), )?, - height_to_vbytes: EagerVec::forced_import_compressed( - &db, - "vbytes", - version + Version::ZERO, - )?, - height_to_24h_block_count: EagerVec::forced_import_compressed( + height_to_vbytes: EagerVec::forced_import(&db, "vbytes", version + Version::ZERO)?, + height_to_24h_block_count: EagerVec::forced_import( &db, "24h_block_count", version + Version::ZERO, )?, - height_to_24h_coinbase_sum: EagerVec::forced_import_compressed( + height_to_24h_coinbase_sum: EagerVec::forced_import( &db, "24h_coinbase_sum", version + Version::ZERO, )?, - height_to_24h_coinbase_usd_sum: EagerVec::forced_import_compressed( + height_to_24h_coinbase_usd_sum: EagerVec::forced_import( &db, "24h_coinbase_usd_sum", version + Version::ZERO, @@ -378,23 +369,23 @@ impl Vecs { .add_percentiles() .add_cumulative(), )?, - difficultyepoch_to_timestamp: EagerVec::forced_import_compressed( + difficultyepoch_to_timestamp: EagerVec::forced_import( &db, "timestamp", version + Version::ZERO, )?, - halvingepoch_to_timestamp: EagerVec::forced_import_compressed( + halvingepoch_to_timestamp: EagerVec::forced_import( &db, "timestamp", version + Version::ZERO, )?, - dateindex_to_fee_dominance: EagerVec::forced_import_compressed( + dateindex_to_fee_dominance: EagerVec::forced_import( &db, "fee_dominance", version + Version::ZERO, )?, - dateindex_to_subsidy_dominance: EagerVec::forced_import_compressed( + dateindex_to_subsidy_dominance: EagerVec::forced_import( &db, "subsidy_dominance", version + Version::ZERO, @@ -1022,7 +1013,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index 885d904b1..bb8989e1e 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -279,7 +279,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index d7b813140..d98525949 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -162,7 +162,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index d84fd9cc7..3905777c3 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -6,8 +6,8 @@ use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{DateIndex, Height, OHLCCents, Version}; use vecdb::{ - AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, IterableVec, PAGE_SIZE, RawVec, - TypedVecIterator, VecIndex, + AnyStoredVec, AnyVec, BytesVec, Database, Exit, GenericStoredVec, Importable, IterableVec, + PAGE_SIZE, TypedVecIterator, VecIndex, }; use super::{Indexes, indexes}; @@ -17,8 +17,8 @@ pub struct Vecs { db: Database, fetcher: Fetcher, - pub dateindex_to_price_ohlc_in_cents: RawVec, - pub height_to_price_ohlc_in_cents: RawVec, + pub dateindex_to_price_ohlc_in_cents: BytesVec, + pub height_to_price_ohlc_in_cents: BytesVec, } impl Vecs { @@ -29,12 +29,12 @@ impl Vecs { let this = Self { fetcher, - dateindex_to_price_ohlc_in_cents: RawVec::forced_import( + dateindex_to_price_ohlc_in_cents: BytesVec::forced_import( &db, "price_ohlc_in_cents", version + Version::ZERO, )?, - height_to_price_ohlc_in_cents: RawVec::forced_import( + height_to_price_ohlc_in_cents: BytesVec::forced_import( &db, "price_ohlc_in_cents", version + Version::ZERO, @@ -44,7 +44,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index 9cf7a4af5..65d9cf98a 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -2,8 +2,8 @@ use brk_error::{Error, Result}; use brk_traversable::Traversable; use brk_types::{CheckedSub, StoredU64, Version}; use vecdb::{ - AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, GenericStoredVec, IterableVec, - VecIndex, VecValue, + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, Importable, IterableVec, + PcoVec, VecIndex, VecValue, }; use crate::utils::get_percentile; @@ -16,18 +16,18 @@ where I: VecIndex, T: ComputedVecValue, { - pub first: Option>>, - pub average: Option>>, - pub sum: Option>>, - pub max: Option>>, - pub pct90: Option>>, - pub pct75: Option>>, - pub median: Option>>, - pub pct25: Option>>, - pub pct10: Option>>, - pub min: Option>>, - pub last: Option>>, - pub cumulative: Option>>, + pub first: Option>>>, + pub average: Option>>>, + pub sum: Option>>>, + pub max: Option>>>, + pub pct90: Option>>>, + pub pct75: Option>>>, + pub median: Option>>>, + pub pct25: Option>>>, + pub pct10: Option>>>, + pub min: Option>>>, + pub last: Option>>>, + pub cumulative: Option>>>, } const VERSION: Version = Version::ZERO; @@ -37,20 +37,10 @@ where I: VecIndex, T: ComputedVecValue, { - pub fn forced_import_compressed( - db: &Database, - name: &str, - version: Version, - options: VecBuilderOptions, - ) -> Result { - Self::forced_import(db, name, version, Format::Compressed, options) - } - pub fn forced_import( db: &Database, name: &str, version: Version, - format: Format, options: VecBuilderOptions, ) -> Result { let only_one_active = options.is_only_one_active(); @@ -72,15 +62,12 @@ where db, &maybe_suffix("first"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) }), last: options.last.then(|| { - Box::new( - EagerVec::forced_import(db, name, version + Version::ZERO, format).unwrap(), - ) + Box::new(EagerVec::forced_import(db, name, version + Version::ZERO).unwrap()) }), min: options.min.then(|| { Box::new( @@ -88,7 +75,6 @@ where db, &maybe_suffix("min"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -99,7 +85,6 @@ where db, &maybe_suffix("max"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -110,7 +95,6 @@ where db, &maybe_suffix("median"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -121,7 +105,6 @@ where db, &maybe_suffix("avg"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -136,7 +119,6 @@ where maybe_suffix("sum") }), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -147,7 +129,6 @@ where db, &suffix("cumulative"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -158,7 +139,6 @@ where db, &maybe_suffix("pct90"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -169,7 +149,6 @@ where db, &maybe_suffix("pct75"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -180,7 +159,6 @@ where db, &maybe_suffix("pct25"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -191,7 +169,6 @@ where db, &maybe_suffix("pct10"), version + VERSION + Version::ZERO, - format, ) .unwrap(), ) @@ -535,51 +512,51 @@ where pub fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( - self.iter_any_writable().map(|v| v.len()).min().unwrap(), + self.iter_any_exportable().map(|v| v.len()).min().unwrap(), )) } - pub fn unwrap_first(&self) -> &EagerVec { + pub fn unwrap_first(&self) -> &EagerVec> { self.first.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_average(&self) -> &EagerVec { + pub fn unwrap_average(&self) -> &EagerVec> { self.average.as_ref().unwrap() } - pub fn unwrap_sum(&self) -> &EagerVec { + pub fn unwrap_sum(&self) -> &EagerVec> { self.sum.as_ref().unwrap() } - pub fn unwrap_max(&self) -> &EagerVec { + pub fn unwrap_max(&self) -> &EagerVec> { self.max.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_pct90(&self) -> &EagerVec { + pub fn unwrap_pct90(&self) -> &EagerVec> { self.pct90.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_pct75(&self) -> &EagerVec { + pub fn unwrap_pct75(&self) -> &EagerVec> { self.pct75.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_median(&self) -> &EagerVec { + pub fn unwrap_median(&self) -> &EagerVec> { self.median.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_pct25(&self) -> &EagerVec { + pub fn unwrap_pct25(&self) -> &EagerVec> { self.pct25.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_pct10(&self) -> &EagerVec { + pub fn unwrap_pct10(&self) -> &EagerVec> { self.pct10.as_ref().unwrap() } - pub fn unwrap_min(&self) -> &EagerVec { + pub fn unwrap_min(&self) -> &EagerVec> { self.min.as_ref().unwrap() } - pub fn unwrap_last(&self) -> &EagerVec { + pub fn unwrap_last(&self) -> &EagerVec> { self.last.as_ref().unwrap() } #[allow(unused)] - pub fn unwrap_cumulative(&self) -> &EagerVec { + pub fn unwrap_cumulative(&self) -> &EagerVec> { self.cumulative.as_ref().unwrap() } diff --git a/crates/brk_computer/src/grouped/builder_lazy.rs b/crates/brk_computer/src/grouped/builder_lazy.rs index 0ce5fbad9..27dbf793e 100644 --- a/crates/brk_computer/src/grouped/builder_lazy.rs +++ b/crates/brk_computer/src/grouped/builder_lazy.rs @@ -209,7 +209,7 @@ where pub fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( - self.iter_any_writable().map(|v| v.len()).min().unwrap(), + self.iter_any_exportable().map(|v| v.len()).min().unwrap(), )) } diff --git a/crates/brk_computer/src/grouped/computed.rs b/crates/brk_computer/src/grouped/computed.rs index 9b15558d3..d0cf21d97 100644 --- a/crates/brk_computer/src/grouped/computed.rs +++ b/crates/brk_computer/src/grouped/computed.rs @@ -1,5 +1,6 @@ use std::ops::{Add, AddAssign, Div}; +use serde::Serialize; use vecdb::{Formattable, PcoVecValue}; pub trait ComputedVecValue @@ -10,7 +11,8 @@ where + Add + AddAssign + Ord - + Formattable, + + Formattable + + Serialize, { } impl ComputedVecValue for T where @@ -21,5 +23,6 @@ impl ComputedVecValue for T where + AddAssign + Ord + Formattable + + Serialize { } diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index 1109e6ae6..607bd667a 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -4,7 +4,10 @@ use brk_traversable::Traversable; use brk_types::{ DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{AnyWritableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec}; +use vecdb::{ + AnyExportableVec, Database, EagerVec, Exit, Importable, IterableCloneableVec, IterableVec, + PcoVec, +}; use crate::{Indexes, grouped::LazyVecsBuilder, indexes}; @@ -15,7 +18,7 @@ pub struct ComputedVecsFromDateIndex where T: ComputedVecValue + PartialOrd, { - pub dateindex: Option>, + pub dateindex: Option>>, pub dateindex_extra: EagerVecsBuilder, pub weekindex: LazyVecsBuilder, pub monthindex: LazyVecsBuilder, @@ -40,11 +43,11 @@ where indexes: &indexes::Vecs, options: VecBuilderOptions, ) -> Result { - let dateindex = source.is_compute().then(|| { - EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO).unwrap() - }); + let dateindex = source + .is_compute() + .then(|| EagerVec::forced_import(db, name, version + VERSION + Version::ZERO).unwrap()); - let dateindex_extra = EagerVecsBuilder::forced_import_compressed( + let dateindex_extra = EagerVecsBuilder::forced_import( db, name, version + VERSION + Version::ZERO, @@ -116,11 +119,11 @@ where mut compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { compute(self.dateindex.as_mut().unwrap())?; - let dateindex: Option<&EagerVec> = None; + let dateindex: Option<&EagerVec>> = None; self.compute_rest(starting_indexes, exit, dateindex) } @@ -178,17 +181,17 @@ where .unwrap() } - fn iter_any_writable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.dateindex_extra.iter_any_writable()); - regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable())); + fn iter_any_exportable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.dateindex_extra.iter_any_exportable()); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_exportable())); if let Some(ref x) = self.dateindex { - regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_exportable())); } regular_iter } diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 7a96c69a7..284076181 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -5,7 +5,10 @@ use brk_types::{ DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{AnyWritableVec, Database, EagerVec, Exit, IterableCloneableVec, IterableVec}; +use vecdb::{ + AnyExportableVec, Database, EagerVec, Exit, Importable, IterableCloneableVec, IterableVec, + PcoVec, +}; use crate::{ Indexes, @@ -20,7 +23,7 @@ pub struct ComputedVecsFromHeight where T: ComputedVecValue + PartialOrd, { - pub height: Option>, + pub height: Option>>, pub height_extra: EagerVecsBuilder, pub dateindex: EagerVecsBuilder, pub weekindex: LazyVecsBuilder, @@ -49,23 +52,19 @@ where indexes: &indexes::Vecs, options: VecBuilderOptions, ) -> Result { - let height = source.is_compute().then(|| { - EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO).unwrap() - }); + let height = source + .is_compute() + .then(|| EagerVec::forced_import(db, name, version + VERSION + Version::ZERO).unwrap()); - let height_extra = EagerVecsBuilder::forced_import_compressed( + let height_extra = EagerVecsBuilder::forced_import( db, name, version + VERSION + Version::ZERO, options.copy_self_extra(), )?; - let dateindex = EagerVecsBuilder::forced_import_compressed( - db, - name, - version + VERSION + Version::ZERO, - options, - )?; + let dateindex = + EagerVecsBuilder::forced_import(db, name, version + VERSION + Version::ZERO, options)?; let options = options.remove_percentiles(); @@ -122,7 +121,7 @@ where height, height_extra, dateindex, - difficultyepoch: EagerVecsBuilder::forced_import_compressed( + difficultyepoch: EagerVecsBuilder::forced_import( db, name, version + VERSION + Version::ZERO, @@ -139,11 +138,11 @@ where mut compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { compute(self.height.as_mut().unwrap())?; - let height: Option<&EagerVec> = None; + let height: Option<&EagerVec>> = None; self.compute_rest(indexes, starting_indexes, exit, height) } @@ -239,19 +238,19 @@ where .unwrap() } - fn iter_any_writable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.height_extra.iter_any_writable()); - regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable())); + fn iter_any_exportable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height_extra.iter_any_exportable()); + regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_exportable())); if let Some(ref x) = self.height { - regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_exportable())); } regular_iter } diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index d3989cbe3..71521e5d8 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -2,7 +2,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{DifficultyEpoch, Height, Version}; -use vecdb::{AnyWritableVec, Database, EagerVec, Exit}; +use vecdb::{AnyExportableVec, Database, EagerVec, Exit, Importable, PcoVec}; use crate::{Indexes, indexes}; @@ -13,7 +13,7 @@ pub struct ComputedVecsFromHeightStrict where T: ComputedVecValue + PartialOrd, { - pub height: EagerVec, + pub height: EagerVec>, pub height_extra: EagerVecsBuilder, pub difficultyepoch: EagerVecsBuilder, // TODO: pub halvingepoch: StorableVecGeneator, @@ -32,10 +32,9 @@ where version: Version, options: VecBuilderOptions, ) -> Result { - let height = - EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO)?; + let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO)?; - let height_extra = EagerVecsBuilder::forced_import_compressed( + let height_extra = EagerVecsBuilder::forced_import( db, name, version + VERSION + Version::ZERO, @@ -47,7 +46,7 @@ where Ok(Self { height, height_extra, - difficultyepoch: EagerVecsBuilder::forced_import_compressed( + difficultyepoch: EagerVecsBuilder::forced_import( db, name, version + VERSION + Version::ZERO, @@ -65,7 +64,7 @@ where mut compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { compute(&mut self.height)?; @@ -111,11 +110,11 @@ where .unwrap() } - fn iter_any_writable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.height.iter_any_writable()); - regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable())); + fn iter_any_exportable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height.iter_any_exportable()); + regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_exportable())); regular_iter } } diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index 40cc0a2c1..90182e888 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -6,8 +6,8 @@ use brk_types::{ Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex, }; use vecdb::{ - AnyVec, AnyWritableVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, - IterableCloneableVec, TypedVecIterator, VecIndex, + AnyExportableVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, + Importable, IterableCloneableVec, PcoVec, TypedVecIterator, VecIndex, }; use crate::{ @@ -23,7 +23,7 @@ pub struct ComputedVecsFromTxindex where T: ComputedVecValue + PartialOrd, { - pub txindex: Option>>, + pub txindex: Option>>>, pub height: EagerVecsBuilder, pub dateindex: EagerVecsBuilder, pub weekindex: LazyVecsBuilder, @@ -53,27 +53,16 @@ where options: VecBuilderOptions, ) -> Result { let txindex = source.is_compute().then(|| { - Box::new( - EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO) - .unwrap(), - ) + Box::new(EagerVec::forced_import(db, name, version + VERSION + Version::ZERO).unwrap()) }); - let height = EagerVecsBuilder::forced_import_compressed( - db, - name, - version + VERSION + Version::ZERO, - options, - )?; + let height = + EagerVecsBuilder::forced_import(db, name, version + VERSION + Version::ZERO, options)?; let options = options.remove_percentiles(); - let dateindex = EagerVecsBuilder::forced_import_compressed( - db, - name, - version + VERSION + Version::ZERO, - options, - )?; + let dateindex = + EagerVecsBuilder::forced_import(db, name, version + VERSION + Version::ZERO, options)?; Ok(Self { weekindex: LazyVecsBuilder::forced_import( @@ -128,7 +117,7 @@ where txindex, height, dateindex, - difficultyepoch: EagerVecsBuilder::forced_import_compressed( + difficultyepoch: EagerVecsBuilder::forced_import( db, name, version + VERSION + Version::ZERO, @@ -149,7 +138,7 @@ where // ) -> Result<()> // where // F: FnMut( - // &mut EagerVec, + // &mut EagerVec>, // &Indexer, // &indexes::Vecs, // &Indexes, @@ -497,19 +486,19 @@ where .unwrap() } - fn iter_any_writable(&self) -> impl Iterator { - let mut regular_iter: Box> = - Box::new(self.height.iter_any_writable()); - regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_writable())); - regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_writable())); + fn iter_any_exportable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height.iter_any_exportable()); + regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_exportable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_exportable())); if let Some(ref x) = self.txindex { - regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_exportable())); } regular_iter } diff --git a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs index 11a8bac35..37db62389 100644 --- a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Date, DateIndex, Dollars, StoredF32, Version}; -use vecdb::{ +use vecdb::{PcoVec, AnyStoredVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, IterableVec, TypedVecIterator, VecIndex, }; @@ -290,14 +290,14 @@ impl ComputedRatioVecsFromDateIndex { compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { self.price .as_mut() .unwrap() .compute_all(starting_indexes, exit, compute)?; - let date_to_price_opt: Option<&EagerVec> = None; + let date_to_price_opt: Option<&EagerVec>> = None; self.compute_rest(price, starting_indexes, exit, date_to_price_opt) } @@ -501,32 +501,32 @@ impl ComputedRatioVecsFromDateIndex { self.ratio_pct1.as_mut().unwrap().compute_rest( starting_indexes, exit, - None as Option<&EagerVec<_, _>>, + None as Option<&EagerVec>>, )?; self.ratio_pct2.as_mut().unwrap().compute_rest( starting_indexes, exit, - None as Option<&EagerVec<_, _>>, + None as Option<&EagerVec>>, )?; self.ratio_pct5.as_mut().unwrap().compute_rest( starting_indexes, exit, - None as Option<&EagerVec<_, _>>, + None as Option<&EagerVec>>, )?; self.ratio_pct95.as_mut().unwrap().compute_rest( starting_indexes, exit, - None as Option<&EagerVec<_, _>>, + None as Option<&EagerVec>>, )?; self.ratio_pct98.as_mut().unwrap().compute_rest( starting_indexes, exit, - None as Option<&EagerVec<_, _>>, + None as Option<&EagerVec>>, )?; self.ratio_pct99.as_mut().unwrap().compute_rest( starting_indexes, exit, - None as Option<&EagerVec<_, _>>, + None as Option<&EagerVec>>, )?; let date_to_price = price_opt.unwrap_or_else(|| unsafe { @@ -610,7 +610,7 @@ impl ComputedRatioVecsFromDateIndex { Ok(()) } - fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec> { + fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec>> { let mut vecs = Vec::with_capacity(6); if let Some(v) = self.ratio_pct1.as_mut() { vecs.push(v.dateindex.as_mut().unwrap()); diff --git a/crates/brk_computer/src/grouped/sd_from_dateindex.rs b/crates/brk_computer/src/grouped/sd_from_dateindex.rs index 182196b33..01b734d76 100644 --- a/crates/brk_computer/src/grouped/sd_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/sd_from_dateindex.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Date, DateIndex, Dollars, StoredF32, Version}; -use vecdb::{ +use vecdb::{PcoVec, AnyStoredVec, AnyVec, BoxedVecIterator, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, IterableVec, VecIndex, }; @@ -440,7 +440,7 @@ impl ComputedStandardDeviationVecsFromDateIndex { Ok(()) })?; - let sma_opt: Option<&EagerVec> = None; + let sma_opt: Option<&EagerVec>> = None; self.compute_rest(starting_indexes, exit, sma_opt, source, price_opt) } @@ -615,7 +615,7 @@ impl ComputedStandardDeviationVecsFromDateIndex { .try_for_each(|v| v.safe_flush(exit))?; self.mut_stateful_computed().try_for_each(|v| { - v.compute_rest(starting_indexes, exit, None as Option<&EagerVec<_, _>>) + v.compute_rest(starting_indexes, exit, None as Option<&EagerVec>>) })?; if let Some(zscore) = self.zscore.as_mut() { @@ -805,7 +805,7 @@ impl ComputedStandardDeviationVecsFromDateIndex { fn mut_stateful_date_vecs( &mut self, - ) -> impl Iterator> { + ) -> impl Iterator>> { self.mut_stateful_computed() .map(|c| c.dateindex.as_mut().unwrap()) } diff --git a/crates/brk_computer/src/grouped/value_from_dateindex.rs b/crates/brk_computer/src/grouped/value_from_dateindex.rs index 67772b0d0..d882fa14f 100644 --- a/crates/brk_computer/src/grouped/value_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/value_from_dateindex.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Bitcoin, DateIndex, Dollars, Sats, Version}; -use vecdb::{CollectableVec, Database, EagerVec, Exit, StoredVec}; +use vecdb::{CollectableVec, Database, EagerVec, Exit, PcoVec}; use crate::{ Indexes, @@ -71,11 +71,11 @@ impl ComputedValueVecsFromDateIndex { mut compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { compute(self.sats.dateindex.as_mut().unwrap())?; - let dateindex: Option<&StoredVec> = None; + let dateindex: Option<&PcoVec> = None; self.compute_rest(price, starting_indexes, exit, dateindex)?; Ok(()) @@ -96,7 +96,7 @@ impl ComputedValueVecsFromDateIndex { v.compute_from_sats(starting_indexes.dateindex, dateindex, exit) })?; } else { - let dateindex: Option<&StoredVec> = None; + let dateindex: Option<&PcoVec> = None; self.sats.compute_rest(starting_indexes, exit, dateindex)?; diff --git a/crates/brk_computer/src/grouped/value_from_height.rs b/crates/brk_computer/src/grouped/value_from_height.rs index ce1b771dd..c13b02c9f 100644 --- a/crates/brk_computer/src/grouped/value_from_height.rs +++ b/crates/brk_computer/src/grouped/value_from_height.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Bitcoin, Dollars, Height, Sats, Version}; -use vecdb::{CollectableVec, Database, EagerVec, Exit, StoredVec}; +use vecdb::{CollectableVec, Database, EagerVec, Exit, PcoVec}; use crate::{ Indexes, @@ -72,11 +72,11 @@ impl ComputedValueVecsFromHeight { mut compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { compute(self.sats.height.as_mut().unwrap())?; - let height: Option<&StoredVec> = None; + let height: Option<&PcoVec> = None; self.compute_rest(indexes, price, starting_indexes, exit, height)?; Ok(()) @@ -99,7 +99,7 @@ impl ComputedValueVecsFromHeight { v.compute_from_sats(starting_indexes.height, height, exit) })?; } else { - let height: Option<&StoredVec> = None; + let height: Option<&PcoVec> = None; self.sats .compute_rest(indexes, starting_indexes, exit, height)?; diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index aaacacd0f..b949428c5 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -3,7 +3,7 @@ use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; use vecdb::{ - CollectableVec, Database, Exit, IterableCloneableVec, LazyVecFrom1, LazyVecFrom3, StoredVec, + CollectableVec, Database, Exit, IterableCloneableVec, LazyVecFrom1, LazyVecFrom3, PcoVec, VecIndex, }; @@ -122,7 +122,7 @@ impl ComputedValueVecsFromTxindex { // ) -> Result<()> // where // F: FnMut( - // &mut EagerVec, + // &mut EagerVec>, // &Indexer, // &indexes::Vecs, // &Indexes, @@ -137,7 +137,7 @@ impl ComputedValueVecsFromTxindex { // exit, // )?; - // let txindex: Option<&StoredVec> = None; + // let txindex: Option<&PcoVec> = None; // self.compute_rest( // indexer, // indexes, @@ -163,7 +163,7 @@ impl ComputedValueVecsFromTxindex { self.sats .compute_rest(indexer, indexes, starting_indexes, exit, Some(txindex))?; } else { - let txindex: Option<&StoredVec> = None; + let txindex: Option<&PcoVec> = None; self.sats .compute_rest(indexer, indexes, starting_indexes, exit, txindex)?; } diff --git a/crates/brk_computer/src/grouped/value_height.rs b/crates/brk_computer/src/grouped/value_height.rs index c13ae5a0b..14903f418 100644 --- a/crates/brk_computer/src/grouped/value_height.rs +++ b/crates/brk_computer/src/grouped/value_height.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Bitcoin, Dollars, Height, Sats, Version}; -use vecdb::{CollectableVec, Database, EagerVec, Exit, Format, StoredVec}; +use vecdb::{CollectableVec, Database, EagerVec, Exit, Importable, PcoVec}; use crate::{ Indexes, @@ -12,9 +12,9 @@ use crate::{ #[derive(Clone, Traversable)] pub struct ComputedHeightValueVecs { - pub sats: Option>, - pub bitcoin: EagerVec, - pub dollars: Option>, + pub sats: Option>>, + pub bitcoin: EagerVec>, + pub dollars: Option>>, } const VERSION: Version = Version::ZERO; @@ -25,26 +25,22 @@ impl ComputedHeightValueVecs { name: &str, source: Source, version: Version, - format: Format, compute_dollars: bool, ) -> Result { Ok(Self { sats: source.is_compute().then(|| { - EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format) - .unwrap() + EagerVec::forced_import(db, name, version + VERSION + Version::ZERO).unwrap() }), bitcoin: EagerVec::forced_import( db, &format!("{name}_btc"), version + VERSION + Version::ZERO, - format, )?, dollars: compute_dollars.then(|| { EagerVec::forced_import( db, &format!("{name}_usd"), version + VERSION + Version::ZERO, - format, ) .unwrap() }), @@ -59,11 +55,11 @@ impl ComputedHeightValueVecs { mut compute: F, ) -> Result<()> where - F: FnMut(&mut EagerVec) -> Result<()>, + F: FnMut(&mut EagerVec>) -> Result<()>, { compute(self.sats.as_mut().unwrap())?; - let height: Option<&StoredVec> = None; + let height: Option<&PcoVec> = None; self.compute_rest(price, starting_indexes, exit, height)?; Ok(()) diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index 704babd86..cd6d8762b 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -13,8 +13,8 @@ use brk_types::{ YearIndex, }; use vecdb::{ - Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, LazyVecFrom1, PAGE_SIZE, - TypedVecIterator, unlikely, + Database, EagerVec, Exit, GenericStoredVec, Importable, IterableCloneableVec, LazyVecFrom1, + PAGE_SIZE, PcoVec, TypedVecIterator, unlikely, }; const VERSION: Version = Version::ZERO; @@ -23,36 +23,36 @@ const VERSION: Version = Version::ZERO; pub struct Vecs { db: Database, - pub dateindex_to_date: EagerVec, - pub dateindex_to_dateindex: EagerVec, - pub dateindex_to_first_height: EagerVec, - pub dateindex_to_height_count: EagerVec, - pub dateindex_to_monthindex: EagerVec, - pub dateindex_to_weekindex: EagerVec, - pub decadeindex_to_decadeindex: EagerVec, - pub decadeindex_to_first_yearindex: EagerVec, - pub decadeindex_to_yearindex_count: EagerVec, - pub difficultyepoch_to_difficultyepoch: EagerVec, - pub difficultyepoch_to_first_height: EagerVec, - pub difficultyepoch_to_height_count: EagerVec, + pub dateindex_to_date: EagerVec>, + pub dateindex_to_dateindex: EagerVec>, + pub dateindex_to_first_height: EagerVec>, + pub dateindex_to_height_count: EagerVec>, + pub dateindex_to_monthindex: EagerVec>, + pub dateindex_to_weekindex: EagerVec>, + pub decadeindex_to_decadeindex: EagerVec>, + pub decadeindex_to_first_yearindex: EagerVec>, + pub decadeindex_to_yearindex_count: EagerVec>, + pub difficultyepoch_to_difficultyepoch: EagerVec>, + pub difficultyepoch_to_first_height: EagerVec>, + pub difficultyepoch_to_height_count: EagerVec>, pub emptyoutputindex_to_emptyoutputindex: LazyVecFrom1, - pub halvingepoch_to_first_height: EagerVec, - pub halvingepoch_to_halvingepoch: EagerVec, - pub height_to_date: EagerVec, - pub height_to_date_fixed: EagerVec, - pub height_to_dateindex: EagerVec, - pub height_to_difficultyepoch: EagerVec, - pub height_to_halvingepoch: EagerVec, - pub height_to_height: EagerVec, - pub height_to_timestamp_fixed: EagerVec, - pub height_to_txindex_count: EagerVec, - pub monthindex_to_dateindex_count: EagerVec, - pub monthindex_to_first_dateindex: EagerVec, - pub monthindex_to_monthindex: EagerVec, - pub monthindex_to_quarterindex: EagerVec, - pub monthindex_to_semesterindex: EagerVec, - pub monthindex_to_yearindex: EagerVec, + pub halvingepoch_to_first_height: EagerVec>, + pub halvingepoch_to_halvingepoch: EagerVec>, + pub height_to_date: EagerVec>, + pub height_to_date_fixed: EagerVec>, + pub height_to_dateindex: EagerVec>, + pub height_to_difficultyepoch: EagerVec>, + pub height_to_halvingepoch: EagerVec>, + pub height_to_height: EagerVec>, + pub height_to_timestamp_fixed: EagerVec>, + pub height_to_txindex_count: EagerVec>, + pub monthindex_to_dateindex_count: EagerVec>, + pub monthindex_to_first_dateindex: EagerVec>, + pub monthindex_to_monthindex: EagerVec>, + pub monthindex_to_quarterindex: EagerVec>, + pub monthindex_to_semesterindex: EagerVec>, + pub monthindex_to_yearindex: EagerVec>, pub opreturnindex_to_opreturnindex: LazyVecFrom1, pub p2aaddressindex_to_p2aaddressindex: @@ -73,27 +73,27 @@ pub struct Vecs { LazyVecFrom1, pub p2wshaddressindex_to_p2wshaddressindex: LazyVecFrom1, - pub quarterindex_to_first_monthindex: EagerVec, - pub quarterindex_to_monthindex_count: EagerVec, - pub quarterindex_to_quarterindex: EagerVec, - pub semesterindex_to_first_monthindex: EagerVec, - pub semesterindex_to_monthindex_count: EagerVec, - pub semesterindex_to_semesterindex: EagerVec, - pub txindex_to_input_count: EagerVec, - pub txindex_to_output_count: EagerVec, + pub quarterindex_to_first_monthindex: EagerVec>, + pub quarterindex_to_monthindex_count: EagerVec>, + pub quarterindex_to_quarterindex: EagerVec>, + pub semesterindex_to_first_monthindex: EagerVec>, + pub semesterindex_to_monthindex_count: EagerVec>, + pub semesterindex_to_semesterindex: EagerVec>, + pub txindex_to_input_count: EagerVec>, + pub txindex_to_output_count: EagerVec>, pub txindex_to_txindex: LazyVecFrom1, pub txinindex_to_txinindex: LazyVecFrom1, - pub txinindex_to_txoutindex: EagerVec, + pub txinindex_to_txoutindex: EagerVec>, pub txoutindex_to_txoutindex: LazyVecFrom1, pub unknownoutputindex_to_unknownoutputindex: LazyVecFrom1, - pub weekindex_to_dateindex_count: EagerVec, - pub weekindex_to_first_dateindex: EagerVec, - pub weekindex_to_weekindex: EagerVec, - pub yearindex_to_decadeindex: EagerVec, - pub yearindex_to_first_monthindex: EagerVec, - pub yearindex_to_monthindex_count: EagerVec, - pub yearindex_to_yearindex: EagerVec, + pub weekindex_to_dateindex_count: EagerVec>, + pub weekindex_to_first_dateindex: EagerVec>, + pub weekindex_to_weekindex: EagerVec>, + pub yearindex_to_decadeindex: EagerVec>, + pub yearindex_to_first_monthindex: EagerVec>, + pub yearindex_to_monthindex_count: EagerVec>, + pub yearindex_to_yearindex: EagerVec>, } impl Vecs { @@ -108,11 +108,7 @@ impl Vecs { let version = parent_version + VERSION; let this = Self { - txinindex_to_txoutindex: EagerVec::forced_import_compressed( - &db, - "txoutindex", - version, - )?, + txinindex_to_txoutindex: EagerVec::forced_import(&db, "txoutindex", version)?, txoutindex_to_txoutindex: LazyVecFrom1::init( "txoutindex", version + Version::ZERO, @@ -203,217 +199,205 @@ impl Vecs { indexer.vecs.txindex_to_txid.boxed_clone(), |index, _| Some(index), ), - txindex_to_input_count: EagerVec::forced_import_compressed( + txindex_to_input_count: EagerVec::forced_import( &db, "input_count", version + Version::ZERO, )?, - txindex_to_output_count: EagerVec::forced_import_compressed( + txindex_to_output_count: EagerVec::forced_import( &db, "output_count", version + Version::ZERO, )?, - dateindex_to_date: EagerVec::forced_import_compressed( - &db, - "date", - version + Version::ZERO, - )?, - dateindex_to_dateindex: EagerVec::forced_import_compressed( + dateindex_to_date: EagerVec::forced_import(&db, "date", version + Version::ZERO)?, + dateindex_to_dateindex: EagerVec::forced_import( &db, "dateindex", version + Version::ZERO, )?, - dateindex_to_first_height: EagerVec::forced_import_compressed( + dateindex_to_first_height: EagerVec::forced_import( &db, "first_height", version + Version::ZERO, )?, - dateindex_to_monthindex: EagerVec::forced_import_compressed( + dateindex_to_monthindex: EagerVec::forced_import( &db, "monthindex", version + Version::ZERO, )?, - dateindex_to_weekindex: EagerVec::forced_import_compressed( + dateindex_to_weekindex: EagerVec::forced_import( &db, "weekindex", version + Version::ZERO, )?, - decadeindex_to_decadeindex: EagerVec::forced_import_compressed( + decadeindex_to_decadeindex: EagerVec::forced_import( &db, "decadeindex", version + Version::ZERO, )?, - decadeindex_to_first_yearindex: EagerVec::forced_import_compressed( + decadeindex_to_first_yearindex: EagerVec::forced_import( &db, "first_yearindex", version + Version::ZERO, )?, - difficultyepoch_to_difficultyepoch: EagerVec::forced_import_compressed( + difficultyepoch_to_difficultyepoch: EagerVec::forced_import( &db, "difficultyepoch", version + Version::ZERO, )?, - difficultyepoch_to_first_height: EagerVec::forced_import_compressed( + difficultyepoch_to_first_height: EagerVec::forced_import( &db, "first_height", version + Version::ZERO, )?, - halvingepoch_to_first_height: EagerVec::forced_import_compressed( + halvingepoch_to_first_height: EagerVec::forced_import( &db, "first_height", version + Version::ZERO, )?, - halvingepoch_to_halvingepoch: EagerVec::forced_import_compressed( + halvingepoch_to_halvingepoch: EagerVec::forced_import( &db, "halvingepoch", version + Version::ZERO, )?, - height_to_date: EagerVec::forced_import_compressed( - &db, - "date", - version + Version::ZERO, - )?, - height_to_difficultyepoch: EagerVec::forced_import_compressed( + height_to_date: EagerVec::forced_import(&db, "date", version + Version::ZERO)?, + height_to_difficultyepoch: EagerVec::forced_import( &db, "difficultyepoch", version + Version::ZERO, )?, - height_to_halvingepoch: EagerVec::forced_import_compressed( + height_to_halvingepoch: EagerVec::forced_import( &db, "halvingepoch", version + Version::ZERO, )?, - height_to_height: EagerVec::forced_import_compressed( - &db, - "height", - version + Version::ZERO, - )?, - monthindex_to_first_dateindex: EagerVec::forced_import_compressed( + height_to_height: EagerVec::forced_import(&db, "height", version + Version::ZERO)?, + monthindex_to_first_dateindex: EagerVec::forced_import( &db, "first_dateindex", version + Version::ZERO, )?, - monthindex_to_monthindex: EagerVec::forced_import_compressed( + monthindex_to_monthindex: EagerVec::forced_import( &db, "monthindex", version + Version::ZERO, )?, - monthindex_to_quarterindex: EagerVec::forced_import_compressed( + monthindex_to_quarterindex: EagerVec::forced_import( &db, "quarterindex", version + Version::ZERO, )?, - monthindex_to_semesterindex: EagerVec::forced_import_compressed( + monthindex_to_semesterindex: EagerVec::forced_import( &db, "semesterindex", version + Version::ZERO, )?, - monthindex_to_yearindex: EagerVec::forced_import_compressed( + monthindex_to_yearindex: EagerVec::forced_import( &db, "yearindex", version + Version::ZERO, )?, - quarterindex_to_first_monthindex: EagerVec::forced_import_compressed( + quarterindex_to_first_monthindex: EagerVec::forced_import( &db, "first_monthindex", version + Version::ZERO, )?, - semesterindex_to_first_monthindex: EagerVec::forced_import_compressed( + semesterindex_to_first_monthindex: EagerVec::forced_import( &db, "first_monthindex", version + Version::ZERO, )?, - weekindex_to_first_dateindex: EagerVec::forced_import_compressed( + weekindex_to_first_dateindex: EagerVec::forced_import( &db, "first_dateindex", version + Version::ZERO, )?, - yearindex_to_first_monthindex: EagerVec::forced_import_compressed( + yearindex_to_first_monthindex: EagerVec::forced_import( &db, "first_monthindex", version + Version::ZERO, )?, - quarterindex_to_quarterindex: EagerVec::forced_import_compressed( + quarterindex_to_quarterindex: EagerVec::forced_import( &db, "quarterindex", version + Version::ZERO, )?, - semesterindex_to_semesterindex: EagerVec::forced_import_compressed( + semesterindex_to_semesterindex: EagerVec::forced_import( &db, "semesterindex", version + Version::ZERO, )?, - weekindex_to_weekindex: EagerVec::forced_import_compressed( + weekindex_to_weekindex: EagerVec::forced_import( &db, "weekindex", version + Version::ZERO, )?, - yearindex_to_decadeindex: EagerVec::forced_import_compressed( + yearindex_to_decadeindex: EagerVec::forced_import( &db, "decadeindex", version + Version::ZERO, )?, - yearindex_to_yearindex: EagerVec::forced_import_compressed( + yearindex_to_yearindex: EagerVec::forced_import( &db, "yearindex", version + Version::ZERO, )?, - height_to_date_fixed: EagerVec::forced_import_compressed( + height_to_date_fixed: EagerVec::forced_import( &db, "date_fixed", version + Version::ZERO, )?, - height_to_dateindex: EagerVec::forced_import_compressed( + height_to_dateindex: EagerVec::forced_import( &db, "dateindex", version + Version::ZERO, )?, - height_to_timestamp_fixed: EagerVec::forced_import_compressed( + height_to_timestamp_fixed: EagerVec::forced_import( &db, "timestamp_fixed", version + Version::ZERO, )?, - height_to_txindex_count: EagerVec::forced_import_compressed( + height_to_txindex_count: EagerVec::forced_import( &db, "txindex_count", version + Version::ZERO, )?, - dateindex_to_height_count: EagerVec::forced_import_compressed( + dateindex_to_height_count: EagerVec::forced_import( &db, "height_count", version + Version::ZERO, )?, - weekindex_to_dateindex_count: EagerVec::forced_import_compressed( + weekindex_to_dateindex_count: EagerVec::forced_import( &db, "dateindex_count", version + Version::ZERO, )?, - difficultyepoch_to_height_count: EagerVec::forced_import_compressed( + difficultyepoch_to_height_count: EagerVec::forced_import( &db, "height_count", version + Version::ZERO, )?, - monthindex_to_dateindex_count: EagerVec::forced_import_compressed( + monthindex_to_dateindex_count: EagerVec::forced_import( &db, "dateindex_count", version + Version::ZERO, )?, - quarterindex_to_monthindex_count: EagerVec::forced_import_compressed( + quarterindex_to_monthindex_count: EagerVec::forced_import( &db, "monthindex_count", version + Version::ZERO, )?, - semesterindex_to_monthindex_count: EagerVec::forced_import_compressed( + semesterindex_to_monthindex_count: EagerVec::forced_import( &db, "monthindex_count", version + Version::ZERO, )?, - yearindex_to_monthindex_count: EagerVec::forced_import_compressed( + yearindex_to_monthindex_count: EagerVec::forced_import( &db, "monthindex_count", version + Version::ZERO, )?, - decadeindex_to_yearindex_count: EagerVec::forced_import_compressed( + decadeindex_to_yearindex_count: EagerVec::forced_import( &db, "yearindex_count", version + Version::ZERO, @@ -422,7 +406,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index c71395f75..4afc9a791 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -9,7 +9,7 @@ use brk_reader::Reader; use brk_traversable::Traversable; use brk_types::Version; use log::info; -use vecdb::{Exit, Format}; +use vecdb::Exit; mod blks; mod chain; @@ -125,13 +125,8 @@ impl Computer { })?; // Threads inside - let stateful = stateful::Vecs::forced_import( - &computed_path, - VERSION, - Format::Compressed, - &indexes, - price.as_ref(), - )?; + let stateful = + stateful::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())?; Ok(Self { constants, diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index e92c68ae1..34c1ac7c1 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -3,7 +3,10 @@ use std::{path::Path, thread}; use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version}; -use vecdb::{Database, EagerVec, Exit, GenericStoredVec, PAGE_SIZE, TypedVecIterator, VecIndex}; +use vecdb::{ + Database, EagerVec, Exit, GenericStoredVec, Importable, PAGE_SIZE, PcoVec, TypedVecIterator, + VecIndex, +}; use crate::{ grouped::{ComputedStandardDeviationVecsFromDateIndex, Source, StandardDeviationVecsOptions}, @@ -21,8 +24,8 @@ use super::{ pub struct Vecs { db: Database, - pub height_to_price_ath: EagerVec, - pub height_to_price_drawdown: EagerVec, + pub height_to_price_ath: EagerVec>, + pub height_to_price_drawdown: EagerVec>, pub indexes_to_price_ath: ComputedVecsFromDateIndex, pub indexes_to_price_drawdown: ComputedVecsFromDateIndex, pub indexes_to_days_since_price_ath: ComputedVecsFromDateIndex, @@ -45,8 +48,8 @@ pub struct Vecs { pub indexes_to_price_1y_min: ComputedVecsFromDateIndex, pub indexes_to_price_1y_max: ComputedVecsFromDateIndex, - pub dateindex_to_price_true_range: EagerVec, - pub dateindex_to_price_true_range_2w_sum: EagerVec, + pub dateindex_to_price_true_range: EagerVec>, + pub dateindex_to_price_true_range_2w_sum: EagerVec>, pub indexes_to_price_2w_choppiness_index: ComputedVecsFromDateIndex, pub indexes_to_price_1w_sma: ComputedRatioVecsFromDateIndex, @@ -210,12 +213,12 @@ impl Vecs { let version = parent_version + Version::ZERO; let this = Self { - height_to_price_ath: EagerVec::forced_import_compressed( + height_to_price_ath: EagerVec::forced_import( &db, "price_ath", version + Version::ZERO, )?, - height_to_price_drawdown: EagerVec::forced_import_compressed( + height_to_price_drawdown: EagerVec::forced_import( &db, "price_drawdown", version + Version::ZERO, @@ -1431,12 +1434,12 @@ impl Vecs { indexes, VecBuilderOptions::default().add_last(), )?, - dateindex_to_price_true_range: EagerVec::forced_import_compressed( + dateindex_to_price_true_range: EagerVec::forced_import( &db, "price_true_range", version + Version::ZERO, )?, - dateindex_to_price_true_range_2w_sum: EagerVec::forced_import_compressed( + dateindex_to_price_true_range_2w_sum: EagerVec::forced_import( &db, "price_true_range_2w_sum", version + Version::ZERO, @@ -1517,7 +1520,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; @@ -1593,7 +1596,7 @@ impl Vecs { if prev.is_none() { let i = i.to_usize(); prev.replace(if i > 0 { - slf.get_or_read_at_once(i - 1).unwrap().unwrap() + slf.get_pushed_or_read_at_unwrap_once(i - 1) } else { StoredU16::default() }); @@ -1624,7 +1627,7 @@ impl Vecs { if prev.is_none() { let i = i.to_usize(); prev.replace(if i > 0 { - slf.get_or_read_at_unwrap_once(i - 1) + slf.get_pushed_or_read_at_unwrap_once(i - 1) } else { StoredU16::ZERO }); @@ -2137,19 +2140,19 @@ impl Vecs { starting_indexes, exit, self._1d_price_returns.dateindex.as_ref().unwrap(), - None as Option<&EagerVec>, + None as Option<&EagerVec>>, )?; self.indexes_to_1d_returns_1m_sd.compute_all( starting_indexes, exit, self._1d_price_returns.dateindex.as_ref().unwrap(), - None as Option<&EagerVec>, + None as Option<&EagerVec>>, )?; self.indexes_to_1d_returns_1y_sd.compute_all( starting_indexes, exit, self._1d_price_returns.dateindex.as_ref().unwrap(), - None as Option<&EagerVec>, + None as Option<&EagerVec>>, )?; self.indexes_to_price_1w_volatility diff --git a/crates/brk_computer/src/pools/mod.rs b/crates/brk_computer/src/pools/mod.rs index 212198577..b9512ec33 100644 --- a/crates/brk_computer/src/pools/mod.rs +++ b/crates/brk_computer/src/pools/mod.rs @@ -7,8 +7,8 @@ use brk_traversable::Traversable; use brk_types::{Address, AddressBytes, Height, OutputType, PoolId, Pools, TxOutIndex, pools}; use rayon::prelude::*; use vecdb::{ - AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, IterableVec, PAGE_SIZE, RawVec, - TypedVecIterator, VecIndex, Version, + AnyStoredVec, AnyVec, BytesVec, Database, Exit, GenericStoredVec, Importable, IterableVec, + PAGE_SIZE, TypedVecIterator, VecIndex, Version, }; mod vecs; @@ -24,7 +24,7 @@ pub struct Vecs { db: Database, pools: &'static Pools, - pub height_to_pool: RawVec, + pub height_to_pool: BytesVec, pub vecs: BTreeMap, } @@ -42,7 +42,7 @@ impl Vecs { let version = parent_version + Version::new(3) + Version::new(pools.len() as u64); let this = Self { - height_to_pool: RawVec::forced_import(&db, "pool", version + Version::ZERO)?, + height_to_pool: BytesVec::forced_import(&db, "pool", version + Version::ZERO)?, vecs: pools .iter() .map(|pool| { @@ -62,7 +62,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/pools/vecs.rs b/crates/brk_computer/src/pools/vecs.rs index 50b23e547..086dc698a 100644 --- a/crates/brk_computer/src/pools/vecs.rs +++ b/crates/brk_computer/src/pools/vecs.rs @@ -353,7 +353,7 @@ impl Vecs { if prev.is_none() { let i = i.to_usize(); prev.replace(if i > 0 { - slf.get_or_read_at_unwrap_once(i - 1) + slf.get_pushed_or_read_at_unwrap_once(i - 1) } else { StoredU16::ZERO }); diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs index 24f4b23e8..c6872dfb8 100644 --- a/crates/brk_computer/src/price.rs +++ b/crates/brk_computer/src/price.rs @@ -6,7 +6,7 @@ use brk_types::{ Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{Database, EagerVec, Exit, PAGE_SIZE}; +use vecdb::{BytesVec, Database, EagerVec, Exit, Importable, PAGE_SIZE, PcoVec}; use crate::{fetched, grouped::Source}; @@ -20,18 +20,18 @@ use super::{ pub struct Vecs { db: Database, - pub dateindex_to_price_close_in_cents: EagerVec>, - pub dateindex_to_price_high_in_cents: EagerVec>, - pub dateindex_to_price_low_in_cents: EagerVec>, - pub dateindex_to_price_ohlc: EagerVec, - pub dateindex_to_price_ohlc_in_sats: EagerVec, - pub dateindex_to_price_open_in_cents: EagerVec>, - pub height_to_price_close_in_cents: EagerVec>, - pub height_to_price_high_in_cents: EagerVec>, - pub height_to_price_low_in_cents: EagerVec>, - pub height_to_price_ohlc: EagerVec, - pub height_to_price_ohlc_in_sats: EagerVec, - pub height_to_price_open_in_cents: EagerVec>, + pub dateindex_to_price_close_in_cents: EagerVec>>, + pub dateindex_to_price_high_in_cents: EagerVec>>, + pub dateindex_to_price_low_in_cents: EagerVec>>, + pub dateindex_to_price_ohlc: EagerVec>, + pub dateindex_to_price_ohlc_in_sats: EagerVec>, + pub dateindex_to_price_open_in_cents: EagerVec>>, + pub height_to_price_close_in_cents: EagerVec>>, + pub height_to_price_high_in_cents: EagerVec>>, + pub height_to_price_low_in_cents: EagerVec>>, + pub height_to_price_ohlc: EagerVec>, + pub height_to_price_ohlc_in_sats: EagerVec>, + pub height_to_price_open_in_cents: EagerVec>>, pub timeindexes_to_price_close: ComputedVecsFromDateIndex>, pub timeindexes_to_price_high: ComputedVecsFromDateIndex>, pub timeindexes_to_price_low: ComputedVecsFromDateIndex>, @@ -48,22 +48,22 @@ pub struct Vecs { pub chainindexes_to_price_high_in_sats: ComputedVecsFromHeightStrict>, pub chainindexes_to_price_low_in_sats: ComputedVecsFromHeightStrict>, pub chainindexes_to_price_close_in_sats: ComputedVecsFromHeightStrict>, - pub weekindex_to_price_ohlc: EagerVec, - pub weekindex_to_price_ohlc_in_sats: EagerVec, - pub difficultyepoch_to_price_ohlc: EagerVec, - pub difficultyepoch_to_price_ohlc_in_sats: EagerVec, - pub monthindex_to_price_ohlc: EagerVec, - pub monthindex_to_price_ohlc_in_sats: EagerVec, - pub quarterindex_to_price_ohlc: EagerVec, - pub quarterindex_to_price_ohlc_in_sats: EagerVec, - pub semesterindex_to_price_ohlc: EagerVec, - pub semesterindex_to_price_ohlc_in_sats: EagerVec, - pub yearindex_to_price_ohlc: EagerVec, - pub yearindex_to_price_ohlc_in_sats: EagerVec, + pub weekindex_to_price_ohlc: EagerVec>, + pub weekindex_to_price_ohlc_in_sats: EagerVec>, + pub difficultyepoch_to_price_ohlc: EagerVec>, + pub difficultyepoch_to_price_ohlc_in_sats: EagerVec>, + pub monthindex_to_price_ohlc: EagerVec>, + pub monthindex_to_price_ohlc_in_sats: EagerVec>, + pub quarterindex_to_price_ohlc: EagerVec>, + pub quarterindex_to_price_ohlc_in_sats: EagerVec>, + pub semesterindex_to_price_ohlc: EagerVec>, + pub semesterindex_to_price_ohlc_in_sats: EagerVec>, + pub yearindex_to_price_ohlc: EagerVec>, + pub yearindex_to_price_ohlc_in_sats: EagerVec>, // pub halvingepoch_to_price_ohlc: StorableVec, // pub halvingepoch_to_price_ohlc_in_sats: StorableVec, - pub decadeindex_to_price_ohlc: EagerVec, - pub decadeindex_to_price_ohlc_in_sats: EagerVec, + pub decadeindex_to_price_ohlc: EagerVec>, + pub decadeindex_to_price_ohlc_in_sats: EagerVec>, } const VERSION: Version = Version::ZERO; @@ -75,62 +75,62 @@ impl Vecs { db.set_min_len(PAGE_SIZE * 1_000_000)?; let this = Self { - dateindex_to_price_ohlc: EagerVec::forced_import_raw( + dateindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - dateindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + dateindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - dateindex_to_price_close_in_cents: EagerVec::forced_import_compressed( + dateindex_to_price_close_in_cents: EagerVec::forced_import( &db, "price_close_in_cents", version + VERSION + Version::ZERO, )?, - dateindex_to_price_high_in_cents: EagerVec::forced_import_compressed( + dateindex_to_price_high_in_cents: EagerVec::forced_import( &db, "price_high_in_cents", version + VERSION + Version::ZERO, )?, - dateindex_to_price_low_in_cents: EagerVec::forced_import_compressed( + dateindex_to_price_low_in_cents: EagerVec::forced_import( &db, "price_low_in_cents", version + VERSION + Version::ZERO, )?, - dateindex_to_price_open_in_cents: EagerVec::forced_import_compressed( + dateindex_to_price_open_in_cents: EagerVec::forced_import( &db, "price_open_in_cents", version + VERSION + Version::ZERO, )?, - height_to_price_ohlc: EagerVec::forced_import_raw( + height_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - height_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + height_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - height_to_price_close_in_cents: EagerVec::forced_import_compressed( + height_to_price_close_in_cents: EagerVec::forced_import( &db, "price_close_in_cents", version + VERSION + Version::ZERO, )?, - height_to_price_high_in_cents: EagerVec::forced_import_compressed( + height_to_price_high_in_cents: EagerVec::forced_import( &db, "price_high_in_cents", version + VERSION + Version::ZERO, )?, - height_to_price_low_in_cents: EagerVec::forced_import_compressed( + height_to_price_low_in_cents: EagerVec::forced_import( &db, "price_low_in_cents", version + VERSION + Version::ZERO, )?, - height_to_price_open_in_cents: EagerVec::forced_import_compressed( + height_to_price_open_in_cents: EagerVec::forced_import( &db, "price_open_in_cents", version + VERSION + Version::ZERO, @@ -247,74 +247,74 @@ impl Vecs { version + VERSION + VERSION_IN_SATS + Version::ZERO, VecBuilderOptions::default().add_last(), )?, - weekindex_to_price_ohlc: EagerVec::forced_import_raw( + weekindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - weekindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + weekindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - difficultyepoch_to_price_ohlc: EagerVec::forced_import_raw( + difficultyepoch_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - difficultyepoch_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + difficultyepoch_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - monthindex_to_price_ohlc: EagerVec::forced_import_raw( + monthindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - monthindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + monthindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - quarterindex_to_price_ohlc: EagerVec::forced_import_raw( + quarterindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - quarterindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + quarterindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - semesterindex_to_price_ohlc: EagerVec::forced_import_raw( + semesterindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - semesterindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + semesterindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - yearindex_to_price_ohlc: EagerVec::forced_import_raw( + yearindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - yearindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + yearindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, // halvingepoch_to_price_ohlc: StorableVec::forced_import(db, // "halvingepoch_to_price_ohlc"), version + VERSION + Version::ZERO, format)?, - decadeindex_to_price_ohlc: EagerVec::forced_import_raw( + decadeindex_to_price_ohlc: EagerVec::forced_import( &db, "price_ohlc", version + VERSION + Version::ZERO, )?, - decadeindex_to_price_ohlc_in_sats: EagerVec::forced_import_raw( + decadeindex_to_price_ohlc_in_sats: EagerVec::forced_import( &db, "price_ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -324,7 +324,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_computer/src/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs index c0e747680..ed892db6d 100644 --- a/crates/brk_computer/src/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -4,8 +4,8 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version}; use vecdb::{ - AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, GenericStoredVec, IterableVec, - TypedVecIterator, + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, Importable, IterableVec, + PcoVec, TypedVecIterator, }; use crate::{ @@ -31,7 +31,7 @@ pub struct Vecs { #[traversable(flatten)] pub inner: common::Vecs, - pub height_to_addr_count: EagerVec, + pub height_to_addr_count: EagerVec>, pub indexes_to_addr_count: ComputedVecsFromHeight, } @@ -40,7 +40,6 @@ impl Vecs { pub fn forced_import( db: &Database, cohort_name: Option<&str>, - format: Format, version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, @@ -64,7 +63,6 @@ impl Vecs { db, &suffix("addr_count"), version + VERSION + Version::ZERO, - format, )?, indexes_to_addr_count: ComputedVecsFromHeight::forced_import( db, @@ -77,7 +75,6 @@ impl Vecs { inner: common::Vecs::forced_import( db, cohort_name, - format, version, indexes, price, diff --git a/crates/brk_computer/src/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs index 5dcc22b61..b05cd9250 100644 --- a/crates/brk_computer/src/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -5,7 +5,7 @@ use brk_grouper::{AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanA use brk_traversable::Traversable; use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; use derive_deref::{Deref, DerefMut}; -use vecdb::{Database, Exit, Format, IterableVec}; +use vecdb::{Database, Exit, IterableVec}; use crate::{ Indexes, indexes, price, @@ -24,7 +24,6 @@ impl Vecs { pub fn forced_import( db: &Database, version: Version, - format: Format, indexes: &indexes::Vecs, price: Option<&price::Vecs>, states_path: &Path, @@ -35,7 +34,6 @@ impl Vecs { _0sats: address_cohort::Vecs::forced_import( db, Some("addrs_with_0sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -45,7 +43,6 @@ impl Vecs { _1sat_to_10sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1sat_under_10sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -55,7 +52,6 @@ impl Vecs { _10sats_to_100sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10sats_under_100sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -65,7 +61,6 @@ impl Vecs { _100sats_to_1k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100sats_under_1k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -75,7 +70,6 @@ impl Vecs { _1k_sats_to_10k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_sats_under_10k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -85,7 +79,6 @@ impl Vecs { _10k_sats_to_100k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_sats_under_100k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -95,7 +88,6 @@ impl Vecs { _100k_sats_to_1m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100k_sats_under_1m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -105,7 +97,6 @@ impl Vecs { _1m_sats_to_10m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1m_sats_under_10m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -115,7 +106,6 @@ impl Vecs { _10m_sats_to_1btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10m_sats_under_1btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -125,7 +115,6 @@ impl Vecs { _1btc_to_10btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1btc_under_10btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -135,7 +124,6 @@ impl Vecs { _10btc_to_100btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10btc_under_100btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -145,7 +133,6 @@ impl Vecs { _100btc_to_1k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_100btc_under_1k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -155,7 +142,6 @@ impl Vecs { _1k_btc_to_10k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_btc_under_10k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -165,7 +151,6 @@ impl Vecs { _10k_btc_to_100k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_btc_under_100k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -175,7 +160,6 @@ impl Vecs { _100k_btc_or_more: address_cohort::Vecs::forced_import( db, Some("addrs_above_100k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -187,7 +171,6 @@ impl Vecs { _10sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_10sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -197,7 +180,6 @@ impl Vecs { _100sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_100sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -207,7 +189,6 @@ impl Vecs { _1k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_1k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -217,7 +198,6 @@ impl Vecs { _10k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_10k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -227,7 +207,6 @@ impl Vecs { _100k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_100k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -237,7 +216,6 @@ impl Vecs { _1m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_1m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -247,7 +225,6 @@ impl Vecs { _10m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_10m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -257,7 +234,6 @@ impl Vecs { _1btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_1btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -267,7 +243,6 @@ impl Vecs { _10btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_10btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -277,7 +252,6 @@ impl Vecs { _100btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_100btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -287,7 +261,6 @@ impl Vecs { _1k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_1k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -297,7 +270,6 @@ impl Vecs { _10k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_10k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -307,7 +279,6 @@ impl Vecs { _100k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_100k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -319,7 +290,6 @@ impl Vecs { _1sat: address_cohort::Vecs::forced_import( db, Some("addrs_above_1sat"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -329,7 +299,6 @@ impl Vecs { _10sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -339,7 +308,6 @@ impl Vecs { _100sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -349,7 +317,6 @@ impl Vecs { _1k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -359,7 +326,6 @@ impl Vecs { _10k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -369,7 +335,6 @@ impl Vecs { _100k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -379,7 +344,6 @@ impl Vecs { _1m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -389,7 +353,6 @@ impl Vecs { _10m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -399,7 +362,6 @@ impl Vecs { _1btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -409,7 +371,6 @@ impl Vecs { _10btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -419,7 +380,6 @@ impl Vecs { _100btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_100btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -429,7 +389,6 @@ impl Vecs { _1k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -439,7 +398,6 @@ impl Vecs { _10k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, diff --git a/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs index 4e5276154..b9e753ea2 100644 --- a/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype/height_to_addresscount.rs @@ -3,16 +3,16 @@ use brk_grouper::ByAddressType; use brk_traversable::Traversable; use brk_types::{Height, StoredU64}; use derive_deref::{Deref, DerefMut}; -use vecdb::{EagerVec, GenericStoredVec}; +use vecdb::{PcoVec, EagerVec, GenericStoredVec}; use super::AddressTypeToAddressCount; #[derive(Debug, Clone, Deref, DerefMut, Traversable)] -pub struct AddressTypeToHeightToAddressCount(ByAddressType>); +pub struct AddressTypeToHeightToAddressCount(ByAddressType>>); -impl From>> for AddressTypeToHeightToAddressCount { +impl From>>> for AddressTypeToHeightToAddressCount { #[inline] - fn from(value: ByAddressType>) -> Self { + fn from(value: ByAddressType>>) -> Self { Self(value) } } diff --git a/crates/brk_computer/src/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs index a61d0a0d1..f89adf57a 100644 --- a/crates/brk_computer/src/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -4,8 +4,8 @@ use brk_types::{ Bitcoin, DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64, Version, }; use vecdb::{ - AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, GenericStoredVec, IterableCloneableVec, - IterableVec, TypedVecIterator, + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, Importable, + IterableCloneableVec, IterableVec, PcoVec, TypedVecIterator, }; use crate::{ @@ -21,41 +21,41 @@ use crate::{ #[derive(Clone, Traversable)] pub struct Vecs { // Cumulative - pub height_to_realized_cap: Option>, - pub height_to_supply: EagerVec, - pub height_to_utxo_count: EagerVec, + pub height_to_realized_cap: Option>>, + pub height_to_supply: EagerVec>, + pub height_to_utxo_count: EagerVec>, // Single - pub dateindex_to_supply_in_loss: Option>, - pub dateindex_to_supply_in_profit: Option>, - pub dateindex_to_unrealized_loss: Option>, - pub dateindex_to_unrealized_profit: Option>, - pub height_to_adjusted_value_created: Option>, - pub height_to_adjusted_value_destroyed: Option>, - pub height_to_max_price_paid: Option>, - pub height_to_min_price_paid: Option>, - pub height_to_realized_loss: Option>, - pub height_to_realized_profit: Option>, - pub height_to_supply_in_loss: Option>, - pub height_to_supply_in_profit: Option>, - pub height_to_unrealized_loss: Option>, - pub height_to_unrealized_profit: Option>, - pub height_to_value_created: Option>, - pub height_to_value_destroyed: Option>, - pub height_to_satblocks_destroyed: EagerVec, - pub height_to_satdays_destroyed: EagerVec, + pub dateindex_to_supply_in_loss: Option>>, + pub dateindex_to_supply_in_profit: Option>>, + pub dateindex_to_unrealized_loss: Option>>, + pub dateindex_to_unrealized_profit: Option>>, + pub height_to_adjusted_value_created: Option>>, + pub height_to_adjusted_value_destroyed: Option>>, + pub height_to_max_price_paid: Option>>, + pub height_to_min_price_paid: Option>>, + pub height_to_realized_loss: Option>>, + pub height_to_realized_profit: Option>>, + pub height_to_supply_in_loss: Option>>, + pub height_to_supply_in_profit: Option>>, + pub height_to_unrealized_loss: Option>>, + pub height_to_unrealized_profit: Option>>, + pub height_to_value_created: Option>>, + pub height_to_value_destroyed: Option>>, + pub height_to_satblocks_destroyed: EagerVec>, + pub height_to_satdays_destroyed: EagerVec>, pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight, pub indexes_to_coindays_destroyed: ComputedVecsFromHeight, - pub dateindex_to_sopr: Option>, - pub dateindex_to_sopr_7d_ema: Option>, - pub dateindex_to_sopr_30d_ema: Option>, - pub dateindex_to_adjusted_sopr: Option>, - pub dateindex_to_adjusted_sopr_7d_ema: Option>, - pub dateindex_to_adjusted_sopr_30d_ema: Option>, + pub dateindex_to_sopr: Option>>, + pub dateindex_to_sopr_7d_ema: Option>>, + pub dateindex_to_sopr_30d_ema: Option>>, + pub dateindex_to_adjusted_sopr: Option>>, + pub dateindex_to_adjusted_sopr_7d_ema: Option>>, + pub dateindex_to_adjusted_sopr_30d_ema: Option>>, pub indexes_to_realized_cap_30d_delta: Option>, - pub dateindex_to_sell_side_risk_ratio: Option>, - pub dateindex_to_sell_side_risk_ratio_7d_ema: Option>, - pub dateindex_to_sell_side_risk_ratio_30d_ema: Option>, + pub dateindex_to_sell_side_risk_ratio: Option>>, + pub dateindex_to_sell_side_risk_ratio_7d_ema: Option>>, + pub dateindex_to_sell_side_risk_ratio_30d_ema: Option>>, pub indexes_to_adjusted_value_created: Option>, pub indexes_to_adjusted_value_destroyed: Option>, pub indexes_to_neg_realized_loss: Option>, @@ -73,22 +73,23 @@ pub struct Vecs { pub indexes_to_value_destroyed: Option>, pub indexes_to_unrealized_profit: Option>, pub indexes_to_unrealized_loss: Option>, - pub height_to_total_unrealized_pnl: Option>, + pub height_to_total_unrealized_pnl: Option>>, pub indexes_to_total_unrealized_pnl: Option>, - pub height_to_total_realized_pnl: Option>, + pub height_to_total_realized_pnl: Option>>, pub indexes_to_total_realized_pnl: Option>, pub indexes_to_min_price_paid: Option>, pub indexes_to_max_price_paid: Option>, pub height_to_supply_half_value: ComputedHeightValueVecs, pub indexes_to_supply_half: ComputedValueVecsFromDateIndex, - pub height_to_neg_unrealized_loss: Option>, + pub height_to_neg_unrealized_loss: Option>>, pub indexes_to_neg_unrealized_loss: Option>, - pub height_to_net_unrealized_pnl: Option>, + pub height_to_net_unrealized_pnl: Option>>, pub indexes_to_net_unrealized_pnl: Option>, - pub height_to_unrealized_profit_rel_to_market_cap: Option>, - pub height_to_unrealized_loss_rel_to_market_cap: Option>, - pub height_to_neg_unrealized_loss_rel_to_market_cap: Option>, - pub height_to_net_unrealized_pnl_rel_to_market_cap: Option>, + pub height_to_unrealized_profit_rel_to_market_cap: Option>>, + pub height_to_unrealized_loss_rel_to_market_cap: Option>>, + pub height_to_neg_unrealized_loss_rel_to_market_cap: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_market_cap: Option>>, pub indexes_to_unrealized_profit_rel_to_market_cap: Option>, pub indexes_to_unrealized_loss_rel_to_market_cap: Option>, @@ -96,10 +97,14 @@ pub struct Vecs { Option>, pub indexes_to_net_unrealized_pnl_rel_to_market_cap: Option>, - pub height_to_unrealized_profit_rel_to_own_market_cap: Option>, - pub height_to_unrealized_loss_rel_to_own_market_cap: Option>, - pub height_to_neg_unrealized_loss_rel_to_own_market_cap: Option>, - pub height_to_net_unrealized_pnl_rel_to_own_market_cap: Option>, + pub height_to_unrealized_profit_rel_to_own_market_cap: + Option>>, + pub height_to_unrealized_loss_rel_to_own_market_cap: + Option>>, + pub height_to_neg_unrealized_loss_rel_to_own_market_cap: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_own_market_cap: + Option>>, pub indexes_to_unrealized_profit_rel_to_own_market_cap: Option>, pub indexes_to_unrealized_loss_rel_to_own_market_cap: @@ -109,13 +114,13 @@ pub struct Vecs { pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap: Option>, pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: - Option>, + Option>>, pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: - Option>, + Option>>, pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: - Option>, + Option>>, pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: - Option>, + Option>>, pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: Option>, pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: @@ -132,13 +137,15 @@ pub struct Vecs { pub height_to_supply_in_profit_value: Option, pub indexes_to_supply_in_loss: Option, pub indexes_to_supply_in_profit: Option, - pub height_to_supply_in_loss_rel_to_own_supply: Option>, - pub height_to_supply_in_profit_rel_to_own_supply: Option>, + pub height_to_supply_in_loss_rel_to_own_supply: Option>>, + pub height_to_supply_in_profit_rel_to_own_supply: Option>>, pub indexes_to_supply_in_loss_rel_to_own_supply: Option>, pub indexes_to_supply_in_profit_rel_to_own_supply: Option>, pub indexes_to_supply_rel_to_circulating_supply: Option>, - pub height_to_supply_in_loss_rel_to_circulating_supply: Option>, - pub height_to_supply_in_profit_rel_to_circulating_supply: Option>, + pub height_to_supply_in_loss_rel_to_circulating_supply: + Option>>, + pub height_to_supply_in_profit_rel_to_circulating_supply: + Option>>, pub indexes_to_supply_in_loss_rel_to_circulating_supply: Option>, pub indexes_to_supply_in_profit_rel_to_circulating_supply: @@ -149,7 +156,7 @@ pub struct Vecs { Option>, pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: Option>, - pub dateindex_to_realized_profit_to_loss_ratio: Option>, + pub dateindex_to_realized_profit_to_loss_ratio: Option>>, } impl Vecs { @@ -157,7 +164,6 @@ impl Vecs { pub fn forced_import( db: &Database, cohort_name: Option<&str>, - format: Format, parent_version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, @@ -174,54 +180,28 @@ impl Vecs { let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); let dateindex_to_supply_in_profit = compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("supply_in_profit"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("supply_in_profit"), version + Version::ZERO) + .unwrap() }); let dateindex_to_supply_in_loss = compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("supply_in_loss"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("supply_in_loss"), version + Version::ZERO).unwrap() }); let dateindex_to_unrealized_profit = compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("unrealized_profit"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("unrealized_profit"), version + Version::ZERO) + .unwrap() }); let dateindex_to_unrealized_loss = compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("unrealized_loss"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("unrealized_loss"), version + Version::ZERO) + .unwrap() }); Ok(Self { height_to_supply_in_profit: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("supply_in_profit"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("supply_in_profit"), version + Version::ZERO) + .unwrap() }), indexes_to_supply_in_profit: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( @@ -240,13 +220,8 @@ impl Vecs { }), dateindex_to_supply_in_profit, height_to_supply_in_loss: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("supply_in_loss"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("supply_in_loss"), version + Version::ZERO) + .unwrap() }), indexes_to_supply_in_loss: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( @@ -265,13 +240,8 @@ impl Vecs { }), dateindex_to_supply_in_loss, height_to_unrealized_profit: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("unrealized_profit"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("unrealized_profit"), version + Version::ZERO) + .unwrap() }), indexes_to_unrealized_profit: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( @@ -289,31 +259,16 @@ impl Vecs { }), dateindex_to_unrealized_profit, height_to_unrealized_loss: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("unrealized_loss"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("unrealized_loss"), version + Version::ZERO) + .unwrap() }), height_to_min_price_paid: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("min_price_paid"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("min_price_paid"), version + Version::ZERO) + .unwrap() }), height_to_max_price_paid: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("max_price_paid"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("max_price_paid"), version + Version::ZERO) + .unwrap() }), indexes_to_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( @@ -330,7 +285,7 @@ impl Vecs { .unwrap() }), height_to_total_unrealized_pnl: compute_dollars.then(|| { - EagerVec::forced_import_compressed( + EagerVec::forced_import( db, &suffix("total_unrealized_pnl"), version + Version::ZERO, @@ -349,12 +304,8 @@ impl Vecs { .unwrap() }), height_to_total_realized_pnl: compute_dollars.then(|| { - EagerVec::forced_import_compressed( - db, - &suffix("total_realized_pnl"), - version + Version::ZERO, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("total_realized_pnl"), version + Version::ZERO) + .unwrap() }), indexes_to_total_realized_pnl: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( @@ -369,13 +320,8 @@ impl Vecs { }), dateindex_to_unrealized_loss, height_to_realized_cap: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("realized_cap"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("realized_cap"), version + Version::ZERO) + .unwrap() }), indexes_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( @@ -414,14 +360,12 @@ impl Vecs { db, &suffix("supply"), version + Version::ZERO, - format, )?, height_to_supply_value: ComputedHeightValueVecs::forced_import( db, &suffix("supply"), Source::None, version + Version::ZERO, - format, compute_dollars, )?, indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( @@ -437,7 +381,6 @@ impl Vecs { db, &suffix("utxo_count"), version + Version::ZERO, - format, )?, indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( db, @@ -483,13 +426,8 @@ impl Vecs { }, ), height_to_realized_profit: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("realized_profit"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("realized_profit"), version + Version::ZERO) + .unwrap() }), indexes_to_realized_profit: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( @@ -503,13 +441,8 @@ impl Vecs { .unwrap() }), height_to_realized_loss: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("realized_loss"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("realized_loss"), version + Version::ZERO) + .unwrap() }), indexes_to_realized_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( @@ -534,13 +467,8 @@ impl Vecs { .unwrap() }), height_to_value_created: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("value_created"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("value_created"), version + Version::ZERO) + .unwrap() }), indexes_to_value_created: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( @@ -569,7 +497,6 @@ impl Vecs { db, &suffix("adjusted_value_created"), version + Version::ZERO, - format, ) .unwrap() }), @@ -585,13 +512,8 @@ impl Vecs { .unwrap() }), height_to_value_destroyed: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("value_destroyed"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("value_destroyed"), version + Version::ZERO) + .unwrap() }), indexes_to_value_destroyed: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( @@ -609,7 +531,6 @@ impl Vecs { db, &suffix("adjusted_value_destroyed"), version + Version::ZERO, - format, ) .unwrap() }), @@ -647,20 +568,14 @@ impl Vecs { .unwrap() }), dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("sell_side_risk_ratio"), - version + Version::ONE, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("sell_side_risk_ratio"), version + Version::ONE) + .unwrap() }), dateindex_to_sell_side_risk_ratio_7d_ema: compute_dollars.then(|| { EagerVec::forced_import( db, &suffix("sell_side_risk_ratio_7d_ema"), version + Version::ONE, - format, ) .unwrap() }), @@ -669,46 +584,32 @@ impl Vecs { db, &suffix("sell_side_risk_ratio_30d_ema"), version + Version::ONE, - format, ) .unwrap() }), dateindex_to_sopr: compute_dollars.then(|| { - EagerVec::forced_import(db, &suffix("sopr"), version + Version::ONE, format) - .unwrap() + EagerVec::forced_import(db, &suffix("sopr"), version + Version::ONE).unwrap() }), dateindex_to_sopr_7d_ema: compute_dollars.then(|| { - EagerVec::forced_import(db, &suffix("sopr_7d_ema"), version + Version::ONE, format) - .unwrap() + EagerVec::forced_import(db, &suffix("sopr_7d_ema"), version + Version::ONE).unwrap() }), dateindex_to_sopr_30d_ema: compute_dollars.then(|| { - EagerVec::forced_import(db, &suffix("sopr_30d_ema"), version + Version::ONE, format) + EagerVec::forced_import(db, &suffix("sopr_30d_ema"), version + Version::ONE) .unwrap() }), dateindex_to_adjusted_sopr: (compute_dollars && compute_adjusted).then(|| { - EagerVec::forced_import( - db, - &suffix("adjusted_sopr"), - version + Version::ONE, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("adjusted_sopr"), version + Version::ONE) + .unwrap() }), dateindex_to_adjusted_sopr_7d_ema: (compute_dollars && compute_adjusted).then(|| { - EagerVec::forced_import( - db, - &suffix("adjusted_sopr_7d_ema"), - version + Version::ONE, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("adjusted_sopr_7d_ema"), version + Version::ONE) + .unwrap() }), dateindex_to_adjusted_sopr_30d_ema: (compute_dollars && compute_adjusted).then(|| { EagerVec::forced_import( db, &suffix("adjusted_sopr_30d_ema"), version + Version::ONE, - format, ) .unwrap() }), @@ -717,7 +618,6 @@ impl Vecs { &suffix("supply_half"), Source::Compute, version + Version::ZERO, - format, compute_dollars, )?, indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import( @@ -730,13 +630,8 @@ impl Vecs { indexes, )?, height_to_neg_unrealized_loss: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("neg_unrealized_loss"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("neg_unrealized_loss"), version + Version::ZERO) + .unwrap() }), indexes_to_neg_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( @@ -750,13 +645,8 @@ impl Vecs { .unwrap() }), height_to_net_unrealized_pnl: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &suffix("net_unrealized_pnl"), - version + Version::ZERO, - format, - ) - .unwrap() + EagerVec::forced_import(db, &suffix("net_unrealized_pnl"), version + Version::ZERO) + .unwrap() }), indexes_to_net_unrealized_pnl: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( @@ -774,7 +664,6 @@ impl Vecs { db, &suffix("unrealized_profit_rel_to_market_cap"), version + Version::ZERO, - format, ) .unwrap() }), @@ -783,7 +672,6 @@ impl Vecs { db, &suffix("unrealized_loss_rel_to_market_cap"), version + Version::ZERO, - format, ) .unwrap() }), @@ -792,7 +680,6 @@ impl Vecs { db, &suffix("neg_unrealized_loss_rel_to_market_cap"), version + Version::ZERO, - format, ) .unwrap() }), @@ -801,7 +688,6 @@ impl Vecs { db, &suffix("net_unrealized_pnl_rel_to_market_cap"), version + Version::ONE, - format, ) .unwrap() }), @@ -857,7 +743,6 @@ impl Vecs { db, &suffix("unrealized_profit_rel_to_own_market_cap"), version + Version::ONE, - format, ) .unwrap() }), @@ -869,7 +754,6 @@ impl Vecs { db, &suffix("unrealized_loss_rel_to_own_market_cap"), version + Version::ONE, - format, ) .unwrap() }), @@ -881,7 +765,6 @@ impl Vecs { db, &suffix("neg_unrealized_loss_rel_to_own_market_cap"), version + Version::ONE, - format, ) .unwrap() }), @@ -893,7 +776,6 @@ impl Vecs { db, &suffix("net_unrealized_pnl_rel_to_own_market_cap"), version + Version::TWO, - format, ) .unwrap() }), @@ -960,7 +842,6 @@ impl Vecs { db, &suffix("unrealized_profit_rel_to_own_total_unrealized_pnl"), version + Version::ZERO, - format, ) .unwrap() }), @@ -971,7 +852,6 @@ impl Vecs { db, &suffix("unrealized_loss_rel_to_own_total_unrealized_pnl"), version + Version::ZERO, - format, ) .unwrap() }), @@ -982,7 +862,6 @@ impl Vecs { db, &suffix("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"), version + Version::ZERO, - format, ) .unwrap() }), @@ -993,7 +872,6 @@ impl Vecs { db, &suffix("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"), version + Version::ONE, - format, ) .unwrap() }), @@ -1088,7 +966,6 @@ impl Vecs { &suffix("supply_in_loss"), Source::None, version + Version::ZERO, - format, compute_dollars, ) .unwrap() @@ -1099,7 +976,6 @@ impl Vecs { &suffix("supply_in_profit"), Source::None, version + Version::ZERO, - format, compute_dollars, ) .unwrap() @@ -1109,7 +985,6 @@ impl Vecs { db, &suffix("supply_in_loss_rel_to_own_supply"), version + Version::ONE, - format, ) .unwrap() }), @@ -1118,7 +993,6 @@ impl Vecs { db, &suffix("supply_in_profit_rel_to_own_supply"), version + Version::ONE, - format, ) .unwrap() }), @@ -1162,7 +1036,6 @@ impl Vecs { db, &suffix("supply_in_loss_rel_to_circulating_supply"), version + Version::ONE, - format, ) .unwrap() }), @@ -1173,7 +1046,6 @@ impl Vecs { db, &suffix("supply_in_profit_rel_to_circulating_supply"), version + Version::ONE, - format, ) .unwrap() }), @@ -1207,13 +1079,11 @@ impl Vecs { db, &suffix("satblocks_destroyed"), version + Version::ZERO, - format, )?, height_to_satdays_destroyed: EagerVec::forced_import( db, &suffix("satdays_destroyed"), version + Version::ZERO, - format, )?, indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import( db, @@ -1267,7 +1137,7 @@ impl Vecs { .unwrap() }), dateindex_to_realized_profit_to_loss_ratio: (compute_dollars && extended).then(|| { - EagerVec::forced_import_compressed( + EagerVec::forced_import( db, &suffix("realized_profit_to_loss_ratio"), version + Version::ONE, diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index f8d32c591..7c4635b12 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -16,9 +16,9 @@ use rayon::prelude::*; use rustc_hash::FxHashMap; use smallvec::SmallVec; use vecdb::{ - AnyStoredVec, AnyVec, BoxedVecIterator, CollectableVec, Database, EagerVec, Exit, Format, - GenericStoredVec, ImportOptions, IterableCloneableVec, IterableVec, LazyVecFrom1, PAGE_SIZE, - RawVec, Reader, Stamp, TypedVecIterator, VecIndex, + AnyStoredVec, AnyVec, BoxedVecIterator, BytesVec, CollectableVec, Database, EagerVec, Exit, + GenericStoredVec, ImportOptions, Importable, IterableCloneableVec, IterableVec, LazyVecFrom1, + PAGE_SIZE, PcoVec, Reader, Stamp, TypedVecIterator, VecIndex, }; use crate::{ @@ -56,14 +56,14 @@ pub struct Vecs { // --- // States // --- - pub chain_state: RawVec, + pub chain_state: BytesVec, pub any_address_indexes: AnyAddressIndexes, pub addresses_data: AddressesData, pub utxo_cohorts: utxo_cohorts::Vecs, pub address_cohorts: address_cohorts::Vecs, - pub height_to_unspendable_supply: EagerVec, - pub height_to_opreturn_supply: EagerVec, + pub height_to_unspendable_supply: EagerVec>, + pub height_to_opreturn_supply: EagerVec>, pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount, pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount, @@ -90,7 +90,6 @@ impl Vecs { pub fn forced_import( parent: &Path, version: Version, - format: Format, indexes: &indexes::Vecs, price: Option<&price::Vecs>, ) -> Result { @@ -104,13 +103,13 @@ impl Vecs { let compute_dollars = price.is_some(); let utxo_cohorts = - utxo_cohorts::Vecs::forced_import(&db, version, format, indexes, price, &states_path)?; + utxo_cohorts::Vecs::forced_import(&db, version, indexes, price, &states_path)?; - let loadedaddressindex_to_loadedaddressdata = RawVec::forced_import_with( + let loadedaddressindex_to_loadedaddressdata = BytesVec::forced_import_with( ImportOptions::new(&db, "loadedaddressdata", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?; - let emptyaddressindex_to_emptyaddressdata = RawVec::forced_import_with( + let emptyaddressindex_to_emptyaddressdata = BytesVec::forced_import_with( ImportOptions::new(&db, "emptyaddressdata", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?; @@ -128,12 +127,12 @@ impl Vecs { ); let this = Self { - chain_state: RawVec::forced_import_with( + chain_state: BytesVec::forced_import_with( ImportOptions::new(&db, "chain", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - height_to_unspendable_supply: EagerVec::forced_import_compressed( + height_to_unspendable_supply: EagerVec::forced_import( &db, "unspendable_supply", version + VERSION + Version::ZERO, @@ -147,7 +146,7 @@ impl Vecs { compute_dollars, indexes, )?, - height_to_opreturn_supply: EagerVec::forced_import_compressed( + height_to_opreturn_supply: EagerVec::forced_import( &db, "opreturn_supply", version + VERSION + Version::ZERO, @@ -205,42 +204,42 @@ impl Vecs { }), addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::from( ByAddressType { - p2pk65: EagerVec::forced_import_compressed( + p2pk65: EagerVec::forced_import( &db, "p2pk65_addr_count", version + VERSION + Version::ZERO, )?, - p2pk33: EagerVec::forced_import_compressed( + p2pk33: EagerVec::forced_import( &db, "p2pk33_addr_count", version + VERSION + Version::ZERO, )?, - p2pkh: EagerVec::forced_import_compressed( + p2pkh: EagerVec::forced_import( &db, "p2pkh_addr_count", version + VERSION + Version::ZERO, )?, - p2sh: EagerVec::forced_import_compressed( + p2sh: EagerVec::forced_import( &db, "p2sh_addr_count", version + VERSION + Version::ZERO, )?, - p2wpkh: EagerVec::forced_import_compressed( + p2wpkh: EagerVec::forced_import( &db, "p2wpkh_addr_count", version + VERSION + Version::ZERO, )?, - p2wsh: EagerVec::forced_import_compressed( + p2wsh: EagerVec::forced_import( &db, "p2wsh_addr_count", version + VERSION + Version::ZERO, )?, - p2tr: EagerVec::forced_import_compressed( + p2tr: EagerVec::forced_import( &db, "p2tr_addr_count", version + VERSION + Version::ZERO, )?, - p2a: EagerVec::forced_import_compressed( + p2a: EagerVec::forced_import( &db, "p2a_addr_count", version + VERSION + Version::ZERO, @@ -249,42 +248,42 @@ impl Vecs { ), addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount::from( ByAddressType { - p2pk65: EagerVec::forced_import_compressed( + p2pk65: EagerVec::forced_import( &db, "p2pk65_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2pk33: EagerVec::forced_import_compressed( + p2pk33: EagerVec::forced_import( &db, "p2pk33_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2pkh: EagerVec::forced_import_compressed( + p2pkh: EagerVec::forced_import( &db, "p2pkh_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2sh: EagerVec::forced_import_compressed( + p2sh: EagerVec::forced_import( &db, "p2sh_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2wpkh: EagerVec::forced_import_compressed( + p2wpkh: EagerVec::forced_import( &db, "p2wpkh_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2wsh: EagerVec::forced_import_compressed( + p2wsh: EagerVec::forced_import( &db, "p2wsh_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2tr: EagerVec::forced_import_compressed( + p2tr: EagerVec::forced_import( &db, "p2tr_empty_addr_count", version + VERSION + Version::ZERO, )?, - p2a: EagerVec::forced_import_compressed( + p2a: EagerVec::forced_import( &db, "p2a_empty_addr_count", version + VERSION + Version::ZERO, @@ -431,42 +430,41 @@ impl Vecs { address_cohorts: address_cohorts::Vecs::forced_import( &db, version, - format, indexes, price, &states_path, )?, any_address_indexes: AnyAddressIndexes { - p2a: RawVec::forced_import_with( + p2a: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2pk33: RawVec::forced_import_with( + p2pk33: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2pk65: RawVec::forced_import_with( + p2pk65: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2pkh: RawVec::forced_import_with( + p2pkh: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2sh: RawVec::forced_import_with( + p2sh: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2tr: RawVec::forced_import_with( + p2tr: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2wpkh: RawVec::forced_import_with( + p2wpkh: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - p2wsh: RawVec::forced_import_with( + p2wsh: BytesVec::forced_import_with( ImportOptions::new(&db, "anyaddressindex", version + VERSION + Version::ZERO) .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, @@ -482,7 +480,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; @@ -1518,9 +1516,7 @@ impl Vecs { let loadedaddressdata = addresses_data .loaded - .get_or_read(loadedaddressindex, reader) - .unwrap() - .unwrap(); + .get_pushed_or_read_unwrap(loadedaddressindex, reader); WithAddressDataSource::FromLoadedAddressDataVec(( loadedaddressindex, @@ -1532,9 +1528,7 @@ impl Vecs { let emptyaddressdata = addresses_data .empty - .get_or_read(emtpyaddressindex, reader) - .unwrap() - .unwrap(); + .get_pushed_or_read_unwrap(emtpyaddressindex, reader); WithAddressDataSource::FromEmptyAddressDataVec(( emtpyaddressindex, @@ -1923,14 +1917,14 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { #[derive(Clone, Traversable)] pub struct AnyAddressIndexes { - pub p2pk33: RawVec, - pub p2pk65: RawVec, - pub p2pkh: RawVec, - pub p2sh: RawVec, - pub p2tr: RawVec, - pub p2wpkh: RawVec, - pub p2wsh: RawVec, - pub p2a: RawVec, + pub p2pk33: BytesVec, + pub p2pk65: BytesVec, + pub p2pkh: BytesVec, + pub p2sh: BytesVec, + pub p2tr: BytesVec, + pub p2wpkh: BytesVec, + pub p2wsh: BytesVec, + pub p2a: BytesVec, } impl AnyAddressIndexes { @@ -1975,20 +1969,35 @@ impl AnyAddressIndexes { &self, address_type: OutputType, typeindex: TypeIndex, - reader: &Reader<'static>, + reader: &Reader, ) -> AnyAddressIndex { - let result = match address_type { - OutputType::P2PK33 => self.p2pk33.get_or_read(typeindex.into(), reader), - OutputType::P2PK65 => self.p2pk65.get_or_read(typeindex.into(), reader), - OutputType::P2PKH => self.p2pkh.get_or_read(typeindex.into(), reader), - OutputType::P2SH => self.p2sh.get_or_read(typeindex.into(), reader), - OutputType::P2TR => self.p2tr.get_or_read(typeindex.into(), reader), - OutputType::P2WPKH => self.p2wpkh.get_or_read(typeindex.into(), reader), - OutputType::P2WSH => self.p2wsh.get_or_read(typeindex.into(), reader), - OutputType::P2A => self.p2a.get_or_read(typeindex.into(), reader), + match address_type { + OutputType::P2PK33 => self + .p2pk33 + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2PK65 => self + .p2pk65 + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2PKH => self + .p2pkh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2SH => self + .p2sh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2TR => self + .p2tr + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2WPKH => self + .p2wpkh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2WSH => self + .p2wsh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2A => self + .p2a + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), _ => unreachable!(), - }; - result.unwrap().unwrap() + } } fn update_or_push( @@ -2040,8 +2049,8 @@ impl AnyAddressIndexes { #[derive(Clone, Traversable)] pub struct AddressesData { - pub loaded: RawVec, - pub empty: RawVec, + pub loaded: BytesVec, + pub empty: BytesVec, } impl AddressesData { @@ -2073,16 +2082,16 @@ impl AddressesData { } } -struct IndexerReaders<'a> { - txinindex_to_outpoint: Reader<'a>, - txindex_to_first_txoutindex: Reader<'a>, - txoutindex_to_value: Reader<'a>, - txoutindex_to_outputtype: Reader<'a>, - txoutindex_to_typeindex: Reader<'a>, +struct IndexerReaders { + txinindex_to_outpoint: Reader, + txindex_to_first_txoutindex: Reader, + txoutindex_to_value: Reader, + txoutindex_to_outputtype: Reader, + txoutindex_to_typeindex: Reader, } -impl<'a> IndexerReaders<'a> { - fn new(indexer: &'a Indexer) -> Self { +impl IndexerReaders { + fn new(indexer: &Indexer) -> Self { Self { txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(), txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(), @@ -2094,31 +2103,31 @@ impl<'a> IndexerReaders<'a> { } struct VecsReaders { - addresstypeindex_to_anyaddressindex: ByAddressType>, - anyaddressindex_to_anyaddressdata: ByAnyAddress>, + addresstypeindex_to_anyaddressindex: ByAddressType, + anyaddressindex_to_anyaddressdata: ByAnyAddress, } impl VecsReaders { fn new(vecs: &Vecs) -> Self { Self { addresstypeindex_to_anyaddressindex: ByAddressType { - p2pk33: vecs.any_address_indexes.p2pk33.create_static_reader(), - p2pk65: vecs.any_address_indexes.p2pk65.create_static_reader(), - p2pkh: vecs.any_address_indexes.p2pkh.create_static_reader(), - p2sh: vecs.any_address_indexes.p2sh.create_static_reader(), - p2tr: vecs.any_address_indexes.p2tr.create_static_reader(), - p2wpkh: vecs.any_address_indexes.p2wpkh.create_static_reader(), - p2wsh: vecs.any_address_indexes.p2wsh.create_static_reader(), - p2a: vecs.any_address_indexes.p2a.create_static_reader(), + p2pk33: vecs.any_address_indexes.p2pk33.create_reader(), + p2pk65: vecs.any_address_indexes.p2pk65.create_reader(), + p2pkh: vecs.any_address_indexes.p2pkh.create_reader(), + p2sh: vecs.any_address_indexes.p2sh.create_reader(), + p2tr: vecs.any_address_indexes.p2tr.create_reader(), + p2wpkh: vecs.any_address_indexes.p2wpkh.create_reader(), + p2wsh: vecs.any_address_indexes.p2wsh.create_reader(), + p2a: vecs.any_address_indexes.p2a.create_reader(), }, anyaddressindex_to_anyaddressdata: ByAnyAddress { - loaded: vecs.addresses_data.loaded.create_static_reader(), - empty: vecs.addresses_data.empty.create_static_reader(), + loaded: vecs.addresses_data.loaded.create_reader(), + empty: vecs.addresses_data.empty.create_reader(), }, } } - fn get_anyaddressindex_reader(&self, address_type: OutputType) -> &Reader<'static> { + fn get_anyaddressindex_reader(&self, address_type: OutputType) -> &Reader { self.addresstypeindex_to_anyaddressindex .get_unwrap(address_type) } diff --git a/crates/brk_computer/src/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs index a78f3302e..cb0938c3c 100644 --- a/crates/brk_computer/src/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -3,7 +3,7 @@ use std::{ops::Deref, path::Path}; use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; -use vecdb::{Database, Exit, Format, IterableVec}; +use vecdb::{Database, Exit, IterableVec}; use crate::{ Indexes, UTXOCohortState, indexes, price, @@ -29,7 +29,6 @@ impl Vecs { pub fn forced_import( db: &Database, cohort_name: Option<&str>, - format: Format, version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, @@ -54,7 +53,6 @@ impl Vecs { inner: common::Vecs::forced_import( db, cohort_name, - format, version, indexes, price, diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index 9257b4431..09210e054 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -11,7 +11,7 @@ use brk_types::{ }; use derive_deref::{Deref, DerefMut}; use rustc_hash::FxHashMap; -use vecdb::{Database, Exit, Format, IterableVec, VecIndex}; +use vecdb::{Database, Exit, IterableVec, VecIndex}; use crate::{ Indexes, indexes, price, @@ -30,7 +30,6 @@ impl Vecs { pub fn forced_import( db: &Database, version: Version, - format: Format, indexes: &indexes::Vecs, price: Option<&price::Vecs>, states_path: &Path, @@ -40,7 +39,6 @@ impl Vecs { all: utxo_cohort::Vecs::forced_import( db, None, - format, version + VERSION + Version::ONE, indexes, price, @@ -53,7 +51,6 @@ impl Vecs { short: utxo_cohort::Vecs::forced_import( db, Some("sth"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -65,7 +62,6 @@ impl Vecs { long: utxo_cohort::Vecs::forced_import( db, Some("lth"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -79,7 +75,6 @@ impl Vecs { _0: utxo_cohort::Vecs::forced_import( db, Some("epoch_0"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -91,7 +86,6 @@ impl Vecs { _1: utxo_cohort::Vecs::forced_import( db, Some("epoch_1"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -103,7 +97,6 @@ impl Vecs { _2: utxo_cohort::Vecs::forced_import( db, Some("epoch_2"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -115,7 +108,6 @@ impl Vecs { _3: utxo_cohort::Vecs::forced_import( db, Some("epoch_3"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -127,7 +119,6 @@ impl Vecs { _4: utxo_cohort::Vecs::forced_import( db, Some("epoch_4"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -141,7 +132,6 @@ impl Vecs { p2pk65: utxo_cohort::Vecs::forced_import( db, Some("p2pk65"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -153,7 +143,6 @@ impl Vecs { p2pk33: utxo_cohort::Vecs::forced_import( db, Some("p2pk33"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -165,7 +154,6 @@ impl Vecs { p2pkh: utxo_cohort::Vecs::forced_import( db, Some("p2pkh"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -177,7 +165,6 @@ impl Vecs { p2sh: utxo_cohort::Vecs::forced_import( db, Some("p2sh"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -189,7 +176,6 @@ impl Vecs { p2wpkh: utxo_cohort::Vecs::forced_import( db, Some("p2wpkh"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -201,7 +187,6 @@ impl Vecs { p2wsh: utxo_cohort::Vecs::forced_import( db, Some("p2wsh"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -213,7 +198,6 @@ impl Vecs { p2tr: utxo_cohort::Vecs::forced_import( db, Some("p2tr"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -225,7 +209,6 @@ impl Vecs { p2a: utxo_cohort::Vecs::forced_import( db, Some("p2a"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -237,7 +220,6 @@ impl Vecs { p2ms: utxo_cohort::Vecs::forced_import( db, Some("p2ms_outputs"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -249,7 +231,6 @@ impl Vecs { empty: utxo_cohort::Vecs::forced_import( db, Some("empty_outputs"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -261,7 +242,6 @@ impl Vecs { unknown: utxo_cohort::Vecs::forced_import( db, Some("unknown_outputs"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -275,7 +255,6 @@ impl Vecs { _1w: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1w_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -287,7 +266,6 @@ impl Vecs { _1m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -299,7 +277,6 @@ impl Vecs { _2m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_2m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -311,7 +288,6 @@ impl Vecs { _3m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_3m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -323,7 +299,6 @@ impl Vecs { _4m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_4m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -335,7 +310,6 @@ impl Vecs { _5m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_5m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -347,7 +321,6 @@ impl Vecs { _6m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_6m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -359,7 +332,6 @@ impl Vecs { _1y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -371,7 +343,6 @@ impl Vecs { _2y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_2y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -383,7 +354,6 @@ impl Vecs { _3y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_3y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -395,7 +365,6 @@ impl Vecs { _4y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_4y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -407,7 +376,6 @@ impl Vecs { _5y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_5y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -419,7 +387,6 @@ impl Vecs { _6y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_6y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -431,7 +398,6 @@ impl Vecs { _7y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_7y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -443,7 +409,6 @@ impl Vecs { _8y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_8y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -455,7 +420,6 @@ impl Vecs { _10y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_10y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -467,7 +431,6 @@ impl Vecs { _12y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_12y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -479,7 +442,6 @@ impl Vecs { _15y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_15y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -493,7 +455,6 @@ impl Vecs { _1d: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1d_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -505,7 +466,6 @@ impl Vecs { _1w: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1w_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -517,7 +477,6 @@ impl Vecs { _1m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -529,7 +488,6 @@ impl Vecs { _2m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -541,7 +499,6 @@ impl Vecs { _3m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -553,7 +510,6 @@ impl Vecs { _4m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -565,7 +521,6 @@ impl Vecs { _5m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -577,7 +532,6 @@ impl Vecs { _6m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -589,7 +543,6 @@ impl Vecs { _1y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -601,7 +554,6 @@ impl Vecs { _2y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -613,7 +565,6 @@ impl Vecs { _3y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -625,7 +576,6 @@ impl Vecs { _4y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -637,7 +587,6 @@ impl Vecs { _5y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -649,7 +598,6 @@ impl Vecs { _6y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -661,7 +609,6 @@ impl Vecs { _7y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_7y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -673,7 +620,6 @@ impl Vecs { _8y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_8y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -685,7 +631,6 @@ impl Vecs { _10y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_10y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -697,7 +642,6 @@ impl Vecs { _12y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_12y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -711,7 +655,6 @@ impl Vecs { up_to_1d: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1d_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -723,7 +666,6 @@ impl Vecs { _1d_to_1w: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1d_up_to_1w_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -735,7 +677,6 @@ impl Vecs { _1w_to_1m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1w_up_to_1m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -747,7 +688,6 @@ impl Vecs { _1m_to_2m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1m_up_to_2m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -759,7 +699,6 @@ impl Vecs { _2m_to_3m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2m_up_to_3m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -771,7 +710,6 @@ impl Vecs { _3m_to_4m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3m_up_to_4m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -783,7 +721,6 @@ impl Vecs { _4m_to_5m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4m_up_to_5m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -795,7 +732,6 @@ impl Vecs { _5m_to_6m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5m_up_to_6m_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -807,7 +743,6 @@ impl Vecs { _6m_to_1y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6m_up_to_1y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -819,7 +754,6 @@ impl Vecs { _1y_to_2y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1y_up_to_2y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -831,7 +765,6 @@ impl Vecs { _2y_to_3y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2y_up_to_3y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -843,7 +776,6 @@ impl Vecs { _3y_to_4y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3y_up_to_4y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -855,7 +787,6 @@ impl Vecs { _4y_to_5y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4y_up_to_5y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -867,7 +798,6 @@ impl Vecs { _5y_to_6y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5y_up_to_6y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -879,7 +809,6 @@ impl Vecs { _6y_to_7y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6y_up_to_7y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -891,7 +820,6 @@ impl Vecs { _7y_to_8y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_7y_up_to_8y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -903,7 +831,6 @@ impl Vecs { _8y_to_10y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_8y_up_to_10y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -915,7 +842,6 @@ impl Vecs { _10y_to_12y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_10y_up_to_12y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -927,7 +853,6 @@ impl Vecs { _12y_to_15y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_12y_up_to_15y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -939,7 +864,6 @@ impl Vecs { from_15y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_15y_old"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -953,7 +877,6 @@ impl Vecs { _0sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_with_0sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -965,7 +888,6 @@ impl Vecs { _1sat_to_10sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1sat_under_10sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -977,7 +899,6 @@ impl Vecs { _10sats_to_100sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10sats_under_100sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -989,7 +910,6 @@ impl Vecs { _100sats_to_1k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100sats_under_1k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1001,7 +921,6 @@ impl Vecs { _1k_sats_to_10k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_sats_under_10k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1013,7 +932,6 @@ impl Vecs { _10k_sats_to_100k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_sats_under_100k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1025,7 +943,6 @@ impl Vecs { _100k_sats_to_1m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100k_sats_under_1m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1037,7 +954,6 @@ impl Vecs { _1m_sats_to_10m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1m_sats_under_10m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1049,7 +965,6 @@ impl Vecs { _10m_sats_to_1btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10m_sats_under_1btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1061,7 +976,6 @@ impl Vecs { _1btc_to_10btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1btc_under_10btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1073,7 +987,6 @@ impl Vecs { _10btc_to_100btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10btc_under_100btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1085,7 +998,6 @@ impl Vecs { _100btc_to_1k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100btc_under_1k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1097,7 +1009,6 @@ impl Vecs { _1k_btc_to_10k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_btc_under_10k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1109,7 +1020,6 @@ impl Vecs { _10k_btc_to_100k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_btc_under_100k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1121,7 +1031,6 @@ impl Vecs { _100k_btc_or_more: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1135,7 +1044,6 @@ impl Vecs { _10sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1147,7 +1055,6 @@ impl Vecs { _100sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1159,7 +1066,6 @@ impl Vecs { _1k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1171,7 +1077,6 @@ impl Vecs { _10k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1183,7 +1088,6 @@ impl Vecs { _100k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1195,7 +1099,6 @@ impl Vecs { _1m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1207,7 +1110,6 @@ impl Vecs { _10m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1219,7 +1121,6 @@ impl Vecs { _1btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1231,7 +1132,6 @@ impl Vecs { _10btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1243,7 +1143,6 @@ impl Vecs { _100btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1255,7 +1154,6 @@ impl Vecs { _1k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1267,7 +1165,6 @@ impl Vecs { _10k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1279,7 +1176,6 @@ impl Vecs { _100k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1293,7 +1189,6 @@ impl Vecs { _1sat: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1sat"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1305,7 +1200,6 @@ impl Vecs { _10sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1317,7 +1211,6 @@ impl Vecs { _100sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1329,7 +1222,6 @@ impl Vecs { _1k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1341,7 +1233,6 @@ impl Vecs { _10k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1353,7 +1244,6 @@ impl Vecs { _100k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100k_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1365,7 +1255,6 @@ impl Vecs { _1m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1377,7 +1266,6 @@ impl Vecs { _10m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10m_sats"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1389,7 +1277,6 @@ impl Vecs { _1btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1401,7 +1288,6 @@ impl Vecs { _10btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1413,7 +1299,6 @@ impl Vecs { _100btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1425,7 +1310,6 @@ impl Vecs { _1k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, @@ -1437,7 +1321,6 @@ impl Vecs { _10k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_btc"), - format, version + VERSION + Version::ZERO, indexes, price, diff --git a/crates/brk_computer/src/states/supply.rs b/crates/brk_computer/src/states/supply.rs index d9aa61cf1..134eda781 100644 --- a/crates/brk_computer/src/states/supply.rs +++ b/crates/brk_computer/src/states/supply.rs @@ -2,7 +2,7 @@ use std::ops::{Add, AddAssign, SubAssign}; use brk_types::{CheckedSub, LoadedAddressData, Sats}; use serde::Serialize; -use vecdb::Formattable; +use vecdb::{Bytes, Formattable}; #[derive(Debug, Default, Clone, Serialize)] pub struct SupplyState { @@ -62,3 +62,20 @@ impl Formattable for SupplyState { true } } + +impl Bytes for SupplyState { + fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&self.utxo_count.to_bytes()); + bytes.extend_from_slice(&self.value.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let mut offset = 0; + let utxo_count = u64::from_bytes(&bytes[offset..])?; + offset += utxo_count.to_bytes().len(); + let value = Sats::from_bytes(&bytes[offset..])?; + Ok(Self { utxo_count, value }) + } +} diff --git a/crates/brk_computer/src/traits.rs b/crates/brk_computer/src/traits.rs index 5f2c09c86..14ab37430 100644 --- a/crates/brk_computer/src/traits.rs +++ b/crates/brk_computer/src/traits.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_types::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32}; use vecdb::{ - AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, IterableVec, VecIndex, Version, + AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, IterableVec, PcoVec, VecIndex, Version, }; const DCA_AMOUNT: Dollars = Dollars::mint(100.0); @@ -24,7 +24,7 @@ pub trait ComputeDCAStackViaLen { ) -> Result<()>; } -impl ComputeDCAStackViaLen for EagerVec { +impl ComputeDCAStackViaLen for EagerVec> { fn compute_dca_stack_via_len( &mut self, max_from: DateIndex, @@ -142,7 +142,7 @@ pub trait ComputeDCAAveragePriceViaLen { ) -> Result<()>; } -impl ComputeDCAAveragePriceViaLen for EagerVec { +impl ComputeDCAAveragePriceViaLen for EagerVec> { fn compute_dca_avg_price_via_len( &mut self, max_from: DateIndex, @@ -223,7 +223,7 @@ pub trait ComputeFromSats { ) -> Result<()>; } -impl ComputeFromSats for EagerVec +impl ComputeFromSats for EagerVec> where I: VecIndex, { @@ -253,7 +253,7 @@ pub trait ComputeFromBitcoin { ) -> Result<()>; } -impl ComputeFromBitcoin for EagerVec +impl ComputeFromBitcoin for EagerVec> where I: VecIndex, { @@ -285,7 +285,7 @@ pub trait ComputeDrawdown { ) -> Result<()>; } -impl ComputeDrawdown for EagerVec +impl ComputeDrawdown for EagerVec> where I: VecIndex, { diff --git a/crates/brk_grouper/src/address.rs b/crates/brk_grouper/src/address.rs index 1a908d5b3..125133d60 100644 --- a/crates/brk_grouper/src/address.rs +++ b/crates/brk_grouper/src/address.rs @@ -1,6 +1,6 @@ use brk_traversable::Traversable; use rayon::prelude::*; -use vecdb::AnyWritableVec; +use vecdb::AnyExportableVec; use crate::Filtered; @@ -78,14 +78,14 @@ where ) } - fn iter_any_writable(&self) -> impl Iterator { + fn iter_any_exportable(&self) -> impl Iterator { [ - Box::new(self.ge_amount.iter_any_writable()) - as Box>, - Box::new(self.amount_range.iter_any_writable()) - as Box>, - Box::new(self.lt_amount.iter_any_writable()) - as Box>, + Box::new(self.ge_amount.iter_any_exportable()) + as Box>, + Box::new(self.amount_range.iter_any_exportable()) + as Box>, + Box::new(self.lt_amount.iter_any_exportable()) + as Box>, ] .into_iter() .flatten() diff --git a/crates/brk_grouper/src/by_address_type.rs b/crates/brk_grouper/src/by_address_type.rs index 1b6a09e95..5bcb60454 100644 --- a/crates/brk_grouper/src/by_address_type.rs +++ b/crates/brk_grouper/src/by_address_type.rs @@ -4,7 +4,7 @@ use brk_error::Result; use brk_traversable::{Traversable, TreeNode}; use brk_types::OutputType; use rayon::prelude::*; -use vecdb::AnyWritableVec; +use vecdb::AnyExportableVec; use super::{Filter, Filtered}; @@ -308,16 +308,16 @@ impl Traversable for ByAddressType { ) } - fn iter_any_writable(&self) -> impl Iterator { - let mut iter: Box> = - Box::new(self.p2pk65.iter_any_writable()); - iter = Box::new(iter.chain(self.p2pk33.iter_any_writable())); - iter = Box::new(iter.chain(self.p2pkh.iter_any_writable())); - iter = Box::new(iter.chain(self.p2sh.iter_any_writable())); - iter = Box::new(iter.chain(self.p2wpkh.iter_any_writable())); - iter = Box::new(iter.chain(self.p2wsh.iter_any_writable())); - iter = Box::new(iter.chain(self.p2tr.iter_any_writable())); - iter = Box::new(iter.chain(self.p2a.iter_any_writable())); + fn iter_any_exportable(&self) -> impl Iterator { + let mut iter: Box> = + Box::new(self.p2pk65.iter_any_exportable()); + iter = Box::new(iter.chain(self.p2pk33.iter_any_exportable())); + iter = Box::new(iter.chain(self.p2pkh.iter_any_exportable())); + iter = Box::new(iter.chain(self.p2sh.iter_any_exportable())); + iter = Box::new(iter.chain(self.p2wpkh.iter_any_exportable())); + iter = Box::new(iter.chain(self.p2wsh.iter_any_exportable())); + iter = Box::new(iter.chain(self.p2tr.iter_any_exportable())); + iter = Box::new(iter.chain(self.p2a.iter_any_exportable())); iter } } diff --git a/crates/brk_grouper/src/filter.rs b/crates/brk_grouper/src/filter.rs index 890c6a1b4..73a1e2dd2 100644 --- a/crates/brk_grouper/src/filter.rs +++ b/crates/brk_grouper/src/filter.rs @@ -2,7 +2,7 @@ use std::ops::Range; use brk_traversable::{Traversable, TreeNode}; use brk_types::{HalvingEpoch, OutputType}; -use vecdb::AnyWritableVec; +use vecdb::AnyExportableVec; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Filter { @@ -82,7 +82,7 @@ impl Traversable for Filtered { self.1.to_tree_node() } - fn iter_any_writable(&self) -> impl Iterator { - self.1.iter_any_writable() + fn iter_any_exportable(&self) -> impl Iterator { + self.1.iter_any_exportable() } } diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index 3cf04f84c..0886c1d05 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -187,7 +187,7 @@ impl Vecs { }; this.db.retain_regions( - this.iter_any_writable() + this.iter_any_exportable() .flat_map(|v| v.region_names()) .collect(), )?; diff --git a/crates/brk_query/src/async.rs b/crates/brk_query/src/async.rs index 04fbbc4d2..680d84628 100644 --- a/crates/brk_query/src/async.rs +++ b/crates/brk_query/src/async.rs @@ -53,14 +53,14 @@ impl AsyncQuery { // metric: &str, // index: Index, // // params: &Params, - // ) -> Result> { + // ) -> Result> { // let query = self.0.clone(); // spawn_blocking(move || query.search_metric_with_index(metric, index)).await? // } // pub async fn format( // &self, - // metrics: Vec<(String, &&dyn AnyWritableVec)>, + // metrics: Vec<(String, &&dyn AnyExportableVec)>, // params: &ParamsOpt, // ) -> Result { // let query = self.0.clone(); diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index fefd2de4b..67775a939 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -11,7 +11,7 @@ use brk_types::{ Address, AddressStats, Format, Height, Index, IndexInfo, Limit, Metric, MetricCount, Transaction, TxidPath, }; -use vecdb::{AnyStoredVec, AnyWritableVec}; +use vecdb::{AnyExportableVec, AnyStoredVec}; mod r#async; mod chain; @@ -77,7 +77,7 @@ impl Query { metric: &str, index: Index, // params: &Params, - ) -> Result> { + ) -> Result> { todo!(); // let all_metrics = &self.vecs.metrics; @@ -121,7 +121,7 @@ impl Query { } fn columns_to_csv( - columns: &[&&dyn AnyWritableVec], + columns: &[&&dyn AnyExportableVec], from: Option, to: Option, ) -> Result { @@ -163,7 +163,11 @@ impl Query { Ok(csv) } - pub fn format(&self, metrics: Vec<&&dyn AnyWritableVec>, params: &ParamsOpt) -> Result { + pub fn format( + &self, + metrics: Vec<&&dyn AnyExportableVec>, + params: &ParamsOpt, + ) -> Result { let from = params.from().map(|from| { metrics .iter() diff --git a/crates/brk_query/src/vecs.rs b/crates/brk_query/src/vecs.rs index 4908a26ac..2ebb3c524 100644 --- a/crates/brk_query/src/vecs.rs +++ b/crates/brk_query/src/vecs.rs @@ -6,7 +6,7 @@ use brk_traversable::{Traversable, TreeNode}; use brk_types::{Index, IndexInfo, Limit, Metric}; use derive_deref::{Deref, DerefMut}; use quickmatch::{QuickMatch, QuickMatchConfig}; -use vecdb::AnyWritableVec; +use vecdb::AnyExportableVec; use crate::pagination::{PaginatedIndexParam, PaginatedMetrics, PaginationParam}; @@ -31,11 +31,11 @@ impl<'a> Vecs<'a> { indexer .vecs - .iter_any_writable() + .iter_any_exportable() .for_each(|vec| this.insert(vec)); computer - .iter_any_writable() + .iter_any_exportable() .for_each(|vec| this.insert(vec)); let mut ids = this @@ -108,7 +108,7 @@ impl<'a> Vecs<'a> { } // Not the most performant or type safe but only built once so that's okay - fn insert(&mut self, vec: &'a dyn AnyWritableVec) { + fn insert(&mut self, vec: &'a dyn AnyExportableVec) { let name = vec.name(); let serialized_index = vec.index_type_to_string(); let index = Index::try_from(serialized_index) @@ -181,7 +181,7 @@ impl<'a> Vecs<'a> { } #[derive(Default, Deref, DerefMut)] -pub struct IndexToVec<'a>(BTreeMap); +pub struct IndexToVec<'a>(BTreeMap); #[derive(Default, Deref, DerefMut)] -pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyWritableVec>); +pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyExportableVec>); diff --git a/crates/brk_traversable/Cargo.toml b/crates/brk_traversable/Cargo.toml index c5a848bd5..5aca32eb1 100644 --- a/crates/brk_traversable/Cargo.toml +++ b/crates/brk_traversable/Cargo.toml @@ -19,4 +19,5 @@ zstd = ["vecdb/zstd"] [dependencies] brk_types = { workspace = true } brk_traversable_derive = { workspace = true, optional = true } +serde = { workspace = true } vecdb = { workspace = true } diff --git a/crates/brk_traversable/src/lib.rs b/crates/brk_traversable/src/lib.rs index 03e864754..c5be26485 100644 --- a/crates/brk_traversable/src/lib.rs +++ b/crates/brk_traversable/src/lib.rs @@ -4,24 +4,25 @@ pub use brk_types::TreeNode; #[cfg(feature = "derive")] pub use brk_traversable_derive::Traversable; +use serde::Serialize; use vecdb::{ - AnyVec, AnyWritableVec, BytesVec, BytesVecValue, EagerVec, Formattable, LazyVecFrom1, + AnyExportableVec, AnyVec, BytesVec, BytesVecValue, EagerVec, Formattable, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, StoredVec, VecIndex, VecValue, }; pub trait Traversable { fn to_tree_node(&self) -> TreeNode; - fn iter_any_writable(&self) -> impl Iterator; + fn iter_any_exportable(&self) -> impl Iterator; } // BytesVec implementation impl Traversable for BytesVec where I: VecIndex, - T: BytesVecValue + Formattable, + T: BytesVecValue + Formattable + Serialize, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -36,8 +37,8 @@ where I: VecIndex, T: vecdb::ZeroCopyVecValue + Formattable, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -50,10 +51,10 @@ where impl Traversable for vecdb::PcoVec where I: VecIndex, - T: vecdb::PcoVecValue + Formattable, + T: vecdb::PcoVecValue + Formattable + Serialize, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -68,8 +69,8 @@ where I: VecIndex, T: vecdb::LZ4VecValue + Formattable, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -84,8 +85,8 @@ where I: VecIndex, T: vecdb::ZstdVecValue + Formattable, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -97,10 +98,10 @@ where impl Traversable for EagerVec where V: StoredVec, - V::T: Formattable, + V::T: Formattable + Serialize, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -111,12 +112,12 @@ where impl Traversable for LazyVecFrom1 where I: VecIndex, - T: VecValue + Formattable, + T: VecValue + Formattable + Serialize, S1I: VecIndex, S1T: VecValue, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -127,14 +128,14 @@ where impl Traversable for LazyVecFrom2 where I: VecIndex, - T: VecValue + Formattable, + T: VecValue + Formattable + Serialize, S1I: VecIndex, S1T: VecValue, S2I: VecIndex, S2T: VecValue, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -146,7 +147,7 @@ impl Traversable for LazyVecFrom3 where I: VecIndex, - T: VecValue + Formattable, + T: VecValue + Formattable + Serialize, S1I: VecIndex, S1T: VecValue, S2I: VecIndex, @@ -154,8 +155,8 @@ where S3I: VecIndex, S3T: VecValue, { - fn iter_any_writable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyWritableVec) + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) } fn to_tree_node(&self) -> TreeNode { @@ -168,8 +169,8 @@ impl Traversable for Box { (**self).to_tree_node() } - fn iter_any_writable(&self) -> impl Iterator { - (**self).iter_any_writable() + fn iter_any_exportable(&self) -> impl Iterator { + (**self).iter_any_exportable() } } @@ -181,11 +182,10 @@ impl Traversable for Option { } } - fn iter_any_writable(&self) -> impl Iterator { + fn iter_any_exportable(&self) -> impl Iterator { match self { - Some(inner) => { - Box::new(inner.iter_any_writable()) as Box> - } + Some(inner) => Box::new(inner.iter_any_exportable()) + as Box>, None => Box::new(std::iter::empty()), } } @@ -200,10 +200,11 @@ impl Traversable for BTreeMap { TreeNode::Branch(children) } - fn iter_any_writable(&self) -> impl Iterator { - let mut iter: Box> = Box::new(std::iter::empty()); + fn iter_any_exportable(&self) -> impl Iterator { + let mut iter: Box> = + Box::new(std::iter::empty()); for v in self.values() { - iter = Box::new(iter.chain(v.iter_any_writable())); + iter = Box::new(iter.chain(v.iter_any_exportable())); } iter } diff --git a/crates/brk_traversable_derive/src/lib.rs b/crates/brk_traversable_derive/src/lib.rs index 63cb69c94..03a25f5de 100644 --- a/crates/brk_traversable_derive/src/lib.rs +++ b/crates/brk_traversable_derive/src/lib.rs @@ -29,8 +29,8 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { self.0.to_tree_node() } - fn iter_any_writable(&self) -> impl Iterator { - self.0.iter_any_writable() + fn iter_any_exportable(&self) -> impl Iterator { + self.0.iter_any_exportable() } } }); @@ -44,7 +44,7 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { brk_traversable::TreeNode::Branch(std::collections::BTreeMap::new()) } - fn iter_any_writable(&self) -> impl Iterator { + fn iter_any_exportable(&self) -> impl Iterator { std::iter::empty() } } @@ -281,7 +281,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { if regular_fields.is_empty() && option_fields.is_empty() { return quote! { - fn iter_any_writable(&self) -> impl Iterator { + fn iter_any_exportable(&self) -> impl Iterator { std::iter::empty() } }; @@ -290,17 +290,17 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { let (init_part, chain_part) = if let Some((&first, rest)) = regular_fields.split_first() { ( quote! { - let mut regular_iter: Box> = - Box::new(self.#first.iter_any_writable()); + let mut regular_iter: Box> = + Box::new(self.#first.iter_any_exportable()); }, quote! { - #(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_writable()));)* + #(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_exportable()));)* }, ) } else { ( quote! { - let mut regular_iter: Box> = + let mut regular_iter: Box> = Box::new(std::iter::empty()); }, quote! {}, @@ -311,7 +311,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { let chains = option_fields.iter().map(|f| { quote! { if let Some(ref x) = self.#f { - regular_iter = Box::new(regular_iter.chain(x.iter_any_writable())); + regular_iter = Box::new(regular_iter.chain(x.iter_any_exportable())); } } }); @@ -321,7 +321,7 @@ fn generate_iterator_impl(infos: &[FieldInfo]) -> proc_macro2::TokenStream { }; quote! { - fn iter_any_writable(&self) -> impl Iterator { + fn iter_any_exportable(&self) -> impl Iterator { #init_part #chain_part #option_part diff --git a/crates/brk_types/src/anyaddressindex.rs b/crates/brk_types/src/anyaddressindex.rs index fdff6eb19..2eb7bee36 100644 --- a/crates/brk_types/src/anyaddressindex.rs +++ b/crates/brk_types/src/anyaddressindex.rs @@ -1,11 +1,11 @@ use serde::Serialize; -use vecdb::Formattable; +use vecdb::{Bytes, Formattable}; use crate::{EmptyAddressIndex, LoadedAddressIndex, TypeIndex}; const MIN_EMPTY_INDEX: u32 = u32::MAX - 4_000_000_000; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Bytes)] pub struct AnyAddressIndex(TypeIndex); impl AnyAddressIndex { diff --git a/crates/brk_types/src/emptyaddressdata.rs b/crates/brk_types/src/emptyaddressdata.rs index a2fd4c996..64a549f9a 100644 --- a/crates/brk_types/src/emptyaddressdata.rs +++ b/crates/brk_types/src/emptyaddressdata.rs @@ -1,5 +1,5 @@ use serde::Serialize; -use vecdb::Formattable; +use vecdb::{Bytes, Formattable}; use crate::{LoadedAddressData, Sats}; @@ -53,3 +53,27 @@ impl Formattable for EmptyAddressData { true } } + +impl Bytes for EmptyAddressData { + fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&self.tx_count.to_bytes()); + bytes.extend_from_slice(&self.funded_txo_count.to_bytes()); + bytes.extend_from_slice(&self.transfered.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let mut offset = 0; + let tx_count = u32::from_bytes(&bytes[offset..])?; + offset += tx_count.to_bytes().len(); + let funded_txo_count = u32::from_bytes(&bytes[offset..])?; + offset += funded_txo_count.to_bytes().len(); + let transfered = Sats::from_bytes(&bytes[offset..])?; + Ok(Self { + tx_count, + funded_txo_count, + transfered, + }) + } +} diff --git a/crates/brk_types/src/loadedaddressdata.rs b/crates/brk_types/src/loadedaddressdata.rs index 444f24759..27b8fce7e 100644 --- a/crates/brk_types/src/loadedaddressdata.rs +++ b/crates/brk_types/src/loadedaddressdata.rs @@ -1,6 +1,6 @@ use brk_error::{Error, Result}; use serde::Serialize; -use vecdb::{CheckedSub, Formattable}; +use vecdb::{Bytes, CheckedSub, Formattable}; use crate::{Bitcoin, Dollars, EmptyAddressData, Sats}; @@ -145,3 +145,43 @@ impl Formattable for LoadedAddressData { true } } + +impl Bytes for LoadedAddressData { + fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&self.tx_count.to_bytes()); + bytes.extend_from_slice(&self.funded_txo_count.to_bytes()); + bytes.extend_from_slice(&self.spent_txo_count.to_bytes()); + bytes.extend_from_slice(&self.padding.to_bytes()); + bytes.extend_from_slice(&self.received.to_bytes()); + bytes.extend_from_slice(&self.sent.to_bytes()); + bytes.extend_from_slice(&self.realized_cap.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let mut offset = 0; + let tx_count = u32::from_bytes(&bytes[offset..])?; + offset += tx_count.to_bytes().len(); + let funded_txo_count = u32::from_bytes(&bytes[offset..])?; + offset += funded_txo_count.to_bytes().len(); + let spent_txo_count = u32::from_bytes(&bytes[offset..])?; + offset += spent_txo_count.to_bytes().len(); + let padding = u32::from_bytes(&bytes[offset..])?; + offset += padding.to_bytes().len(); + let received = Sats::from_bytes(&bytes[offset..])?; + offset += received.to_bytes().len(); + let sent = Sats::from_bytes(&bytes[offset..])?; + offset += sent.to_bytes().len(); + let realized_cap = Dollars::from_bytes(&bytes[offset..])?; + Ok(Self { + tx_count, + funded_txo_count, + spent_txo_count, + padding, + received, + sent, + realized_cap, + }) + } +} diff --git a/crates/brk_types/src/ohlc.rs b/crates/brk_types/src/ohlc.rs index 1fb2ff5f7..af300d7c8 100644 --- a/crates/brk_types/src/ohlc.rs +++ b/crates/brk_types/src/ohlc.rs @@ -6,7 +6,7 @@ use std::{ use derive_deref::{Deref, DerefMut}; use serde::{Serialize, Serializer, ser::SerializeTuple}; -use vecdb::{Formattable, Pco, TransparentPco}; +use vecdb::{Bytes, Formattable, Pco, TransparentPco}; use crate::StoredF64; @@ -76,6 +76,34 @@ impl Formattable for OHLCCents { } } +impl Bytes for OHLCCents { + fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&self.open.to_bytes()); + bytes.extend_from_slice(&self.high.to_bytes()); + bytes.extend_from_slice(&self.low.to_bytes()); + bytes.extend_from_slice(&self.close.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let mut offset = 0; + let open = Open::::from_bytes(&bytes[offset..])?; + offset += open.to_bytes().len(); + let high = High::::from_bytes(&bytes[offset..])?; + offset += high.to_bytes().len(); + let low = Low::::from_bytes(&bytes[offset..])?; + offset += low.to_bytes().len(); + let close = Close::::from_bytes(&bytes[offset..])?; + Ok(Self { + open, + high, + low, + close, + }) + } +} + #[derive(Debug, Default, Clone, Copy)] #[repr(C)] pub struct OHLCDollars { @@ -166,6 +194,34 @@ impl Formattable for OHLCDollars { } } +impl Bytes for OHLCDollars { + fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&self.open.to_bytes()); + bytes.extend_from_slice(&self.high.to_bytes()); + bytes.extend_from_slice(&self.low.to_bytes()); + bytes.extend_from_slice(&self.close.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let mut offset = 0; + let open = Open::::from_bytes(&bytes[offset..])?; + offset += open.to_bytes().len(); + let high = High::::from_bytes(&bytes[offset..])?; + offset += high.to_bytes().len(); + let low = Low::::from_bytes(&bytes[offset..])?; + offset += low.to_bytes().len(); + let close = Close::::from_bytes(&bytes[offset..])?; + Ok(Self { + open, + high, + low, + close, + }) + } +} + #[derive(Debug, Default, Clone, Copy)] #[repr(C)] pub struct OHLCSats { @@ -237,6 +293,34 @@ impl Formattable for OHLCSats { } } +impl Bytes for OHLCSats { + fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&self.open.to_bytes()); + bytes.extend_from_slice(&self.high.to_bytes()); + bytes.extend_from_slice(&self.low.to_bytes()); + bytes.extend_from_slice(&self.close.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let mut offset = 0; + let open = Open::::from_bytes(&bytes[offset..])?; + offset += open.to_bytes().len(); + let high = High::::from_bytes(&bytes[offset..])?; + offset += high.to_bytes().len(); + let low = Low::::from_bytes(&bytes[offset..])?; + offset += low.to_bytes().len(); + let close = Close::::from_bytes(&bytes[offset..])?; + Ok(Self { + open, + high, + low, + close, + }) + } +} + #[derive( Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Serialize, Pco, )] diff --git a/crates/brk_types/src/poolid.rs b/crates/brk_types/src/poolid.rs index f90d01c1f..f7476ca6c 100644 --- a/crates/brk_types/src/poolid.rs +++ b/crates/brk_types/src/poolid.rs @@ -1,7 +1,7 @@ use num_enum::{FromPrimitive, IntoPrimitive}; use serde::{Deserialize, Serialize}; use strum::Display; -use vecdb::Formattable; +use vecdb::{Bytes, Formattable}; // Created from the list in `pools.rs` // Can be used as index for said list @@ -289,3 +289,21 @@ impl Formattable for PoolId { false } } + +impl Bytes for PoolId { + #[inline] + fn to_bytes(&self) -> Vec { + vec![*self as u8] + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + if bytes.len() != 1 { + return Err(vecdb::Error::WrongLength); + } + // SAFETY: OutputType is repr(u8) and we're transmuting from u8 + // All values 0-255 are valid (includes dummy variants) + let s: Self = unsafe { std::mem::transmute(bytes[0]) }; + Ok(s) + } +}