diff --git a/Cargo.lock b/Cargo.lock index ec09946b4..00f1d55fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,6 +1020,7 @@ dependencies = [ "brk_core", "byteview", "fjall", + "log", ] [[package]] diff --git a/crates/brk_computer/src/states/cohorts/address.rs b/crates/brk_computer/src/states/cohorts/address.rs index 2933e694d..fd1a2f3fb 100644 --- a/crates/brk_computer/src/states/cohorts/address.rs +++ b/crates/brk_computer/src/states/cohorts/address.rs @@ -20,31 +20,80 @@ impl AddressCohortState { }) } + pub fn height(&self) -> Option { + self.inner.height() + } + + pub fn reset_price_to_amount(&mut self) -> Result<()> { + self.inner.reset_price_to_amount() + } + pub fn reset_single_iteration_values(&mut self) { self.inner.reset_single_iteration_values(); } + #[allow(clippy::too_many_arguments)] pub fn send( &mut self, + addressdata: &mut AddressData, value: Sats, current_price: Option, prev_price: Option, blocks_old: usize, days_old: f64, older_than_hour: bool, - ) { - self.inner.send( + ) -> Result<()> { + let compute_price = current_price.is_some(); + + let prev_realized_price = compute_price.then(|| addressdata.realized_price()); + let prev_supply_state = SupplyState { + utxos: addressdata.outputs_len as usize, + value: addressdata.amount(), + }; + + addressdata.send(value, prev_price)?; + + let supply_state = SupplyState { + utxos: addressdata.outputs_len as usize, + value: addressdata.amount(), + }; + + self.inner.send_( &SupplyState { utxos: 1, value }, current_price, prev_price, blocks_old, days_old, older_than_hour, + compute_price.then(|| (addressdata.realized_price(), &supply_state)), + prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)), ); + + Ok(()) } - pub fn receive(&mut self, value: Sats, price: Option) { - self.inner.receive(&SupplyState { utxos: 1, value }, price); + pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option) { + let compute_price = price.is_some(); + + let prev_realized_price = compute_price.then(|| address_data.realized_price()); + let prev_supply_state = SupplyState { + utxos: address_data.outputs_len as usize, + value: address_data.amount(), + }; + + address_data.receive(value, price); + + let supply_state = SupplyState { + utxos: address_data.outputs_len as usize, + value: address_data.amount(), + }; + + self.inner.receive_( + &SupplyState { utxos: 1, value }, + price, + compute_price.then(|| (address_data.realized_price(), &supply_state)), + prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)), + ); } pub fn add(&mut self, addressdata: &AddressData) { @@ -54,7 +103,7 @@ impl AddressCohortState { } pub fn subtract(&mut self, addressdata: &AddressData) { - self.address_count.checked_sub(1).unwrap(); + self.address_count = self.address_count.checked_sub(1).unwrap(); self.inner .decrement_(&addressdata.into(), addressdata.realized_cap); } @@ -63,39 +112,3 @@ impl AddressCohortState { self.inner.commit(height) } } - -// fn decrement(&mut self, supply_state: &SupplyState, price: Option) { -// self.inner.decrement(supply_state, price); -// } - -// fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) { -// self.inner.decrement_price_to_amount(supply_state, price); -// } - -// fn receive(&mut self, supply_state: &SupplyState, price: Option) { -// self.inner.receive(supply_state, price); -// } - -// fn compute_unrealized_states( -// &self, -// height_price: Dollars, -// date_price: Option, -// ) -> (UnrealizedState, Option) { -// self.inner -// .compute_unrealized_states(height_price, date_price) -// } - -// } - -// impl Deref for AddressCohortState { -// type Target = CohortState; -// fn deref(&self) -> &Self::Target { -// &self.inner -// } -// } - -// impl DerefMut for AddressCohortState { -// fn deref_mut(&mut self) -> &mut Self::Target { -// &mut self.inner -// } -// } diff --git a/crates/brk_computer/src/states/cohorts/common.rs b/crates/brk_computer/src/states/cohorts/common.rs index 7dba4bf26..7d2fe4cca 100644 --- a/crates/brk_computer/src/states/cohorts/common.rs +++ b/crates/brk_computer/src/states/cohorts/common.rs @@ -10,7 +10,8 @@ pub struct CohortState { pub realized: Option, pub satblocks_destroyed: Sats, pub satdays_destroyed: Sats, - pub price_to_amount: PriceToAmount, + + price_to_amount: PriceToAmount, } impl CohortState { @@ -24,6 +25,22 @@ impl CohortState { }) } + pub fn height(&self) -> Option { + self.price_to_amount.height() + } + + pub fn reset_price_to_amount(&mut self) -> Result<()> { + self.price_to_amount.reset() + } + + pub fn price_to_amount_first_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.price_to_amount.first_key_value() + } + + pub fn price_to_amount_last_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.price_to_amount.last_key_value() + } + pub fn reset_single_iteration_values(&mut self) { self.satdays_destroyed = Sats::ZERO; self.satblocks_destroyed = Sats::ZERO; @@ -39,7 +56,7 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.increment(supply_state, price); - *self.price_to_amount.entry(price).or_default() += supply_state.value; + self.price_to_amount.increment(price, supply_state); } } } @@ -50,10 +67,10 @@ impl CohortState { if supply_state.value > Sats::ZERO { if let Some(realized) = self.realized.as_mut() { realized.increment_(realized_cap); - *self - .price_to_amount - .entry(realized_cap / Bitcoin::from(supply_state.value)) - .or_default() += supply_state.value; + self.price_to_amount.increment( + realized_cap / Bitcoin::from(supply_state.value), + supply_state, + ); } } } @@ -65,7 +82,7 @@ impl CohortState { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.decrement(supply_state, price); - self.decrement_price_to_amount(supply_state, price); + self.price_to_amount.decrement(price, supply_state); } } } @@ -76,30 +93,47 @@ impl CohortState { if supply_state.value > Sats::ZERO { if let Some(realized) = self.realized.as_mut() { realized.decrement_(realized_cap); - self.decrement_price_to_amount( + self.price_to_amount.decrement( + (realized_cap / Bitcoin::from(supply_state.value)).round_nearest_cent(), supply_state, - realized_cap / Bitcoin::from(supply_state.value), ); } } } - fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) { - let amount = self.price_to_amount.get_mut(&price).unwrap(); - *amount -= supply_state.value; - if *amount == Sats::ZERO { - self.price_to_amount.remove(&price); - } + pub fn receive(&mut self, supply_state: &SupplyState, price: Option) { + self.receive_( + supply_state, + price, + price.map(|price| (price, supply_state)), + None, + ); } - pub fn receive(&mut self, supply_state: &SupplyState, price: Option) { + pub fn receive_( + &mut self, + supply_state: &SupplyState, + price: Option, + price_to_amount_increment: Option<(Dollars, &SupplyState)>, + price_to_amount_decrement: Option<(Dollars, &SupplyState)>, + ) { self.supply += supply_state; if supply_state.value > Sats::ZERO { if let Some(realized) = self.realized.as_mut() { let price = price.unwrap(); realized.receive(supply_state, price); - *self.price_to_amount.entry(price).or_default() += supply_state.value; + + if let Some((price, supply)) = price_to_amount_increment + && supply.value.is_not_zero() + { + self.price_to_amount.increment(price, supply); + } + if let Some((price, supply)) = price_to_amount_decrement + && supply.value.is_not_zero() + { + self.price_to_amount.decrement(price, supply); + } } } } @@ -112,6 +146,30 @@ impl CohortState { blocks_old: usize, days_old: f64, older_than_hour: bool, + ) { + self.send_( + supply_state, + current_price, + prev_price, + blocks_old, + days_old, + older_than_hour, + None, + prev_price.map(|prev_price| (prev_price, supply_state)), + ); + } + + #[allow(clippy::too_many_arguments)] + pub fn send_( + &mut self, + supply_state: &SupplyState, + current_price: Option, + prev_price: Option, + blocks_old: usize, + days_old: f64, + older_than_hour: bool, + price_to_amount_increment: Option<(Dollars, &SupplyState)>, + price_to_amount_decrement: Option<(Dollars, &SupplyState)>, ) { if supply_state.utxos == 0 { return; @@ -129,7 +187,16 @@ impl CohortState { let current_price = current_price.unwrap(); let prev_price = prev_price.unwrap(); realized.send(supply_state, current_price, prev_price, older_than_hour); - self.decrement_price_to_amount(supply_state, prev_price); + if let Some((price, supply)) = price_to_amount_increment + && supply.value.is_not_zero() + { + self.price_to_amount.increment(price, supply); + } + if let Some((price, supply)) = price_to_amount_decrement + && supply.value.is_not_zero() + { + self.price_to_amount.decrement(price, supply); + } } } } diff --git a/crates/brk_computer/src/states/cohorts/utxo.rs b/crates/brk_computer/src/states/cohorts/utxo.rs index 89987d2ab..15ccd2836 100644 --- a/crates/brk_computer/src/states/cohorts/utxo.rs +++ b/crates/brk_computer/src/states/cohorts/utxo.rs @@ -1,6 +1,6 @@ use std::path::Path; -use brk_core::Result; +use brk_core::{Height, Result}; use derive_deref::{Deref, DerefMut}; use super::CohortState; @@ -16,4 +16,12 @@ impl UTXOCohortState { compute_dollars, )?)) } + + pub fn height(&self) -> Option { + self.0.height() + } + + pub fn reset_price_to_amount(&mut self) -> Result<()> { + self.0.reset_price_to_amount() + } } diff --git a/crates/brk_computer/src/states/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs index 1b68adca5..493b76336 100644 --- a/crates/brk_computer/src/states/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -2,7 +2,6 @@ use std::{ collections::BTreeMap, fs::{self, File}, io::{BufReader, BufWriter}, - ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; @@ -11,6 +10,8 @@ use brk_core::{Dollars, Height, Result, Sats}; use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; +use crate::states::SupplyState; + #[derive(Clone, Debug)] pub struct PriceToAmount { pathbuf: PathBuf, @@ -46,8 +47,36 @@ impl PriceToAmount { }) } + pub fn iter(&self) -> impl Iterator { + self.state.iter() + } + + pub fn is_empty(&self) -> bool { + self.state.is_empty() + } + + pub fn first_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.state.first_key_value() + } + + pub fn last_key_value(&self) -> Option<(&Dollars, &Sats)> { + self.state.last_key_value() + } + + pub fn increment(&mut self, price: Dollars, supply_state: &SupplyState) { + *self.state.entry(price).or_default() += supply_state.value; + } + + pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) { + let amount = self.state.get_mut(&price).unwrap(); + *amount -= supply_state.value; + if *amount == Sats::ZERO { + self.state.remove(&price); + } + } + pub fn reset(&mut self) -> Result<()> { - self.clear(); + self.state.clear(); self.height = None; fs::remove_dir_all(&self.pathbuf)?; fs::create_dir_all(&self.pathbuf)?; @@ -89,16 +118,3 @@ impl PriceToAmount { path.join("height") } } - -impl Deref for PriceToAmount { - type Target = BTreeMap; - fn deref(&self) -> &Self::Target { - &self.state - } -} - -impl DerefMut for PriceToAmount { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.state - } -} diff --git a/crates/brk_computer/src/stores.rs b/crates/brk_computer/src/stores.rs index 71875aa73..ca0862e79 100644 --- a/crates/brk_computer/src/stores.rs +++ b/crates/brk_computer/src/stores.rs @@ -10,7 +10,7 @@ use brk_core::{ }; use brk_store::{AnyStore, Store}; use fjall::{PersistMode, TransactionalKeyspace}; -use rayon::prelude::*; +use log::info; use crate::{ GroupedByAddressType, @@ -484,10 +484,15 @@ impl Stores { } pub fn reset(&mut self) -> Result<()> { + info!("Resetting stores..."); + info!("> If it gets stuck here, stop the program and start it again"); + self.as_mut_slice() - .into_par_iter() + .into_iter() .try_for_each(|store| store.reset())?; + info!("Persisting stores..."); + self.keyspace .persist(PersistMode::SyncAll) .map_err(|e| e.into()) diff --git a/crates/brk_computer/src/vecs/stateful/address_cohort.rs b/crates/brk_computer/src/vecs/stateful/address_cohort.rs index 1e8006164..aa1d31f15 100644 --- a/crates/brk_computer/src/vecs/stateful/address_cohort.rs +++ b/crates/brk_computer/src/vecs/stateful/address_cohort.rs @@ -71,11 +71,7 @@ impl CohortVecs for Vecs { fn starting_height(&self) -> Height { [ - self.state - .inner - .price_to_amount - .height() - .map_or(Height::MAX, |h| h.incremented()), + self.state.height().map_or(Height::MAX, |h| h.incremented()), self.height_to_address_count.len().into(), self.inner.starting_height(), ] diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs b/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs index f8c2fef36..970d62680 100644 --- a/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs +++ b/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs @@ -1,8 +1,10 @@ use std::mem; -use brk_core::TypeIndex; +use brk_core::{AddressData, Dollars, OutputIndex, Sats, TypeIndex}; use derive_deref::{Deref, DerefMut}; +use crate::vecs::stateful::WithAddressDataSource; + use super::GroupedByAddressType; #[derive(Debug, Default, Deref, DerefMut)] @@ -33,3 +35,86 @@ impl AddressTypeToTypeIndexVec { self.0 } } + +impl AddressTypeToTypeIndexVec { + #[allow(clippy::type_complexity)] + pub fn extend_from_sent( + &mut self, + other: &AddressTypeToTypeIndexVec<( + OutputIndex, + Sats, + Option>, + Option, + usize, + f64, + bool, + )>, + ) { + Self::extend_from_sent_(&mut self.p2pk33, &other.p2pk33); + Self::extend_from_sent_(&mut self.p2pkh, &other.p2pkh); + Self::extend_from_sent_(&mut self.p2sh, &other.p2sh); + Self::extend_from_sent_(&mut self.p2wpkh, &other.p2wpkh); + Self::extend_from_sent_(&mut self.p2wsh, &other.p2wsh); + Self::extend_from_sent_(&mut self.p2tr, &other.p2tr); + Self::extend_from_sent_(&mut self.p2a, &other.p2a); + } + + #[allow(clippy::type_complexity)] + fn extend_from_sent_( + own: &mut Vec<(TypeIndex, OutputIndex)>, + other: &[( + TypeIndex, + ( + OutputIndex, + Sats, + Option>, + Option, + usize, + f64, + bool, + ), + )], + ) { + own.extend( + other + .iter() + .map(|(type_index, (output_index, ..))| (*type_index, *output_index)), + ); + } + + pub fn extend_from_received( + &mut self, + other: &AddressTypeToTypeIndexVec<( + OutputIndex, + Sats, + Option>, + )>, + ) { + Self::extend_from_received_(&mut self.p2pk33, &other.p2pk33); + Self::extend_from_received_(&mut self.p2pkh, &other.p2pkh); + Self::extend_from_received_(&mut self.p2sh, &other.p2sh); + Self::extend_from_received_(&mut self.p2wpkh, &other.p2wpkh); + Self::extend_from_received_(&mut self.p2wsh, &other.p2wsh); + Self::extend_from_received_(&mut self.p2tr, &other.p2tr); + Self::extend_from_received_(&mut self.p2a, &other.p2a); + } + + #[allow(clippy::type_complexity)] + fn extend_from_received_( + own: &mut Vec<(TypeIndex, OutputIndex)>, + other: &[( + TypeIndex, + ( + OutputIndex, + Sats, + Option>, + ), + )], + ) { + own.extend( + other + .iter() + .map(|(type_index, (output_index, ..))| (*type_index, *output_index)), + ); + } +} diff --git a/crates/brk_computer/src/vecs/stateful/common.rs b/crates/brk_computer/src/vecs/stateful/common.rs index 6467c11d3..58154e1b2 100644 --- a/crates/brk_computer/src/vecs/stateful/common.rs +++ b/crates/brk_computer/src/vecs/stateful/common.rs @@ -1341,8 +1341,7 @@ impl Vecs { .forced_push_at( height, state - .price_to_amount - .first_key_value() + .price_to_amount_first_key_value() .map(|(&dollars, _)| dollars) .unwrap_or(Dollars::NAN), exit, @@ -1353,8 +1352,7 @@ impl Vecs { .forced_push_at( height, state - .price_to_amount - .last_key_value() + .price_to_amount_last_key_value() .map(|(&dollars, _)| dollars) .unwrap_or(Dollars::NAN), exit, diff --git a/crates/brk_computer/src/vecs/stateful/mod.rs b/crates/brk_computer/src/vecs/stateful/mod.rs index 4c16a2ad1..612308d09 100644 --- a/crates/brk_computer/src/vecs/stateful/mod.rs +++ b/crates/brk_computer/src/vecs/stateful/mod.rs @@ -38,7 +38,7 @@ pub use addresstype_to_typeindex_vec::*; use r#trait::CohortVecs; pub use withaddressdatasource::WithAddressDataSource; -const VERSION: Version = Version::new(5); +const VERSION: Version = Version::new(9); #[derive(Clone)] pub struct Vecs { @@ -401,13 +401,17 @@ impl Vecs { stores.reset()?; + info!("Resetting utxo price maps..."); + separate_utxo_vecs .par_iter_mut() - .try_for_each(|(_, v)| v.state.price_to_amount.reset())?; + .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; + + info!("Resetting address price maps..."); separate_address_vecs .par_iter_mut() - .try_for_each(|(_, v)| v.state.inner.price_to_amount.reset())?; + .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; } if starting_height < Height::from(height_to_date_fixed.len()) { @@ -485,11 +489,13 @@ impl Vecs { let output_count = height_to_output_count_iter.unwrap_get_inner(height); let input_count = height_to_input_count_iter.unwrap_get_inner(height); - let ((mut height_to_sent, new_addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_data), (mut received, new_addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_data)) = thread::scope(|s| { + let ( + (mut height_to_sent, addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_data), + (mut received, addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_data), + ) = thread::scope(|s| { if chain_state_starting_height <= height { s.spawn(|| { - self.utxo_vecs - .tick_tock_next_block(&chain_state, timestamp); + self.utxo_vecs.tick_tock_next_block(&chain_state, timestamp); }); } @@ -527,8 +533,17 @@ impl Vecs { unreachable!() } - let addressdata_opt = if input_type.is_address() && !addresstype_to_typeindex_to_addressdata.get(input_type).unwrap().contains_key(&typeindex) { - Some(WithAddressDataSource::FromAddressDataStore( stores.get_addressdata(input_type, typeindex).unwrap().unwrap())) + let addressdata_opt = if input_type.is_address() + && !addresstype_to_typeindex_to_addressdata + .get(input_type) + .unwrap() + .contains_key(&typeindex) + && let Some(address_data) = + stores.get_addressdata(input_type, typeindex).unwrap() + { + Some(WithAddressDataSource::FromAddressDataStore( + address_data, + )) } else { None }; @@ -545,65 +560,126 @@ impl Vecs { .unwrap() .into_owned(); - let prev_price = height_to_close.map(|m| *m.get_or_read(prev_height, height_to_close_mmap.as_ref().unwrap()).unwrap() - .unwrap() - .into_owned()); + let prev_price = height_to_close.map(|m| { + *m.get_or_read( + prev_height, + height_to_close_mmap.as_ref().unwrap(), + ) + .unwrap() + .unwrap() + .into_owned() + }); - let prev_timestamp = height_to_timestamp_fixed.get_or_read(prev_height, &height_to_timestamp_fixed_mmap) - .unwrap() - .unwrap() - .into_owned(); + let prev_timestamp = height_to_timestamp_fixed + .get_or_read(prev_height, &height_to_timestamp_fixed_mmap) + .unwrap() + .unwrap() + .into_owned(); - let blocks_old = height.unwrap_to_usize() - prev_height.unwrap_to_usize(); + let blocks_old = + height.unwrap_to_usize() - prev_height.unwrap_to_usize(); - let days_old = prev_timestamp - .difference_in_days_between_float(timestamp); + let days_old = + prev_timestamp.difference_in_days_between_float(timestamp); let older_than_hour = timestamp .checked_sub(prev_timestamp) .unwrap() .is_more_than_hour(); - (prev_height, value, input_type, typeindex, outputindex, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour) + ( + prev_height, + value, + input_type, + typeindex, + outputindex, + addressdata_opt, + prev_price, + blocks_old, + days_old, + older_than_hour, + ) }) .fold( || { ( BTreeMap::::default(), AddressTypeToTypeIndexVec::::default(), - AddressTypeToTypeIndexVec::<(Sats, Option>, Option, usize, f64, bool)>::default(), + AddressTypeToTypeIndexVec::<( + Sats, + Option>, + Option, + usize, + f64, + bool, + )>::default( + ), ) }, - |(mut tree, mut vecs, mut vecs2), (height, value, input_type, typeindex, outputindex, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour)| { + |(mut tree, mut vecs, mut vecs2), + ( + height, + value, + input_type, + typeindex, + outputindex, + addressdata_opt, + prev_price, + blocks_old, + days_old, + older_than_hour, + )| { tree.entry(height).or_default().iterate(value, input_type); if let Some(vec) = vecs.get_mut(input_type) { vec.push((typeindex, outputindex)); } if let Some(vec) = vecs2.get_mut(input_type) { - vec.push((typeindex, (value, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour))); + vec.push(( + typeindex, + ( + value, + addressdata_opt, + prev_price, + blocks_old, + days_old, + older_than_hour, + ), + )); } (tree, vecs, vecs2) }, ) - .reduce( || { - ( - BTreeMap::::default(), - AddressTypeToTypeIndexVec::::default(), - AddressTypeToTypeIndexVec::<(Sats, Option>, Option, usize, f64, bool)>::default(), - ) - }, |(first_tree, mut source_vecs,mut source_vecs2), (second_tree, other_vecs, other_vecs2)| { - let (mut tree_source, tree_to_consume) = if first_tree.len() > second_tree.len() { - (first_tree, second_tree) - } else { - (second_tree, first_tree) - }; - tree_to_consume.into_iter().for_each(|(k, v)| { - *tree_source.entry(k).or_default() += v; - }); - source_vecs.merge(other_vecs); - source_vecs2.merge(other_vecs2); - (tree_source, source_vecs, source_vecs2) - }) + .reduce( + || { + ( + BTreeMap::::default(), + AddressTypeToTypeIndexVec::::default(), + AddressTypeToTypeIndexVec::<( + Sats, + Option>, + Option, + usize, + f64, + bool, + )>::default( + ), + ) + }, + |(first_tree, mut source_vecs, mut source_vecs2), (second_tree, other_vecs, other_vecs2)| { + let (mut tree_source, tree_to_consume) = + if first_tree.len() > second_tree.len() { + (first_tree, second_tree) + } else { + (second_tree, first_tree) + }; + tree_to_consume.into_iter().for_each(|(k, v)| { + *tree_source.entry(k).or_default() += v; + }); + source_vecs.merge(other_vecs); + source_vecs2.merge(other_vecs2); + (tree_source, source_vecs, source_vecs2) + }, + ) }); let received_handle = s.spawn(|| { @@ -629,14 +705,35 @@ impl Vecs { .unwrap() .into_owned(); - let addressdata_opt = if output_type.is_address() && !addresstype_to_typeindex_to_addressdata.get(output_type).unwrap().contains_key(&typeindex) && !addresstype_to_typeindex_to_emptyaddressdata.get(output_type).unwrap().contains_key(&typeindex) { - Some(if let Some(addressdata) = stores.get_addressdata(output_type, typeindex).unwrap() { - WithAddressDataSource::FromAddressDataStore(addressdata) - } else if let Some(emptyaddressdata) = stores.get_emptyaddressdata(output_type, typeindex).unwrap() { - WithAddressDataSource::FromEmptyAddressDataStore(emptyaddressdata.into()) - } else { - WithAddressDataSource::New(AddressData::default()) - }) + let addressdata_opt = if output_type.is_address() + && !addresstype_to_typeindex_to_addressdata + .get(output_type) + .unwrap() + .contains_key(&typeindex) + && !addresstype_to_typeindex_to_emptyaddressdata + .get(output_type) + .unwrap() + .contains_key(&typeindex) + { + Some( + if let Some(addressdata) = stores + .get_addressdata(output_type, typeindex) + .unwrap() + { + WithAddressDataSource::FromAddressDataStore( + addressdata, + ) + } else if let Some(emptyaddressdata) = stores + .get_emptyaddressdata(output_type, typeindex) + .unwrap() + { + WithAddressDataSource::FromEmptyAddressDataStore( + emptyaddressdata.into(), + ) + } else { + WithAddressDataSource::New(AddressData::default()) + }, + ) } else { None }; @@ -644,52 +741,68 @@ impl Vecs { (value, output_type, typeindex, outputindex, addressdata_opt) }) .fold( - || (Transacted::default(), AddressTypeToTypeIndexVec::::default(), - AddressTypeToTypeIndexVec::<(Sats, Option>)>::default(), - ), - |(mut transacted, mut vecs, mut vecs2), (value, output_type, typeindex, outputindex, addressdata_opt)| { + || { + ( + Transacted::default(), + AddressTypeToTypeIndexVec::::default(), + AddressTypeToTypeIndexVec::<( + Sats, + Option>, + )>::default( + ), + ) + }, + |(mut transacted, mut vecs, mut vecs2), + ( + value, + output_type, + typeindex, + outputindex, + addressdata_opt, + )| { transacted.iterate(value, output_type); if let Some(vec) = vecs.get_mut(output_type) { - vec.push((typeindex, outputindex)); + vec.push(( + typeindex, + outputindex, + )); } if let Some(vec) = vecs2.get_mut(output_type) { - vec.push((typeindex, (value, addressdata_opt))); + vec.push(( + typeindex, + (value, addressdata_opt), + )); } (transacted, vecs, vecs2) }, ) - .reduce(|| (Transacted::default(), AddressTypeToTypeIndexVec::::default(), - AddressTypeToTypeIndexVec::<(Sats, Option>)>::default(), - ), |(transacted, mut vecs, mut vecs2), (other_transacted, other_vecs, other_vecs2)| { - vecs.merge(other_vecs); - vecs2.merge(other_vecs2); - (transacted + other_transacted, vecs, vecs2) - }) + .reduce( + || { + ( + Transacted::default(), + AddressTypeToTypeIndexVec::::default(), + AddressTypeToTypeIndexVec::<( + Sats, + Option>, + )>::default( + ), + ) + }, + |(transacted, mut vecs, mut vecs2), (other_transacted, other_vecs, other_vecs2)| { + vecs.merge(other_vecs); + vecs2.merge(other_vecs2); + (transacted + other_transacted, vecs, vecs2) + }, + ) }); (sent_handle.join().unwrap(), received_handle.join().unwrap()) }); - addresstype_to_typeindex_to_sent_outputindex.merge(new_addresstype_to_typedindex_to_sent_outputindex); - addresstype_to_typeindex_to_received_outputindex.merge(new_addresstype_to_typedindex_to_received_outputindex); - - addresstype_to_typedindex_to_received_data.process_received( - &mut self.address_vecs, - &mut addresstype_to_typeindex_to_addressdata, - &mut addresstype_to_typeindex_to_emptyaddressdata, - price, - &mut addresstype_to_address_count, - &mut addresstype_to_empty_address_count - ); - - addresstype_to_typedindex_to_sent_data.process_sent( - &mut self.address_vecs, - &mut addresstype_to_typeindex_to_addressdata, - &mut addresstype_to_typeindex_to_emptyaddressdata, - price, - &mut addresstype_to_address_count, - &mut addresstype_to_empty_address_count - )?; + if chain_state_starting_height > height { + dbg!(chain_state_starting_height, height); + panic!("temp, just making sure") + } unspendable_supply += received .by_type @@ -715,28 +828,53 @@ impl Vecs { .iterate(Sats::FIFTY_BTC, OutputType::P2PK65); }; - if chain_state_starting_height <= height { - // Push current block state before processing sends and receives - chain_state.push(BlockState { - supply: received.spendable_supply.clone(), + thread::scope(|scope| -> Result<()> { + scope.spawn(|| addresstype_to_typeindex_to_sent_outputindex + .merge(addresstype_to_typedindex_to_sent_outputindex)); + + scope.spawn(|| addresstype_to_typeindex_to_received_outputindex + .merge(addresstype_to_typedindex_to_received_outputindex)); + + scope.spawn(|| { + // Push current block state before processing sends and receives + chain_state.push(BlockState { + supply: received.spendable_supply.clone(), + price, + timestamp, + }); + + self.utxo_vecs.receive(received, height, price); + + let unsafe_chain_state = UnsafeSlice::new(&mut chain_state); + + height_to_sent.par_iter().for_each(|(height, sent)| unsafe { + (*unsafe_chain_state.get(height.unwrap_to_usize())).supply -= + &sent.spendable_supply; + }); + + self.utxo_vecs.send(height_to_sent, chain_state.as_slice()); + }); + + addresstype_to_typedindex_to_received_data.process_received( + &mut self.address_vecs, + &mut addresstype_to_typeindex_to_addressdata, + &mut addresstype_to_typeindex_to_emptyaddressdata, price, - timestamp, - }); + &mut addresstype_to_address_count, + &mut addresstype_to_empty_address_count, + ); - self.utxo_vecs.receive(received, height, price); + addresstype_to_typedindex_to_sent_data.process_sent( + &mut self.address_vecs, + &mut addresstype_to_typeindex_to_addressdata, + &mut addresstype_to_typeindex_to_emptyaddressdata, + price, + &mut addresstype_to_address_count, + &mut addresstype_to_empty_address_count, + )?; - let unsafe_chain_state = UnsafeSlice::new(&mut chain_state); - - height_to_sent.par_iter().for_each(|(height, sent)| unsafe { - (*unsafe_chain_state.get(height.unwrap_to_usize())).supply -= - &sent.spendable_supply; - }); - - self.utxo_vecs.send(height_to_sent, chain_state.as_slice()); - } else { - dbg!(chain_state_starting_height, height); - panic!("temp, just making sure") - } + Ok(()) + })?; let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); @@ -759,9 +897,14 @@ impl Vecs { self.height_to_opreturn_supply .forced_push_at(height, opreturn_supply, exit)?; - self.addresstype_to_height_to_address_count.forced_push_at(height, &addresstype_to_address_count, exit)?; + self.addresstype_to_height_to_address_count.forced_push_at( + height, + &addresstype_to_address_count, + exit, + )?; - self.addresstype_to_height_to_empty_address_count.forced_push_at(height, &addresstype_to_empty_address_count, exit)?; + self.addresstype_to_height_to_empty_address_count + .forced_push_at(height, &addresstype_to_empty_address_count, exit)?; let date = height_to_date_fixed_iter.unwrap_get_inner(height); let dateindex = DateIndex::try_from(date).unwrap(); @@ -776,45 +919,47 @@ impl Vecs { .as_mut() .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex))); - // thread::scope(|scope| { - // scope.spawn(|| { - separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| { - v.compute_then_force_push_unrealized_states( - height, - price, - is_date_last_height.then_some(dateindex), - date_price, - exit, - ) - })?; - // }); - // scope.spawn(|| { - separate_address_vecs.par_iter_mut().try_for_each(|(_, v)| { - v.compute_then_force_push_unrealized_states( - height, - price, - is_date_last_height.then_some(dateindex), - date_price, - exit, - ) - })?; - // }); - // }); - + thread::scope(|scope| { + scope.spawn(|| { + separate_utxo_vecs + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_then_force_push_unrealized_states( + height, + price, + is_date_last_height.then_some(dateindex), + date_price, + exit, + ) + }) + .unwrap(); + }); + scope.spawn(|| { + separate_address_vecs + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_then_force_push_unrealized_states( + height, + price, + is_date_last_height.then_some(dateindex), + date_price, + exit, + ) + }) + .unwrap(); + }); + }); if height != Height::ZERO && height.unwrap_to_usize() % 10_000 == 0 { info!("Flushing..."); exit.block(); - self.flush_states( - height, - &chain_state, - exit, - )?; + self.flush_states(height, &chain_state, exit)?; stores.commit( height, mem::take(&mut addresstype_to_typeindex_to_sent_outputindex), mem::take(&mut addresstype_to_typeindex_to_received_outputindex), - mem::take(&mut addresstype_to_typeindex_to_addressdata), mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata) + mem::take(&mut addresstype_to_typeindex_to_addressdata), + mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), )?; exit.release(); } @@ -840,39 +985,41 @@ impl Vecs { info!("Computing overlapping..."); - // thread::scope(|scope| { - // scope.spawn(|| { - self.utxo_vecs - .compute_overlapping_vecs(starting_indexes, exit)?; - // }); - // scope.spawn(|| { - self.address_vecs - .compute_overlapping_vecs(starting_indexes, exit)?; - // }); - // }); + thread::scope(|scope| { + scope.spawn(|| { + self.utxo_vecs + .compute_overlapping_vecs(starting_indexes, exit) + .unwrap(); + }); + scope.spawn(|| { + self.address_vecs + .compute_overlapping_vecs(starting_indexes, exit) + .unwrap(); + }); + }); info!("Computing rest part 1..."); - // thread::scope(|scope| { - // scope.spawn(|| { - self.utxo_vecs - .as_mut_vecs() - .par_iter_mut() - .try_for_each(|(_, v)| { - v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) - }) - .unwrap(); - // }); - // scope.spawn(|| { - self.address_vecs - .as_mut_vecs() - .par_iter_mut() - .try_for_each(|(_, v)| { - v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) - }) - .unwrap(); - // }); - // }); + thread::scope(|scope| { + scope.spawn(|| { + self.utxo_vecs + .as_mut_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + }) + .unwrap(); + }); + scope.spawn(|| { + self.address_vecs + .as_mut_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + }) + .unwrap(); + }); + }); info!("Computing rest part 2..."); @@ -894,46 +1041,48 @@ impl Vecs { .as_ref() .map(|v| v.dateindex.unwrap_last().clone()); - // thread::scope(|scope| { - // scope.spawn(|| { - self.utxo_vecs - .as_mut_vecs() - .par_iter_mut() - .try_for_each(|(_, v)| { - v.compute_rest_part2( - indexer, - indexes, - fetched, - starting_indexes, - market, - &height_to_supply, - dateindex_to_supply.as_ref().unwrap(), - height_to_realized_cap.as_ref(), - dateindex_to_realized_cap.as_ref(), - exit, - ) - })?; - // }); - // scope.spawn(|| { - self.address_vecs - .as_mut_vecs() - .par_iter_mut() - .try_for_each(|(_, v)| { - v.compute_rest_part2( - indexer, - indexes, - fetched, - starting_indexes, - market, - &height_to_supply, - dateindex_to_supply.as_ref().unwrap(), - height_to_realized_cap.as_ref(), - dateindex_to_realized_cap.as_ref(), - exit, - ) - })?; - // }); - // }); + thread::scope(|scope| { + scope.spawn(|| { + self.utxo_vecs + .as_mut_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_rest_part2( + indexer, + indexes, + fetched, + starting_indexes, + market, + &height_to_supply, + dateindex_to_supply.as_ref().unwrap(), + height_to_realized_cap.as_ref(), + dateindex_to_realized_cap.as_ref(), + exit, + ) + }) + .unwrap(); + }); + scope.spawn(|| { + self.address_vecs + .as_mut_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_rest_part2( + indexer, + indexes, + fetched, + starting_indexes, + market, + &height_to_supply, + dateindex_to_supply.as_ref().unwrap(), + height_to_realized_cap.as_ref(), + dateindex_to_realized_cap.as_ref(), + exit, + ) + }) + .unwrap(); + }); + }); self.indexes_to_unspendable_supply.compute_rest( indexer, indexes, @@ -1084,28 +1233,27 @@ impl AddressTypeToTypeIndexVec<(Sats, Option> let amount = prev_amount + value; if is_new + || from_any_empty || vecs.by_size_range.get_mut(amount).0.clone() != vecs.by_size_range.get_mut(prev_amount).0.clone() { - if !is_new { - vecs.by_size_range - .get_mut(prev_amount) - .1 - .state - .subtract(addressdata); + // dbg!((prev_amount, amount, is_new)); + + if !is_new && !from_any_empty { + let state = &mut vecs.by_size_range.get_mut(prev_amount).1.state; + // dbg!((prev_amount, &state.address_count, &addressdata)); + state.subtract(addressdata); } addressdata.receive(value, price); vecs.by_size_range.get_mut(amount).1.state.add(addressdata); } else { - addressdata.receive(value, price); - - vecs.by_size_range - .get_mut(amount) - .1 - .state - .receive(value, price); + vecs.by_size_range.get_mut(amount).1.state.receive( + addressdata, + value, + price, + ); } }); }); @@ -1149,7 +1297,7 @@ impl let addressdata_withsource = typeindex_to_addressdata .entry(type_index) - .or_insert(addressdata_opt.unwrap()); + .or_insert_with(|| addressdata_opt.unwrap()); let addressdata = addressdata_withsource.deref_mut(); @@ -1159,6 +1307,8 @@ impl let will_be_empty = addressdata.outputs_len - 1 == 0; + // dbg!((prev_amount, amount, will_be_empty)); + if will_be_empty || vecs.by_size_range.get_mut(amount).0.clone() != vecs.by_size_range.get_mut(prev_amount).0.clone() @@ -1172,6 +1322,10 @@ impl addressdata.send(value, prev_price)?; if will_be_empty { + if amount.is_not_zero() { + unreachable!() + } + (*addresstype_to_address_count.get_mut(_type).unwrap()) -= 1; (*addresstype_to_empty_address_count.get_mut(_type).unwrap()) += 1; @@ -1186,16 +1340,15 @@ impl vecs.by_size_range.get_mut(amount).1.state.add(addressdata); } } else { - addressdata.send(value, prev_price)?; - vecs.by_size_range.get_mut(amount).1.state.send( + addressdata, value, price, prev_price, blocks_old, days_old, older_than_hour, - ); + )?; } Ok(()) diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs b/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs index 7b9d02480..9ab8c9b82 100644 --- a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs @@ -59,10 +59,7 @@ impl CohortVecs for Vecs { fn starting_height(&self) -> Height { [ - self.state - .price_to_amount - .height() - .map_or(Height::MAX, |h| h.incremented()), + self.state.height().map_or(Height::MAX, |h| h.incremented()), self.inner.starting_height(), ] .into_iter() diff --git a/crates/brk_core/src/structs/addressdata.rs b/crates/brk_core/src/structs/addressdata.rs index 88665a5df..dc26502a7 100644 --- a/crates/brk_core/src/structs/addressdata.rs +++ b/crates/brk_core/src/structs/addressdata.rs @@ -1,10 +1,9 @@ use byteview::ByteView; use zerocopy::{FromBytes, IntoBytes}; -use zerocopy_derive::{FromBytes, Immutable, KnownLayout}; -use crate::{CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats}; +use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats}; -#[derive(Debug, Default, Clone, FromBytes, Immutable, KnownLayout)] +#[derive(Debug, Default, Clone)] #[repr(C)] pub struct AddressData { pub sent: Sats, @@ -18,6 +17,10 @@ impl AddressData { (u64::from(self.received) - u64::from(self.sent)).into() } + pub fn realized_price(&self) -> Dollars { + (self.realized_cap / Bitcoin::from(self.amount())).round_nearest_cent() + } + #[inline(always)] pub fn has_0_sats(&self) -> bool { self.amount() == Sats::ZERO @@ -70,7 +73,13 @@ impl From<&EmptyAddressData> for AddressData { impl From for AddressData { fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() + Self { + // MUST be same order as impl From<&AddressData> for ByteView + sent: Sats::read_from_bytes(&value[..8]).unwrap(), + received: Sats::read_from_bytes(&value[8..16]).unwrap(), + realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(), + outputs_len: u32::read_from_bytes(&value[24..]).unwrap(), + } } } impl From for ByteView { diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs index 0cbb62303..e67172359 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_core/src/structs/dollars.rs @@ -38,6 +38,10 @@ impl Dollars { pub const fn mint(dollars: f64) -> Self { Self(dollars) } + + pub fn round_nearest_cent(self) -> Self { + Dollars((self.0 * 100.0).round() / 100.0) + } } impl From for Dollars { diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index 778c3897e..100c29bae 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -278,6 +278,8 @@ impl Stores { self.txidprefix_to_txindex.reset()?; } + self.keyspace.persist(PersistMode::SyncAll)?; + self.commit(starting_indexes.height.decremented().unwrap_or_default())?; Ok(()) diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 59de7ea9d..915309dad 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -85,9 +85,19 @@ impl<'a> Interface<'a> { vecs: Vec<(String, &&dyn AnyCollectableVec)>, params: &ParamsOpt, ) -> color_eyre::Result { - let from = params.from(); - let to = params.to(); - let format = params.format(); + let from = params.from().map(|from| { + vecs.iter() + .map(|(_, v)| v.i64_to_usize(from)) + .min() + .unwrap_or_default() + }); + + let to = params.to().map(|to| { + vecs.iter() + .map(|(_, v)| v.i64_to_usize(to)) + .min() + .unwrap_or_default() + }); let mut values = vecs .iter() @@ -96,6 +106,8 @@ impl<'a> Interface<'a> { }) .collect::>>()?; + let format = params.format(); + if values.is_empty() { return Ok(Output::default(format)); } diff --git a/crates/brk_server/src/api/interface/mod.rs b/crates/brk_server/src/api/interface/mod.rs index cf92afe3c..0b1b276b5 100644 --- a/crates/brk_server/src/api/interface/mod.rs +++ b/crates/brk_server/src/api/interface/mod.rs @@ -4,9 +4,7 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; -use brk_core::DateIndex; use brk_interface::{Format, Output, Params}; -use brk_vec::{CollectableVec, StoredVec}; use color_eyre::eyre::eyre; use crate::traits::{HeaderMapExtended, ResponseExtended}; @@ -52,11 +50,7 @@ fn req_to_response_res( let weight = vecs .iter() - .map(|(_, v)| { - let len = v.len(); - let count = StoredVec::::range_count(from, to, len); - count * v.value_type_to_size_of() - }) + .map(|(_, v)| v.range_weight(from, to)) .sum::(); if weight > MAX_WEIGHT { diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 687831ee4..0c1e88c30 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -13,3 +13,4 @@ repository.workspace = true brk_core = { workspace = true } byteview = { workspace = true } fjall = { workspace = true } +log = { workspace = true } diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 704b6ea7f..137553eca 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -14,13 +14,13 @@ use std::{ use brk_core::{Height, Result, Version}; use byteview::ByteView; use fjall::{ - PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace, - TransactionalPartitionHandle, + PartitionCreateOptions, ReadTransaction, TransactionalKeyspace, TransactionalPartitionHandle, }; mod meta; mod r#trait; +use log::info; use meta::*; pub use r#trait::*; @@ -241,12 +241,12 @@ where } fn reset(&mut self) -> Result<()> { + info!("Resetting {}...", self.name); + let partition: TransactionalPartitionHandle = self.partition.take().unwrap(); self.keyspace.delete_partition(partition)?; - self.keyspace.persist(PersistMode::SyncAll)?; - self.meta.reset(); let partition = diff --git a/crates/brk_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs index e9d35c1e2..8e50da293 100644 --- a/crates/brk_vec/src/traits/any.rs +++ b/crates/brk_vec/src/traits/any.rs @@ -2,6 +2,15 @@ use brk_core::Version; use super::{BoxedVecIterator, StoredIndex, StoredType}; +pub fn i64_to_usize(i: i64, len: usize) -> usize { + if i >= 0 { + (i as usize).min(len) + } else { + let v = len as i64 + i; + if v < 0 { 0 } else { v as usize } + } +} + pub trait AnyVec: Send + Sync { fn version(&self) -> Version; fn name(&self) -> &str; @@ -26,6 +35,12 @@ pub trait AnyVec: Send + Sync { self.version() ) } + + #[inline] + fn i64_to_usize(&self, i: i64) -> usize { + let len = self.len(); + i64_to_usize(i, len) + } } pub trait AnyIterableVec: AnyVec { diff --git a/crates/brk_vec/src/traits/collectable.rs b/crates/brk_vec/src/traits/collectable.rs index ed8f51b1f..a9aa2cc7e 100644 --- a/crates/brk_vec/src/traits/collectable.rs +++ b/crates/brk_vec/src/traits/collectable.rs @@ -1,5 +1,7 @@ use brk_core::{Error, Result}; +use crate::i64_to_usize; + use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType}; pub trait CollectableVec: AnyVec + AnyIterableVec @@ -25,7 +27,7 @@ where } #[inline] - fn i64_to_usize(i: i64, len: usize) -> usize { + fn i64_to_usize_(i: i64, len: usize) -> usize { if i >= 0 { (i as usize).min(len) } else { @@ -34,27 +36,19 @@ where } } - fn range_count(from: Option, to: Option, len: usize) -> usize { - let from = from.map(|i| Self::i64_to_usize(i, len)); - let to = to.map(|i| Self::i64_to_usize(i, len)); - (from.unwrap_or_default()..to.unwrap_or(len)).count() - } - - #[doc(hidden)] fn collect_signed_range(&self, from: Option, to: Option) -> Result> { - let len = self.len(); - let from = from.map(|i| Self::i64_to_usize(i, len)); - let to = to.map(|i| Self::i64_to_usize(i, len)); + let from = from.map(|i| self.i64_to_usize(i)); + let to = to.map(|i| self.i64_to_usize(i)); self.collect_range(from, to) } #[inline] fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { - self.collect_signed_range(from, to)? + self.collect_range(from, to)? .into_iter() .map(|v| serde_json::to_value(v).map_err(Error::from)) .collect::>>() @@ -72,7 +66,18 @@ where pub trait AnyCollectableVec: AnyVec { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result>; + + fn range_count(&self, from: Option, to: Option) -> usize { + let len = self.len(); + let from = from.map(|i| i64_to_usize(i, len)); + let to = to.map(|i| i64_to_usize(i, len)); + (from.unwrap_or_default()..to.unwrap_or(len)).count() + } + + fn range_weight(&self, from: Option, to: Option) -> usize { + self.range_count(from, to) * self.value_type_to_size_of() + } } diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 422431bc9..29d9b5905 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -528,8 +528,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/computed.rs b/crates/brk_vec/src/variants/computed.rs index 0a9cb68e9..3b876efb1 100644 --- a/crates/brk_vec/src/variants/computed.rs +++ b/crates/brk_vec/src/variants/computed.rs @@ -397,8 +397,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/eager.rs b/crates/brk_vec/src/variants/eager.rs index 70b83dc9d..befdff8a2 100644 --- a/crates/brk_vec/src/variants/eager.rs +++ b/crates/brk_vec/src/variants/eager.rs @@ -1336,8 +1336,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/indexed.rs b/crates/brk_vec/src/variants/indexed.rs index 6d4a2eeee..8be6e1b8c 100644 --- a/crates/brk_vec/src/variants/indexed.rs +++ b/crates/brk_vec/src/variants/indexed.rs @@ -166,8 +166,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/lazy1.rs b/crates/brk_vec/src/variants/lazy1.rs index 852681ee7..f14fa9d89 100644 --- a/crates/brk_vec/src/variants/lazy1.rs +++ b/crates/brk_vec/src/variants/lazy1.rs @@ -179,8 +179,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/lazy2.rs b/crates/brk_vec/src/variants/lazy2.rs index baf3d86c8..2d5bb249b 100644 --- a/crates/brk_vec/src/variants/lazy2.rs +++ b/crates/brk_vec/src/variants/lazy2.rs @@ -230,8 +230,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/lazy3.rs b/crates/brk_vec/src/variants/lazy3.rs index ed7ce5c01..8bf3d8594 100644 --- a/crates/brk_vec/src/variants/lazy3.rs +++ b/crates/brk_vec/src/variants/lazy3.rs @@ -272,8 +272,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 4bb3432c6..94a400a9e 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -359,8 +359,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) } diff --git a/crates/brk_vec/src/variants/stored.rs b/crates/brk_vec/src/variants/stored.rs index 465eb0a16..ffd6d06f6 100644 --- a/crates/brk_vec/src/variants/stored.rs +++ b/crates/brk_vec/src/variants/stored.rs @@ -272,8 +272,8 @@ where { fn collect_range_serde_json( &self, - from: Option, - to: Option, + from: Option, + to: Option, ) -> Result> { CollectableVec::collect_range_serde_json(self, from, to) }