diff --git a/crates/brk_computer/src/stateful/addresstype_to_vec.rs b/crates/brk_computer/src/stateful/addresstype_to_vec.rs index c629ca6c2..2bc15f5e5 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_vec.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_vec.rs @@ -1,3 +1,5 @@ +use std::mem; + use derive_deref::{Deref, DerefMut}; use super::ByAddressType; @@ -5,6 +7,40 @@ use super::ByAddressType; #[derive(Debug, Deref, DerefMut)] pub struct AddressTypeToVec(ByAddressType>); +impl AddressTypeToVec { + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + self + } + + pub fn merge_mut(&mut self, mut other: Self) { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + } + + fn merge_(own: &mut Vec, other: &mut Vec) { + if own.len() >= other.len() { + own.append(other); + } else { + other.append(own); + mem::swap(own, other); + } + } +} + impl Default for AddressTypeToVec { fn default() -> Self { Self(ByAddressType { diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 97b40f22e..3ae01ea2a 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1,4 +1,4 @@ -use std::{cmp::Ordering, collections::BTreeMap, mem, ops::ControlFlow, path::Path, thread}; +use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; use brk_core::{ AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex, @@ -567,11 +567,11 @@ impl Vecs { let dateindex_to_first_height = &indexes.dateindex_to_first_height; let dateindex_to_height_count = &indexes.dateindex_to_height_count; + let inputindex_to_outputindex_mmap = inputindex_to_outputindex.create_mmap()?; let outputindex_to_value_mmap = outputindex_to_value.create_mmap()?; let outputindex_to_outputtype_mmap = outputindex_to_outputtype.create_mmap()?; let outputindex_to_typeindex_mmap = outputindex_to_typeindex.create_mmap()?; - let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter(); let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); let mut height_to_first_p2aaddressindex_iter = height_to_first_p2aaddressindex.into_iter(); @@ -902,87 +902,96 @@ impl Vecs { .tick_tock_next_block(&chain_state, timestamp); }); - let received = scope.spawn(|| { - let mut transacted = Transacted::default(); + let (transacted, addresstype_to_typedindex_to_received_data, receiving_addresstype_to_typeindex_to_addressdatawithsource) = (first_outputindex..first_outputindex + *output_count) + .into_par_iter() + .map(OutputIndex::from) + .map(|outputindex| { + let value = outputindex_to_value + .unwrap_read(outputindex, &outputindex_to_value_mmap); - let mut addresstype_to_typedindex_to_received_data = - AddressTypeToVec::<(TypeIndex, Sats)>::default(); + let output_type = outputindex_to_outputtype + .unwrap_read(outputindex, &outputindex_to_outputtype_mmap); - let mut addresstype_to_typeindex_to_addressdatawithsource = - AddressTypeToTypeIndexTree::default(); + if output_type.is_not_address() { + return (value, output_type, None); + } - let _ = (first_outputindex..first_outputindex + *output_count) - .map(OutputIndex::from) - .try_for_each(|outputindex| -> ControlFlow<()> { - let value = outputindex_to_value - .unwrap_read(outputindex, &outputindex_to_value_mmap); + let typeindex = outputindex_to_typeindex + .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); - let output_type = outputindex_to_outputtype - .unwrap_read(outputindex, &outputindex_to_outputtype_mmap); + let addressdata_opt = Self::get_addressdatawithsource( + output_type, + typeindex, + &first_addressindexes, + &addresstype_to_typeindex_to_loadedaddressdata, + &addresstype_to_typeindex_to_emptyaddressdata, + &addresstypeindex_to_anyaddressindex_mmap_opt, + &anyaddressindex_to_anyaddressdata_mmap_opt, + &self.p2pk33addressindex_to_anyaddressindex, + &self.p2pk65addressindex_to_anyaddressindex, + &self.p2pkhaddressindex_to_anyaddressindex, + &self.p2shaddressindex_to_anyaddressindex, + &self.p2traddressindex_to_anyaddressindex, + &self.p2wpkhaddressindex_to_anyaddressindex, + &self.p2wshaddressindex_to_anyaddressindex, + &self.p2aaddressindex_to_anyaddressindex, + &self.loadedaddressindex_to_loadedaddressdata, + &self.emptyaddressindex_to_emptyaddressdata, + ); - transacted.iterate(value, output_type); + (value, output_type, Some((typeindex, addressdata_opt))) + }).fold( + || { + ( + Transacted::default(), + AddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexTree::default() + ) + }, + |(mut transacted, mut addresstype_to_typedindex_to_data, mut addresstype_to_typeindex_to_addressdatawithsource), + ( + value, + output_type, + typeindex_with_addressdata_opt, + )| { + transacted.iterate(value, output_type); - if output_type.is_not_address() { - return ControlFlow::Continue(()); - } - - let typeindex = outputindex_to_typeindex - .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); - - if let Some(addressdata) = Self::get_addressdatawithsource( - output_type, - typeindex, - &first_addressindexes, - &addresstype_to_typeindex_to_loadedaddressdata, - &addresstype_to_typeindex_to_emptyaddressdata, - &addresstypeindex_to_anyaddressindex_mmap_opt, - &anyaddressindex_to_anyaddressdata_mmap_opt, - &self.p2pk33addressindex_to_anyaddressindex, - &self.p2pk65addressindex_to_anyaddressindex, - &self.p2pkhaddressindex_to_anyaddressindex, - &self.p2shaddressindex_to_anyaddressindex, - &self.p2traddressindex_to_anyaddressindex, - &self.p2wpkhaddressindex_to_anyaddressindex, - &self.p2wshaddressindex_to_anyaddressindex, - &self.p2aaddressindex_to_anyaddressindex, - &self.loadedaddressindex_to_loadedaddressdata, - &self.emptyaddressindex_to_emptyaddressdata, - ) { + if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt { + if let Some(addressdata) = addressdata_opt + { addresstype_to_typeindex_to_addressdatawithsource .get_mut(output_type) .unwrap() .insert(typeindex, addressdata); } - addresstype_to_typedindex_to_received_data + addresstype_to_typedindex_to_data .get_mut(output_type) .unwrap() .push((typeindex, value)); + } - ControlFlow::Continue(()) - }); - - ( - transacted, - addresstype_to_typedindex_to_received_data, - addresstype_to_typeindex_to_addressdatawithsource, - ) - }); - - let mut height_to_sent = BTreeMap::::default(); - - let mut addresstype_to_typedindex_to_sent_data = - HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(); - - let mut sending_addresstype_to_typeindex_to_addressdatawithsource = - AddressTypeToTypeIndexTree::default(); + (transacted, addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource) + }).reduce( + || { + ( + Transacted::default(), + AddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexTree::default() + ) + }, + |(transacted, addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource), (transacted2, addresstype_to_typedindex_to_data2, addresstype_to_typeindex_to_addressdatawithsource2)| { + (transacted + transacted2, addresstype_to_typedindex_to_data.merge(addresstype_to_typedindex_to_data2), addresstype_to_typeindex_to_addressdatawithsource.merge(addresstype_to_typeindex_to_addressdatawithsource2)) + }, + ); // Skip coinbase - let _ = (first_inputindex + 1..first_inputindex + *input_count) + let (height_to_sent, addresstype_to_typedindex_to_sent_data, sending_addresstype_to_typeindex_to_addressdatawithsource) = (first_inputindex + 1..first_inputindex + *input_count) + .into_par_iter() .map(InputIndex::from) - .try_for_each(|inputindex| -> ControlFlow<()> { + .map(|inputindex| { let outputindex = - inputindex_to_outputindex_iter.unwrap_get_inner(inputindex); + inputindex_to_outputindex.unwrap_read(inputindex, &inputindex_to_outputindex_mmap); let value = outputindex_to_value .unwrap_read(outputindex, &outputindex_to_value_mmap); @@ -993,19 +1002,14 @@ impl Vecs { let prev_height = *outputindex_range_to_height.get(outputindex).unwrap(); - height_to_sent - .entry(prev_height) - .or_default() - .iterate(value, input_type); - if input_type.is_not_address() { - return ControlFlow::Continue(()); + return (prev_height, value, input_type, None); } let typeindex = outputindex_to_typeindex .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); - if let Some(addressdata) = Self::get_addressdatawithsource( + let addressdata_opt = Self::get_addressdatawithsource( input_type, typeindex, &first_addressindexes, @@ -1023,28 +1027,77 @@ impl Vecs { &self.p2aaddressindex_to_anyaddressindex, &self.loadedaddressindex_to_loadedaddressdata, &self.emptyaddressindex_to_emptyaddressdata, - ) { - sending_addresstype_to_typeindex_to_addressdatawithsource - .get_mut(input_type) - .unwrap() - .insert(typeindex, addressdata); - } + ); - addresstype_to_typedindex_to_sent_data + (prev_height, value, input_type, Some((typeindex, addressdata_opt))) + }).fold( + || { + ( + BTreeMap::::default(), + HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexTree::default() + ) + }, + |(mut height_to_transacted, mut height_to_addresstype_to_typedindex_to_data, mut addresstype_to_typeindex_to_addressdatawithsource), + ( + prev_height, + value, + output_type, + typeindex_with_addressdata_opt, + )| { + height_to_transacted .entry(prev_height) .or_default() - .get_mut(input_type) - .unwrap() - .push((typeindex, value)); + .iterate(value, output_type); - ControlFlow::Continue(()) - }); + if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt { + if let Some(addressdata) = addressdata_opt + { + addresstype_to_typeindex_to_addressdatawithsource + .get_mut(output_type) + .unwrap() + .insert(typeindex, addressdata); + } - let ( - transacted, - addresstype_to_typedindex_to_received_data, - receiving_addresstype_to_typeindex_to_addressdatawithsource, - ) = received.join().unwrap(); + height_to_addresstype_to_typedindex_to_data + .entry(height) + .or_default() + .get_mut(output_type) + .unwrap() + .push((typeindex, value)); + } + + (height_to_transacted, height_to_addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource) + }).reduce( + || { + ( + BTreeMap::::default(), + HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexTree::default() + ) + }, + |(height_to_transacted, addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource), (height_to_transacted2, addresstype_to_typedindex_to_data2, addresstype_to_typeindex_to_addressdatawithsource2)| { + let (mut height_to_transacted, height_to_transacted_consumed) = if height_to_transacted.len() > height_to_transacted2.len() { + (height_to_transacted, height_to_transacted2) + } else { + (height_to_transacted2, height_to_transacted) + }; + height_to_transacted_consumed.into_iter().for_each(|(k, v)| { + *height_to_transacted.entry(k).or_default() += v; + }); + + let (mut addresstype_to_typedindex_to_data, addresstype_to_typedindex_to_data_consumed) = if addresstype_to_typedindex_to_data.len() > addresstype_to_typedindex_to_data2.len() { + (addresstype_to_typedindex_to_data, addresstype_to_typedindex_to_data2) + } else { + (addresstype_to_typedindex_to_data2, addresstype_to_typedindex_to_data) + }; + addresstype_to_typedindex_to_data_consumed.0.into_iter().for_each(|(k, v)| { + addresstype_to_typedindex_to_data.entry(k).or_default().merge_mut(v); + }); + + (height_to_transacted, addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource.merge(addresstype_to_typeindex_to_addressdatawithsource2)) + }, + ); let addresstype_to_typeindex_to_addressdatawithsource = receiving_addresstype_to_typeindex_to_addressdatawithsource @@ -1116,6 +1169,7 @@ impl Vecs { } .iterate(Sats::FIFTY_BTC, OutputType::P2PK65); }; + // Push current block state before processing sends and receives chain_state.push(BlockState { supply: transacted.spendable_supply.clone(), @@ -1128,18 +1182,6 @@ impl Vecs { self.utxo_cohorts.send(height_to_sent, &mut chain_state); }); - let mut separate_utxo_vecs = self.utxo_cohorts.as_mut_separate_vecs(); - - separate_utxo_vecs - .iter_mut() - .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; - - let mut separate_address_vecs = self.address_cohorts.as_mut_separate_vecs(); - - separate_address_vecs - .iter_mut() - .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; - self.height_to_unspendable_supply.forced_push_at( height, unspendable_supply, @@ -1172,15 +1214,17 @@ impl Vecs { .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex))); let dateindex = is_date_last_height.then_some(dateindex); - separate_utxo_vecs + + self.utxo_cohorts.as_mut_separate_vecs() .into_par_iter() .map(|(_, v)| v as &mut dyn DynCohortVecs) .chain( - separate_address_vecs + self.address_cohorts.as_mut_separate_vecs() .into_par_iter() .map(|(_, v)| v as &mut dyn DynCohortVecs), ) .try_for_each(|v| { + v.forced_pushed_at(height, exit)?; v.compute_then_force_push_unrealized_states( height, price, dateindex, date_price, exit, ) diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index b8c1e9957..e73fdcbe1 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -1506,13 +1506,16 @@ impl Vecs { let last_timestamp = chain_state.last().unwrap().timestamp; let current_price = chain_state.last().unwrap().price; + let chain_state_len = chain_state.len(); + height_to_sent.into_iter().for_each(|(height, sent)| { chain_state[height.unwrap_to_usize()].supply -= &sent.spendable_supply; let block_state = chain_state.get(height.unwrap_to_usize()).unwrap(); + let prev_price = block_state.price; - let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize(); + let blocks_old = chain_state_len - 1 - height.unwrap_to_usize(); let days_old = last_timestamp.difference_in_days_between(block_state.timestamp); let days_old_float = diff --git a/crates/brk_computer/src/states/cohorts/common.rs b/crates/brk_computer/src/states/cohorts/common.rs index ec63241d6..c2630e14f 100644 --- a/crates/brk_computer/src/states/cohorts/common.rs +++ b/crates/brk_computer/src/states/cohorts/common.rs @@ -223,17 +223,15 @@ impl CohortState { let update_state = |price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| { match price.cmp(¤t_price) { - Ordering::Equal => { - state.supply_even += sats; - } Ordering::Less => { state.supply_in_profit += sats; if price > Dollars::ZERO && current_price > Dollars::ZERO { let diff = current_price.checked_sub(price).unwrap(); - if diff <= Dollars::ZERO { - dbg!(price, current_price, diff, sats); - panic!(); - } + // Add back once in a while to verify, but generally not needed + // if diff <= Dollars::ZERO { + // dbg!(price, current_price, diff, sats); + // panic!(); + // } state.unrealized_profit += diff * sats; } } @@ -241,13 +239,17 @@ impl CohortState { state.supply_in_loss += sats; if price > Dollars::ZERO && current_price > Dollars::ZERO { let diff = price.checked_sub(current_price).unwrap(); - if diff <= Dollars::ZERO { - dbg!(price, current_price, diff, sats); - panic!(); - } + // Add back once in a while to verify, but generally not needed + // if diff <= Dollars::ZERO { + // dbg!(price, current_price, diff, sats); + // panic!(); + // } state.unrealized_loss += diff * sats; } } + Ordering::Equal => { + state.supply_even += sats; + } } }; diff --git a/crates/brk_core/src/groups/filter.rs b/crates/brk_core/src/groups/filter.rs index d6a738b4e..73098b54a 100644 --- a/crates/brk_core/src/groups/filter.rs +++ b/crates/brk_core/src/groups/filter.rs @@ -15,12 +15,11 @@ pub enum GroupFilter { impl GroupFilter { pub fn contains(&self, value: usize) -> bool { match self { - GroupFilter::All => true, + GroupFilter::Range(r) => r.contains(&value), GroupFilter::LowerThan(max) => *max > value, GroupFilter::GreaterOrEqual(min) => *min <= value, - GroupFilter::Range(r) => r.contains(&value), - GroupFilter::Epoch(_) => false, - GroupFilter::Type(_) => false, + GroupFilter::All => true, + GroupFilter::Epoch(_) | GroupFilter::Type(_) => false, } } @@ -28,24 +27,22 @@ impl GroupFilter { match self { GroupFilter::All => true, GroupFilter::LowerThan(max) => match other { - GroupFilter::All => false, GroupFilter::LowerThan(max2) => max >= max2, GroupFilter::Range(range) => range.end <= *max, - GroupFilter::GreaterOrEqual(_) => false, - GroupFilter::Epoch(_) => false, - GroupFilter::Type(_) => false, + GroupFilter::All + | GroupFilter::GreaterOrEqual(_) + | GroupFilter::Epoch(_) + | GroupFilter::Type(_) => false, }, GroupFilter::GreaterOrEqual(min) => match other { - GroupFilter::All => false, - GroupFilter::LowerThan(_) => false, GroupFilter::Range(range) => range.start >= *min, GroupFilter::GreaterOrEqual(min2) => min <= min2, - GroupFilter::Epoch(_) => false, - GroupFilter::Type(_) => false, + GroupFilter::All + | GroupFilter::LowerThan(_) + | GroupFilter::Epoch(_) + | GroupFilter::Type(_) => false, }, - GroupFilter::Range(_) => false, - GroupFilter::Epoch(_) => false, - GroupFilter::Type(_) => false, + GroupFilter::Range(_) | GroupFilter::Epoch(_) | GroupFilter::Type(_) => false, } } }