Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 15:19:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: store part 8

Browse Source
This commit is contained in:
nym21
2025-07-03 18:19:36 +02:00
parent 5810276156
commit be4e693a27
29 changed files with 745 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
crates/brk_computer/src/states/cohorts/address.rs
View File

@@ -20,31 +20,80 @@ impl AddressCohortState {
})
}
pub fn height(&self) -> Option<Height> {
self.inner.height()
}
pub fn reset_price_to_amount(&mut self) -> Result<()> {
self.inner.reset_price_to_amount()
}
pub fn reset_single_iteration_values(&mut self) {
self.inner.reset_single_iteration_values();
}
#[allow(clippy::too_many_arguments)]
pub fn send(
&mut self,
addressdata: &mut AddressData,
value: Sats,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
) {
self.inner.send(
) -> Result<()> {
let compute_price = current_price.is_some();
let prev_realized_price = compute_price.then(|| addressdata.realized_price());
let prev_supply_state = SupplyState {
utxos: addressdata.outputs_len as usize,
value: addressdata.amount(),
};
addressdata.send(value, prev_price)?;
let supply_state = SupplyState {
utxos: addressdata.outputs_len as usize,
value: addressdata.amount(),
};
self.inner.send_(
&SupplyState { utxos: 1, value },
current_price,
prev_price,
blocks_old,
days_old,
older_than_hour,
compute_price.then(|| (addressdata.realized_price(), &supply_state)),
prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)),
);
Ok(())
}
pub fn receive(&mut self, value: Sats, price: Option<Dollars>) {
self.inner.receive(&SupplyState { utxos: 1, value }, price);
pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) {
let compute_price = price.is_some();
let prev_realized_price = compute_price.then(|| address_data.realized_price());
let prev_supply_state = SupplyState {
utxos: address_data.outputs_len as usize,
value: address_data.amount(),
};
address_data.receive(value, price);
let supply_state = SupplyState {
utxos: address_data.outputs_len as usize,
value: address_data.amount(),
};
self.inner.receive_(
&SupplyState { utxos: 1, value },
price,
compute_price.then(|| (address_data.realized_price(), &supply_state)),
prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)),
);
}
pub fn add(&mut self, addressdata: &AddressData) {
@@ -54,7 +103,7 @@ impl AddressCohortState {
}
pub fn subtract(&mut self, addressdata: &AddressData) {
self.address_count.checked_sub(1).unwrap();
self.address_count = self.address_count.checked_sub(1).unwrap();
self.inner
.decrement_(&addressdata.into(), addressdata.realized_cap);
}
@@ -63,39 +112,3 @@ impl AddressCohortState {
self.inner.commit(height)
}
}
// fn decrement(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
// self.inner.decrement(supply_state, price);
// }
// fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) {
// self.inner.decrement_price_to_amount(supply_state, price);
// }
// fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
// self.inner.receive(supply_state, price);
// }
// fn compute_unrealized_states(
// &self,
// height_price: Dollars,
// date_price: Option<Dollars>,
// ) -> (UnrealizedState, Option<UnrealizedState>) {
// self.inner
// .compute_unrealized_states(height_price, date_price)
// }
// }
// impl Deref for AddressCohortState {
// type Target = CohortState;
// fn deref(&self) -> &Self::Target {
// &self.inner
// }
// }
// impl DerefMut for AddressCohortState {
// fn deref_mut(&mut self) -> &mut Self::Target {
// &mut self.inner
// }
// }

103
crates/brk_computer/src/states/cohorts/common.rs
View File

@@ -10,7 +10,8 @@ pub struct CohortState {
pub realized: Option<RealizedState>,
pub satblocks_destroyed: Sats,
pub satdays_destroyed: Sats,
pub price_to_amount: PriceToAmount,
price_to_amount: PriceToAmount,
}
impl CohortState {
@@ -24,6 +25,22 @@ impl CohortState {
})
}
pub fn height(&self) -> Option<Height> {
self.price_to_amount.height()
}
pub fn reset_price_to_amount(&mut self) -> Result<()> {
self.price_to_amount.reset()
}
pub fn price_to_amount_first_key_value(&self) -> Option<(&Dollars, &Sats)> {
self.price_to_amount.first_key_value()
}
pub fn price_to_amount_last_key_value(&self) -> Option<(&Dollars, &Sats)> {
self.price_to_amount.last_key_value()
}
pub fn reset_single_iteration_values(&mut self) {
self.satdays_destroyed = Sats::ZERO;
self.satblocks_destroyed = Sats::ZERO;
@@ -39,7 +56,7 @@ impl CohortState {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.increment(supply_state, price);
*self.price_to_amount.entry(price).or_default() += supply_state.value;
self.price_to_amount.increment(price, supply_state);
}
}
}
@@ -50,10 +67,10 @@ impl CohortState {
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
realized.increment_(realized_cap);
*self
.price_to_amount
.entry(realized_cap / Bitcoin::from(supply_state.value))
.or_default() += supply_state.value;
self.price_to_amount.increment(
realized_cap / Bitcoin::from(supply_state.value),
supply_state,
);
}
}
}
@@ -65,7 +82,7 @@ impl CohortState {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.decrement(supply_state, price);
self.decrement_price_to_amount(supply_state, price);
self.price_to_amount.decrement(price, supply_state);
}
}
}
@@ -76,30 +93,47 @@ impl CohortState {
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
realized.decrement_(realized_cap);
self.decrement_price_to_amount(
self.price_to_amount.decrement(
(realized_cap / Bitcoin::from(supply_state.value)).round_nearest_cent(),
supply_state,
realized_cap / Bitcoin::from(supply_state.value),
);
}
}
}
fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) {
let amount = self.price_to_amount.get_mut(&price).unwrap();
*amount -= supply_state.value;
if *amount == Sats::ZERO {
self.price_to_amount.remove(&price);
}
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.receive_(
supply_state,
price,
price.map(|price| (price, supply_state)),
None,
);
}
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
pub fn receive_(
&mut self,
supply_state: &SupplyState,
price: Option<Dollars>,
price_to_amount_increment: Option<(Dollars, &SupplyState)>,
price_to_amount_decrement: Option<(Dollars, &SupplyState)>,
) {
self.supply += supply_state;
if supply_state.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.receive(supply_state, price);
*self.price_to_amount.entry(price).or_default() += supply_state.value;
if let Some((price, supply)) = price_to_amount_increment
&& supply.value.is_not_zero()
{
self.price_to_amount.increment(price, supply);
}
if let Some((price, supply)) = price_to_amount_decrement
&& supply.value.is_not_zero()
{
self.price_to_amount.decrement(price, supply);
}
}
}
}
@@ -112,6 +146,30 @@ impl CohortState {
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
) {
self.send_(
supply_state,
current_price,
prev_price,
blocks_old,
days_old,
older_than_hour,
None,
prev_price.map(|prev_price| (prev_price, supply_state)),
);
}
#[allow(clippy::too_many_arguments)]
pub fn send_(
&mut self,
supply_state: &SupplyState,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
price_to_amount_increment: Option<(Dollars, &SupplyState)>,
price_to_amount_decrement: Option<(Dollars, &SupplyState)>,
) {
if supply_state.utxos == 0 {
return;
@@ -129,7 +187,16 @@ impl CohortState {
let current_price = current_price.unwrap();
let prev_price = prev_price.unwrap();
realized.send(supply_state, current_price, prev_price, older_than_hour);
self.decrement_price_to_amount(supply_state, prev_price);
if let Some((price, supply)) = price_to_amount_increment
&& supply.value.is_not_zero()
{
self.price_to_amount.increment(price, supply);
}
if let Some((price, supply)) = price_to_amount_decrement
&& supply.value.is_not_zero()
{
self.price_to_amount.decrement(price, supply);
}
}
}
}

10
crates/brk_computer/src/states/cohorts/utxo.rs
View File

@@ -1,6 +1,6 @@
use std::path::Path;
use brk_core::Result;
use brk_core::{Height, Result};
use derive_deref::{Deref, DerefMut};
use super::CohortState;
@@ -16,4 +16,12 @@ impl UTXOCohortState {
compute_dollars,
)?))
}
pub fn height(&self) -> Option<Height> {
self.0.height()
}
pub fn reset_price_to_amount(&mut self) -> Result<()> {
self.0.reset_price_to_amount()
}
}

46
crates/brk_computer/src/states/price_to_amount.rs
View File

@@ -2,7 +2,6 @@ use std::{
collections::BTreeMap,
fs::{self, File},
io::{BufReader, BufWriter},
ops::{Deref, DerefMut},
path::{Path, PathBuf},
};
@@ -11,6 +10,8 @@ use brk_core::{Dollars, Height, Result, Sats};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use crate::states::SupplyState;
#[derive(Clone, Debug)]
pub struct PriceToAmount {
pathbuf: PathBuf,
@@ -46,8 +47,36 @@ impl PriceToAmount {
})
}
pub fn iter(&self) -> impl Iterator<Item = (&Dollars, &Sats)> {
self.state.iter()
}
pub fn is_empty(&self) -> bool {
self.state.is_empty()
}
pub fn first_key_value(&self) -> Option<(&Dollars, &Sats)> {
self.state.first_key_value()
}
pub fn last_key_value(&self) -> Option<(&Dollars, &Sats)> {
self.state.last_key_value()
}
pub fn increment(&mut self, price: Dollars, supply_state: &SupplyState) {
*self.state.entry(price).or_default() += supply_state.value;
}
pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) {
let amount = self.state.get_mut(&price).unwrap();
*amount -= supply_state.value;
if *amount == Sats::ZERO {
self.state.remove(&price);
}
}
pub fn reset(&mut self) -> Result<()> {
self.clear();
self.state.clear();
self.height = None;
fs::remove_dir_all(&self.pathbuf)?;
fs::create_dir_all(&self.pathbuf)?;
@@ -89,16 +118,3 @@ impl PriceToAmount {
path.join("height")
}
}
impl Deref for PriceToAmount {
type Target = BTreeMap<Dollars, Sats>;
fn deref(&self) -> &Self::Target {
&self.state
}
}
impl DerefMut for PriceToAmount {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.state
}
}

9
crates/brk_computer/src/stores.rs
View File

@@ -10,7 +10,7 @@ use brk_core::{
};
use brk_store::{AnyStore, Store};
use fjall::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*;
use log::info;
use crate::{
GroupedByAddressType,
@@ -484,10 +484,15 @@ impl Stores {
}
pub fn reset(&mut self) -> Result<()> {
info!("Resetting stores...");
info!("> If it gets stuck here, stop the program and start it again");
self.as_mut_slice()
.into_par_iter()
.into_iter()
.try_for_each(|store| store.reset())?;
info!("Persisting stores...");
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())

6
crates/brk_computer/src/vecs/stateful/address_cohort.rs
View File

@@ -71,11 +71,7 @@ impl CohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state
.inner
.price_to_amount
.height()
.map_or(Height::MAX, |h| h.incremented()),
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.height_to_address_count.len().into(),
self.inner.starting_height(),
]

87
crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs
View File

@@ -1,8 +1,10 @@
use std::mem;
use brk_core::TypeIndex;
use brk_core::{AddressData, Dollars, OutputIndex, Sats, TypeIndex};
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::WithAddressDataSource;
use super::GroupedByAddressType;
#[derive(Debug, Default, Deref, DerefMut)]
@@ -33,3 +35,86 @@ impl<T> AddressTypeToTypeIndexVec<T> {
self.0
}
}
impl AddressTypeToTypeIndexVec<OutputIndex> {
#[allow(clippy::type_complexity)]
pub fn extend_from_sent(
&mut self,
other: &AddressTypeToTypeIndexVec<(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
)>,
) {
Self::extend_from_sent_(&mut self.p2pk33, &other.p2pk33);
Self::extend_from_sent_(&mut self.p2pkh, &other.p2pkh);
Self::extend_from_sent_(&mut self.p2sh, &other.p2sh);
Self::extend_from_sent_(&mut self.p2wpkh, &other.p2wpkh);
Self::extend_from_sent_(&mut self.p2wsh, &other.p2wsh);
Self::extend_from_sent_(&mut self.p2tr, &other.p2tr);
Self::extend_from_sent_(&mut self.p2a, &other.p2a);
}
#[allow(clippy::type_complexity)]
fn extend_from_sent_(
own: &mut Vec<(TypeIndex, OutputIndex)>,
other: &[(
TypeIndex,
(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
),
)],
) {
own.extend(
other
.iter()
.map(|(type_index, (output_index, ..))| (*type_index, *output_index)),
);
}
pub fn extend_from_received(
&mut self,
other: &AddressTypeToTypeIndexVec<(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
)>,
) {
Self::extend_from_received_(&mut self.p2pk33, &other.p2pk33);
Self::extend_from_received_(&mut self.p2pkh, &other.p2pkh);
Self::extend_from_received_(&mut self.p2sh, &other.p2sh);
Self::extend_from_received_(&mut self.p2wpkh, &other.p2wpkh);
Self::extend_from_received_(&mut self.p2wsh, &other.p2wsh);
Self::extend_from_received_(&mut self.p2tr, &other.p2tr);
Self::extend_from_received_(&mut self.p2a, &other.p2a);
}
#[allow(clippy::type_complexity)]
fn extend_from_received_(
own: &mut Vec<(TypeIndex, OutputIndex)>,
other: &[(
TypeIndex,
(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
),
)],
) {
own.extend(
other
.iter()
.map(|(type_index, (output_index, ..))| (*type_index, *output_index)),
);
}
}

6
crates/brk_computer/src/vecs/stateful/common.rs
View File

@@ -1341,8 +1341,7 @@ impl Vecs {
.forced_push_at(
height,
state
.price_to_amount
.first_key_value()
.price_to_amount_first_key_value()
.map(|(&dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
exit,
@@ -1353,8 +1352,7 @@ impl Vecs {
.forced_push_at(
height,
state
.price_to_amount
.last_key_value()
.price_to_amount_last_key_value()
.map(|(&dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
exit,

595
crates/brk_computer/src/vecs/stateful/mod.rs
View File

@@ -38,7 +38,7 @@ pub use addresstype_to_typeindex_vec::*;
use r#trait::CohortVecs;
pub use withaddressdatasource::WithAddressDataSource;
const VERSION: Version = Version::new(5);
const VERSION: Version = Version::new(9);
#[derive(Clone)]
pub struct Vecs {
@@ -401,13 +401,17 @@ impl Vecs {
stores.reset()?;
info!("Resetting utxo price maps...");
separate_utxo_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
info!("Resetting address price maps...");
separate_address_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.inner.price_to_amount.reset())?;
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
}
if starting_height < Height::from(height_to_date_fixed.len()) {
@@ -485,11 +489,13 @@ impl Vecs {
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let ((mut height_to_sent, new_addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_data), (mut received, new_addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_data)) = thread::scope(|s| {
let (
(mut height_to_sent, addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_data),
(mut received, addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_data),
) = thread::scope(|s| {
if chain_state_starting_height <= height {
s.spawn(|| {
self.utxo_vecs
.tick_tock_next_block(&chain_state, timestamp);
self.utxo_vecs.tick_tock_next_block(&chain_state, timestamp);
});
}
@@ -527,8 +533,17 @@ impl Vecs {
unreachable!()
}
let addressdata_opt = if input_type.is_address() && !addresstype_to_typeindex_to_addressdata.get(input_type).unwrap().contains_key(&typeindex) {
Some(WithAddressDataSource::FromAddressDataStore( stores.get_addressdata(input_type, typeindex).unwrap().unwrap()))
let addressdata_opt = if input_type.is_address()
&& !addresstype_to_typeindex_to_addressdata
.get(input_type)
.unwrap()
.contains_key(&typeindex)
&& let Some(address_data) =
stores.get_addressdata(input_type, typeindex).unwrap()
{
Some(WithAddressDataSource::FromAddressDataStore(
address_data,
))
} else {
None
};
@@ -545,65 +560,126 @@ impl Vecs {
.unwrap()
.into_owned();
let prev_price = height_to_close.map(|m| *m.get_or_read(prev_height, height_to_close_mmap.as_ref().unwrap()).unwrap()
.unwrap()
.into_owned());
let prev_price = height_to_close.map(|m| {
*m.get_or_read(
prev_height,
height_to_close_mmap.as_ref().unwrap(),
)
.unwrap()
.unwrap()
.into_owned()
});
let prev_timestamp = height_to_timestamp_fixed.get_or_read(prev_height, &height_to_timestamp_fixed_mmap)
.unwrap()
.unwrap()
.into_owned();
let prev_timestamp = height_to_timestamp_fixed
.get_or_read(prev_height, &height_to_timestamp_fixed_mmap)
.unwrap()
.unwrap()
.into_owned();
let blocks_old = height.unwrap_to_usize() - prev_height.unwrap_to_usize();
let blocks_old =
height.unwrap_to_usize() - prev_height.unwrap_to_usize();
let days_old = prev_timestamp
.difference_in_days_between_float(timestamp);
let days_old =
prev_timestamp.difference_in_days_between_float(timestamp);
let older_than_hour = timestamp
.checked_sub(prev_timestamp)
.unwrap()
.is_more_than_hour();
(prev_height, value, input_type, typeindex, outputindex, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour)
(
prev_height,
value,
input_type,
typeindex,
outputindex,
addressdata_opt,
prev_price,
blocks_old,
days_old,
older_than_hour,
)
})
.fold(
|| {
(
BTreeMap::<Height, Transacted>::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(Sats, Option<WithAddressDataSource<AddressData>>, Option<Dollars>, usize, f64, bool)>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
)>::default(
),
)
},
|(mut tree, mut vecs, mut vecs2), (height, value, input_type, typeindex, outputindex, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour)| {
|(mut tree, mut vecs, mut vecs2),
(
height,
value,
input_type,
typeindex,
outputindex,
addressdata_opt,
prev_price,
blocks_old,
days_old,
older_than_hour,
)| {
tree.entry(height).or_default().iterate(value, input_type);
if let Some(vec) = vecs.get_mut(input_type) {
vec.push((typeindex, outputindex));
}
if let Some(vec) = vecs2.get_mut(input_type) {
vec.push((typeindex, (value, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour)));
vec.push((
typeindex,
(
value,
addressdata_opt,
prev_price,
blocks_old,
days_old,
older_than_hour,
),
));
}
(tree, vecs, vecs2)
},
)
.reduce( || {
(
BTreeMap::<Height, Transacted>::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(Sats, Option<WithAddressDataSource<AddressData>>, Option<Dollars>, usize, f64, bool)>::default(),
)
}, |(first_tree, mut source_vecs,mut source_vecs2), (second_tree, other_vecs, other_vecs2)| {
let (mut tree_source, tree_to_consume) = if first_tree.len() > second_tree.len() {
(first_tree, second_tree)
} else {
(second_tree, first_tree)
};
tree_to_consume.into_iter().for_each(|(k, v)| {
*tree_source.entry(k).or_default() += v;
});
source_vecs.merge(other_vecs);
source_vecs2.merge(other_vecs2);
(tree_source, source_vecs, source_vecs2)
})
.reduce(
|| {
(
BTreeMap::<Height, Transacted>::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
)>::default(
),
)
},
|(first_tree, mut source_vecs, mut source_vecs2), (second_tree, other_vecs, other_vecs2)| {
let (mut tree_source, tree_to_consume) =
if first_tree.len() > second_tree.len() {
(first_tree, second_tree)
} else {
(second_tree, first_tree)
};
tree_to_consume.into_iter().for_each(|(k, v)| {
*tree_source.entry(k).or_default() += v;
});
source_vecs.merge(other_vecs);
source_vecs2.merge(other_vecs2);
(tree_source, source_vecs, source_vecs2)
},
)
});
let received_handle = s.spawn(|| {
@@ -629,14 +705,35 @@ impl Vecs {
.unwrap()
.into_owned();
let addressdata_opt = if output_type.is_address() && !addresstype_to_typeindex_to_addressdata.get(output_type).unwrap().contains_key(&typeindex) && !addresstype_to_typeindex_to_emptyaddressdata.get(output_type).unwrap().contains_key(&typeindex) {
Some(if let Some(addressdata) = stores.get_addressdata(output_type, typeindex).unwrap() {
WithAddressDataSource::FromAddressDataStore(addressdata)
} else if let Some(emptyaddressdata) = stores.get_emptyaddressdata(output_type, typeindex).unwrap() {
WithAddressDataSource::FromEmptyAddressDataStore(emptyaddressdata.into())
} else {
WithAddressDataSource::New(AddressData::default())
})
let addressdata_opt = if output_type.is_address()
&& !addresstype_to_typeindex_to_addressdata
.get(output_type)
.unwrap()
.contains_key(&typeindex)
&& !addresstype_to_typeindex_to_emptyaddressdata
.get(output_type)
.unwrap()
.contains_key(&typeindex)
{
Some(
if let Some(addressdata) = stores
.get_addressdata(output_type, typeindex)
.unwrap()
{
WithAddressDataSource::FromAddressDataStore(
addressdata,
)
} else if let Some(emptyaddressdata) = stores
.get_emptyaddressdata(output_type, typeindex)
.unwrap()
{
WithAddressDataSource::FromEmptyAddressDataStore(
emptyaddressdata.into(),
)
} else {
WithAddressDataSource::New(AddressData::default())
},
)
} else {
None
};
@@ -644,52 +741,68 @@ impl Vecs {
(value, output_type, typeindex, outputindex, addressdata_opt)
})
.fold(
|| (Transacted::default(), AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(Sats, Option<WithAddressDataSource<AddressData>>)>::default(),
),
|(mut transacted, mut vecs, mut vecs2), (value, output_type, typeindex, outputindex, addressdata_opt)| {
|| {
(
Transacted::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
)>::default(
),
)
},
|(mut transacted, mut vecs, mut vecs2),
(
value,
output_type,
typeindex,
outputindex,
addressdata_opt,
)| {
transacted.iterate(value, output_type);
if let Some(vec) = vecs.get_mut(output_type) {
vec.push((typeindex, outputindex));
vec.push((
typeindex,
outputindex,
));
}
if let Some(vec) = vecs2.get_mut(output_type) {
vec.push((typeindex, (value, addressdata_opt)));
vec.push((
typeindex,
(value, addressdata_opt),
));
}
(transacted, vecs, vecs2)
},
)
.reduce(|| (Transacted::default(), AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(Sats, Option<WithAddressDataSource<AddressData>>)>::default(),
), |(transacted, mut vecs, mut vecs2), (other_transacted, other_vecs, other_vecs2)| {
vecs.merge(other_vecs);
vecs2.merge(other_vecs2);
(transacted + other_transacted, vecs, vecs2)
})
.reduce(
|| {
(
Transacted::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
)>::default(
),
)
},
|(transacted, mut vecs, mut vecs2), (other_transacted, other_vecs, other_vecs2)| {
vecs.merge(other_vecs);
vecs2.merge(other_vecs2);
(transacted + other_transacted, vecs, vecs2)
},
)
});
(sent_handle.join().unwrap(), received_handle.join().unwrap())
});
addresstype_to_typeindex_to_sent_outputindex.merge(new_addresstype_to_typedindex_to_sent_outputindex);
addresstype_to_typeindex_to_received_outputindex.merge(new_addresstype_to_typedindex_to_received_outputindex);
addresstype_to_typedindex_to_received_data.process_received(
&mut self.address_vecs,
&mut addresstype_to_typeindex_to_addressdata,
&mut addresstype_to_typeindex_to_emptyaddressdata,
price,
&mut addresstype_to_address_count,
&mut addresstype_to_empty_address_count
);
addresstype_to_typedindex_to_sent_data.process_sent(
&mut self.address_vecs,
&mut addresstype_to_typeindex_to_addressdata,
&mut addresstype_to_typeindex_to_emptyaddressdata,
price,
&mut addresstype_to_address_count,
&mut addresstype_to_empty_address_count
)?;
if chain_state_starting_height > height {
dbg!(chain_state_starting_height, height);
panic!("temp, just making sure")
}
unspendable_supply += received
.by_type
@@ -715,28 +828,53 @@ impl Vecs {
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
};
if chain_state_starting_height <= height {
// Push current block state before processing sends and receives
chain_state.push(BlockState {
supply: received.spendable_supply.clone(),
thread::scope(|scope| -> Result<()> {
scope.spawn(|| addresstype_to_typeindex_to_sent_outputindex
.merge(addresstype_to_typedindex_to_sent_outputindex));
scope.spawn(|| addresstype_to_typeindex_to_received_outputindex
.merge(addresstype_to_typedindex_to_received_outputindex));
scope.spawn(|| {
// Push current block state before processing sends and receives
chain_state.push(BlockState {
supply: received.spendable_supply.clone(),
price,
timestamp,
});
self.utxo_vecs.receive(received, height, price);
let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
height_to_sent.par_iter().for_each(|(height, sent)| unsafe {
(*unsafe_chain_state.get(height.unwrap_to_usize())).supply -=
&sent.spendable_supply;
});
self.utxo_vecs.send(height_to_sent, chain_state.as_slice());
});
addresstype_to_typedindex_to_received_data.process_received(
&mut self.address_vecs,
&mut addresstype_to_typeindex_to_addressdata,
&mut addresstype_to_typeindex_to_emptyaddressdata,
price,
timestamp,
});
&mut addresstype_to_address_count,
&mut addresstype_to_empty_address_count,
);
self.utxo_vecs.receive(received, height, price);
addresstype_to_typedindex_to_sent_data.process_sent(
&mut self.address_vecs,
&mut addresstype_to_typeindex_to_addressdata,
&mut addresstype_to_typeindex_to_emptyaddressdata,
price,
&mut addresstype_to_address_count,
&mut addresstype_to_empty_address_count,
)?;
let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
height_to_sent.par_iter().for_each(|(height, sent)| unsafe {
(*unsafe_chain_state.get(height.unwrap_to_usize())).supply -=
&sent.spendable_supply;
});
self.utxo_vecs.send(height_to_sent, chain_state.as_slice());
} else {
dbg!(chain_state_starting_height, height);
panic!("temp, just making sure")
}
Ok(())
})?;
let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs();
@@ -759,9 +897,14 @@ impl Vecs {
self.height_to_opreturn_supply
.forced_push_at(height, opreturn_supply, exit)?;
self.addresstype_to_height_to_address_count.forced_push_at(height, &addresstype_to_address_count, exit)?;
self.addresstype_to_height_to_address_count.forced_push_at(
height,
&addresstype_to_address_count,
exit,
)?;
self.addresstype_to_height_to_empty_address_count.forced_push_at(height, &addresstype_to_empty_address_count, exit)?;
self.addresstype_to_height_to_empty_address_count
.forced_push_at(height, &addresstype_to_empty_address_count, exit)?;
let date = height_to_date_fixed_iter.unwrap_get_inner(height);
let dateindex = DateIndex::try_from(date).unwrap();
@@ -776,45 +919,47 @@ impl Vecs {
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
// thread::scope(|scope| {
// scope.spawn(|| {
separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})?;
// });
// scope.spawn(|| {
separate_address_vecs.par_iter_mut().try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})?;
// });
// });
thread::scope(|scope| {
scope.spawn(|| {
separate_utxo_vecs
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})
.unwrap();
});
scope.spawn(|| {
separate_address_vecs
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})
.unwrap();
});
});
if height != Height::ZERO && height.unwrap_to_usize() % 10_000 == 0 {
info!("Flushing...");
exit.block();
self.flush_states(
height,
&chain_state,
exit,
)?;
self.flush_states(height, &chain_state, exit)?;
stores.commit(
height,
mem::take(&mut addresstype_to_typeindex_to_sent_outputindex),
mem::take(&mut addresstype_to_typeindex_to_received_outputindex),
mem::take(&mut addresstype_to_typeindex_to_addressdata), mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata)
mem::take(&mut addresstype_to_typeindex_to_addressdata),
mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata),
)?;
exit.release();
}
@@ -840,39 +985,41 @@ impl Vecs {
info!("Computing overlapping...");
// thread::scope(|scope| {
// scope.spawn(|| {
self.utxo_vecs
.compute_overlapping_vecs(starting_indexes, exit)?;
// });
// scope.spawn(|| {
self.address_vecs
.compute_overlapping_vecs(starting_indexes, exit)?;
// });
// });
thread::scope(|scope| {
scope.spawn(|| {
self.utxo_vecs
.compute_overlapping_vecs(starting_indexes, exit)
.unwrap();
});
scope.spawn(|| {
self.address_vecs
.compute_overlapping_vecs(starting_indexes, exit)
.unwrap();
});
});
info!("Computing rest part 1...");
// thread::scope(|scope| {
// scope.spawn(|| {
self.utxo_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
})
.unwrap();
// });
// scope.spawn(|| {
self.address_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
})
.unwrap();
// });
// });
thread::scope(|scope| {
scope.spawn(|| {
self.utxo_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
})
.unwrap();
});
scope.spawn(|| {
self.address_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
})
.unwrap();
});
});
info!("Computing rest part 2...");
@@ -894,46 +1041,48 @@ impl Vecs {
.as_ref()
.map(|v| v.dateindex.unwrap_last().clone());
// thread::scope(|scope| {
// scope.spawn(|| {
self.utxo_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
fetched,
starting_indexes,
market,
&height_to_supply,
dateindex_to_supply.as_ref().unwrap(),
height_to_realized_cap.as_ref(),
dateindex_to_realized_cap.as_ref(),
exit,
)
})?;
// });
// scope.spawn(|| {
self.address_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
fetched,
starting_indexes,
market,
&height_to_supply,
dateindex_to_supply.as_ref().unwrap(),
height_to_realized_cap.as_ref(),
dateindex_to_realized_cap.as_ref(),
exit,
)
})?;
// });
// });
thread::scope(|scope| {
scope.spawn(|| {
self.utxo_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
fetched,
starting_indexes,
market,
&height_to_supply,
dateindex_to_supply.as_ref().unwrap(),
height_to_realized_cap.as_ref(),
dateindex_to_realized_cap.as_ref(),
exit,
)
})
.unwrap();
});
scope.spawn(|| {
self.address_vecs
.as_mut_vecs()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest_part2(
indexer,
indexes,
fetched,
starting_indexes,
market,
&height_to_supply,
dateindex_to_supply.as_ref().unwrap(),
height_to_realized_cap.as_ref(),
dateindex_to_realized_cap.as_ref(),
exit,
)
})
.unwrap();
});
});
self.indexes_to_unspendable_supply.compute_rest(
indexer,
indexes,
@@ -1084,28 +1233,27 @@ impl AddressTypeToTypeIndexVec<(Sats, Option<WithAddressDataSource<AddressData>>
let amount = prev_amount + value;
if is_new
|| from_any_empty
|| vecs.by_size_range.get_mut(amount).0.clone()
!= vecs.by_size_range.get_mut(prev_amount).0.clone()
{
if !is_new {
vecs.by_size_range
.get_mut(prev_amount)
.1
.state
.subtract(addressdata);
// dbg!((prev_amount, amount, is_new));
if !is_new && !from_any_empty {
let state = &mut vecs.by_size_range.get_mut(prev_amount).1.state;
// dbg!((prev_amount, &state.address_count, &addressdata));
state.subtract(addressdata);
}
addressdata.receive(value, price);
vecs.by_size_range.get_mut(amount).1.state.add(addressdata);
} else {
addressdata.receive(value, price);
vecs.by_size_range
.get_mut(amount)
.1
.state
.receive(value, price);
vecs.by_size_range.get_mut(amount).1.state.receive(
addressdata,
value,
price,
);
}
});
});
@@ -1149,7 +1297,7 @@ impl
let addressdata_withsource = typeindex_to_addressdata
.entry(type_index)
.or_insert(addressdata_opt.unwrap());
.or_insert_with(|| addressdata_opt.unwrap());
let addressdata = addressdata_withsource.deref_mut();
@@ -1159,6 +1307,8 @@ impl
let will_be_empty = addressdata.outputs_len - 1 == 0;
// dbg!((prev_amount, amount, will_be_empty));
if will_be_empty
|| vecs.by_size_range.get_mut(amount).0.clone()
!= vecs.by_size_range.get_mut(prev_amount).0.clone()
@@ -1172,6 +1322,10 @@ impl
addressdata.send(value, prev_price)?;
if will_be_empty {
if amount.is_not_zero() {
unreachable!()
}
(*addresstype_to_address_count.get_mut(_type).unwrap()) -= 1;
(*addresstype_to_empty_address_count.get_mut(_type).unwrap()) += 1;
@@ -1186,16 +1340,15 @@ impl
vecs.by_size_range.get_mut(amount).1.state.add(addressdata);
}
} else {
addressdata.send(value, prev_price)?;
vecs.by_size_range.get_mut(amount).1.state.send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
);
)?;
}
Ok(())

5
crates/brk_computer/src/vecs/stateful/utxo_cohort.rs
View File

@@ -59,10 +59,7 @@ impl CohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state
.price_to_amount
.height()
.map_or(Height::MAX, |h| h.incremented()),
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.inner.starting_height(),
]
.into_iter()