Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-13 08:58:36 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: stateful: perf improvements

Browse Source
This commit is contained in:
nym21
2025-07-11 11:43:53 +02:00
parent 4489920cbf
commit 8c289df336
7 changed files with 220 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs → crates/brk_computer/src/vecs/stateful/addresstype_to_vec.rs
View File

9
crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs Normal file
View File

@@ -0,0 +1,9 @@
use std::collections::BTreeMap;
use brk_core::Height;
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::AddressTypeToVec;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);

274
crates/brk_computer/src/vecs/stateful/mod.rs
View File

@@ -2,7 +2,7 @@ use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_core::{ use brk_core::{
AddressData, ByAddressType, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height, AddressData, ByAddressType, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height,
InputIndex, OutputIndex, OutputType, Result, Sats, StoredUsize, TypeIndex, Version, InputIndex, OutputIndex, OutputType, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version,
}; };
use brk_exit::Exit; use brk_exit::Exit;
use brk_indexer::Indexer; use brk_indexer::Indexer;
@@ -41,15 +41,19 @@ mod addresstype_to_addresscount;
mod addresstype_to_height_to_addresscount; mod addresstype_to_height_to_addresscount;
mod addresstype_to_indexes_to_addresscount; mod addresstype_to_indexes_to_addresscount;
mod addresstype_to_typeindex_tree; mod addresstype_to_typeindex_tree;
mod addresstype_to_typeindex_vec; mod addresstype_to_vec;
mod common; mod common;
mod height_to_addresstype_to_vec;
mod range_map;
mod r#trait; mod r#trait;
mod utxo_cohort; mod utxo_cohort;
mod utxo_cohorts; mod utxo_cohorts;
mod withaddressdatasource; mod withaddressdatasource;
pub use addresstype_to_typeindex_tree::*; pub use addresstype_to_typeindex_tree::*;
pub use addresstype_to_typeindex_vec::*; pub use addresstype_to_vec::*;
use height_to_addresstype_to_vec::*;
use range_map::*;
use r#trait::CohortVecs; use r#trait::CohortVecs;
pub use withaddressdatasource::WithAddressDataSource; pub use withaddressdatasource::WithAddressDataSource;
@@ -497,8 +501,6 @@ impl Vecs {
let outputindex_to_value_mmap = outputindex_to_value.mmap().load(); let outputindex_to_value_mmap = outputindex_to_value.mmap().load();
let outputindex_to_outputtype_mmap = outputindex_to_outputtype.mmap().load(); let outputindex_to_outputtype_mmap = outputindex_to_outputtype.mmap().load();
let outputindex_to_typeindex_mmap = outputindex_to_typeindex.mmap().load(); let outputindex_to_typeindex_mmap = outputindex_to_typeindex.mmap().load();
let outputindex_to_txindex_mmap = outputindex_to_txindex.mmap().load();
let txindex_to_height_mmap = txindex_to_height.mmap().load();
let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter();
let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter();
@@ -662,6 +664,12 @@ impl Vecs {
height_to_close.map(|height_to_close| height_to_close.collect().unwrap()); height_to_close.map(|height_to_close| height_to_close.collect().unwrap());
let height_to_timestamp_fixed_vec = height_to_timestamp_fixed.collect().unwrap(); let height_to_timestamp_fixed_vec = height_to_timestamp_fixed.collect().unwrap();
let outputindex_range_to_height = RangeMap::from(height_to_first_outputindex);
let mut outputindex_to_value_type_typeindex: BTreeMap<
OutputIndex,
(Sats, OutputType, Option<TypeIndex>),
> = BTreeMap::default();
let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() { let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_unspendable_supply self.height_to_unspendable_supply
@@ -752,8 +760,8 @@ impl Vecs {
}; };
let ( let (
(mut height_to_sent, addresstype_to_typedindex_to_sent_data), (sent_outputindexes, mut height_to_sent, addresstype_to_typedindex_to_sent_data),
(mut received, addresstype_to_typedindex_to_received_data), (mut received_outputindexes, mut transacted, addresstype_to_typedindex_to_received_data),
) = thread::scope(|s| { ) = thread::scope(|s| {
if chain_state_starting_height <= height { if chain_state_starting_height <= height {
s.spawn(|| { s.spawn(|| {
@@ -773,34 +781,28 @@ impl Vecs {
.unwrap() .unwrap()
.into_owned(); .into_owned();
let value = outputindex_to_value let cached = outputindex_to_value_type_typeindex.get(&outputindex).cloned();
.get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap()
.unwrap()
.into_owned();
let input_type = outputindex_to_outputtype let (value, input_type) = cached.map_or_else(|| (
.get_or_read(outputindex, &outputindex_to_outputtype_mmap) outputindex_to_value
.unwrap() .get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap() .unwrap()
.into_owned(); .unwrap()
.into_owned(),
outputindex_to_outputtype
.get_or_read(outputindex, &outputindex_to_outputtype_mmap)
.unwrap()
.unwrap()
.into_owned()
), |(value,_type,..)| (value, _type));
let input_txindex = outputindex_to_txindex let prev_height = *outputindex_range_to_height.get(outputindex).unwrap();
.get_or_read(outputindex, &outputindex_to_txindex_mmap)
.unwrap()
.unwrap()
.into_owned();
let prev_height = txindex_to_height
.get_or_read(input_txindex, &txindex_to_height_mmap)
.unwrap()
.unwrap()
.into_owned();
if input_type.is_unspendable() { if input_type.is_unspendable() {
unreachable!() unreachable!()
} else if input_type.is_not_address() { } else if input_type.is_not_address() {
return ( return (
outputindex,
prev_height, prev_height,
value, value,
input_type, input_type,
@@ -808,11 +810,11 @@ impl Vecs {
) )
} }
let typeindex = outputindex_to_typeindex let typeindex = cached.map_or_else(|| outputindex_to_typeindex
.get_or_read(outputindex, &outputindex_to_typeindex_mmap) .get_or_read(outputindex, &outputindex_to_typeindex_mmap)
.unwrap() .unwrap()
.unwrap() .unwrap()
.into_owned(); .into_owned(), |(_, _, typeindex)| typeindex.unwrap());
let addressdata_opt = if input_type.is_address() let addressdata_opt = if input_type.is_address()
&& *first_addressindexes.get(input_type).unwrap() && *first_addressindexes.get(input_type).unwrap()
@@ -832,97 +834,70 @@ impl Vecs {
None None
}; };
let prev_price = height_to_close_vec
.as_ref()
.map(|v| **v.get(prev_height.unwrap_to_usize()).unwrap());
let prev_timestamp = *height_to_timestamp_fixed_vec
.get(prev_height.unwrap_to_usize())
.unwrap();
let blocks_old =
height.unwrap_to_usize() - prev_height.unwrap_to_usize();
let days_old =
prev_timestamp.difference_in_days_between_float(timestamp);
let older_than_hour = timestamp
.checked_sub(prev_timestamp)
.unwrap()
.is_more_than_hour();
( (
outputindex,
prev_height, prev_height,
value, value,
input_type, input_type,
Some((typeindex, Some((typeindex,
addressdata_opt, addressdata_opt
prev_price,
blocks_old,
days_old,
older_than_hour
)) ))
) )
}) })
.fold( .fold(
|| { || {
( (
Vec::<OutputIndex>::default(),
BTreeMap::<Height, Transacted>::default(), BTreeMap::<Height, Transacted>::default(),
AddressTypeToVec::<( HeightToAddressTypeToVec::<(
TypeIndex, TypeIndex,
Sats, Sats,
Option<WithAddressDataSource<AddressData>>, Option<WithAddressDataSource<AddressData>>,
Option<Dollars>, )>::default(),
usize,
f64,
bool
)>::default(
),
) )
}, },
|(mut tree, mut vecs), |(mut vec, mut tree, mut tree2),
( (
outputindex,
height, height,
value, value,
input_type, input_type,
address_data_opt address_data_opt
)| { )| {
vec.push(outputindex);
tree.entry(height).or_default().iterate(value, input_type); tree.entry(height).or_default().iterate(value, input_type);
if let Some((typeindex, if let Some((typeindex,
addressdata_opt, addressdata_opt)) = address_data_opt {
prev_price, tree2.entry(height).or_default().get_mut(input_type).unwrap().push((
blocks_old,
days_old,
older_than_hour)) = address_data_opt {
vecs.get_mut(input_type).unwrap().push((
typeindex, typeindex,
value, value,
addressdata_opt, addressdata_opt,
prev_price,
blocks_old,
days_old,
older_than_hour,
)); ));
} }
(tree, vecs) (vec, tree, tree2)
}, },
) )
.reduce( .reduce(
|| { || {
( (
Vec::<OutputIndex>::default(),
BTreeMap::<Height, Transacted>::default(), BTreeMap::<Height, Transacted>::default(),
AddressTypeToVec::<( HeightToAddressTypeToVec::<(
TypeIndex, TypeIndex,
Sats, Sats,
Option<WithAddressDataSource<AddressData>>, Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
)>::default(), )>::default(),
) )
}, },
|(first_tree, mut source_vecs), (second_tree, other_vecs)| { |(mut first_vec, first_tree, first_tree2), (mut second_vec, second_tree, second_tree2)| {
let vec = if first_vec.len() > second_vec.len() {
first_vec.append(&mut second_vec);
first_vec
} else {
second_vec.append(&mut first_vec);
second_vec
};
let (mut tree_source, tree_to_consume) = let (mut tree_source, tree_to_consume) =
if first_tree.len() > second_tree.len() { if first_tree.len() > second_tree.len() {
(first_tree, second_tree) (first_tree, second_tree)
@@ -932,13 +907,22 @@ impl Vecs {
tree_to_consume.into_iter().for_each(|(k, v)| { tree_to_consume.into_iter().for_each(|(k, v)| {
*tree_source.entry(k).or_default() += v; *tree_source.entry(k).or_default() += v;
}); });
source_vecs.merge(other_vecs);
(tree_source, source_vecs) let (mut tree_source2, tree_to_consume2) =
if first_tree2.len() > second_tree2.len() {
(first_tree2, second_tree2)
} else {
(second_tree2, first_tree2)
};
tree_to_consume2.0.into_iter().for_each(|(k, v)| {
tree_source2.entry(k).or_default().merge(v);
});
(vec, tree_source, tree_source2)
}, },
) )
}); });
// let received_handle = s.spawn(|| {
let received_output = (first_outputindex..first_outputindex + *output_count) let received_output = (first_outputindex..first_outputindex + *output_count)
.into_par_iter() .into_par_iter()
.map(OutputIndex::from) .map(OutputIndex::from)
@@ -956,7 +940,7 @@ impl Vecs {
.into_owned(); .into_owned();
if output_type.is_not_address() { if output_type.is_not_address() {
return (value, output_type, None); return (outputindex, value, output_type, None);
} }
let typeindex = outputindex_to_typeindex let typeindex = outputindex_to_typeindex
@@ -999,11 +983,15 @@ impl Vecs {
None None
}; };
(value, output_type, Some((typeindex, addressdata_opt))) (outputindex, value, output_type, Some((typeindex, addressdata_opt)))
}) })
.fold( .fold(
|| { || {
( (
BTreeMap::<
OutputIndex,
(Sats, OutputType, Option<TypeIndex>),
>::default(),
Transacted::default(), Transacted::default(),
AddressTypeToVec::<( AddressTypeToVec::<(
TypeIndex, TypeIndex,
@@ -1013,12 +1001,17 @@ impl Vecs {
), ),
) )
}, },
|(mut transacted, mut vecs), |(mut received_outputindexes, mut transacted, mut vecs),
( (
outputindex,
value, value,
output_type, output_type,
typeindex_with_addressdata_opt, typeindex_with_addressdata_opt,
)| { )| {
if output_type.is_address() {
received_outputindexes.insert(outputindex, (value, output_type, typeindex_with_addressdata_opt.as_ref().map(|(v, ..)| *v)));
}
transacted.iterate(value, output_type); transacted.iterate(value, output_type);
if let Some(vec) = vecs.get_mut(output_type) { if let Some(vec) = vecs.get_mut(output_type) {
let (typeindex, let (typeindex,
@@ -1029,12 +1022,17 @@ impl Vecs {
addressdata_opt, addressdata_opt,
)); ));
} }
(transacted, vecs)
(received_outputindexes, transacted, vecs)
}, },
) )
.reduce( .reduce(
|| { || {
( (
BTreeMap::<
OutputIndex,
(Sats, OutputType, Option<TypeIndex>),
>::default(),
Transacted::default(), Transacted::default(),
AddressTypeToVec::<( AddressTypeToVec::<(
TypeIndex, TypeIndex,
@@ -1043,12 +1041,20 @@ impl Vecs {
)>::default(), )>::default(),
) )
}, },
|(transacted, mut vecs), (other_transacted, other_vecs)| { |(received_outputindexes, transacted, mut vecs), (other_received_outputindexes, other_transacted, other_vecs)| {
let (mut tree_source, mut tree_to_consume) =
if received_outputindexes.len() > other_received_outputindexes.len() {
(received_outputindexes, other_received_outputindexes)
} else {
(other_received_outputindexes, received_outputindexes)
};
tree_source.append(&mut tree_to_consume);
vecs.merge(other_vecs); vecs.merge(other_vecs);
(transacted + other_transacted, vecs)
(tree_source, transacted + other_transacted, vecs)
}, },
); );
// });
(sent_handle.join().unwrap(), received_output) (sent_handle.join().unwrap(), received_output)
}); });
@@ -1058,7 +1064,7 @@ impl Vecs {
panic!("temp, just making sure") panic!("temp, just making sure")
} }
unspendable_supply += received unspendable_supply += transacted
.by_type .by_type
.unspendable .unspendable
.as_vec() .as_vec()
@@ -1067,10 +1073,10 @@ impl Vecs {
.sum::<Sats>() .sum::<Sats>()
+ height_to_unclaimed_rewards_iter.unwrap_get_inner(height); + height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
opreturn_supply += received.by_type.unspendable.opreturn.value; opreturn_supply += transacted.by_type.unspendable.opreturn.value;
if height == Height::new(0) { if height == Height::new(0) {
received = Transacted::default(); transacted = Transacted::default();
unspendable_supply += Sats::FIFTY_BTC; unspendable_supply += Sats::FIFTY_BTC;
} else if height == Height::new(91_842) || height == Height::new(91_880) { } else if height == Height::new(91_842) || height == Height::new(91_880) {
// Need to destroy invalid coinbases due to duplicate txids // Need to destroy invalid coinbases due to duplicate txids
@@ -1082,16 +1088,35 @@ impl Vecs {
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65); .iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
}; };
thread::scope(|scope| -> Result<()> { thread::scope(|scope| -> Result<()> {
scope.spawn(|| {
let min_outputindex = if let Some(min_height) = height.checked_sub(Height::new(5000)) {
height_to_first_outputindex.into_iter().unwrap_get_inner(min_height)
} else {
OutputIndex::ZERO
};
outputindex_to_value_type_typeindex.retain(|k, _| *k >= min_outputindex);
sent_outputindexes
.into_iter()
.filter(|k| *k >= min_outputindex)
.for_each(|outputindex| {
outputindex_to_value_type_typeindex.remove(&outputindex);
});
outputindex_to_value_type_typeindex.append(&mut received_outputindexes);
});
scope.spawn(|| { scope.spawn(|| {
// Push current block state before processing sends and receives // Push current block state before processing sends and receives
chain_state.push(BlockState { chain_state.push(BlockState {
supply: received.spendable_supply.clone(), supply: transacted.spendable_supply.clone(),
price, price,
timestamp, timestamp,
}); });
self.utxo_vecs.receive(received, height, price); self.utxo_vecs.receive(transacted, height, price);
let unsafe_chain_state = UnsafeSlice::new(&mut chain_state); let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
@@ -1119,6 +1144,10 @@ impl Vecs {
price, price,
&mut addresstype_to_address_count, &mut addresstype_to_address_count,
&mut addresstype_to_empty_address_count, &mut addresstype_to_empty_address_count,
height_to_close_vec.as_ref(),
&height_to_timestamp_fixed_vec,
height,
timestamp
)?; )?;
Ok(()) Ok(())
@@ -1556,19 +1585,10 @@ impl AddressTypeToVec<(TypeIndex, Sats, Option<WithAddressDataSource<AddressData
} }
} }
impl impl HeightToAddressTypeToVec<(TypeIndex, Sats, Option<WithAddressDataSource<AddressData>>)> {
AddressTypeToVec<( #[allow(clippy::too_many_arguments)]
TypeIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
)>
{
fn process_sent( fn process_sent(
mut self, self,
vecs: &mut address_cohorts::Vecs, vecs: &mut address_cohorts::Vecs,
addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree< addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree<
WithAddressDataSource<AddressData>, WithAddressDataSource<AddressData>,
@@ -1579,20 +1599,32 @@ impl
price: Option<Dollars>, price: Option<Dollars>,
addresstype_to_address_count: &mut ByAddressType<usize>, addresstype_to_address_count: &mut ByAddressType<usize>,
addresstype_to_empty_address_count: &mut ByAddressType<usize>, addresstype_to_empty_address_count: &mut ByAddressType<usize>,
height_to_close_vec: Option<&Vec<brk_core::Close<Dollars>>>,
height_to_timestamp_fixed_vec: &[Timestamp],
height: Height,
timestamp: Timestamp,
) -> Result<()> { ) -> Result<()> {
self.into_typed_vec() self.0.into_iter().try_for_each(|(prev_height, mut v)| {
.into_iter() let prev_price = height_to_close_vec
.try_for_each(|(_type, vec)| { .as_ref()
vec.into_iter().try_for_each( .map(|v| **v.get(prev_height.unwrap_to_usize()).unwrap());
|(
type_index, let prev_timestamp = *height_to_timestamp_fixed_vec
value, .get(prev_height.unwrap_to_usize())
addressdata_opt, .unwrap();
prev_price,
blocks_old, let blocks_old = height.unwrap_to_usize() - prev_height.unwrap_to_usize();
days_old,
older_than_hour, let days_old = prev_timestamp.difference_in_days_between_float(timestamp);
)| {
let older_than_hour = timestamp
.checked_sub(prev_timestamp)
.unwrap()
.is_more_than_hour();
v.into_typed_vec().into_iter().try_for_each(|(_type, vec)| {
vec.into_iter()
.try_for_each(|(type_index, value, addressdata_opt)| {
let typeindex_to_addressdata = addresstype_to_typeindex_to_addressdata let typeindex_to_addressdata = addresstype_to_typeindex_to_addressdata
.get_mut(_type) .get_mut(_type)
.unwrap(); .unwrap();
@@ -1654,8 +1686,8 @@ impl
} }
Ok(()) Ok(())
}, })
)
}) })
})
} }
} }

35
crates/brk_computer/src/vecs/stateful/range_map.rs Normal file
View File

@@ -0,0 +1,35 @@
use std::collections::BTreeMap;
use brk_vec::{IndexedVec, StoredIndex, StoredType};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
impl<I, T> RangeMap<I, T>
where
I: StoredIndex,
T: StoredIndex,
{
pub fn get(&self, key: I) -> Option<&T> {
self.0.range(..=key).next_back().map(|(&min, value)| {
if min > key {
unreachable!()
}
value
})
}
}
impl<I, T> From<&IndexedVec<I, T>> for RangeMap<T, I>
where
I: StoredIndex,
T: StoredIndex + StoredType,
{
fn from(vec: &IndexedVec<I, T>) -> Self {
Self(
vec.into_iter()
.map(|(i, v)| (v.into_owned(), i))
.collect::<BTreeMap<_, _>>(),
)
}
}

2
crates/brk_core/src/structs/addressdata.rs
View File

@@ -18,7 +18,7 @@ impl AddressData {
} }
pub fn realized_price(&self) -> Dollars { pub fn realized_price(&self) -> Dollars {
(self.realized_cap / Bitcoin::from(self.amount())).round_nearest_cent() (self.realized_cap / Bitcoin::from(self.amount())).round_to_4_digits()
} }
#[inline(always)] #[inline(always)]

18
crates/brk_core/src/structs/cents.rs
View File

@@ -24,10 +24,28 @@ use super::Dollars;
)] )]
pub struct Cents(i64); pub struct Cents(i64);
const SIGNIFICANT_DIGITS: i32 = 4;
impl Cents { impl Cents {
pub const fn mint(value: i64) -> Self { pub const fn mint(value: i64) -> Self {
Self(value) Self(value)
} }
pub fn round_to_4_digits(self) -> Self {
let v = self.0;
let ilog10 = v.checked_ilog10().unwrap_or(0) as i32;
Self::from(if ilog10 >= SIGNIFICANT_DIGITS {
let log_diff = ilog10 - SIGNIFICANT_DIGITS + 1;
let pow = 10.0_f64.powi(log_diff);
((v as f64 / pow).round() * pow) as i64
} else {
v
})
}
} }
impl From<Dollars> for Cents { impl From<Dollars> for Cents {

4
crates/brk_core/src/structs/dollars.rs
View File

@@ -42,6 +42,10 @@ impl Dollars {
pub fn round_nearest_cent(self) -> Self { pub fn round_nearest_cent(self) -> Self {
Dollars((self.0 * 100.0).round() / 100.0) Dollars((self.0 * 100.0).round() / 100.0)
} }
pub fn round_to_4_digits(self) -> Self {
Self::from(Cents::from(self).round_to_4_digits())
}
} }
impl From<f32> for Dollars { impl From<f32> for Dollars {