global: MASSIVE snapshot

This commit is contained in:
nym21
2026-01-02 19:08:20 +01:00
parent ac6175688d
commit 3e9b1cc2b2
462 changed files with 34975 additions and 20072 deletions
@@ -0,0 +1,236 @@
use brk_cohort::ByAddressType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
IterableCloneableVec, PcoVec, TypedVecIterator,
};
use crate::{
ComputeIndexes, indexes,
internal::{ComputedVecsFromHeight, Source, VecBuilderOptions},
};
/// Address count per address type (runtime state).
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
#[inline]
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(ByAddressType {
p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(),
p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(),
p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(),
p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(),
p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(),
p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(),
p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(),
p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(),
})
} else {
Default::default()
}
}
}
/// Address count per address type, indexed by height.
#[derive(Debug, Clone, Deref, DerefMut, Traversable)]
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>);
impl From<ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>>
for AddressTypeToHeightToAddressCount
{
#[inline]
fn from(value: ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>) -> Self {
Self(value)
}
}
impl AddressTypeToHeightToAddressCount {
pub fn min_len(&self) -> usize {
self.p2pk65
.len()
.min(self.p2pk33.len())
.min(self.p2pkh.len())
.min(self.p2sh.len())
.min(self.p2wpkh.len())
.min(self.p2wsh.len())
.min(self.p2tr.len())
.min(self.p2a.len())
}
pub fn forced_import(db: &Database, name: &str, version: Version) -> Result<Self> {
Ok(Self::from(ByAddressType::new_with_name(|type_name| {
Ok(EagerVec::forced_import(
db,
&format!("{type_name}_{name}"),
version,
)?)
})?))
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let inner = &mut self.0;
[
&mut inner.p2pk65 as &mut dyn AnyStoredVec,
&mut inner.p2pk33 as &mut dyn AnyStoredVec,
&mut inner.p2pkh as &mut dyn AnyStoredVec,
&mut inner.p2sh as &mut dyn AnyStoredVec,
&mut inner.p2wpkh as &mut dyn AnyStoredVec,
&mut inner.p2wsh as &mut dyn AnyStoredVec,
&mut inner.p2tr as &mut dyn AnyStoredVec,
&mut inner.p2a as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
pub fn write(&mut self) -> Result<()> {
self.p2pk65.write()?;
self.p2pk33.write()?;
self.p2pkh.write()?;
self.p2sh.write()?;
self.p2wpkh.write()?;
self.p2wsh.write()?;
self.p2tr.write()?;
self.p2a.write()?;
Ok(())
}
pub fn truncate_push(
&mut self,
height: Height,
addresstype_to_usize: &AddressTypeToAddressCount,
) -> Result<()> {
self.p2pk65
.truncate_push(height, addresstype_to_usize.p2pk65.into())?;
self.p2pk33
.truncate_push(height, addresstype_to_usize.p2pk33.into())?;
self.p2pkh
.truncate_push(height, addresstype_to_usize.p2pkh.into())?;
self.p2sh
.truncate_push(height, addresstype_to_usize.p2sh.into())?;
self.p2wpkh
.truncate_push(height, addresstype_to_usize.p2wpkh.into())?;
self.p2wsh
.truncate_push(height, addresstype_to_usize.p2wsh.into())?;
self.p2tr
.truncate_push(height, addresstype_to_usize.p2tr.into())?;
self.p2a
.truncate_push(height, addresstype_to_usize.p2a.into())?;
Ok(())
}
pub fn reset(&mut self) -> Result<()> {
use vecdb::GenericStoredVec;
self.p2pk65.reset()?;
self.p2pk33.reset()?;
self.p2pkh.reset()?;
self.p2sh.reset()?;
self.p2wpkh.reset()?;
self.p2wsh.reset()?;
self.p2tr.reset()?;
self.p2a.reset()?;
Ok(())
}
}
/// Address count per address type, indexed by various indexes (dateindex, etc.).
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
#[inline]
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
Self(value)
}
}
impl AddressTypeToIndexesToAddressCount {
pub fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
sources: &AddressTypeToHeightToAddressCount,
) -> Result<Self> {
Ok(Self::from(ByAddressType::<
ComputedVecsFromHeight<StoredU64>,
>::try_zip_with_name(
sources,
|type_name, source| {
ComputedVecsFromHeight::forced_import(
db,
&format!("{type_name}_{name}"),
Source::Vec(source.boxed_clone()),
version,
indexes,
VecBuilderOptions::default().add_last(),
)
},
)?))
}
pub fn compute(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
) -> Result<()> {
self.p2pk65.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk65),
)?;
self.p2pk33.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk33),
)?;
self.p2pkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pkh),
)?;
self.p2sh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2sh),
)?;
self.p2wpkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wpkh),
)?;
self.p2wsh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wsh),
)?;
self.p2tr.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2tr),
)?;
self.p2a.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2a),
)?;
Ok(())
}
}
@@ -0,0 +1,74 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{
EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, Version,
};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Stamp,
};
const SAVED_STAMPED_CHANGES: u16 = 10;
/// Storage for both loaded and empty address data.
#[derive(Clone, Traversable)]
pub struct AddressesDataVecs {
pub loaded: BytesVec<LoadedAddressIndex, LoadedAddressData>,
pub empty: BytesVec<EmptyAddressIndex, EmptyAddressData>,
}
impl AddressesDataVecs {
/// Import from database.
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
loaded: BytesVec::forced_import_with(
ImportOptions::new(db, "loadedaddressdata", version)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
empty: BytesVec::forced_import_with(
ImportOptions::new(db, "emptyaddressdata", version)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
})
}
/// Get minimum stamped height across loaded and empty data.
pub fn min_stamped_height(&self) -> Height {
Height::from(self.loaded.stamp())
.incremented()
.min(Height::from(self.empty.stamp()).incremented())
}
/// Rollback both loaded and empty data to before the given stamp.
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> {
Ok([
self.loaded.rollback_before(stamp)?,
self.empty.rollback_before(stamp)?,
])
}
/// Reset both loaded and empty data.
pub fn reset(&mut self) -> Result<()> {
self.loaded.reset()?;
self.empty.reset()?;
Ok(())
}
/// Flush both loaded and empty data with stamp.
pub fn write(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> {
self.loaded
.stamped_write_maybe_with_changes(stamp, with_changes)?;
self.empty
.stamped_write_maybe_with_changes(stamp, with_changes)?;
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.loaded as &mut dyn AnyStoredVec,
&mut self.empty as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
}
@@ -0,0 +1,102 @@
use brk_error::{Error, Result};
use brk_traversable::Traversable;
use brk_types::{
AnyAddressIndex, Height, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex,
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
TypeIndex, Version,
};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Reader, Stamp,
};
const SAVED_STAMPED_CHANGES: u16 = 10;
/// Macro to define AnyAddressIndexesVecs and its methods.
macro_rules! define_any_address_indexes_vecs {
($(($field:ident, $variant:ident, $index:ty)),* $(,)?) => {
#[derive(Clone, Traversable)]
pub struct AnyAddressIndexesVecs {
$(pub $field: BytesVec<$index, AnyAddressIndex>,)*
}
impl AnyAddressIndexesVecs {
/// Import from database.
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
$($field: BytesVec::forced_import_with(
ImportOptions::new(db, "anyaddressindex", version)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,)*
})
}
/// Get minimum stamped height across all address types.
pub fn min_stamped_height(&self) -> Height {
[$(Height::from(self.$field.stamp()).incremented()),*]
.into_iter()
.min()
.unwrap_or_default()
}
/// Rollback all address types to before the given stamp.
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<Vec<Stamp>> {
Ok(vec![$(self.$field.rollback_before(stamp)?),*])
}
/// Reset all address types.
pub fn reset(&mut self) -> Result<()> {
$(self.$field.reset()?;)*
Ok(())
}
/// Get address index for a given type and typeindex.
pub fn get(&self, address_type: OutputType, typeindex: TypeIndex, reader: &Reader) -> AnyAddressIndex {
match address_type {
$(OutputType::$variant => self.$field.get_pushed_or_read_at_unwrap(typeindex.into(), reader),)*
_ => unreachable!("Invalid address type: {:?}", address_type),
}
}
/// Get address index with single read (no caching).
pub fn get_once(&self, address_type: OutputType, typeindex: TypeIndex) -> Result<AnyAddressIndex> {
match address_type {
$(OutputType::$variant => self.$field.read_at_once(typeindex.into()).map_err(Into::into),)*
_ => Err(Error::UnsupportedType(address_type.to_string())),
}
}
/// Update or push address index for a given type.
pub fn update_or_push(&mut self, address_type: OutputType, typeindex: TypeIndex, index: AnyAddressIndex) -> Result<()> {
match address_type {
$(OutputType::$variant => self.$field.update_or_push(typeindex.into(), index)?,)*
_ => unreachable!("Invalid address type: {:?}", address_type),
}
Ok(())
}
/// Write all address types with stamp.
pub fn write(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> {
$(self.$field.stamped_write_maybe_with_changes(stamp, with_changes)?;)*
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![$(&mut self.$field as &mut dyn AnyStoredVec),*].into_par_iter()
}
}
};
}
// Generate the struct and methods
define_any_address_indexes_vecs!(
(p2a, P2A, P2AAddressIndex),
(p2pk33, P2PK33, P2PK33AddressIndex),
(p2pk65, P2PK65, P2PK65AddressIndex),
(p2pkh, P2PKH, P2PKHAddressIndex),
(p2sh, P2SH, P2SHAddressIndex),
(p2tr, P2TR, P2TRAddressIndex),
(p2wpkh, P2WPKH, P2WPKHAddressIndex),
(p2wsh, P2WSH, P2WSHAddressIndex),
);
@@ -0,0 +1,3 @@
mod any;
pub use any::*;
@@ -0,0 +1,12 @@
mod address_count;
mod data;
mod indexes;
mod type_map;
pub use address_count::{
AddressTypeToAddressCount, AddressTypeToHeightToAddressCount,
AddressTypeToIndexesToAddressCount,
};
pub use data::AddressesDataVecs;
pub use indexes::AnyAddressIndexesVecs;
pub use type_map::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec};
@@ -0,0 +1,26 @@
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;
use super::vec::AddressTypeToVec;
/// Hashmap from Height to AddressTypeToVec.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct HeightToAddressTypeToVec<T>(FxHashMap<Height, AddressTypeToVec<T>>);
impl<T> HeightToAddressTypeToVec<T> {
/// Create with pre-allocated capacity for unique heights.
pub fn with_capacity(capacity: usize) -> Self {
Self(FxHashMap::with_capacity_and_hasher(
capacity,
Default::default(),
))
}
}
impl<T> HeightToAddressTypeToVec<T> {
/// Consume and iterate over (Height, AddressTypeToVec) pairs.
pub fn into_iter(self) -> impl Iterator<Item = (Height, AddressTypeToVec<T>)> {
self.0.into_iter()
}
}
@@ -0,0 +1,126 @@
use std::{collections::hash_map::Entry, mem};
use brk_cohort::ByAddressType;
use brk_types::{OutputType, TypeIndex};
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;
use smallvec::{Array, SmallVec};
/// A hashmap for each address type, keyed by TypeIndex.
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct AddressTypeToTypeIndexMap<T>(ByAddressType<FxHashMap<TypeIndex, T>>);
impl<T> Default for AddressTypeToTypeIndexMap<T> {
fn default() -> Self {
Self(ByAddressType {
p2a: FxHashMap::default(),
p2pk33: FxHashMap::default(),
p2pk65: FxHashMap::default(),
p2pkh: FxHashMap::default(),
p2sh: FxHashMap::default(),
p2tr: FxHashMap::default(),
p2wpkh: FxHashMap::default(),
p2wsh: FxHashMap::default(),
})
}
}
impl<T> AddressTypeToTypeIndexMap<T> {
/// Create with pre-allocated capacity per address type.
pub fn with_capacity(capacity: usize) -> Self {
Self(ByAddressType {
p2a: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2pk33: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2pk65: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2pkh: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2sh: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2tr: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2wpkh: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
p2wsh: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
})
}
/// Merge two maps, consuming other and extending self.
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_single(&mut self.p2a, &mut other.p2a);
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
self
}
fn merge_single(own: &mut FxHashMap<TypeIndex, T>, other: &mut FxHashMap<TypeIndex, T>) {
if own.len() < other.len() {
mem::swap(own, other);
}
own.extend(other.drain());
}
/// Merge another map into self, consuming other.
pub fn merge_mut(&mut self, mut other: Self) {
Self::merge_single(&mut self.p2a, &mut other.p2a);
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
}
/// Insert a value for a specific address type and typeindex.
pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) {
self.get_mut(address_type).unwrap().insert(typeindex, value);
}
/// Iterate over sorted entries by address type.
pub fn into_sorted_iter(self) -> impl Iterator<Item = (OutputType, Vec<(TypeIndex, T)>)> {
self.0.into_iter().map(|(output_type, map)| {
let mut sorted: Vec<_> = map.into_iter().collect();
sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex);
(output_type, sorted)
})
}
/// Consume and iterate over entries by address type.
#[allow(clippy::should_implement_trait)]
pub fn into_iter(self) -> impl Iterator<Item = (OutputType, FxHashMap<TypeIndex, T>)> {
self.0.into_iter()
}
/// Iterate mutably over entries by address type.
pub fn iter_mut(&mut self) -> impl Iterator<Item = (OutputType, &mut FxHashMap<TypeIndex, T>)> {
self.0.iter_mut()
}
}
impl<T> AddressTypeToTypeIndexMap<SmallVec<T>>
where
T: Array,
{
/// Merge two maps of SmallVec values, concatenating vectors.
pub fn merge_vec(mut self, other: Self) -> Self {
for (address_type, other_map) in other.0.into_iter() {
let self_map = self.0.get_mut_unwrap(address_type);
for (typeindex, mut other_vec) in other_map {
match self_map.entry(typeindex) {
Entry::Occupied(mut entry) => {
let self_vec = entry.get_mut();
if other_vec.len() > self_vec.len() {
mem::swap(self_vec, &mut other_vec);
}
self_vec.extend(other_vec);
}
Entry::Vacant(entry) => {
entry.insert(other_vec);
}
}
}
}
self
}
}
@@ -0,0 +1,7 @@
mod height_vec;
mod index_map;
mod vec;
pub use height_vec::*;
pub use index_map::*;
pub use vec::*;
@@ -0,0 +1,44 @@
use brk_cohort::ByAddressType;
use derive_deref::{Deref, DerefMut};
/// A vector for each address type.
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2a: vec![],
p2pk33: vec![],
p2pk65: vec![],
p2pkh: vec![],
p2sh: vec![],
p2tr: vec![],
p2wpkh: vec![],
p2wsh: vec![],
})
}
}
impl<T> AddressTypeToVec<T> {
/// Create with pre-allocated capacity per address type.
pub fn with_capacity(capacity: usize) -> Self {
Self(ByAddressType {
p2a: Vec::with_capacity(capacity),
p2pk33: Vec::with_capacity(capacity),
p2pk65: Vec::with_capacity(capacity),
p2pkh: Vec::with_capacity(capacity),
p2sh: Vec::with_capacity(capacity),
p2tr: Vec::with_capacity(capacity),
p2wpkh: Vec::with_capacity(capacity),
p2wsh: Vec::with_capacity(capacity),
})
}
}
impl<T> AddressTypeToVec<T> {
/// Unwrap the inner ByAddressType.
pub fn unwrap(self) -> ByAddressType<Vec<T>> {
self.0
}
}
@@ -0,0 +1,128 @@
use brk_cohort::ByAddressType;
use brk_types::{AnyAddressDataIndexEnum, LoadedAddressData, OutputType, TypeIndex};
use vecdb::GenericStoredVec;
use crate::distribution::{
address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs},
compute::VecsReaders,
};
use super::super::cohort::{
EmptyAddressDataWithSource, LoadedAddressDataWithSource, TxIndexVec, WithAddressDataSource,
update_tx_counts,
};
use super::lookup::AddressLookup;
/// Cache for address data within a flush interval.
pub struct AddressCache {
/// Addresses with non-zero balance
loaded: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
/// Addresses that became empty (zero balance)
empty: AddressTypeToTypeIndexMap<EmptyAddressDataWithSource>,
}
impl Default for AddressCache {
fn default() -> Self {
Self::new()
}
}
impl AddressCache {
pub fn new() -> Self {
Self {
loaded: AddressTypeToTypeIndexMap::default(),
empty: AddressTypeToTypeIndexMap::default(),
}
}
/// Check if address is in cache (either loaded or empty).
#[inline]
pub fn contains(&self, address_type: OutputType, typeindex: TypeIndex) -> bool {
self.loaded
.get(address_type)
.is_some_and(|m| m.contains_key(&typeindex))
|| self
.empty
.get(address_type)
.is_some_and(|m| m.contains_key(&typeindex))
}
/// Merge address data into loaded cache.
#[inline]
pub fn merge_loaded(&mut self, data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>) {
self.loaded.merge_mut(data);
}
/// Create an AddressLookup view into this cache.
#[inline]
pub fn as_lookup(&mut self) -> AddressLookup<'_> {
AddressLookup {
loaded: &mut self.loaded,
empty: &mut self.empty,
}
}
/// Update transaction counts for addresses.
pub fn update_tx_counts(&mut self, txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>) {
update_tx_counts(&mut self.loaded, &mut self.empty, txindex_vecs);
}
/// Take the cache contents for flushing, leaving empty caches.
pub fn take(
&mut self,
) -> (
AddressTypeToTypeIndexMap<EmptyAddressDataWithSource>,
AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
) {
(
std::mem::take(&mut self.empty),
std::mem::take(&mut self.loaded),
)
}
}
/// Load address data from storage or create new.
///
/// Returns None if address is already in cache (loaded or empty).
#[allow(clippy::too_many_arguments)]
pub fn load_uncached_address_data(
address_type: OutputType,
typeindex: TypeIndex,
first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache,
vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> Option<LoadedAddressDataWithSource> {
// Check if this is a new address (typeindex >= first for this height)
let first = *first_addressindexes.get(address_type).unwrap();
if first <= typeindex {
return Some(WithAddressDataSource::New(LoadedAddressData::default()));
}
// Skip if already in cache
if cache.contains(address_type, typeindex) {
return None;
}
// Read from storage
let reader = vr.address_reader(address_type);
let anyaddressindex = any_address_indexes.get(address_type, typeindex, reader);
Some(match anyaddressindex.to_enum() {
AnyAddressDataIndexEnum::Loaded(loaded_index) => {
let reader = &vr.anyaddressindex_to_anyaddressdata.loaded;
let loaded_data = addresses_data
.loaded
.get_pushed_or_read_unwrap(loaded_index, reader);
WithAddressDataSource::FromLoaded(loaded_index, loaded_data)
}
AnyAddressDataIndexEnum::Empty(empty_index) => {
let reader = &vr.anyaddressindex_to_anyaddressdata.empty;
let empty_data = addresses_data
.empty
.get_pushed_or_read_unwrap(empty_index, reader);
WithAddressDataSource::FromEmpty(empty_index, empty_data.into())
}
})
}
@@ -0,0 +1,108 @@
use brk_types::{LoadedAddressData, OutputType, TypeIndex};
use crate::distribution::address::AddressTypeToTypeIndexMap;
use super::super::cohort::{
EmptyAddressDataWithSource, LoadedAddressDataWithSource, WithAddressDataSource,
};
/// Tracking status of an address - determines cohort update strategy.
#[derive(Clone, Copy)]
pub enum TrackingStatus {
/// Brand new address (never seen before)
New,
/// Already tracked in a cohort (has existing balance)
Tracked,
/// Was in empty cache, now rejoining a cohort
WasEmpty,
}
/// Context for looking up and storing address data during block processing.
pub struct AddressLookup<'a> {
pub loaded: &'a mut AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
pub empty: &'a mut AddressTypeToTypeIndexMap<EmptyAddressDataWithSource>,
}
impl<'a> AddressLookup<'a> {
pub fn get_or_create_for_receive(
&mut self,
output_type: OutputType,
type_index: TypeIndex,
) -> (&mut LoadedAddressDataWithSource, TrackingStatus) {
use std::collections::hash_map::Entry;
let map = self.loaded.get_mut(output_type).unwrap();
match map.entry(type_index) {
Entry::Occupied(entry) => {
// Address is in cache. Need to determine if it's been processed
// by process_received (added to a cohort) or just loaded this block.
//
// - If wrapper is New AND funded_txo_count == 0: hasn't received yet,
// was just created in process_outputs this block → New
// - If wrapper is New AND funded_txo_count > 0: received in previous
// block but still in cache (no flush) → Tracked
// - If wrapper is FromLoaded: loaded from storage → Tracked
// - If wrapper is FromEmpty AND utxo_count == 0: still empty → WasEmpty
// - If wrapper is FromEmpty AND utxo_count > 0: already received → Tracked
let status = match entry.get() {
WithAddressDataSource::New(data) => {
if data.funded_txo_count == 0 {
TrackingStatus::New
} else {
TrackingStatus::Tracked
}
}
WithAddressDataSource::FromLoaded(..) => TrackingStatus::Tracked,
WithAddressDataSource::FromEmpty(_, data) => {
if data.utxo_count() == 0 {
TrackingStatus::WasEmpty
} else {
TrackingStatus::Tracked
}
}
};
(entry.into_mut(), status)
}
Entry::Vacant(entry) => {
if let Some(empty_data) =
self.empty.get_mut(output_type).unwrap().remove(&type_index)
{
return (entry.insert(empty_data.into()), TrackingStatus::WasEmpty);
}
(
entry.insert(WithAddressDataSource::New(LoadedAddressData::default())),
TrackingStatus::New,
)
}
}
}
/// Get address data for a send operation (must exist in cache).
pub fn get_for_send(
&mut self,
output_type: OutputType,
type_index: TypeIndex,
) -> &mut LoadedAddressDataWithSource {
self.loaded
.get_mut(output_type)
.unwrap()
.get_mut(&type_index)
.expect("Address must exist for send")
}
/// Move address from loaded to empty set.
pub fn move_to_empty(&mut self, output_type: OutputType, type_index: TypeIndex) {
let data = self
.loaded
.get_mut(output_type)
.unwrap()
.remove(&type_index)
.unwrap();
self.empty
.get_mut(output_type)
.unwrap()
.insert(type_index, data.into());
}
}
@@ -0,0 +1,5 @@
mod address;
mod lookup;
pub use address::*;
pub use lookup::*;
@@ -0,0 +1,121 @@
use brk_error::Result;
use brk_types::{
AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex,
OutputType, TypeIndex,
};
use crate::distribution::{AddressTypeToTypeIndexMap, AddressesDataVecs};
use super::with_source::{EmptyAddressDataWithSource, LoadedAddressDataWithSource};
/// Process loaded address data updates.
///
/// Handles:
/// - New loaded address: push to loaded storage
/// - Updated loaded address (was loaded): update in place
/// - Transition empty -> loaded: delete from empty, push to loaded
pub fn process_loaded_addresses(
addresses_data: &mut AddressesDataVecs,
loaded_updates: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
) -> Result<AddressTypeToTypeIndexMap<AnyAddressIndex>> {
let total: usize = loaded_updates.iter().map(|(_, m)| m.len()).sum();
let mut updates: Vec<(LoadedAddressIndex, LoadedAddressData)> = Vec::with_capacity(total);
let mut deletes: Vec<EmptyAddressIndex> = Vec::with_capacity(total);
let mut pushes: Vec<(OutputType, TypeIndex, LoadedAddressData)> = Vec::with_capacity(total);
for (address_type, items) in loaded_updates.into_iter() {
for (typeindex, source) in items {
match source {
LoadedAddressDataWithSource::New(data) => {
pushes.push((address_type, typeindex, data));
}
LoadedAddressDataWithSource::FromLoaded(index, data) => {
updates.push((index, data));
}
LoadedAddressDataWithSource::FromEmpty(empty_index, data) => {
deletes.push(empty_index);
pushes.push((address_type, typeindex, data));
}
}
}
}
// Phase 1: Deletes (creates holes)
for empty_index in deletes {
addresses_data.empty.delete(empty_index);
}
// Phase 2: Updates (in-place)
for (index, data) in updates {
addresses_data.loaded.update(index, data)?;
}
// Phase 3: Pushes (fills holes, then grows)
let mut result = AddressTypeToTypeIndexMap::default();
for (address_type, typeindex, data) in pushes {
let index = addresses_data.loaded.fill_first_hole_or_push(data)?;
result
.get_mut(address_type)
.unwrap()
.insert(typeindex, AnyAddressIndex::from(index));
}
Ok(result)
}
/// Process empty address data updates.
///
/// Handles:
/// - New empty address: push to empty storage
/// - Updated empty address (was empty): update in place
/// - Transition loaded -> empty: delete from loaded, push to empty
pub fn process_empty_addresses(
addresses_data: &mut AddressesDataVecs,
empty_updates: AddressTypeToTypeIndexMap<EmptyAddressDataWithSource>,
) -> Result<AddressTypeToTypeIndexMap<AnyAddressIndex>> {
let total: usize = empty_updates.iter().map(|(_, m)| m.len()).sum();
let mut updates: Vec<(EmptyAddressIndex, EmptyAddressData)> = Vec::with_capacity(total);
let mut deletes: Vec<LoadedAddressIndex> = Vec::with_capacity(total);
let mut pushes: Vec<(OutputType, TypeIndex, EmptyAddressData)> = Vec::with_capacity(total);
for (address_type, items) in empty_updates.into_iter() {
for (typeindex, source) in items {
match source {
EmptyAddressDataWithSource::New(data) => {
pushes.push((address_type, typeindex, data));
}
EmptyAddressDataWithSource::FromEmpty(index, data) => {
updates.push((index, data));
}
EmptyAddressDataWithSource::FromLoaded(loaded_index, data) => {
deletes.push(loaded_index);
pushes.push((address_type, typeindex, data));
}
}
}
}
// Phase 1: Deletes (creates holes)
for loaded_index in deletes {
addresses_data.loaded.delete(loaded_index);
}
// Phase 2: Updates (in-place)
for (index, data) in updates {
addresses_data.empty.update(index, data)?;
}
// Phase 3: Pushes (fills holes, then grows)
let mut result = AddressTypeToTypeIndexMap::default();
for (address_type, typeindex, data) in pushes {
let index = addresses_data.empty.fill_first_hole_or_push(data)?;
result
.get_mut(address_type)
.unwrap()
.insert(typeindex, AnyAddressIndex::from(index));
}
Ok(result)
}
@@ -0,0 +1,11 @@
mod address_updates;
mod received;
mod sent;
mod tx_counts;
mod with_source;
pub use address_updates::*;
pub use received::*;
pub use sent::*;
pub use tx_counts::*;
pub use with_source::*;
@@ -0,0 +1,115 @@
use brk_cohort::{AmountBucket, ByAddressType};
use brk_types::{Dollars, Sats, TypeIndex};
use rustc_hash::FxHashMap;
use crate::distribution::{address::AddressTypeToVec, cohorts::AddressCohorts};
use super::super::cache::{AddressLookup, TrackingStatus};
pub fn process_received(
received_data: AddressTypeToVec<(TypeIndex, Sats)>,
cohorts: &mut AddressCohorts,
lookup: &mut AddressLookup<'_>,
price: Option<Dollars>,
addr_count: &mut ByAddressType<u64>,
empty_addr_count: &mut ByAddressType<u64>,
) {
for (output_type, vec) in received_data.unwrap().into_iter() {
if vec.is_empty() {
continue;
}
// Cache mutable refs for this address type
let type_addr_count = addr_count.get_mut(output_type).unwrap();
let type_empty_count = empty_addr_count.get_mut(output_type).unwrap();
// Aggregate receives by address - each address processed exactly once
// Track (total_value, output_count) for correct UTXO counting
let mut aggregated: FxHashMap<TypeIndex, (Sats, u32)> = FxHashMap::default();
for (type_index, value) in vec {
let entry = aggregated.entry(type_index).or_default();
entry.0 += value;
entry.1 += 1;
}
for (type_index, (total_value, output_count)) in aggregated {
let (addr_data, status) = lookup.get_or_create_for_receive(output_type, type_index);
match status {
TrackingStatus::New => {
*type_addr_count += 1;
}
TrackingStatus::WasEmpty => {
*type_addr_count += 1;
*type_empty_count -= 1;
}
TrackingStatus::Tracked => {}
}
let is_new_entry = matches!(status, TrackingStatus::New | TrackingStatus::WasEmpty);
if is_new_entry {
// New/was-empty address - just add to cohort
addr_data.receive_outputs(total_value, price, output_count);
let new_bucket = AmountBucket::from(total_value);
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.add(addr_data);
} else {
let prev_balance = addr_data.balance();
let new_balance = prev_balance + total_value;
let prev_bucket = AmountBucket::from(prev_balance);
let new_bucket = AmountBucket::from(new_balance);
if let Some((old_bucket, new_bucket)) = prev_bucket.transition_to(new_bucket) {
// Crossing cohort boundary - subtract from old, add to new
let cohort_state = cohorts
.amount_range
.get_mut_by_bucket(old_bucket)
.state
.as_mut()
.unwrap();
// Debug info for tracking down underflow issues
if cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64 {
panic!(
"process_received: cohort underflow detected!\n\
output_type={:?}, type_index={:?}\n\
prev_balance={}, new_balance={}, total_value={}\n\
Address: {:?}",
output_type,
type_index,
prev_balance,
new_balance,
total_value,
addr_data
);
}
cohort_state.subtract(addr_data);
addr_data.receive_outputs(total_value, price, output_count);
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.add(addr_data);
} else {
// Staying in same cohort - just receive
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.receive_outputs(addr_data, total_value, price, output_count);
}
}
}
}
}
@@ -0,0 +1,141 @@
use brk_cohort::{AmountBucket, ByAddressType};
use brk_error::Result;
use brk_types::{CheckedSub, Dollars, Height, Sats, Timestamp, TypeIndex};
use vecdb::{VecIndex, unlikely};
use crate::distribution::{address::HeightToAddressTypeToVec, cohorts::AddressCohorts};
use super::super::cache::AddressLookup;
/// Process sent outputs for address cohorts.
///
/// For each spent UTXO:
/// 1. Look up address data
/// 2. Calculate age metrics (blocks_old, days_old)
/// 3. Update address balance and cohort membership
/// 4. Handle addresses becoming empty
///
/// Note: Takes separate price/timestamp slices instead of chain_state to allow
/// parallel execution with UTXO cohort processing (which mutates chain_state).
#[allow(clippy::too_many_arguments)]
pub fn process_sent(
sent_data: HeightToAddressTypeToVec<(TypeIndex, Sats)>,
cohorts: &mut AddressCohorts,
lookup: &mut AddressLookup<'_>,
current_price: Option<Dollars>,
addr_count: &mut ByAddressType<u64>,
empty_addr_count: &mut ByAddressType<u64>,
height_to_price: Option<&[Dollars]>,
height_to_timestamp: &[Timestamp],
current_height: Height,
current_timestamp: Timestamp,
) -> Result<()> {
for (prev_height, by_type) in sent_data.into_iter() {
let prev_price = height_to_price.map(|v| v[prev_height.to_usize()]);
let prev_timestamp = height_to_timestamp[prev_height.to_usize()];
let blocks_old = current_height.to_usize() - prev_height.to_usize();
let days_old = current_timestamp.difference_in_days_between_float(prev_timestamp);
let older_than_hour = current_timestamp
.checked_sub(prev_timestamp)
.unwrap()
.is_more_than_hour();
for (output_type, vec) in by_type.unwrap().into_iter() {
// Cache mutable refs for this address type
let type_addr_count = addr_count.get_mut(output_type).unwrap();
let type_empty_count = empty_addr_count.get_mut(output_type).unwrap();
for (type_index, value) in vec {
let addr_data = lookup.get_for_send(output_type, type_index);
let prev_balance = addr_data.balance();
let new_balance = prev_balance.checked_sub(value).unwrap();
let will_be_empty = addr_data.has_1_utxos();
// Compute buckets once
let prev_bucket = AmountBucket::from(prev_balance);
let new_bucket = AmountBucket::from(new_balance);
let crossing_boundary = prev_bucket != new_bucket;
if will_be_empty || crossing_boundary {
// Subtract from old cohort
let cohort_state = cohorts
.amount_range
.get_mut_by_bucket(prev_bucket)
.state
.as_mut()
.unwrap();
// Debug info for tracking down underflow issues
if unlikely(
cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64,
) {
panic!(
"process_sent: cohort underflow detected!\n\
Block context: prev_height={:?}, output_type={:?}, type_index={:?}\n\
prev_balance={}, new_balance={}, value={}\n\
will_be_empty={}, crossing_boundary={}\n\
Address: {:?}",
prev_height,
output_type,
type_index,
prev_balance,
new_balance,
value,
will_be_empty,
crossing_boundary,
addr_data
);
}
cohort_state.subtract(addr_data);
// Update address data
addr_data.send(value, prev_price)?;
if will_be_empty {
// Address becoming empty - invariant check
if new_balance.is_not_zero() {
unreachable!()
}
*type_addr_count -= 1;
*type_empty_count += 1;
// Move from loaded to empty
lookup.move_to_empty(output_type, type_index);
} else {
// Add to new cohort
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.add(addr_data);
}
} else {
// Address staying in same cohort - update in place
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.send(
addr_data,
value,
current_price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
}
}
}
}
Ok(())
}
@@ -0,0 +1,49 @@
use crate::distribution::address::AddressTypeToTypeIndexMap;
use super::with_source::{EmptyAddressDataWithSource, LoadedAddressDataWithSource, TxIndexVec};
/// Update tx_count for addresses based on unique transactions they participated in.
///
/// For each address:
/// 1. Deduplicate transaction indexes (an address may appear in multiple inputs/outputs of same tx)
/// 2. Add the unique count to the address's tx_count field
///
/// Addresses are looked up in loaded_cache first, then empty_cache.
/// NOTE: This should be called AFTER merging parallel-fetched address data into loaded_cache.
pub fn update_tx_counts(
loaded_cache: &mut AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
empty_cache: &mut AddressTypeToTypeIndexMap<EmptyAddressDataWithSource>,
mut txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>,
) {
// First, deduplicate txindex_vecs for addresses that appear multiple times in a block
for (_, map) in txindex_vecs.iter_mut() {
for (_, txindex_vec) in map.iter_mut() {
if txindex_vec.len() > 1 {
txindex_vec.sort_unstable();
txindex_vec.dedup();
}
}
}
// Update tx_count on address data
for (address_type, typeindex, txindex_vec) in txindex_vecs
.into_iter()
.flat_map(|(t, m)| m.into_iter().map(move |(i, v)| (t, i, v)))
{
let tx_count = txindex_vec.len() as u32;
if let Some(addr_data) = loaded_cache
.get_mut(address_type)
.unwrap()
.get_mut(&typeindex)
{
addr_data.tx_count += tx_count;
} else if let Some(addr_data) = empty_cache
.get_mut(address_type)
.unwrap()
.get_mut(&typeindex)
{
addr_data.tx_count += tx_count;
}
}
}
@@ -0,0 +1,65 @@
use brk_types::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex, TxIndex};
use smallvec::SmallVec;
/// Loaded address data with source tracking for flush operations.
pub type LoadedAddressDataWithSource = WithAddressDataSource<LoadedAddressData>;
/// Empty address data with source tracking for flush operations.
pub type EmptyAddressDataWithSource = WithAddressDataSource<EmptyAddressData>;
/// SmallVec for transaction indexes - most addresses have few transactions per block.
pub type TxIndexVec = SmallVec<[TxIndex; 4]>;
/// Address data wrapped with its source location for flush operations.
///
/// This enum tracks where the data came from so it can be correctly
/// updated or created during the flush phase.
#[derive(Debug, Clone)]
pub enum WithAddressDataSource<T> {
/// Brand new address (never seen before)
New(T),
/// Loaded from loaded address storage (with original index)
FromLoaded(LoadedAddressIndex, T),
/// Loaded from empty address storage (with original index)
FromEmpty(EmptyAddressIndex, T),
}
impl<T> std::ops::Deref for WithAddressDataSource<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::New(v) | Self::FromLoaded(_, v) | Self::FromEmpty(_, v) => v,
}
}
}
impl<T> std::ops::DerefMut for WithAddressDataSource<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
Self::New(v) | Self::FromLoaded(_, v) | Self::FromEmpty(_, v) => v,
}
}
}
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
#[inline]
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoaded(i, v) => Self::FromLoaded(i, v.into()),
WithAddressDataSource::FromEmpty(i, v) => Self::FromEmpty(i, v.into()),
}
}
}
impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
#[inline]
fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoaded(i, v) => Self::FromLoaded(i, v.into()),
WithAddressDataSource::FromEmpty(i, v) => Self::FromEmpty(i, v.into()),
}
}
}
@@ -0,0 +1,7 @@
mod cache;
mod cohort;
mod utxo;
pub use cache::*;
pub use cohort::*;
pub use utxo::*;
@@ -0,0 +1,140 @@
use brk_cohort::ByAddressType;
use brk_types::{Height, OutputType, Sats, TxIndex, TypeIndex};
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use crate::distribution::{
address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs},
compute::VecsReaders,
state::Transacted,
};
use crate::distribution::address::HeightToAddressTypeToVec;
use super::super::{
cache::{AddressCache, load_uncached_address_data},
cohort::{LoadedAddressDataWithSource, TxIndexVec},
};
/// Result of processing inputs for a block.
pub struct InputsResult {
/// Map from UTXO creation height -> aggregated sent supply.
pub height_to_sent: FxHashMap<Height, Transacted>,
/// Per-height, per-address-type sent data: (typeindex, value) for each address.
pub sent_data: HeightToAddressTypeToVec<(TypeIndex, Sats)>,
/// Address data looked up during processing, keyed by (address_type, typeindex).
pub address_data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
/// Transaction indexes per address for tx_count tracking.
pub txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>,
}
/// Process inputs (spent UTXOs) for a block.
///
/// For each input:
/// 1. Use pre-collected outpoint (from reusable iterator, avoids PcoVec re-decompression)
/// 2. Resolve outpoint to txoutindex
/// 3. Get the creation height from txoutindex_to_height map
/// 4. Read value and type from the referenced output (random access via mmap)
/// 5. Look up address data if input references an address type
/// 6. Accumulate into height_to_sent map
/// 7. Track address-specific data for address cohort processing
///
/// Uses parallel reads followed by sequential accumulation to avoid
/// expensive merge overhead from rayon's fold/reduce pattern.
#[allow(clippy::too_many_arguments)]
pub fn process_inputs(
input_count: usize,
txinindex_to_txindex: &[TxIndex],
txinindex_to_value: &[Sats],
txinindex_to_outputtype: &[OutputType],
txinindex_to_typeindex: &[TypeIndex],
txinindex_to_prev_height: &[Height],
first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache,
vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> InputsResult {
let items: Vec<_> = (0..input_count)
.into_par_iter()
.map(|local_idx| {
let txindex = txinindex_to_txindex[local_idx];
let prev_height = *txinindex_to_prev_height.get(local_idx).unwrap();
let value = *txinindex_to_value.get(local_idx).unwrap();
let input_type = *txinindex_to_outputtype.get(local_idx).unwrap();
if input_type.is_not_address() {
return (prev_height, value, input_type, None);
}
let typeindex = *txinindex_to_typeindex.get(local_idx).unwrap();
// Look up address data
let addr_data_opt = load_uncached_address_data(
input_type,
typeindex,
first_addressindexes,
cache,
vr,
any_address_indexes,
addresses_data,
);
(
prev_height,
value,
input_type,
Some((typeindex, txindex, value, addr_data_opt)),
)
})
.collect();
// Phase 2: Sequential accumulation - no merge overhead
// Estimate: unique heights bounded by block depth, addresses spread across ~8 types
let estimated_unique_heights = (input_count / 4).max(16);
let estimated_per_type = (input_count / 8).max(8);
let mut height_to_sent = FxHashMap::<Height, Transacted>::with_capacity_and_hasher(
estimated_unique_heights,
Default::default(),
);
let mut sent_data = HeightToAddressTypeToVec::with_capacity(estimated_unique_heights);
let mut address_data =
AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::with_capacity(estimated_per_type);
let mut txindex_vecs =
AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type);
for (prev_height, value, output_type, addr_info) in items {
height_to_sent
.entry(prev_height)
.or_default()
.iterate(value, output_type);
if let Some((typeindex, txindex, value, addr_data_opt)) = addr_info {
sent_data
.entry(prev_height)
.or_default()
.get_mut(output_type)
.unwrap()
.push((typeindex, value));
if let Some(addr_data) = addr_data_opt {
address_data.insert_for_type(output_type, typeindex, addr_data);
}
txindex_vecs
.get_mut(output_type)
.unwrap()
.entry(typeindex)
.or_default()
.push(txindex);
}
}
InputsResult {
height_to_sent,
sent_data,
address_data,
txindex_vecs,
}
}
@@ -0,0 +1,5 @@
mod inputs;
mod outputs;
pub use inputs::*;
pub use outputs::*;
@@ -0,0 +1,104 @@
use brk_cohort::ByAddressType;
use brk_types::{Sats, TxIndex, TypeIndex};
use crate::distribution::{
address::{
AddressTypeToTypeIndexMap, AddressTypeToVec, AddressesDataVecs, AnyAddressIndexesVecs,
},
compute::{TxOutData, VecsReaders},
state::Transacted,
};
use super::super::{
cache::{AddressCache, load_uncached_address_data},
cohort::{LoadedAddressDataWithSource, TxIndexVec},
};
/// Result of processing outputs for a block.
pub struct OutputsResult {
/// Aggregated supply transacted in this block.
pub transacted: Transacted,
/// Per-address-type received data: (typeindex, value) for each address.
pub received_data: AddressTypeToVec<(TypeIndex, Sats)>,
/// Address data looked up during processing, keyed by (address_type, typeindex).
pub address_data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
/// Transaction indexes per address for tx_count tracking.
pub txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>,
}
/// Process outputs (new UTXOs) for a block.
///
/// For each output:
/// 1. Read pre-collected value, output type, and typeindex
/// 2. Accumulate into Transacted by type and amount
/// 3. Look up address data if output is an address type
/// 4. Track address-specific data for address cohort processing
#[allow(clippy::too_many_arguments)]
pub fn process_outputs(
txoutindex_to_txindex: &[TxIndex],
txoutdata_vec: &[TxOutData],
first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache,
vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> OutputsResult {
let output_count = txoutdata_vec.len();
// Pre-allocate result structures
let estimated_per_type = (output_count / 8).max(8);
let mut transacted = Transacted::default();
let mut received_data = AddressTypeToVec::with_capacity(estimated_per_type);
let mut address_data =
AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::with_capacity(estimated_per_type);
let mut txindex_vecs =
AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type);
// Single pass: read from pre-collected vecs and accumulate
for (local_idx, txoutdata) in txoutdata_vec.iter().enumerate() {
let txindex = txoutindex_to_txindex[local_idx];
let value = txoutdata.value;
let output_type = txoutdata.outputtype;
transacted.iterate(value, output_type);
if output_type.is_not_address() {
continue;
}
let typeindex = txoutdata.typeindex;
received_data
.get_mut(output_type)
.unwrap()
.push((typeindex, value));
let addr_data_opt = load_uncached_address_data(
output_type,
typeindex,
first_addressindexes,
cache,
vr,
any_address_indexes,
addresses_data,
);
if let Some(addr_data) = addr_data_opt {
address_data.insert_for_type(output_type, typeindex, addr_data);
}
txindex_vecs
.get_mut(output_type)
.unwrap()
.entry(typeindex)
.or_default()
.push(txindex);
}
OutputsResult {
transacted,
received_data,
address_data,
txindex_vecs,
}
}
@@ -0,0 +1,212 @@
use std::path::Path;
use brk_cohort::{
AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, Filter, Filtered,
};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, IterableVec};
use crate::{ComputeIndexes, indexes, price, distribution::DynCohortVecs};
use crate::distribution::metrics::SupplyMetrics;
use super::{super::traits::CohortVecs, vecs::AddressCohortVecs};
const VERSION: Version = Version::new(0);
/// All Address cohorts organized by filter type.
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct AddressCohorts(AddressGroups<AddressCohortVecs>);
impl AddressCohorts {
/// Import all Address cohorts from database.
///
/// `all_supply` is the supply metrics from the UTXO "all" cohort, used as global
/// sources for `*_rel_to_market_cap` ratios.
pub fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
all_supply: Option<&SupplyMetrics>,
) -> Result<Self> {
let v = version + VERSION + Version::ZERO;
// Helper to create a cohort - only amount_range cohorts have state
let create = |filter: Filter,
name: &'static str,
has_state: bool|
-> Result<AddressCohortVecs> {
let sp = if has_state { Some(states_path) } else { None };
AddressCohortVecs::forced_import(db, filter, name, v, indexes, price, sp, all_supply)
};
let full = |f: Filter, name: &'static str| create(f, name, true);
let none = |f: Filter, name: &'static str| create(f, name, false);
Ok(Self(AddressGroups {
amount_range: ByAmountRange::try_new(&full)?,
lt_amount: ByLowerThanAmount::try_new(&none)?,
ge_amount: ByGreatEqualAmount::try_new(&none)?,
}))
}
/// Compute overlapping cohorts from component amount_range cohorts.
///
/// For example, ">=1 BTC" cohort is computed from sum of amount_range cohorts that match.
pub fn compute_overlapping_vecs(
&mut self,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let by_amount_range = &self.0.amount_range;
// ge_amount cohorts computed from matching amount_range cohorts
[
self.0
.ge_amount
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
// lt_amount cohorts computed from matching amount_range cohorts
self.0
.lt_amount
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
]
.into_iter()
.flatten()
.try_for_each(|(vecs, components)| {
vecs.compute_from_stateful(starting_indexes, &components, exit)
})
}
/// First phase of post-processing: compute index transforms.
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
/// Second phase of post-processing: compute relative metrics.
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2<S, HM, DM>(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &S,
height_to_market_cap: Option<&HM>,
dateindex_to_market_cap: Option<&DM>,
exit: &Exit,
) -> Result<()>
where
S: IterableVec<Height, Bitcoin> + Sync,
HM: IterableVec<Height, Dollars> + Sync,
DM: IterableVec<DateIndex, Dollars> + Sync,
{
self.0.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)
})
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
// Collect all vecs from all cohorts
self.0
.iter_mut()
.flat_map(|v| v.par_iter_vecs_mut().collect::<Vec<_>>())
.collect::<Vec<_>>()
.into_par_iter()
}
/// Commit all states to disk (separate from vec writes for parallelization).
pub fn commit_all_states(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.write_state(height, cleanup))
}
/// Get minimum height from all separate cohorts' height-indexed vectors.
pub fn min_separate_stateful_height_len(&self) -> Height {
self.iter_separate()
.map(|v| Height::from(v.min_stateful_height_len()))
.min()
.unwrap_or_default()
}
/// Get minimum dateindex from all separate cohorts' dateindex-indexed vectors.
pub fn min_separate_stateful_dateindex_len(&self) -> usize {
self.iter_separate()
.map(|v| v.min_stateful_dateindex_len())
.min()
.unwrap_or(usize::MAX)
}
/// Import state for all separate cohorts at or before given height.
/// Returns true if all imports succeeded and returned the expected height.
pub fn import_separate_states(&mut self, height: Height) -> bool {
self.par_iter_separate_mut()
.map(|v| v.import_state(height).unwrap_or_default())
.all(|h| h == height)
}
/// Reset state heights for all separate cohorts.
pub fn reset_separate_state_heights(&mut self) {
self.par_iter_separate_mut().for_each(|v| {
v.reset_state_starting_height();
});
}
/// Reset price_to_amount for all separate cohorts (called during fresh start).
pub fn reset_separate_price_to_amount(&mut self) -> Result<()> {
self.par_iter_separate_mut().try_for_each(|v| {
if let Some(state) = v.state.as_mut() {
state.reset_price_to_amount_if_needed()?;
}
Ok(())
})
}
/// Validate computed versions for all separate cohorts.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.validate_computed_versions(base_version))
}
}
@@ -0,0 +1,4 @@
mod groups;
mod vecs;
pub use groups::*;
@@ -0,0 +1,309 @@
use std::path::Path;
use brk_cohort::{CohortContext, Filter, Filtered};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
IterableCloneableVec, IterableVec, PcoVec,
};
use crate::{
ComputeIndexes,
internal::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, price,
distribution::state::AddressCohortState,
};
use crate::distribution::metrics::{CohortMetrics, ImportConfig, SupplyMetrics};
use super::super::traits::{CohortVecs, DynCohortVecs};
const VERSION: Version = Version::ZERO;
/// Address cohort with metrics and optional runtime state.
#[derive(Clone, Traversable)]
pub struct AddressCohortVecs {
/// Starting height when state was imported
starting_height: Option<Height>,
/// Runtime state for block-by-block processing
#[traversable(skip)]
pub state: Option<AddressCohortState>,
/// Metric vectors
#[traversable(flatten)]
pub metrics: CohortMetrics,
/// Address count at each height
pub height_to_addr_count: EagerVec<PcoVec<Height, StoredU64>>,
/// Address count indexed by various dimensions
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
}
impl AddressCohortVecs {
/// Import address cohort from database.
///
/// `all_supply` is the supply metrics from the "all" cohort, used as global
/// sources for `*_rel_to_market_cap` ratios. Pass `None` if not available.
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
db: &Database,
filter: Filter,
name: &str,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: Option<&Path>,
all_supply: Option<&SupplyMetrics>,
) -> Result<Self> {
let compute_dollars = price.is_some();
let full_name = CohortContext::Address.full_name(&filter, name);
let cfg = ImportConfig {
db,
filter,
full_name: &full_name,
context: CohortContext::Address,
version,
indexes,
price,
};
let height_to_addr_count = EagerVec::forced_import(
db,
&cfg.name("addr_count"),
version + VERSION + Version::ZERO,
)?;
Ok(Self {
starting_height: None,
state: states_path
.map(|path| AddressCohortState::new(path, &full_name, compute_dollars)),
metrics: CohortMetrics::forced_import(&cfg, all_supply)?,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
db,
&cfg.name("addr_count"),
Source::Vec(height_to_addr_count.boxed_clone()),
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
height_to_addr_count,
})
}
/// Get the starting height when state was imported.
pub fn starting_height(&self) -> Option<Height> {
self.starting_height
}
/// Set the starting height.
pub fn set_starting_height(&mut self, height: Height) {
self.starting_height = Some(height);
}
/// Reset starting height to zero.
pub fn reset_starting_height(&mut self) {
self.starting_height = Some(Height::ZERO);
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
rayon::iter::once(&mut self.height_to_addr_count as &mut dyn AnyStoredVec)
.chain(self.metrics.par_iter_mut())
}
/// Commit state to disk (separate from vec writes for parallelization).
pub fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.inner.write(height, cleanup)?;
}
Ok(())
}
}
impl Filtered for AddressCohortVecs {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for AddressCohortVecs {
fn min_stateful_height_len(&self) -> usize {
self.height_to_addr_count
.len()
.min(self.metrics.min_stateful_height_len())
}
fn min_stateful_dateindex_len(&self) -> usize {
self.metrics.min_stateful_dateindex_len()
}
fn reset_state_starting_height(&mut self) {
self.reset_starting_height();
if let Some(state) = self.state.as_mut() {
state.reset();
}
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
use vecdb::GenericStoredVec;
// Import state from runtime state if present
if let Some(state) = self.state.as_mut() {
// State files are saved AT height H, so to resume at H+1 we need to import at H
// Decrement first, then increment result to match expected starting_height
if let Some(mut prev_height) = starting_height.decremented() {
// Import price_to_amount state file (may adjust prev_height to actual file found)
prev_height = state.inner.import_at_or_before(prev_height)?;
// Restore supply state from height-indexed vectors
state.inner.supply.value = self
.metrics
.supply
.height_to_supply
.read_once(prev_height)?;
state.inner.supply.utxo_count = *self
.metrics
.supply
.height_to_utxo_count
.read_once(prev_height)?;
state.addr_count = *self.height_to_addr_count.read_once(prev_height)?;
// Restore realized cap if present
if let Some(realized_metrics) = self.metrics.realized.as_mut()
&& let Some(realized_state) = state.inner.realized.as_mut()
{
realized_state.cap = realized_metrics
.height_to_realized_cap
.read_once(prev_height)?;
}
let result = prev_height.incremented();
self.starting_height = Some(result);
Ok(result)
} else {
// starting_height is 0, nothing to import
self.starting_height = Some(Height::ZERO);
Ok(Height::ZERO)
}
} else {
self.starting_height = Some(starting_height);
Ok(starting_height)
}
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
use vecdb::GenericStoredVec;
self.height_to_addr_count
.validate_computed_version_or_reset(base_version)?;
self.metrics.validate_computed_versions(base_version)?;
Ok(())
}
fn truncate_push(&mut self, height: Height) -> Result<()> {
if self.starting_height.is_some_and(|h| h > height) {
return Ok(());
}
// Push addr_count from state
if let Some(state) = self.state.as_ref() {
self.height_to_addr_count
.truncate_push(height, state.addr_count.into())?;
self.metrics.truncate_push(height, &state.inner)?;
}
Ok(())
}
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()> {
if let Some(state) = self.state.as_mut() {
self.metrics.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
&mut state.inner,
)?;
}
Ok(())
}
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_addr_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_addr_count),
)?;
self.metrics
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
Ok(())
}
}
impl CohortVecs for AddressCohortVecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_addr_count.compute_sum_of_others(
starting_indexes.height,
others
.iter()
.map(|v| &v.height_to_addr_count)
.collect::<Vec<_>>()
.as_slice(),
exit,
)?;
self.metrics.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.metrics.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)?;
Ok(())
}
}
@@ -0,0 +1,7 @@
mod address;
mod traits;
mod utxo;
pub use address::AddressCohorts;
pub use traits::DynCohortVecs;
pub use utxo::UTXOCohorts;
@@ -0,0 +1,71 @@
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Exit, IterableVec};
use crate::{ComputeIndexes, indexes, price};
/// Dynamic dispatch trait for cohort vectors.
///
/// This trait enables heterogeneous cohort processing via trait objects.
pub trait DynCohortVecs: Send + Sync {
/// Get minimum length across height-indexed vectors written in block loop.
fn min_stateful_height_len(&self) -> usize;
/// Get minimum length across dateindex-indexed vectors written in block loop.
fn min_stateful_dateindex_len(&self) -> usize;
/// Reset the starting height for state tracking.
fn reset_state_starting_height(&mut self);
/// Import state from checkpoint at or before the given height.
fn import_state(&mut self, starting_height: Height) -> Result<Height>;
/// Validate that computed vectors have correct versions.
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
/// Push state to height-indexed vectors (truncating if needed).
fn truncate_push(&mut self, height: Height) -> Result<()>;
/// Compute and push unrealized profit/loss states.
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()>;
/// First phase of post-processing computations.
#[allow(clippy::too_many_arguments)]
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()>;
}
/// Static dispatch trait for cohort vectors with additional methods.
pub trait CohortVecs: DynCohortVecs {
/// Compute aggregate cohort from component cohorts.
fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()>;
/// Second phase of post-processing computations.
#[allow(clippy::too_many_arguments)]
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()>;
}
@@ -0,0 +1,430 @@
use std::path::Path;
use brk_cohort::{
ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge, ByMinAge,
BySpendableType, ByTerm, ByYear, Filter, Filtered, StateLevel, UTXOGroups,
};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, IterableVec};
use crate::{
ComputeIndexes,
internal::{PERCENTILES, PERCENTILES_LEN},
indexes, price,
distribution::DynCohortVecs,
};
use super::{super::traits::CohortVecs, vecs::UTXOCohortVecs};
const VERSION: Version = Version::new(0);
/// All UTXO cohorts organized by filter type.
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct UTXOCohorts(pub(crate) UTXOGroups<UTXOCohortVecs>);
impl UTXOCohorts {
/// Import all UTXO cohorts from database.
pub fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
let v = version + VERSION + Version::ZERO;
// Create "all" cohort first - it doesn't need global sources (it IS the global source)
let all = UTXOCohortVecs::forced_import(
db,
Filter::All,
"",
version + VERSION + Version::ONE,
indexes,
price,
states_path,
StateLevel::PriceOnly,
None,
)?;
// Get reference to all's supply for other cohorts to use as global source
let all_supply = Some(&all.metrics.supply);
// Create all cohorts first (while borrowing all_supply), then assemble struct
let price_only = |f: Filter, name: &'static str| {
UTXOCohortVecs::forced_import(
db,
f,
name,
v,
indexes,
price,
states_path,
StateLevel::PriceOnly,
all_supply,
)
};
let term = ByTerm::try_new(&price_only)?;
let full = |f: Filter, name: &'static str| {
UTXOCohortVecs::forced_import(
db,
f,
name,
v,
indexes,
price,
states_path,
StateLevel::Full,
all_supply,
)
};
let none = |f: Filter, name: &'static str| {
UTXOCohortVecs::forced_import(
db,
f,
name,
v,
indexes,
price,
states_path,
StateLevel::None,
all_supply,
)
};
let epoch = ByEpoch::try_new(&full)?;
let year = ByYear::try_new(&full)?;
let type_ = BySpendableType::try_new(&full)?;
let max_age = ByMaxAge::try_new(&none)?;
let min_age = ByMinAge::try_new(&none)?;
let age_range = ByAgeRange::try_new(&full)?;
let amount_range = ByAmountRange::try_new(&full)?;
let lt_amount = ByLowerThanAmount::try_new(&none)?;
let ge_amount = ByGreatEqualAmount::try_new(&none)?;
Ok(Self(UTXOGroups {
all,
term,
epoch,
year,
type_,
max_age,
min_age,
age_range,
amount_range,
lt_amount,
ge_amount,
}))
}
/// Compute overlapping cohorts from component age/amount range cohorts.
pub fn compute_overlapping_vecs(
&mut self,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let by_age_range = &self.0.age_range;
let by_amount_range = &self.0.amount_range;
[(&mut self.0.all, by_age_range.iter().collect::<Vec<_>>())]
.into_par_iter()
.chain(self.0.min_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_age_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.max_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_age_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.term.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_age_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.ge_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.lt_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.try_for_each(|(vecs, components)| {
vecs.compute_from_stateful(starting_indexes, &components, exit)
})
}
/// First phase of post-processing: compute index transforms.
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
/// Second phase of post-processing: compute relative metrics.
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2<S, HM, DM>(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &S,
height_to_market_cap: Option<&HM>,
dateindex_to_market_cap: Option<&DM>,
exit: &Exit,
) -> Result<()>
where
S: IterableVec<Height, Bitcoin> + Sync,
HM: IterableVec<Height, Dollars> + Sync,
DM: IterableVec<DateIndex, Dollars> + Sync,
{
self.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)
})
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
// Collect all vecs from all cohorts (separate + aggregate)
self.0
.iter_mut()
.flat_map(|v| v.par_iter_vecs_mut().collect::<Vec<_>>())
.collect::<Vec<_>>()
.into_par_iter()
}
/// Commit all states to disk (separate from vec writes for parallelization).
pub fn commit_all_states(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.write_state(height, cleanup))
}
/// Get minimum height from all separate cohorts' height-indexed vectors.
pub fn min_separate_stateful_height_len(&self) -> Height {
self.iter_separate()
.map(|v| Height::from(v.min_stateful_height_len()))
.min()
.unwrap_or_default()
}
/// Get minimum dateindex from all separate cohorts' dateindex-indexed vectors.
pub fn min_separate_stateful_dateindex_len(&self) -> usize {
self.iter_separate()
.map(|v| v.min_stateful_dateindex_len())
.min()
.unwrap_or(usize::MAX)
}
/// Get minimum dateindex from all aggregate cohorts' dateindex-indexed vectors.
/// This checks cost_basis percentiles which are only on aggregate cohorts.
pub fn min_aggregate_stateful_dateindex_len(&self) -> usize {
self.0
.iter_aggregate()
.filter_map(|v| v.metrics.cost_basis.as_ref())
.filter_map(|cb| cb.percentiles.as_ref())
.map(|cbp| cbp.min_stateful_dateindex_len())
.min()
.unwrap_or(usize::MAX)
}
/// Import state for all separate cohorts at or before given height.
/// Returns true if all imports succeeded and returned the expected height.
pub fn import_separate_states(&mut self, height: Height) -> bool {
self.par_iter_separate_mut()
.map(|v| v.import_state(height).unwrap_or_default())
.all(|h| h == height)
}
/// Reset state heights for all separate cohorts.
pub fn reset_separate_state_heights(&mut self) {
self.par_iter_separate_mut().for_each(|v| {
v.reset_state_starting_height();
});
}
/// Reset price_to_amount for all separate cohorts (called during fresh start).
pub fn reset_separate_price_to_amount(&mut self) -> Result<()> {
self.par_iter_separate_mut().try_for_each(|v| {
if let Some(state) = v.state.as_mut() {
state.reset_price_to_amount_if_needed()?;
}
Ok(())
})
}
/// Compute and push percentiles for aggregate cohorts (all, sth, lth).
/// Computes on-demand by merging age_range cohorts' price_to_amount data.
/// This avoids maintaining redundant aggregate price_to_amount maps.
pub fn truncate_push_aggregate_percentiles(&mut self, dateindex: DateIndex) -> Result<()> {
use std::cmp::Reverse;
use std::collections::BinaryHeap;
// Collect (filter, supply, price_to_amount as Vec) from age_range cohorts
let age_range_data: Vec<_> = self
.0
.age_range
.iter()
.filter_map(|sub| {
let state = sub.state.as_ref()?;
let entries: Vec<(Dollars, Sats)> = state
.price_to_amount_iter()?
.map(|(p, &a)| (p, a))
.collect();
Some((sub.filter().clone(), state.supply.value, entries))
})
.collect();
// Compute percentiles for each aggregate filter
for aggregate in self.0.iter_aggregate_mut() {
let filter = aggregate.filter().clone();
// Get cost_basis percentiles storage, skip if not configured
let Some(percentiles) = aggregate
.metrics
.cost_basis
.as_mut()
.and_then(|cb| cb.percentiles.as_mut())
else {
continue;
};
// Collect relevant cohort data for this aggregate
let relevant: Vec<_> = age_range_data
.iter()
.filter(|(sub_filter, _, _)| filter.includes(sub_filter))
.collect();
// Calculate total supply
let total_supply: u64 = relevant.iter().map(|(_, s, _)| u64::from(*s)).sum();
if total_supply == 0 {
percentiles.truncate_push(dateindex, &[Dollars::NAN; PERCENTILES_LEN])?;
continue;
}
// K-way merge using min-heap: O(n log k) where k = number of cohorts
// Each heap entry: (price, amount, cohort_idx, entry_idx)
let mut heap: BinaryHeap<Reverse<(Dollars, usize, usize)>> = BinaryHeap::new();
// Initialize heap with first entry from each cohort
for (cohort_idx, (_, _, entries)) in relevant.iter().enumerate() {
if !entries.is_empty() {
heap.push(Reverse((entries[0].0, cohort_idx, 0)));
}
}
let targets = PERCENTILES.map(|p| total_supply * u64::from(p) / 100);
let mut result = [Dollars::NAN; PERCENTILES_LEN];
let mut accumulated = 0u64;
let mut pct_idx = 0;
let mut current_price: Option<Dollars> = None;
let mut amount_at_price = 0u64;
while let Some(Reverse((price, cohort_idx, entry_idx))) = heap.pop() {
let (_, _, entries) = relevant[cohort_idx];
let (_, amount) = entries[entry_idx];
// If price changed, finalize previous price
if let Some(current_price) = current_price
&& current_price != price
{
accumulated += amount_at_price;
while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] {
result[pct_idx] = current_price;
pct_idx += 1;
}
if pct_idx >= PERCENTILES_LEN {
break;
}
amount_at_price = 0;
}
current_price = Some(price);
amount_at_price += u64::from(amount);
// Push next entry from this cohort
let next_idx = entry_idx + 1;
if next_idx < entries.len() {
heap.push(Reverse((entries[next_idx].0, cohort_idx, next_idx)));
}
}
// Finalize last price
if let Some(price) = current_price {
accumulated += amount_at_price;
while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] {
result[pct_idx] = price;
pct_idx += 1;
}
}
percentiles.truncate_push(dateindex, &result)?;
}
Ok(())
}
/// Validate computed versions for all cohorts (separate and aggregate).
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
// Validate separate cohorts
self.par_iter_separate_mut()
.try_for_each(|v| v.validate_computed_versions(base_version))?;
// Validate aggregate cohorts' cost_basis percentiles
for v in self.0.iter_aggregate_mut() {
v.validate_computed_versions(base_version)?;
}
Ok(())
}
}
@@ -0,0 +1,7 @@
mod groups;
mod receive;
mod send;
mod tick_tock;
mod vecs;
pub use groups::*;
@@ -0,0 +1,59 @@
use brk_types::{Dollars, Height, Timestamp};
use crate::distribution::state::Transacted;
use super::groups::UTXOCohorts;
impl UTXOCohorts {
/// Process received outputs for this block.
///
/// New UTXOs are added to:
/// - The "up_to_1d" age cohort (all new UTXOs start at 0 days old)
/// - The appropriate epoch cohort based on block height
/// - The appropriate year cohort based on block timestamp
/// - The appropriate output type cohort (P2PKH, P2SH, etc.)
/// - The appropriate amount range cohort based on value
pub fn receive(
&mut self,
received: Transacted,
height: Height,
timestamp: Timestamp,
price: Option<Dollars>,
) {
let supply_state = received.spendable_supply;
// New UTXOs go into up_to_1d, current epoch, and current year
[
&mut self.0.age_range.up_to_1d,
self.0.epoch.mut_vec_from_height(height),
self.0.year.mut_vec_from_timestamp(timestamp),
]
.into_iter()
.for_each(|v| {
v.state.as_mut().unwrap().receive(&supply_state, price);
});
// Update output type cohorts
self.type_
.iter_typed_mut()
.for_each(|(output_type, vecs)| {
vecs.state
.as_mut()
.unwrap()
.receive(received.by_type.get(output_type), price)
});
// Update amount range cohorts
received
.by_size_group
.iter_typed()
.for_each(|(group, supply_state)| {
self.amount_range
.get_mut(group)
.state
.as_mut()
.unwrap()
.receive(supply_state, price);
});
}
}
@@ -0,0 +1,116 @@
use brk_types::{CheckedSub, Height};
use rustc_hash::FxHashMap;
use vecdb::VecIndex;
use crate::{
distribution::state::{BlockState, Transacted},
utils::OptionExt,
};
use super::groups::UTXOCohorts;
impl UTXOCohorts {
/// Process spent inputs for this block.
///
/// Each input references a UTXO created at some previous height.
/// We need to update the cohort states based on when that UTXO was created.
pub fn send(
&mut self,
height_to_sent: FxHashMap<Height, Transacted>,
chain_state: &mut [BlockState],
) {
if chain_state.is_empty() {
return;
}
let last_block = chain_state.last().unwrap();
let last_timestamp = last_block.timestamp;
let current_price = last_block.price;
let chain_len = chain_state.len();
for (height, sent) in height_to_sent {
// Update chain_state to reflect spent supply
chain_state[height.to_usize()].supply -= &sent.spendable_supply;
let block_state = &chain_state[height.to_usize()];
let prev_price = block_state.price;
let blocks_old = chain_len - 1 - height.to_usize();
let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
let days_old_float =
last_timestamp.difference_in_days_between_float(block_state.timestamp);
let older_than_hour = last_timestamp
.checked_sub(block_state.timestamp)
.unwrap()
.is_more_than_hour();
// Update age range cohort (direct index lookup)
self.0
.age_range
.get_mut_by_days_old(days_old)
.state
.um()
.send(
&sent.spendable_supply,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
// Update epoch cohort (direct lookup by height)
self.0.epoch.mut_vec_from_height(height).state.um().send(
&sent.spendable_supply,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
// Update year cohort (direct lookup by timestamp)
self.0
.year
.mut_vec_from_timestamp(block_state.timestamp)
.state
.um()
.send(
&sent.spendable_supply,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
// Update output type cohorts
sent.by_type
.spendable
.iter_typed()
.for_each(|(output_type, supply_state)| {
self.0.type_.get_mut(output_type).state.um().send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
)
});
// Update amount range cohorts
sent.by_size_group
.iter_typed()
.for_each(|(group, supply_state)| {
self.0.amount_range.get_mut(group).state.um().send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
});
}
}
}
@@ -0,0 +1,79 @@
use brk_cohort::AGE_BOUNDARIES;
use brk_types::{ONE_DAY_IN_SEC, Timestamp};
use crate::distribution::state::BlockState;
use super::groups::UTXOCohorts;
impl UTXOCohorts {
/// Handle age transitions when processing a new block.
///
/// UTXOs age with each block. When they cross day boundaries,
/// they move between age-based cohorts (e.g., from "0-1d" to "1-7d").
///
/// Complexity: O(k * (log n + m)) where:
/// - k = 19 boundaries to check
/// - n = total blocks in chain_state
/// - m = blocks crossing each boundary (typically 0-2 per boundary per block)
pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
if chain_state.is_empty() {
return;
}
let prev_timestamp = chain_state.last().unwrap().timestamp;
let elapsed = (*timestamp).saturating_sub(*prev_timestamp);
// Skip if no time has passed
if elapsed == 0 {
return;
}
// Get age_range cohort states (indexed 0..20)
// Cohort i covers days [BOUNDARIES[i-1], BOUNDARIES[i])
// Cohort 0 covers [0, 1) days
// Cohort 19 covers [15*365, infinity) days
let mut age_cohorts: Vec<_> = self.0.age_range.iter_mut().map(|v| &mut v.state).collect();
// For each boundary, find blocks that just crossed it
for (boundary_idx, &boundary_days) in AGE_BOUNDARIES.iter().enumerate() {
let boundary_seconds = (boundary_days as u32) * ONE_DAY_IN_SEC;
// Blocks crossing boundary B have timestamps in (prev - B*DAY, curr - B*DAY]
// prev_days < B and curr_days >= B
// means: block was younger than B days, now is B days or older
let upper_timestamp = (*timestamp).saturating_sub(boundary_seconds);
let lower_timestamp = (*prev_timestamp).saturating_sub(boundary_seconds);
// Skip if the range is empty (would happen if boundary > chain age)
if upper_timestamp <= lower_timestamp {
continue;
}
// Binary search to find blocks in the timestamp range (lower, upper]
let start_idx = chain_state.partition_point(|b| *b.timestamp <= lower_timestamp);
let end_idx = chain_state.partition_point(|b| *b.timestamp <= upper_timestamp);
// Process blocks that crossed this boundary
for block_state in &chain_state[start_idx..end_idx] {
// Double-check the day boundary was actually crossed
// (handles edge cases with day boundaries)
let prev_days = prev_timestamp.difference_in_days_between(block_state.timestamp);
let curr_days = timestamp.difference_in_days_between(block_state.timestamp);
if prev_days >= boundary_days || curr_days < boundary_days {
continue;
}
// Block crossed from cohort[boundary_idx] to cohort[boundary_idx + 1]
// Decrement from the "younger" cohort
if let Some(state) = age_cohorts[boundary_idx].as_mut() {
state.decrement(&block_state.supply, block_state.price);
}
// Increment in the "older" cohort
if let Some(state) = age_cohorts[boundary_idx + 1].as_mut() {
state.increment(&block_state.supply, block_state.price);
}
}
}
}
}
@@ -0,0 +1,260 @@
use std::path::Path;
use brk_cohort::{CohortContext, Filter, Filtered, StateLevel};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, IterableVec};
use crate::{ComputeIndexes, indexes, price, distribution::state::UTXOCohortState};
use crate::distribution::metrics::{CohortMetrics, ImportConfig, SupplyMetrics};
use super::super::traits::{CohortVecs, DynCohortVecs};
/// UTXO cohort with metrics and optional runtime state.
#[derive(Clone, Traversable)]
pub struct UTXOCohortVecs {
/// Starting height when state was imported
state_starting_height: Option<Height>,
/// Runtime state for block-by-block processing (separate cohorts only)
#[traversable(skip)]
pub state: Option<UTXOCohortState>,
/// Metric vectors
#[traversable(flatten)]
pub metrics: CohortMetrics,
}
impl UTXOCohortVecs {
/// Import UTXO cohort from database.
///
/// `all_supply` is the supply metrics from the "all" cohort, used as global
/// sources for `*_rel_to_market_cap` ratios. Pass `None` for the "all" cohort itself.
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
db: &Database,
filter: Filter,
name: &str,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
state_level: StateLevel,
all_supply: Option<&SupplyMetrics>,
) -> Result<Self> {
let compute_dollars = price.is_some();
let full_name = CohortContext::Utxo.full_name(&filter, name);
let cfg = ImportConfig {
db,
filter,
full_name: &full_name,
context: CohortContext::Utxo,
version,
indexes,
price,
};
Ok(Self {
state_starting_height: None,
state: if state_level.is_full() {
Some(UTXOCohortState::new(
states_path,
&full_name,
compute_dollars,
))
} else {
None
},
metrics: CohortMetrics::forced_import(&cfg, all_supply)?,
})
}
/// Get the starting height when state was imported.
pub fn state_starting_height(&self) -> Option<Height> {
self.state_starting_height
}
/// Set the state starting height.
pub fn set_state_starting_height(&mut self, height: Height) {
self.state_starting_height = Some(height);
}
/// Reset state starting height to zero and reset state values.
pub fn reset_state_starting_height(&mut self) {
self.state_starting_height = Some(Height::ZERO);
if let Some(state) = self.state.as_mut() {
state.reset();
}
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.metrics.par_iter_mut()
}
/// Commit state to disk (separate from vec writes for parallelization).
pub fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.write(height, cleanup)?;
}
Ok(())
}
}
impl Filtered for UTXOCohortVecs {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for UTXOCohortVecs {
fn min_stateful_height_len(&self) -> usize {
self.metrics.min_stateful_height_len()
}
fn min_stateful_dateindex_len(&self) -> usize {
self.metrics.min_stateful_dateindex_len()
}
fn reset_state_starting_height(&mut self) {
self.state_starting_height = Some(Height::ZERO);
if let Some(state) = self.state.as_mut() {
state.reset();
}
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
use vecdb::GenericStoredVec;
// Import state from runtime state if present
if let Some(state) = self.state.as_mut() {
// State files are saved AT height H, so to resume at H+1 we need to import at H
// Decrement first, then increment result to match expected starting_height
if let Some(mut prev_height) = starting_height.decremented() {
// Import price_to_amount state file (may adjust prev_height to actual file found)
prev_height = state.import_at_or_before(prev_height)?;
// Restore supply state from height-indexed vectors
state.supply.value = self
.metrics
.supply
.height_to_supply
.read_once(prev_height)?;
state.supply.utxo_count = *self
.metrics
.supply
.height_to_utxo_count
.read_once(prev_height)?;
// Restore realized cap if present
if let Some(realized_metrics) = self.metrics.realized.as_mut()
&& let Some(realized_state) = state.realized.as_mut()
{
realized_state.cap = realized_metrics
.height_to_realized_cap
.read_once(prev_height)?;
}
let result = prev_height.incremented();
self.state_starting_height = Some(result);
Ok(result)
} else {
// starting_height is 0, nothing to import
self.state_starting_height = Some(Height::ZERO);
Ok(Height::ZERO)
}
} else {
self.state_starting_height = Some(starting_height);
Ok(starting_height)
}
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.metrics.validate_computed_versions(base_version)
}
fn truncate_push(&mut self, height: Height) -> Result<()> {
if self.state_starting_height.is_some_and(|h| h > height) {
return Ok(());
}
// Push from state to metrics
if let Some(state) = self.state.as_ref() {
self.metrics.truncate_push(height, state)?;
}
Ok(())
}
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()> {
if let Some(state) = self.state.as_mut() {
self.metrics.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
state,
)?;
}
Ok(())
}
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(indexes, price, starting_indexes, exit)
}
}
impl CohortVecs for UTXOCohortVecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.metrics.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
exit,
)
}
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.metrics.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)
}
}
@@ -0,0 +1,91 @@
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height};
use log::info;
use vecdb::{Exit, IterableVec};
use crate::{ComputeIndexes, indexes, price};
use super::super::cohorts::{AddressCohorts, UTXOCohorts};
/// Compute overlapping cohorts from component cohorts.
///
/// For example:
/// - ">=1d" UTXO cohort is computed from sum of age_range cohorts that match
/// - ">=1 BTC" address cohort is computed from sum of amount_range cohorts that match
pub fn compute_overlapping(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
info!("Computing overlapping cohorts...");
utxo_cohorts.compute_overlapping_vecs(starting_indexes, exit)?;
address_cohorts.compute_overlapping_vecs(starting_indexes, exit)?;
Ok(())
}
/// First phase of post-processing: compute index transforms.
///
/// Converts height-indexed data to dateindex-indexed data and other transforms.
pub fn compute_rest_part1(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
info!("Computing rest part 1...");
utxo_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?;
address_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?;
Ok(())
}
/// Second phase of post-processing: compute relative metrics.
///
/// Computes supply ratios, market cap ratios, etc. using total references.
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2<S, HM, DM>(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &S,
height_to_market_cap: Option<&HM>,
dateindex_to_market_cap: Option<&DM>,
exit: &Exit,
) -> Result<()>
where
S: IterableVec<Height, Bitcoin> + Sync,
HM: IterableVec<Height, Dollars> + Sync,
DM: IterableVec<DateIndex, Dollars> + Sync,
{
info!("Computing rest part 2...");
utxo_cohorts.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)?;
address_cohorts.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)?;
Ok(())
}
@@ -0,0 +1,485 @@
use std::thread;
use brk_cohort::ByAddressType;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{DateIndex, Height, OutputType, Sats, TxIndex, TypeIndex};
use log::info;
use rayon::prelude::*;
use vecdb::{Exit, IterableVec, TypedVecIterator, VecIndex};
use crate::{
blocks, transactions, indexes, price,
distribution::{
address::AddressTypeToAddressCount,
block::{
AddressCache, InputsResult, process_inputs, process_outputs, process_received,
process_sent,
},
compute::write::{process_address_updates, write},
state::{BlockState, Transacted},
},
inputs, outputs,
utils::OptionExt,
};
use super::{
super::{
RangeMap,
cohorts::{AddressCohorts, DynCohortVecs, UTXOCohorts},
vecs::Vecs,
},
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, TxInIterators, TxOutIterators,
VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex,
};
/// Process all blocks from starting_height to last_height.
#[allow(clippy::too_many_arguments)]
pub fn process_blocks(
vecs: &mut Vecs,
indexer: &Indexer,
indexes: &indexes::Vecs,
inputs: &inputs::Vecs,
outputs: &outputs::Vecs,
transactions: &transactions::Vecs,
blocks: &blocks::Vecs,
price: Option<&price::Vecs>,
starting_height: Height,
last_height: Height,
chain_state: &mut Vec<BlockState>,
exit: &Exit,
) -> Result<()> {
// Create computation context with pre-computed vectors for thread-safe access
let ctx = ComputeContext::new(starting_height, last_height, blocks, price);
if ctx.starting_height > ctx.last_height {
return Ok(());
}
// References to vectors using correct field paths
// From indexer.vecs:
let height_to_first_txindex = &indexer.vecs.tx.height_to_first_txindex;
let height_to_first_txoutindex = &indexer.vecs.txout.height_to_first_txoutindex;
let height_to_first_txinindex = &indexer.vecs.txin.height_to_first_txinindex;
// From transactions and inputs/outputs (via .height.u() or .height.unwrap_sum() patterns):
let height_to_tx_count = transactions.count.indexes_to_tx_count.height.u();
let height_to_output_count = outputs.count.indexes_to_count.height.unwrap_sum();
let height_to_input_count = inputs.count.indexes_to_count.height.unwrap_sum();
// From blocks:
let height_to_timestamp = &blocks.time.height_to_timestamp_fixed;
let height_to_date = &blocks.time.height_to_date_fixed;
let dateindex_to_first_height = &indexes.time.dateindex_to_first_height;
let dateindex_to_height_count = &indexes.time.dateindex_to_height_count;
let txindex_to_output_count = &indexes.transaction.txindex_to_output_count;
let txindex_to_input_count = &indexes.transaction.txindex_to_input_count;
// From price (optional):
let height_to_price = price.map(|p| &p.usd.chainindexes_to_price_close.height);
let dateindex_to_price = price.map(|p| p.usd.timeindexes_to_price_close.dateindex.u());
// Access pre-computed vectors from context for thread-safe access
let height_to_price_vec = &ctx.height_to_price;
let height_to_timestamp_vec = &ctx.height_to_timestamp;
// Create iterators for sequential access
let mut height_to_first_txindex_iter = height_to_first_txindex.into_iter();
let mut height_to_first_txoutindex_iter = height_to_first_txoutindex.into_iter();
let mut height_to_first_txinindex_iter = height_to_first_txinindex.into_iter();
let mut height_to_tx_count_iter = height_to_tx_count.into_iter();
let mut height_to_output_count_iter = height_to_output_count.into_iter();
let mut height_to_input_count_iter = height_to_input_count.into_iter();
let mut height_to_timestamp_iter = height_to_timestamp.into_iter();
let mut height_to_date_iter = height_to_date.into_iter();
let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter();
let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter();
let mut txindex_to_output_count_iter = txindex_to_output_count.iter();
let mut txindex_to_input_count_iter = txindex_to_input_count.iter();
let mut height_to_price_iter = height_to_price.map(|v| v.into_iter());
let mut dateindex_to_price_iter = dateindex_to_price.map(|v| v.into_iter());
let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
// Build txindex -> height lookup map for efficient prev_height computation
let mut txindex_to_height: RangeMap<TxIndex, Height> = {
let mut map = RangeMap::with_capacity(last_height.to_usize() + 1);
for first_txindex in indexer.vecs.tx.height_to_first_txindex.into_iter() {
map.push(first_txindex);
}
map
};
// Create reusable iterators for sequential txout/txin reads (16KB buffered)
let mut txout_iters = TxOutIterators::new(indexer);
let mut txin_iters = TxInIterators::new(indexer, inputs, &mut txindex_to_height);
// Create iterators for first address indexes per type
let mut first_p2a_iter = indexer
.vecs
.address
.height_to_first_p2aaddressindex
.into_iter();
let mut first_p2pk33_iter = indexer
.vecs
.address
.height_to_first_p2pk33addressindex
.into_iter();
let mut first_p2pk65_iter = indexer
.vecs
.address
.height_to_first_p2pk65addressindex
.into_iter();
let mut first_p2pkh_iter = indexer
.vecs
.address
.height_to_first_p2pkhaddressindex
.into_iter();
let mut first_p2sh_iter = indexer
.vecs
.address
.height_to_first_p2shaddressindex
.into_iter();
let mut first_p2tr_iter = indexer
.vecs
.address
.height_to_first_p2traddressindex
.into_iter();
let mut first_p2wpkh_iter = indexer
.vecs
.address
.height_to_first_p2wpkhaddressindex
.into_iter();
let mut first_p2wsh_iter = indexer
.vecs
.address
.height_to_first_p2wshaddressindex
.into_iter();
// Track running totals - recover from previous height if resuming
let (mut addresstype_to_addr_count, mut addresstype_to_empty_addr_count) =
if starting_height > Height::ZERO {
let addr_count = AddressTypeToAddressCount::from((
&vecs.addresstype_to_height_to_addr_count,
starting_height,
));
let empty_addr_count = AddressTypeToAddressCount::from((
&vecs.addresstype_to_height_to_empty_addr_count,
starting_height,
));
(addr_count, empty_addr_count)
} else {
(
AddressTypeToAddressCount::default(),
AddressTypeToAddressCount::default(),
)
};
let mut cache = AddressCache::new();
// Main block iteration
for height in starting_height.to_usize()..=last_height.to_usize() {
let height = Height::from(height);
info!("Processing chain at {}...", height);
// Get block metadata
let first_txindex = height_to_first_txindex_iter.get_unwrap(height);
let tx_count = u64::from(height_to_tx_count_iter.get_unwrap(height));
let first_txoutindex = height_to_first_txoutindex_iter
.get_unwrap(height)
.to_usize();
let output_count = u64::from(height_to_output_count_iter.get_unwrap(height)) as usize;
let first_txinindex = height_to_first_txinindex_iter.get_unwrap(height).to_usize();
let input_count = u64::from(height_to_input_count_iter.get_unwrap(height)) as usize;
let timestamp = height_to_timestamp_iter.get_unwrap(height);
let block_price = height_to_price_iter.as_mut().map(|v| *v.get_unwrap(height));
// Debug validation: verify context methods match iterator values
debug_assert_eq!(ctx.timestamp_at(height), timestamp);
debug_assert_eq!(ctx.price_at(height), block_price);
// Build txindex mappings for this block
let txoutindex_to_txindex =
build_txoutindex_to_txindex(first_txindex, tx_count, &mut txindex_to_output_count_iter);
let txinindex_to_txindex =
build_txinindex_to_txindex(first_txindex, tx_count, &mut txindex_to_input_count_iter);
// Get first address indexes for this height
let first_addressindexes = ByAddressType {
p2a: TypeIndex::from(first_p2a_iter.get_unwrap(height).to_usize()),
p2pk33: TypeIndex::from(first_p2pk33_iter.get_unwrap(height).to_usize()),
p2pk65: TypeIndex::from(first_p2pk65_iter.get_unwrap(height).to_usize()),
p2pkh: TypeIndex::from(first_p2pkh_iter.get_unwrap(height).to_usize()),
p2sh: TypeIndex::from(first_p2sh_iter.get_unwrap(height).to_usize()),
p2tr: TypeIndex::from(first_p2tr_iter.get_unwrap(height).to_usize()),
p2wpkh: TypeIndex::from(first_p2wpkh_iter.get_unwrap(height).to_usize()),
p2wsh: TypeIndex::from(first_p2wsh_iter.get_unwrap(height).to_usize()),
};
// Reset per-block values for all separate cohorts
reset_block_values(&mut vecs.utxo_cohorts, &mut vecs.address_cohorts);
// Collect output/input data using reusable iterators (16KB buffered reads)
// Must be done before thread::scope since iterators aren't Send
let txoutdata_vec = txout_iters.collect_block_outputs(first_txoutindex, output_count);
let (input_values, input_prev_heights, input_outputtypes, input_typeindexes) =
if input_count > 1 {
txin_iters.collect_block_inputs(first_txinindex + 1, input_count - 1, height)
} else {
(Vec::new(), Vec::new(), Vec::new(), Vec::new())
};
// Process outputs and inputs in parallel with tick-tock
let (outputs_result, inputs_result) = thread::scope(|scope| {
// Tick-tock age transitions in background
scope.spawn(|| {
vecs.utxo_cohorts
.tick_tock_next_block(chain_state, timestamp);
});
let outputs_handle = scope.spawn(|| {
// Process outputs (receive)
process_outputs(
&txoutindex_to_txindex,
&txoutdata_vec,
&first_addressindexes,
&cache,
&vr,
&vecs.any_address_indexes,
&vecs.addresses_data,
)
});
// Process inputs (send) - skip coinbase input
let inputs_result = if input_count > 1 {
process_inputs(
input_count - 1,
&txinindex_to_txindex[1..], // Skip coinbase
&input_values,
&input_outputtypes,
&input_typeindexes,
&input_prev_heights,
&first_addressindexes,
&cache,
&vr,
&vecs.any_address_indexes,
&vecs.addresses_data,
)
} else {
InputsResult {
height_to_sent: Default::default(),
sent_data: Default::default(),
address_data: Default::default(),
txindex_vecs: Default::default(),
}
};
let outputs_result = outputs_handle.join().unwrap();
(outputs_result, inputs_result)
});
// Merge new address data into current cache
cache.merge_loaded(outputs_result.address_data);
cache.merge_loaded(inputs_result.address_data);
// Combine txindex_vecs from outputs and inputs, then update tx_count
let combined_txindex_vecs = outputs_result
.txindex_vecs
.merge_vec(inputs_result.txindex_vecs);
cache.update_tx_counts(combined_txindex_vecs);
let mut transacted = outputs_result.transacted;
let mut height_to_sent = inputs_result.height_to_sent;
// Handle special cases
if height == Height::ZERO {
// Genesis block - reset transacted (50 BTC is unspendable, handled in supply module)
transacted = Transacted::default();
} else if height == Height::new(BIP30_DUPLICATE_HEIGHT_1)
|| height == Height::new(BIP30_DUPLICATE_HEIGHT_2)
{
// BIP30: Add 50 BTC to spent from original height
let original_height = if height == Height::new(BIP30_DUPLICATE_HEIGHT_1) {
Height::new(BIP30_ORIGINAL_HEIGHT_1)
} else {
Height::new(BIP30_ORIGINAL_HEIGHT_2)
};
height_to_sent
.entry(original_height)
.or_default()
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
}
// Push current block state before processing cohort updates
chain_state.push(BlockState {
supply: transacted.spendable_supply.clone(),
price: block_price,
timestamp,
});
// Process UTXO cohorts and Address cohorts in parallel
// - Main thread: UTXO cohorts receive/send
// - Spawned thread: Address cohorts process_received/process_sent
thread::scope(|scope| {
// Spawn address cohort processing in background thread
scope.spawn(|| {
let mut lookup = cache.as_lookup();
// Process received outputs (addresses receiving funds)
process_received(
outputs_result.received_data,
&mut vecs.address_cohorts,
&mut lookup,
block_price,
&mut addresstype_to_addr_count,
&mut addresstype_to_empty_addr_count,
);
// Process sent inputs (addresses sending funds)
// Uses separate price/timestamp vecs to avoid borrowing chain_state
process_sent(
inputs_result.sent_data,
&mut vecs.address_cohorts,
&mut lookup,
block_price,
&mut addresstype_to_addr_count,
&mut addresstype_to_empty_addr_count,
height_to_price_vec.as_deref(),
height_to_timestamp_vec,
height,
timestamp,
)
.unwrap();
});
// Main thread: Update UTXO cohorts
vecs.utxo_cohorts
.receive(transacted, height, timestamp, block_price);
vecs.utxo_cohorts.send(height_to_sent, chain_state);
});
// Push to height-indexed vectors
vecs.addresstype_to_height_to_addr_count
.truncate_push(height, &addresstype_to_addr_count)?;
vecs.addresstype_to_height_to_empty_addr_count
.truncate_push(height, &addresstype_to_empty_addr_count)?;
// Get date info for unrealized state computation
let date = height_to_date_iter.get_unwrap(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height_iter.get_unwrap(dateindex);
let date_height_count = dateindex_to_height_count_iter.get_unwrap(dateindex);
let is_date_last_height =
date_first_height + Height::from(date_height_count).decremented().unwrap() == height;
let date_price = dateindex_to_price_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.get_unwrap(dateindex)));
let dateindex_opt = is_date_last_height.then_some(dateindex);
// Push cohort states and compute unrealized
push_cohort_states(
&mut vecs.utxo_cohorts,
&mut vecs.address_cohorts,
height,
block_price,
dateindex_opt,
date_price,
)?;
// Compute and push percentiles for aggregate cohorts (all, sth, lth)
if let Some(dateindex) = dateindex_opt {
vecs.utxo_cohorts
.truncate_push_aggregate_percentiles(dateindex)?;
}
// Periodic checkpoint flush
if height != last_height
&& height != Height::ZERO
&& height.to_usize() % FLUSH_INTERVAL == 0
{
// Drop readers to release mmap handles
drop(vr);
let (empty_updates, loaded_updates) = cache.take();
// Process address updates (mutations)
process_address_updates(
&mut vecs.addresses_data,
&mut vecs.any_address_indexes,
empty_updates,
loaded_updates,
)?;
let _lock = exit.lock();
// Write to disk (pure I/O) - no changes saved for periodic flushes
write(vecs, height, chain_state, false)?;
vecs.flush()?;
// Recreate readers
vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
}
}
// Final write - always save changes for rollback support
let _lock = exit.lock();
drop(vr);
let (empty_updates, loaded_updates) = cache.take();
// Process address updates (mutations)
process_address_updates(
&mut vecs.addresses_data,
&mut vecs.any_address_indexes,
empty_updates,
loaded_updates,
)?;
// Write to disk (pure I/O) - save changes for rollback
write(vecs, last_height, chain_state, true)?;
Ok(())
}
/// Reset per-block values for all separate cohorts.
fn reset_block_values(utxo_cohorts: &mut UTXOCohorts, address_cohorts: &mut AddressCohorts) {
utxo_cohorts.iter_separate_mut().for_each(|v| {
if let Some(state) = v.state.as_mut() {
state.reset_single_iteration_values();
}
});
address_cohorts.iter_separate_mut().for_each(|v| {
if let Some(state) = v.state.as_mut() {
state.inner.reset_single_iteration_values();
}
});
}
/// Push cohort states to height-indexed vectors.
fn push_cohort_states(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
height: Height,
height_price: Option<brk_types::Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<brk_types::Dollars>>,
) -> Result<()> {
// utxo_cohorts.iter_separate_mut().try_for_each(|v| {
utxo_cohorts.par_iter_separate_mut().try_for_each(|v| {
v.truncate_push(height)?;
v.compute_then_truncate_push_unrealized_states(height, height_price, dateindex, date_price)
})?;
// address_cohorts.iter_separate_mut().try_for_each(|v| {
address_cohorts.par_iter_separate_mut().try_for_each(|v| {
v.truncate_push(height)?;
v.compute_then_truncate_push_unrealized_states(height, height_price, dateindex, date_price)
})?;
Ok(())
}
@@ -0,0 +1,56 @@
use brk_types::{Dollars, Height, Timestamp};
use vecdb::VecIndex;
use crate::{blocks, price};
/// Context shared across block processing.
pub struct ComputeContext {
/// Starting height for this computation run
pub starting_height: Height,
/// Last height to process
pub last_height: Height,
/// Pre-computed height -> timestamp mapping
pub height_to_timestamp: Vec<Timestamp>,
/// Pre-computed height -> price mapping (if available)
pub height_to_price: Option<Vec<Dollars>>,
}
impl ComputeContext {
/// Create a new computation context.
pub fn new(
starting_height: Height,
last_height: Height,
blocks: &blocks::Vecs,
price: Option<&price::Vecs>,
) -> Self {
let height_to_timestamp: Vec<Timestamp> =
blocks.time.height_to_timestamp_fixed.into_iter().collect();
let height_to_price: Option<Vec<Dollars>> = price
.map(|p| &p.usd.chainindexes_to_price_close.height)
.map(|v| v.into_iter().map(|d| *d).collect());
Self {
starting_height,
last_height,
height_to_timestamp,
height_to_price,
}
}
/// Get price at height (None if no price data or height out of range).
pub fn price_at(&self, height: Height) -> Option<Dollars> {
self.height_to_price
.as_ref()?
.get(height.to_usize())
.copied()
}
/// Get timestamp at height.
pub fn timestamp_at(&self, height: Height) -> Timestamp {
self.height_to_timestamp[height.to_usize()]
}
}
@@ -0,0 +1,23 @@
pub mod aggregates;
mod block_loop;
mod context;
mod readers;
mod recover;
mod write;
pub use block_loop::process_blocks;
pub use context::ComputeContext;
pub use readers::{
TxInIterators, TxOutData, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
build_txoutindex_to_txindex,
};
pub use recover::{StartMode, determine_start_mode, recover_state, reset_state};
/// Flush checkpoint interval (every N blocks).
pub const FLUSH_INTERVAL: usize = 10_000;
// BIP30 duplicate coinbase heights (special case handling)
pub const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842;
pub const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880;
pub const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812;
pub const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722;
@@ -0,0 +1,198 @@
use brk_cohort::{ByAddressType, ByAnyAddress};
use brk_indexer::Indexer;
use brk_types::{
Height, OutPoint, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutIndex, TypeIndex,
};
use vecdb::{
BoxedVecIterator, BytesVecIterator, GenericStoredVec, PcodecVecIterator, Reader, VecIndex,
VecIterator,
};
use crate::{
distribution::{
RangeMap,
address::{AddressesDataVecs, AnyAddressIndexesVecs},
},
inputs,
};
/// Output data collected from separate vecs.
#[derive(Debug, Clone, Copy)]
pub struct TxOutData {
pub value: Sats,
pub outputtype: OutputType,
pub typeindex: TypeIndex,
}
/// Reusable iterators for txout vectors (16KB buffered reads).
///
/// Iterators are created once and re-positioned each block to avoid
/// creating new file handles repeatedly.
pub struct TxOutIterators<'a> {
value_iter: BytesVecIterator<'a, TxOutIndex, Sats>,
outputtype_iter: BytesVecIterator<'a, TxOutIndex, OutputType>,
typeindex_iter: BytesVecIterator<'a, TxOutIndex, TypeIndex>,
}
impl<'a> TxOutIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self {
Self {
value_iter: indexer.vecs.txout.txoutindex_to_value.into_iter(),
outputtype_iter: indexer.vecs.txout.txoutindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txout.txoutindex_to_typeindex.into_iter(),
}
}
/// Collect output data for a block range using buffered iteration.
pub fn collect_block_outputs(
&mut self,
first_txoutindex: usize,
output_count: usize,
) -> Vec<TxOutData> {
(first_txoutindex..first_txoutindex + output_count)
.map(|i| TxOutData {
value: self.value_iter.get_at_unwrap(i),
outputtype: self.outputtype_iter.get_at_unwrap(i),
typeindex: self.typeindex_iter.get_at_unwrap(i),
})
.collect()
}
}
/// Reusable iterators for txin vectors (PcoVec - avoids repeated page decompression).
pub struct TxInIterators<'a> {
value_iter: PcodecVecIterator<'a, TxInIndex, Sats>,
outpoint_iter: PcodecVecIterator<'a, TxInIndex, OutPoint>,
outputtype_iter: PcodecVecIterator<'a, TxInIndex, OutputType>,
typeindex_iter: PcodecVecIterator<'a, TxInIndex, TypeIndex>,
txindex_to_height: &'a mut RangeMap<TxIndex, Height>,
}
impl<'a> TxInIterators<'a> {
pub fn new(
indexer: &'a Indexer,
txins: &'a inputs::Vecs,
txindex_to_height: &'a mut RangeMap<TxIndex, Height>,
) -> Self {
Self {
value_iter: txins.spent.txinindex_to_value.into_iter(),
outpoint_iter: indexer.vecs.txin.txinindex_to_outpoint.into_iter(),
outputtype_iter: indexer.vecs.txin.txinindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txin.txinindex_to_typeindex.into_iter(),
txindex_to_height,
}
}
/// Collect input data for a block range using buffered iteration.
/// Computes prev_height on-the-fly from outpoint using RangeMap lookup.
pub fn collect_block_inputs(
&mut self,
first_txinindex: usize,
input_count: usize,
current_height: Height,
) -> (Vec<Sats>, Vec<Height>, Vec<OutputType>, Vec<TypeIndex>) {
let mut values = Vec::with_capacity(input_count);
let mut prev_heights = Vec::with_capacity(input_count);
let mut outputtypes = Vec::with_capacity(input_count);
let mut typeindexes = Vec::with_capacity(input_count);
for i in first_txinindex..first_txinindex + input_count {
values.push(self.value_iter.get_at_unwrap(i));
let outpoint = self.outpoint_iter.get_at_unwrap(i);
let prev_height = if outpoint.is_coinbase() {
current_height
} else {
self.txindex_to_height
.get(outpoint.txindex())
.unwrap_or(current_height)
};
prev_heights.push(prev_height);
outputtypes.push(self.outputtype_iter.get_at_unwrap(i));
typeindexes.push(self.typeindex_iter.get_at_unwrap(i));
}
(values, prev_heights, outputtypes, typeindexes)
}
}
/// Cached readers for stateful vectors.
pub struct VecsReaders {
pub addresstypeindex_to_anyaddressindex: ByAddressType<Reader>,
pub anyaddressindex_to_anyaddressdata: ByAnyAddress<Reader>,
}
impl VecsReaders {
pub fn new(
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> Self {
Self {
addresstypeindex_to_anyaddressindex: ByAddressType {
p2a: any_address_indexes.p2a.create_reader(),
p2pk33: any_address_indexes.p2pk33.create_reader(),
p2pk65: any_address_indexes.p2pk65.create_reader(),
p2pkh: any_address_indexes.p2pkh.create_reader(),
p2sh: any_address_indexes.p2sh.create_reader(),
p2tr: any_address_indexes.p2tr.create_reader(),
p2wpkh: any_address_indexes.p2wpkh.create_reader(),
p2wsh: any_address_indexes.p2wsh.create_reader(),
},
anyaddressindex_to_anyaddressdata: ByAnyAddress {
loaded: addresses_data.loaded.create_reader(),
empty: addresses_data.empty.create_reader(),
},
}
}
/// Get reader for specific address type.
pub fn address_reader(&self, address_type: OutputType) -> &Reader {
self.addresstypeindex_to_anyaddressindex
.get_unwrap(address_type)
}
}
/// Build txoutindex -> txindex mapping for a block.
pub fn build_txoutindex_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
build_index_to_txindex(block_first_txindex, block_tx_count, txindex_to_count)
}
/// Build txinindex -> txindex mapping for a block.
pub fn build_txinindex_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
build_index_to_txindex(block_first_txindex, block_tx_count, txindex_to_count)
}
/// Build index -> txindex mapping for a block (shared implementation).
fn build_index_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
let first = block_first_txindex.to_usize();
let counts: Vec<u64> = (0..block_tx_count as usize)
.map(|offset| {
let txindex = TxIndex::from(first + offset);
u64::from(txindex_to_count.get_unwrap(txindex))
})
.collect();
let total: u64 = counts.iter().sum();
let mut result = Vec::with_capacity(total as usize);
for (offset, &count) in counts.iter().enumerate() {
let txindex = TxIndex::from(first + offset);
result.extend(std::iter::repeat_n(txindex, count as usize));
}
result
}
@@ -0,0 +1,156 @@
use std::{cmp::Ordering, collections::BTreeSet};
use brk_error::Result;
use brk_types::Height;
use vecdb::Stamp;
use super::super::{
AddressesDataVecs,
address::AnyAddressIndexesVecs,
cohorts::{AddressCohorts, UTXOCohorts},
};
/// Result of state recovery.
pub struct RecoveredState {
/// Height to start processing from. Zero means fresh start.
pub starting_height: Height,
}
/// Perform state recovery for resuming from checkpoint.
///
/// Rolls back state vectors and imports cohort states.
/// Validates that all rollbacks and imports are consistent.
/// Returns Height::ZERO if any validation fails (triggers fresh start).
pub fn recover_state(
height: Height,
chain_state_rollback: vecdb::Result<Stamp>,
any_address_indexes: &mut AnyAddressIndexesVecs,
addresses_data: &mut AddressesDataVecs,
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
) -> Result<RecoveredState> {
let stamp = Stamp::from(height);
// Rollback address state vectors
let address_indexes_rollback = any_address_indexes.rollback_before(stamp);
let address_data_rollback = addresses_data.rollback_before(stamp);
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
address_indexes_rollback,
address_data_rollback,
);
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Import UTXO cohort states - all must succeed
if !utxo_cohorts.import_separate_states(height) {
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Import address cohort states - all must succeed
if !address_cohorts.import_separate_states(height) {
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
Ok(RecoveredState {
starting_height: height,
})
}
/// Reset all state for fresh start.
///
/// Resets all state vectors and cohort states.
pub fn reset_state(
any_address_indexes: &mut AnyAddressIndexesVecs,
addresses_data: &mut AddressesDataVecs,
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
) -> Result<RecoveredState> {
// Reset address state
any_address_indexes.reset()?;
addresses_data.reset()?;
// Reset cohort state heights
utxo_cohorts.reset_separate_state_heights();
address_cohorts.reset_separate_state_heights();
// Reset price_to_amount for all cohorts
utxo_cohorts.reset_separate_price_to_amount()?;
address_cohorts.reset_separate_price_to_amount()?;
Ok(RecoveredState {
starting_height: Height::ZERO,
})
}
/// Check if we can resume from a checkpoint or need to start fresh.
pub fn determine_start_mode(computed_min: Height, chain_state_height: Height) -> StartMode {
// No data to resume from
if chain_state_height.is_zero() {
return StartMode::Fresh;
}
match computed_min.cmp(&chain_state_height) {
Ordering::Greater => unreachable!("min height > chain state height"),
Ordering::Equal => StartMode::Resume(chain_state_height),
Ordering::Less => StartMode::Fresh,
}
}
/// Whether to resume from checkpoint or start fresh.
pub enum StartMode {
/// Resume from the given height.
Resume(Height),
/// Start from height 0.
Fresh,
}
/// Rollback state vectors to before a given stamp.
///
/// Returns the consistent starting height if ALL rollbacks succeed and agree,
/// otherwise returns Height::ZERO (need fresh start).
fn rollback_states(
chain_state_rollback: vecdb::Result<Stamp>,
address_indexes_rollbacks: Result<Vec<Stamp>>,
address_data_rollbacks: Result<[Stamp; 2]>,
) -> Height {
let mut heights: BTreeSet<Height> = BTreeSet::new();
// All rollbacks must succeed - any error means fresh start
let Ok(s) = chain_state_rollback else {
return Height::ZERO;
};
heights.insert(Height::from(s).incremented());
let Ok(stamps) = address_indexes_rollbacks else {
return Height::ZERO;
};
for s in stamps {
heights.insert(Height::from(s).incremented());
}
let Ok(stamps) = address_data_rollbacks else {
return Height::ZERO;
};
for s in stamps {
heights.insert(Height::from(s).incremented());
}
// All must agree on the same height
if heights.len() == 1 {
heights.pop_first().unwrap()
} else {
Height::ZERO
}
}
@@ -0,0 +1,100 @@
use std::time::Instant;
use brk_error::Result;
use brk_types::Height;
use log::info;
use rayon::prelude::*;
use vecdb::{AnyStoredVec, GenericStoredVec, Stamp};
use crate::distribution::{
Vecs,
block::{
EmptyAddressDataWithSource, LoadedAddressDataWithSource, process_empty_addresses,
process_loaded_addresses,
},
state::BlockState,
};
use super::super::address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs};
/// Process address updates from caches.
///
/// Applies all accumulated address changes to storage structures:
/// - Processes empty address transitions
/// - Processes loaded address transitions
/// - Updates address indexes
///
/// Call this before `flush()` to prepare data for writing.
pub fn process_address_updates(
addresses_data: &mut AddressesDataVecs,
address_indexes: &mut AnyAddressIndexesVecs,
empty_updates: AddressTypeToTypeIndexMap<EmptyAddressDataWithSource>,
loaded_updates: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
) -> Result<()> {
info!("Processing address updates...");
let empty_result = process_empty_addresses(addresses_data, empty_updates)?;
let loaded_result = process_loaded_addresses(addresses_data, loaded_updates)?;
let all_updates = empty_result.merge(loaded_result);
for (address_type, sorted) in all_updates.into_sorted_iter() {
for (typeindex, any_index) in sorted {
address_indexes.update_or_push(address_type, typeindex, any_index)?;
}
}
Ok(())
}
/// Flush checkpoint to disk (pure I/O, no processing).
///
/// Writes all accumulated data in parallel:
/// - Cohort stateful vectors (parallel internally)
/// - Height-indexed vectors
/// - Address indexes and data
/// - Transaction output index mappings
/// - Chain state
///
/// Set `with_changes=true` near chain tip to enable rollback support.
pub fn write(
vecs: &mut Vecs,
height: Height,
chain_state: &[BlockState],
with_changes: bool,
) -> Result<()> {
info!("Writing to disk...");
let i = Instant::now();
let stamp = Stamp::from(height);
// Prepare chain_state before parallel write
vecs.chain_state.truncate_if_needed(Height::ZERO)?;
for block_state in chain_state {
vecs.chain_state.push(block_state.supply.clone());
}
vecs.any_address_indexes
.par_iter_mut()
.chain(vecs.addresses_data.par_iter_mut())
.chain(vecs.addresstype_to_height_to_addr_count.par_iter_mut())
.chain(
vecs.addresstype_to_height_to_empty_addr_count
.par_iter_mut(),
)
.chain(rayon::iter::once(
&mut vecs.chain_state as &mut dyn AnyStoredVec,
))
.chain(vecs.utxo_cohorts.par_iter_vecs_mut())
.chain(vecs.address_cohorts.par_iter_vecs_mut())
.try_for_each(|v| v.any_stamped_write_maybe_with_changes(stamp, with_changes))?;
// Commit states after vec writes
let cleanup = with_changes;
vecs.utxo_cohorts.commit_all_states(height, cleanup)?;
vecs.address_cohorts.commit_all_states(height, cleanup)?;
info!("Wrote in {:?}", i.elapsed());
Ok(())
}
@@ -0,0 +1,214 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Height, Sats, StoredF64, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec,
PcoVec,
};
use crate::{
ComputeIndexes,
internal::{ComputedValueVecsFromHeight, ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, price,
};
use super::ImportConfig;
/// Activity metrics for a cohort.
#[derive(Clone, Traversable)]
pub struct ActivityMetrics {
/// Total satoshis sent at each height
pub height_to_sent: EagerVec<PcoVec<Height, Sats>>,
/// Sent amounts indexed by various dimensions
pub indexes_to_sent: ComputedValueVecsFromHeight,
/// Satoshi-blocks destroyed (supply * blocks_old when spent)
pub height_to_satblocks_destroyed: EagerVec<PcoVec<Height, Sats>>,
/// Satoshi-days destroyed (supply * days_old when spent)
pub height_to_satdays_destroyed: EagerVec<PcoVec<Height, Sats>>,
/// Coin-blocks destroyed (in BTC rather than sats)
pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
/// Coin-days destroyed (in BTC rather than sats)
pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
}
impl ActivityMetrics {
/// Import activity metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let compute_dollars = cfg.compute_dollars();
let sum_cum = VecBuilderOptions::default().add_sum().add_cumulative();
let height_to_sent: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(cfg.db, &cfg.name("sent"), cfg.version + v0)?;
let indexes_to_sent = ComputedValueVecsFromHeight::forced_import(
cfg.db,
&cfg.name("sent"),
Source::Vec(height_to_sent.boxed_clone()),
cfg.version + v0,
sum_cum,
compute_dollars,
cfg.indexes,
)?;
Ok(Self {
height_to_sent,
indexes_to_sent,
height_to_satblocks_destroyed: EagerVec::forced_import(
cfg.db,
&cfg.name("satblocks_destroyed"),
cfg.version + v0,
)?,
height_to_satdays_destroyed: EagerVec::forced_import(
cfg.db,
&cfg.name("satdays_destroyed"),
cfg.version + v0,
)?,
indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("coinblocks_destroyed"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum_cum,
)?,
indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("coindays_destroyed"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum_cum,
)?,
})
}
/// Get minimum length across height-indexed vectors.
pub fn min_len(&self) -> usize {
self.height_to_sent
.len()
.min(self.height_to_satblocks_destroyed.len())
.min(self.height_to_satdays_destroyed.len())
}
/// Push activity state values to height-indexed vectors.
pub fn truncate_push(
&mut self,
height: Height,
sent: Sats,
satblocks_destroyed: Sats,
satdays_destroyed: Sats,
) -> Result<()> {
self.height_to_sent.truncate_push(height, sent)?;
self.height_to_satblocks_destroyed
.truncate_push(height, satblocks_destroyed)?;
self.height_to_satdays_destroyed
.truncate_push(height, satdays_destroyed)?;
Ok(())
}
/// Write height-indexed vectors to disk.
pub fn write(&mut self) -> Result<()> {
self.height_to_sent.write()?;
self.height_to_satblocks_destroyed.write()?;
self.height_to_satdays_destroyed.write()?;
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.height_to_sent as &mut dyn AnyStoredVec,
&mut self.height_to_satblocks_destroyed as &mut dyn AnyStoredVec,
&mut self.height_to_satdays_destroyed as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_sent.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_sent).collect::<Vec<_>>(),
exit,
)?;
self.height_to_satblocks_destroyed.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_satblocks_destroyed)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_satdays_destroyed.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_satdays_destroyed)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_sent.compute_rest(
indexes,
price,
starting_indexes,
exit,
Some(&self.height_to_sent),
)?;
self.indexes_to_coinblocks_destroyed
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_satblocks_destroyed,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)?;
Ok(())
})?;
self.indexes_to_coindays_destroyed
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_satdays_destroyed,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)?;
Ok(())
})?;
Ok(())
}
}
@@ -0,0 +1,49 @@
use brk_cohort::{CohortContext, Filter};
use brk_types::Version;
use vecdb::Database;
use crate::{indexes, price};
/// Configuration for importing metrics.
pub struct ImportConfig<'a> {
pub db: &'a Database,
pub filter: Filter,
pub full_name: &'a str,
pub context: CohortContext,
pub version: Version,
pub indexes: &'a indexes::Vecs,
pub price: Option<&'a price::Vecs>,
}
impl<'a> ImportConfig<'a> {
/// Whether price data is available (enables realized/unrealized metrics).
pub fn compute_dollars(&self) -> bool {
self.price.is_some()
}
/// Whether this is an extended cohort (more relative metrics).
pub fn extended(&self) -> bool {
self.filter.is_extended(self.context)
}
/// Whether to compute relative-to-all metrics.
pub fn compute_rel_to_all(&self) -> bool {
self.filter.compute_rel_to_all()
}
/// Whether to compute adjusted metrics (SOPR, etc.).
pub fn compute_adjusted(&self) -> bool {
self.filter.compute_adjusted(self.context)
}
/// Get full metric name with filter prefix.
pub fn name(&self, suffix: &str) -> String {
if self.full_name.is_empty() {
suffix.to_string()
} else if suffix.is_empty() {
self.full_name.to_string()
} else {
format!("{}_{suffix}", self.full_name)
}
}
}
@@ -0,0 +1,213 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec,
PcoVec,
};
use crate::{
ComputeIndexes,
distribution::state::CohortState,
internal::{ComputedVecsFromHeight, CostBasisPercentiles, Source, VecBuilderOptions},
};
use super::ImportConfig;
/// Cost basis metrics.
#[derive(Clone, Traversable)]
pub struct CostBasisMetrics {
/// Minimum cost basis for any UTXO at this height
pub height_to_min_cost_basis: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_min_cost_basis: ComputedVecsFromHeight<Dollars>,
/// Maximum cost basis for any UTXO at this height
pub height_to_max_cost_basis: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_max_cost_basis: ComputedVecsFromHeight<Dollars>,
/// Cost basis distribution percentiles (median, quartiles, etc.)
pub percentiles: Option<CostBasisPercentiles>,
}
impl CostBasisMetrics {
/// Import cost basis metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let extended = cfg.extended();
let last = VecBuilderOptions::default().add_last();
let height_to_min_cost_basis =
EagerVec::forced_import(cfg.db, &cfg.name("min_cost_basis"), cfg.version + v0)?;
let height_to_max_cost_basis =
EagerVec::forced_import(cfg.db, &cfg.name("max_cost_basis"), cfg.version + v0)?;
Ok(Self {
indexes_to_min_cost_basis: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("min_cost_basis"),
Source::Vec(height_to_min_cost_basis.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
indexes_to_max_cost_basis: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("max_cost_basis"),
Source::Vec(height_to_max_cost_basis.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_min_cost_basis,
height_to_max_cost_basis,
percentiles: extended
.then(|| {
CostBasisPercentiles::forced_import(
cfg.db,
&cfg.name(""),
cfg.version + v0,
cfg.indexes,
true,
)
})
.transpose()?,
})
}
/// Get minimum length across height-indexed vectors written in block loop.
pub fn min_stateful_height_len(&self) -> usize {
self.height_to_min_cost_basis
.len()
.min(self.height_to_max_cost_basis.len())
}
/// Get minimum length across dateindex-indexed vectors written in block loop.
pub fn min_stateful_dateindex_len(&self) -> usize {
self.percentiles
.as_ref()
.map(|p| p.min_stateful_dateindex_len())
.unwrap_or(usize::MAX)
}
/// Push min/max cost basis from state.
pub fn truncate_push_minmax(&mut self, height: Height, state: &CohortState) -> Result<()> {
self.height_to_min_cost_basis.truncate_push(
height,
state
.price_to_amount_first_key_value()
.map(|(dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
)?;
self.height_to_max_cost_basis.truncate_push(
height,
state
.price_to_amount_last_key_value()
.map(|(dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
)?;
Ok(())
}
/// Push cost basis percentiles from state at date boundary.
/// Only called when at the last height of a day.
pub fn truncate_push_percentiles(
&mut self,
dateindex: DateIndex,
state: &CohortState,
) -> Result<()> {
if let Some(percentiles) = self.percentiles.as_mut() {
let percentile_prices = state.compute_percentile_prices();
percentiles.truncate_push(dateindex, &percentile_prices)?;
}
Ok(())
}
/// Write height-indexed vectors to disk.
pub fn write(&mut self) -> Result<()> {
self.height_to_min_cost_basis.write()?;
self.height_to_max_cost_basis.write()?;
if let Some(percentiles) = self.percentiles.as_mut() {
percentiles.write()?;
}
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = vec![
&mut self.height_to_min_cost_basis,
&mut self.height_to_max_cost_basis,
];
if let Some(percentiles) = self.percentiles.as_mut() {
vecs.extend(
percentiles
.vecs
.iter_mut()
.flatten()
.filter_map(|v| v.dateindex.as_mut())
.map(|v| v as &mut dyn AnyStoredVec),
);
}
vecs.into_par_iter()
}
/// Validate computed versions or reset if mismatched.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
if let Some(percentiles) = self.percentiles.as_mut() {
percentiles.validate_computed_version_or_reset(base_version)?;
}
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_min_cost_basis.compute_min_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_min_cost_basis)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_max_cost_basis.compute_max_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_max_cost_basis)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &crate::indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_min_cost_basis.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_min_cost_basis),
)?;
self.indexes_to_max_cost_basis.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_max_cost_basis),
)?;
Ok(())
}
}
@@ -0,0 +1,341 @@
mod activity;
mod config;
mod cost_basis;
mod realized;
mod relative;
mod supply;
mod unrealized;
pub use activity::*;
pub use config::*;
pub use cost_basis::*;
pub use realized::*;
pub use relative::*;
pub use supply::*;
pub use unrealized::*;
use brk_cohort::Filter;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Exit, IterableVec};
use crate::{ComputeIndexes, distribution::state::CohortState, indexes, price as price_vecs};
/// All metrics for a cohort, organized by category.
#[derive(Clone, Traversable)]
pub struct CohortMetrics {
#[traversable(skip)]
pub filter: Filter,
/// Supply and UTXO count (always computed)
pub supply: SupplyMetrics,
/// Transaction activity (always computed)
pub activity: ActivityMetrics,
/// Realized cap and profit/loss (requires price data)
pub realized: Option<RealizedMetrics>,
/// Unrealized profit/loss (requires price data)
pub unrealized: Option<UnrealizedMetrics>,
/// Cost basis metrics (requires price data)
pub cost_basis: Option<CostBasisMetrics>,
/// Relative metrics (requires price data)
pub relative: Option<RelativeMetrics>,
}
impl CohortMetrics {
/// Import all metrics from database.
///
/// `all_supply` is the supply metrics from the "all" cohort, used as global
/// sources for `*_rel_to_market_cap` and `*_rel_to_circulating_supply` ratios.
/// Pass `None` for the "all" cohort itself.
pub fn forced_import(cfg: &ImportConfig, all_supply: Option<&SupplyMetrics>) -> Result<Self> {
let compute_dollars = cfg.compute_dollars();
let supply = SupplyMetrics::forced_import(cfg)?;
let unrealized = compute_dollars
.then(|| UnrealizedMetrics::forced_import(cfg))
.transpose()?;
let relative = unrealized
.as_ref()
.map(|u| RelativeMetrics::forced_import(cfg, u, &supply, all_supply))
.transpose()?;
Ok(Self {
filter: cfg.filter.clone(),
supply,
activity: ActivityMetrics::forced_import(cfg)?,
realized: compute_dollars
.then(|| RealizedMetrics::forced_import(cfg))
.transpose()?,
cost_basis: compute_dollars
.then(|| CostBasisMetrics::forced_import(cfg))
.transpose()?,
relative,
unrealized,
})
}
/// Get minimum length across height-indexed vectors written in block loop.
pub fn min_stateful_height_len(&self) -> usize {
let mut min = self.supply.min_len().min(self.activity.min_len());
if let Some(realized) = &self.realized {
min = min.min(realized.min_stateful_height_len());
}
if let Some(unrealized) = &self.unrealized {
min = min.min(unrealized.min_stateful_height_len());
}
if let Some(cost_basis) = &self.cost_basis {
min = min.min(cost_basis.min_stateful_height_len());
}
min
}
/// Get minimum length across dateindex-indexed vectors written in block loop.
pub fn min_stateful_dateindex_len(&self) -> usize {
let mut min = usize::MAX;
if let Some(unrealized) = &self.unrealized {
min = min.min(unrealized.min_stateful_dateindex_len());
}
if let Some(cost_basis) = &self.cost_basis {
min = min.min(cost_basis.min_stateful_dateindex_len());
}
min
}
/// Push state values to height-indexed vectors.
pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> {
self.supply.truncate_push(height, &state.supply)?;
self.activity.truncate_push(
height,
state.sent,
state.satblocks_destroyed,
state.satdays_destroyed,
)?;
if let (Some(realized), Some(realized_state)) =
(self.realized.as_mut(), state.realized.as_ref())
{
realized.truncate_push(height, realized_state)?;
}
Ok(())
}
/// Write height-indexed vectors to disk.
pub fn write(&mut self) -> Result<()> {
self.supply.write()?;
self.activity.write()?;
if let Some(realized) = self.realized.as_mut() {
realized.write()?;
}
if let Some(unrealized) = self.unrealized.as_mut() {
unrealized.write()?;
}
if let Some(cost_basis) = self.cost_basis.as_mut() {
cost_basis.write()?;
}
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = Vec::new();
vecs.extend(self.supply.par_iter_mut().collect::<Vec<_>>());
vecs.extend(self.activity.par_iter_mut().collect::<Vec<_>>());
if let Some(realized) = self.realized.as_mut() {
vecs.extend(realized.par_iter_mut().collect::<Vec<_>>());
}
if let Some(unrealized) = self.unrealized.as_mut() {
vecs.extend(unrealized.par_iter_mut().collect::<Vec<_>>());
}
if let Some(cost_basis) = self.cost_basis.as_mut() {
vecs.extend(cost_basis.par_iter_mut().collect::<Vec<_>>());
}
vecs.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.supply.validate_computed_versions(base_version)?;
self.activity.validate_computed_versions(base_version)?;
if let Some(realized) = self.realized.as_mut() {
realized.validate_computed_versions(base_version)?;
}
if let Some(cost_basis) = self.cost_basis.as_mut() {
cost_basis.validate_computed_versions(base_version)?;
}
Ok(())
}
/// Compute and push unrealized states.
/// Percentiles are only computed at date boundaries (when dateindex is Some).
pub fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
state: &mut CohortState,
) -> Result<()> {
// Apply pending updates before reading
state.apply_pending();
if let (Some(unrealized), Some(cost_basis), Some(height_price)) = (
self.unrealized.as_mut(),
self.cost_basis.as_mut(),
height_price,
) {
cost_basis.truncate_push_minmax(height, state)?;
let (height_unrealized_state, date_unrealized_state) =
state.compute_unrealized_states(height_price, date_price.unwrap());
unrealized.truncate_push(
height,
dateindex,
&height_unrealized_state,
date_unrealized_state.as_ref(),
)?;
// Only compute expensive percentiles at date boundaries (~144x reduction)
if let Some(dateindex) = dateindex {
cost_basis.truncate_push_percentiles(dateindex, state)?;
}
}
Ok(())
}
/// Compute aggregate cohort values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.supply.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.supply).collect::<Vec<_>>(),
exit,
)?;
self.activity.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.activity).collect::<Vec<_>>(),
exit,
)?;
if let Some(realized) = self.realized.as_mut() {
realized.compute_from_stateful(
starting_indexes,
&others
.iter()
.filter_map(|v| v.realized.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
}
if let Some(unrealized) = self.unrealized.as_mut() {
unrealized.compute_from_stateful(
starting_indexes,
&others
.iter()
.filter_map(|v| v.unrealized.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
}
if let Some(cost_basis) = self.cost_basis.as_mut() {
cost_basis.compute_from_stateful(
starting_indexes,
&others
.iter()
.filter_map(|v| v.cost_basis.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
}
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price_vecs::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.supply
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
self.activity
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
if let Some(realized) = self.realized.as_mut() {
realized.compute_rest_part1(indexes, starting_indexes, exit)?;
}
if let Some(unrealized) = self.unrealized.as_mut() {
unrealized.compute_rest_part1(price, starting_indexes, exit)?;
}
if let Some(cost_basis) = self.cost_basis.as_mut() {
cost_basis.compute_rest_part1(indexes, starting_indexes, exit)?;
}
Ok(())
}
/// Second phase of computed metrics (ratios, relative values).
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price_vecs::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
if let Some(realized) = self.realized.as_mut() {
realized.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,921 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, Ident, ImportableVec,
IterableCloneableVec, IterableVec, Negate, PcoVec,
};
use crate::{
ComputeIndexes,
internal::{
ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
LazyVecsFrom2FromHeight, LazyVecsFromDateIndex, LazyVecsFromHeight, PercentageDollarsF32,
Source, StoredF32Identity, VecBuilderOptions,
},
indexes, price,
distribution::state::RealizedState,
utils::OptionExt,
};
use super::ImportConfig;
/// Realized cap and related metrics.
#[derive(Clone, Traversable)]
pub struct RealizedMetrics {
// === Realized Cap ===
pub height_to_realized_cap: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_realized_cap: ComputedVecsFromHeight<Dollars>,
pub indexes_to_realized_price: ComputedVecsFromHeight<Dollars>,
pub indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex,
pub indexes_to_realized_cap_rel_to_own_market_cap: Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex<Dollars>,
// === MVRV (Market Value to Realized Value) ===
// Proxy for indexes_to_realized_price_extra.ratio (close / realized_price = market_cap / realized_cap)
pub indexes_to_mvrv: LazyVecsFromDateIndex<StoredF32>,
// === Realized Profit/Loss ===
pub height_to_realized_profit: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_realized_profit: ComputedVecsFromHeight<Dollars>,
pub height_to_realized_loss: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_realized_loss: ComputedVecsFromHeight<Dollars>,
pub indexes_to_neg_realized_loss: LazyVecsFromHeight<Dollars>,
pub indexes_to_net_realized_pnl: ComputedVecsFromHeight<Dollars>,
pub indexes_to_realized_value: ComputedVecsFromHeight<Dollars>,
// === Realized vs Realized Cap Ratios (lazy) ===
pub indexes_to_realized_profit_rel_to_realized_cap:
LazyVecsFrom2FromHeight<StoredF32, Dollars, Dollars>,
pub indexes_to_realized_loss_rel_to_realized_cap:
LazyVecsFrom2FromHeight<StoredF32, Dollars, Dollars>,
pub indexes_to_net_realized_pnl_rel_to_realized_cap:
LazyVecsFrom2FromHeight<StoredF32, Dollars, Dollars>,
// === Total Realized PnL ===
pub indexes_to_total_realized_pnl: LazyVecsFromHeight<Dollars>,
pub dateindex_to_realized_profit_to_loss_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
// === Value Created/Destroyed ===
pub height_to_value_created: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_value_created: ComputedVecsFromHeight<Dollars>,
pub height_to_value_destroyed: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_value_destroyed: ComputedVecsFromHeight<Dollars>,
// === Adjusted Value (optional) ===
pub height_to_adjusted_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_adjusted_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_adjusted_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
// === SOPR (Spent Output Profit Ratio) ===
pub dateindex_to_sopr: EagerVec<PcoVec<DateIndex, StoredF64>>,
pub dateindex_to_sopr_7d_ema: EagerVec<PcoVec<DateIndex, StoredF64>>,
pub dateindex_to_sopr_30d_ema: EagerVec<PcoVec<DateIndex, StoredF64>>,
pub dateindex_to_adjusted_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
// === Sell Side Risk ===
pub dateindex_to_sell_side_risk_ratio: EagerVec<PcoVec<DateIndex, StoredF32>>,
pub dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec<PcoVec<DateIndex, StoredF32>>,
pub dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec<PcoVec<DateIndex, StoredF32>>,
// === Net Realized PnL Deltas ===
pub indexes_to_net_realized_pnl_cumulative_30d_delta: ComputedVecsFromDateIndex<Dollars>,
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
ComputedVecsFromDateIndex<StoredF32>,
}
impl RealizedMetrics {
/// Import realized metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let v1 = Version::ONE;
let v3 = Version::new(3);
let extended = cfg.extended();
let compute_adjusted = cfg.compute_adjusted();
let last = VecBuilderOptions::default().add_last();
let sum = VecBuilderOptions::default().add_sum();
let sum_cum = VecBuilderOptions::default().add_sum().add_cumulative();
let height_to_realized_loss: EagerVec<PcoVec<Height, Dollars>> =
EagerVec::forced_import(cfg.db, &cfg.name("realized_loss"), cfg.version + v0)?;
let indexes_to_realized_loss = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_loss"),
Source::Vec(height_to_realized_loss.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum_cum,
)?;
let indexes_to_neg_realized_loss = LazyVecsFromHeight::from_computed::<Negate>(
&cfg.name("neg_realized_loss"),
cfg.version + v1,
height_to_realized_loss.boxed_clone(),
&indexes_to_realized_loss,
);
// realized_value is the source for total_realized_pnl (they're identical)
let indexes_to_realized_value = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_value"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum,
)?;
// total_realized_pnl is a lazy alias to realized_value
let indexes_to_total_realized_pnl = LazyVecsFromHeight::from_computed::<Ident>(
&cfg.name("total_realized_pnl"),
cfg.version + v1,
indexes_to_realized_value
.height
.as_ref()
.unwrap()
.boxed_clone(),
&indexes_to_realized_value,
);
// Extract vecs needed for lazy ratio construction
let height_to_realized_cap: EagerVec<PcoVec<Height, Dollars>> =
EagerVec::forced_import(cfg.db, &cfg.name("realized_cap"), cfg.version + v0)?;
let indexes_to_realized_cap = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_cap"),
Source::Vec(height_to_realized_cap.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?;
let height_to_realized_profit: EagerVec<PcoVec<Height, Dollars>> =
EagerVec::forced_import(cfg.db, &cfg.name("realized_profit"), cfg.version + v0)?;
let indexes_to_realized_profit = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_profit"),
Source::Vec(height_to_realized_profit.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum_cum,
)?;
let indexes_to_net_realized_pnl = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("net_realized_pnl"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum_cum,
)?;
// Construct lazy ratio vecs (before struct assignment to satisfy borrow checker)
let indexes_to_realized_profit_rel_to_realized_cap =
LazyVecsFrom2FromHeight::from_computed::<PercentageDollarsF32>(
&cfg.name("realized_profit_rel_to_realized_cap"),
cfg.version + v1,
height_to_realized_profit.boxed_clone(),
height_to_realized_cap.boxed_clone(),
&indexes_to_realized_profit,
&indexes_to_realized_cap,
);
let indexes_to_realized_loss_rel_to_realized_cap =
LazyVecsFrom2FromHeight::from_computed::<PercentageDollarsF32>(
&cfg.name("realized_loss_rel_to_realized_cap"),
cfg.version + v1,
height_to_realized_loss.boxed_clone(),
height_to_realized_cap.boxed_clone(),
&indexes_to_realized_loss,
&indexes_to_realized_cap,
);
let indexes_to_net_realized_pnl_rel_to_realized_cap =
LazyVecsFrom2FromHeight::from_computed::<PercentageDollarsF32>(
&cfg.name("net_realized_pnl_rel_to_realized_cap"),
cfg.version + v1,
indexes_to_net_realized_pnl
.height
.as_ref()
.unwrap()
.boxed_clone(),
height_to_realized_cap.boxed_clone(),
&indexes_to_net_realized_pnl,
&indexes_to_realized_cap,
);
let indexes_to_realized_price = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_price"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?;
let height_to_value_created =
EagerVec::forced_import(cfg.db, &cfg.name("value_created"), cfg.version + v0)?;
let height_to_value_destroyed =
EagerVec::forced_import(cfg.db, &cfg.name("value_destroyed"), cfg.version + v0)?;
let height_to_adjusted_value_created = compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
cfg.version + v0,
)
})
.transpose()?;
let height_to_adjusted_value_destroyed = compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
cfg.version + v0,
)
})
.transpose()?;
// Create realized_price_extra first so we can reference its ratio for MVRV proxy
let indexes_to_realized_price_extra = ComputedRatioVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("realized_price"),
Some(&indexes_to_realized_price),
cfg.version + v0,
cfg.indexes,
extended,
cfg.price,
)?;
// MVRV is a lazy proxy for realized_price_extra.ratio
// ratio = close / realized_price = market_cap / realized_cap = MVRV
let indexes_to_mvrv = LazyVecsFromDateIndex::from_computed::<StoredF32Identity>(
&cfg.name("mvrv"),
cfg.version + v0,
indexes_to_realized_price_extra
.ratio
.dateindex
.as_ref()
.map(|v| v.boxed_clone()),
&indexes_to_realized_price_extra.ratio,
);
Ok(Self {
// === Realized Cap ===
height_to_realized_cap,
indexes_to_realized_cap,
indexes_to_realized_price_extra,
indexes_to_realized_price,
indexes_to_mvrv,
indexes_to_realized_cap_rel_to_own_market_cap: extended
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_cap_rel_to_own_market_cap"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("realized_cap_30d_delta"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
// === Realized Profit/Loss ===
height_to_realized_profit,
indexes_to_realized_profit,
height_to_realized_loss,
indexes_to_realized_loss,
indexes_to_neg_realized_loss,
indexes_to_net_realized_pnl,
indexes_to_realized_value,
// === Realized vs Realized Cap Ratios (lazy) ===
indexes_to_realized_profit_rel_to_realized_cap,
indexes_to_realized_loss_rel_to_realized_cap,
indexes_to_net_realized_pnl_rel_to_realized_cap,
// === Total Realized PnL ===
indexes_to_total_realized_pnl,
dateindex_to_realized_profit_to_loss_ratio: extended
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("realized_profit_to_loss_ratio"),
cfg.version + v1,
)
})
.transpose()?,
// === Value Created/Destroyed ===
indexes_to_value_created: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("value_created"),
Source::Vec(height_to_value_created.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum,
)?,
indexes_to_value_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("value_destroyed"),
Source::Vec(height_to_value_destroyed.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum,
)?,
height_to_value_created,
height_to_value_destroyed,
// === Adjusted Value (optional) ===
indexes_to_adjusted_value_created: compute_adjusted
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
Source::Vec(
height_to_adjusted_value_created
.as_ref()
.unwrap()
.boxed_clone(),
),
cfg.version + v0,
cfg.indexes,
sum,
)
})
.transpose()?,
indexes_to_adjusted_value_destroyed: compute_adjusted
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
Source::Vec(
height_to_adjusted_value_destroyed
.as_ref()
.unwrap()
.boxed_clone(),
),
cfg.version + v0,
cfg.indexes,
sum,
)
})
.transpose()?,
height_to_adjusted_value_created,
height_to_adjusted_value_destroyed,
// === SOPR ===
dateindex_to_sopr: EagerVec::forced_import(
cfg.db,
&cfg.name("sopr"),
cfg.version + v1,
)?,
dateindex_to_sopr_7d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sopr_7d_ema"),
cfg.version + v1,
)?,
dateindex_to_sopr_30d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sopr_30d_ema"),
cfg.version + v1,
)?,
dateindex_to_adjusted_sopr: compute_adjusted
.then(|| {
EagerVec::forced_import(cfg.db, &cfg.name("adjusted_sopr"), cfg.version + v1)
})
.transpose()?,
dateindex_to_adjusted_sopr_7d_ema: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_sopr_7d_ema"),
cfg.version + v1,
)
})
.transpose()?,
dateindex_to_adjusted_sopr_30d_ema: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_sopr_30d_ema"),
cfg.version + v1,
)
})
.transpose()?,
// === Sell Side Risk ===
dateindex_to_sell_side_risk_ratio: EagerVec::forced_import(
cfg.db,
&cfg.name("sell_side_risk_ratio"),
cfg.version + v1,
)?,
dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sell_side_risk_ratio_7d_ema"),
cfg.version + v1,
)?,
dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sell_side_risk_ratio_30d_ema"),
cfg.version + v1,
)?,
// === Net Realized PnL Deltas ===
indexes_to_net_realized_pnl_cumulative_30d_delta:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_cumulative_30d_delta"),
Source::Compute,
cfg.version + v3,
cfg.indexes,
last,
)?,
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap"),
Source::Compute,
cfg.version + v3,
cfg.indexes,
last,
)?,
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_market_cap"),
Source::Compute,
cfg.version + v3,
cfg.indexes,
last,
)?,
})
}
/// Get minimum length across height-indexed vectors written in block loop.
pub fn min_stateful_height_len(&self) -> usize {
let mut min = self
.height_to_realized_cap
.len()
.min(self.height_to_realized_profit.len())
.min(self.height_to_realized_loss.len())
.min(self.height_to_value_created.len())
.min(self.height_to_value_destroyed.len());
if let Some(v) = &self.height_to_adjusted_value_created {
min = min.min(v.len());
}
if let Some(v) = &self.height_to_adjusted_value_destroyed {
min = min.min(v.len());
}
min
}
/// Push realized state values to height-indexed vectors.
pub fn truncate_push(&mut self, height: Height, state: &RealizedState) -> Result<()> {
self.height_to_realized_cap
.truncate_push(height, state.cap)?;
self.height_to_realized_profit
.truncate_push(height, state.profit)?;
self.height_to_realized_loss
.truncate_push(height, state.loss)?;
self.height_to_value_created
.truncate_push(height, state.value_created)?;
self.height_to_value_destroyed
.truncate_push(height, state.value_destroyed)?;
if let Some(v) = self.height_to_adjusted_value_created.as_mut() {
v.truncate_push(height, state.adj_value_created)?;
}
if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() {
v.truncate_push(height, state.adj_value_destroyed)?;
}
Ok(())
}
/// Write height-indexed vectors to disk.
pub fn write(&mut self) -> Result<()> {
self.height_to_realized_cap.write()?;
self.height_to_realized_profit.write()?;
self.height_to_realized_loss.write()?;
self.height_to_value_created.write()?;
self.height_to_value_destroyed.write()?;
if let Some(v) = self.height_to_adjusted_value_created.as_mut() {
v.write()?;
}
if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() {
v.write()?;
}
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = vec![
&mut self.height_to_realized_cap,
&mut self.height_to_realized_profit,
&mut self.height_to_realized_loss,
&mut self.height_to_value_created,
&mut self.height_to_value_destroyed,
];
if let Some(v) = self.height_to_adjusted_value_created.as_mut() {
vecs.push(v);
}
if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() {
vecs.push(v);
}
vecs.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_realized_cap.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_realized_cap)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_realized_profit.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_realized_profit)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_realized_loss.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_realized_loss)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_value_created.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_value_created)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_value_destroyed.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_value_destroyed)
.collect::<Vec<_>>(),
exit,
)?;
if self.height_to_adjusted_value_created.is_some() {
self.height_to_adjusted_value_created
.um()
.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| {
v.height_to_adjusted_value_created
.as_ref()
.unwrap_or(&v.height_to_value_created)
})
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_adjusted_value_destroyed
.um()
.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| {
v.height_to_adjusted_value_destroyed
.as_ref()
.unwrap_or(&v.height_to_value_destroyed)
})
.collect::<Vec<_>>(),
exit,
)?;
}
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_realized_cap.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_realized_cap),
)?;
self.indexes_to_realized_profit.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_realized_profit),
)?;
self.indexes_to_realized_loss.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_realized_loss),
)?;
// net_realized_pnl = profit - loss
self.indexes_to_net_realized_pnl
.compute_all(indexes, starting_indexes, exit, |vec| {
vec.compute_subtract(
starting_indexes.height,
&self.height_to_realized_profit,
&self.height_to_realized_loss,
exit,
)?;
Ok(())
})?;
// realized_value = profit + loss
// Note: total_realized_pnl is a lazy alias to realized_value since both
// compute profit + loss with sum aggregation, making them identical.
self.indexes_to_realized_value
.compute_all(indexes, starting_indexes, exit, |vec| {
vec.compute_add(
starting_indexes.height,
&self.height_to_realized_profit,
&self.height_to_realized_loss,
exit,
)?;
Ok(())
})?;
self.indexes_to_value_created.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_value_created),
)?;
self.indexes_to_value_destroyed.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_value_destroyed),
)?;
// Optional: adjusted value
if let Some(adjusted_value_created) = self.indexes_to_adjusted_value_created.as_mut() {
adjusted_value_created.compute_rest(
indexes,
starting_indexes,
exit,
self.height_to_adjusted_value_created.as_ref(),
)?;
}
if let Some(adjusted_value_destroyed) = self.indexes_to_adjusted_value_destroyed.as_mut() {
adjusted_value_destroyed.compute_rest(
indexes,
starting_indexes,
exit,
self.height_to_adjusted_value_destroyed.as_ref(),
)?;
}
Ok(())
}
/// Second phase of computed metrics (realized price from realized cap / supply).
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
// realized_price = realized_cap / supply
self.indexes_to_realized_price
.compute_all(indexes, starting_indexes, exit, |vec| {
vec.compute_divide(
starting_indexes.height,
&self.height_to_realized_cap,
height_to_supply,
exit,
)?;
Ok(())
})?;
if let Some(price) = price {
self.indexes_to_realized_price_extra.compute_rest(
price,
starting_indexes,
exit,
Some(self.indexes_to_realized_price.dateindex.unwrap_last()),
)?;
}
// realized_cap_30d_delta
self.indexes_to_realized_cap_30d_delta
.compute_all(starting_indexes, exit, |vec| {
vec.compute_change(
starting_indexes.dateindex,
self.indexes_to_realized_cap.dateindex.unwrap_last(),
30,
exit,
)?;
Ok(())
})?;
// SOPR = value_created / value_destroyed
self.dateindex_to_sopr.compute_divide(
starting_indexes.dateindex,
self.indexes_to_value_created.dateindex.unwrap_sum(),
self.indexes_to_value_destroyed.dateindex.unwrap_sum(),
exit,
)?;
self.dateindex_to_sopr_7d_ema.compute_ema(
starting_indexes.dateindex,
&self.dateindex_to_sopr,
7,
exit,
)?;
self.dateindex_to_sopr_30d_ema.compute_ema(
starting_indexes.dateindex,
&self.dateindex_to_sopr,
30,
exit,
)?;
// Optional: adjusted SOPR
if let (Some(adjusted_sopr), Some(adj_created), Some(adj_destroyed)) = (
self.dateindex_to_adjusted_sopr.as_mut(),
self.indexes_to_adjusted_value_created.as_ref(),
self.indexes_to_adjusted_value_destroyed.as_ref(),
) {
adjusted_sopr.compute_divide(
starting_indexes.dateindex,
adj_created.dateindex.unwrap_sum(),
adj_destroyed.dateindex.unwrap_sum(),
exit,
)?;
if let Some(ema_7d) = self.dateindex_to_adjusted_sopr_7d_ema.as_mut() {
ema_7d.compute_ema(
starting_indexes.dateindex,
self.dateindex_to_adjusted_sopr.as_ref().unwrap(),
7,
exit,
)?;
}
if let Some(ema_30d) = self.dateindex_to_adjusted_sopr_30d_ema.as_mut() {
ema_30d.compute_ema(
starting_indexes.dateindex,
self.dateindex_to_adjusted_sopr.as_ref().unwrap(),
30,
exit,
)?;
}
}
// sell_side_risk_ratio = realized_value / realized_cap
self.dateindex_to_sell_side_risk_ratio.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_realized_value.dateindex.unwrap_sum(),
self.indexes_to_realized_cap.dateindex.unwrap_last(),
exit,
)?;
self.dateindex_to_sell_side_risk_ratio_7d_ema.compute_ema(
starting_indexes.dateindex,
&self.dateindex_to_sell_side_risk_ratio,
7,
exit,
)?;
self.dateindex_to_sell_side_risk_ratio_30d_ema.compute_ema(
starting_indexes.dateindex,
&self.dateindex_to_sell_side_risk_ratio,
30,
exit,
)?;
// Net realized PnL cumulative 30d delta
self.indexes_to_net_realized_pnl_cumulative_30d_delta
.compute_all(starting_indexes, exit, |vec| {
vec.compute_change(
starting_indexes.dateindex,
self.indexes_to_net_realized_pnl
.dateindex
.unwrap_cumulative(),
30,
exit,
)?;
Ok(())
})?;
// Relative to realized cap
self.indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_net_realized_pnl_cumulative_30d_delta
.dateindex
.u(),
self.indexes_to_realized_cap.dateindex.unwrap_last(),
exit,
)?;
Ok(())
})?;
// Relative to market cap
if let Some(dateindex_to_market_cap) = dateindex_to_market_cap {
self.indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_net_realized_pnl_cumulative_30d_delta
.dateindex
.u(),
dateindex_to_market_cap,
exit,
)?;
Ok(())
})?;
}
// Optional: realized_cap_rel_to_own_market_cap
if let (Some(rel_vec), Some(height_to_market_cap)) = (
self.indexes_to_realized_cap_rel_to_own_market_cap.as_mut(),
height_to_market_cap,
) {
rel_vec.compute_all(indexes, starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.height,
&self.height_to_realized_cap,
height_to_market_cap,
exit,
)?;
Ok(())
})?;
}
// Optional: realized_profit_to_loss_ratio
if let Some(ratio) = self.dateindex_to_realized_profit_to_loss_ratio.as_mut() {
ratio.compute_divide(
starting_indexes.dateindex,
self.indexes_to_realized_profit.dateindex.unwrap_sum(),
self.indexes_to_realized_loss.dateindex.unwrap_sum(),
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,467 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Dollars, Height, Sats, StoredF32, StoredF64, Version};
use vecdb::{IterableCloneableVec, LazyVecFrom2};
use crate::internal::{
LazyVecsFrom2FromDateIndex, NegPercentageDollarsF32, NegRatio32, PercentageBtcF64,
PercentageDollarsF32, PercentageSatsF64, Ratio32,
};
use super::{ImportConfig, SupplyMetrics, UnrealizedMetrics};
/// Relative metrics comparing cohort values to global values.
/// All `rel_to_` vecs are lazy - computed on-demand from their sources.
#[derive(Clone, Traversable)]
pub struct RelativeMetrics {
// === Supply Relative to Circulating Supply (lazy from global supply) ===
pub indexes_to_supply_rel_to_circulating_supply:
Option<LazyVecsFrom2FromDateIndex<StoredF64, Sats, Sats>>,
// === Supply in Profit/Loss Relative to Own Supply (lazy) ===
pub height_to_supply_in_profit_rel_to_own_supply:
LazyVecFrom2<Height, StoredF64, Height, Bitcoin, Height, Bitcoin>,
pub height_to_supply_in_loss_rel_to_own_supply:
LazyVecFrom2<Height, StoredF64, Height, Bitcoin, Height, Bitcoin>,
pub indexes_to_supply_in_profit_rel_to_own_supply:
LazyVecsFrom2FromDateIndex<StoredF64, Sats, Sats>,
pub indexes_to_supply_in_loss_rel_to_own_supply:
LazyVecsFrom2FromDateIndex<StoredF64, Sats, Sats>,
// === Supply in Profit/Loss Relative to Circulating Supply (lazy from global supply) ===
pub height_to_supply_in_profit_rel_to_circulating_supply:
Option<LazyVecFrom2<Height, StoredF64, Height, Bitcoin, Height, Bitcoin>>,
pub height_to_supply_in_loss_rel_to_circulating_supply:
Option<LazyVecFrom2<Height, StoredF64, Height, Bitcoin, Height, Bitcoin>>,
pub indexes_to_supply_in_profit_rel_to_circulating_supply:
Option<LazyVecsFrom2FromDateIndex<StoredF64, Sats, Sats>>,
pub indexes_to_supply_in_loss_rel_to_circulating_supply:
Option<LazyVecsFrom2FromDateIndex<StoredF64, Sats, Sats>>,
// === Unrealized vs Market Cap (lazy from global market cap) ===
pub height_to_unrealized_profit_rel_to_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_unrealized_loss_rel_to_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_neg_unrealized_loss_rel_to_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_net_unrealized_pnl_rel_to_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub indexes_to_unrealized_profit_rel_to_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_unrealized_loss_rel_to_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_neg_unrealized_loss_rel_to_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_net_unrealized_pnl_rel_to_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
// === NUPL (Net Unrealized Profit/Loss) ===
// Proxy for indexes_to_net_unrealized_pnl_rel_to_market_cap
pub indexes_to_nupl: Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
// === Unrealized vs Own Market Cap (lazy) ===
pub height_to_unrealized_profit_rel_to_own_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_unrealized_loss_rel_to_own_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_neg_unrealized_loss_rel_to_own_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_net_unrealized_pnl_rel_to_own_market_cap:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub indexes_to_unrealized_profit_rel_to_own_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_unrealized_loss_rel_to_own_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
// === Unrealized vs Own Total Unrealized PnL (lazy) ===
pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
Option<LazyVecFrom2<Height, StoredF32, Height, Dollars, Height, Dollars>>,
pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
Option<LazyVecsFrom2FromDateIndex<StoredF32, Dollars, Dollars>>,
}
impl RelativeMetrics {
/// Import relative metrics from database.
///
/// All `rel_to_` metrics are lazy - computed on-demand from their sources.
/// `all_supply` provides global sources for `*_rel_to_market_cap` and `*_rel_to_circulating_supply`.
pub fn forced_import(
cfg: &ImportConfig,
unrealized: &UnrealizedMetrics,
supply: &SupplyMetrics,
all_supply: Option<&SupplyMetrics>,
) -> Result<Self> {
let v0 = Version::ZERO;
let v1 = Version::ONE;
let v2 = Version::new(2);
let extended = cfg.extended();
let compute_rel_to_all = cfg.compute_rel_to_all();
// Global sources from "all" cohort
let global_supply_sats = all_supply.map(|s| &s.indexes_to_supply.sats);
let global_supply_btc = all_supply.map(|s| &s.height_to_supply_value.bitcoin);
let global_market_cap = all_supply.and_then(|s| s.indexes_to_supply.dollars.as_ref());
let global_market_cap_height =
all_supply.and_then(|s| s.height_to_supply_value.dollars.as_ref());
// Own market cap source
let own_market_cap = supply.indexes_to_supply.dollars.as_ref();
let own_market_cap_height = supply.height_to_supply_value.dollars.as_ref();
Ok(Self {
// === Supply Relative to Circulating Supply (lazy from global supply) ===
indexes_to_supply_rel_to_circulating_supply: (compute_rel_to_all
&& global_supply_sats.is_some())
.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageSatsF64>(
&cfg.name("supply_rel_to_circulating_supply"),
cfg.version + v1,
&supply.indexes_to_supply.sats,
global_supply_sats.unwrap(),
)
}),
// === Supply in Profit/Loss Relative to Own Supply (lazy) ===
height_to_supply_in_profit_rel_to_own_supply: LazyVecFrom2::transformed::<
PercentageBtcF64,
>(
&cfg.name("supply_in_profit_rel_to_own_supply"),
cfg.version + v1,
unrealized
.height_to_supply_in_profit_value
.bitcoin
.boxed_clone(),
supply.height_to_supply_value.bitcoin.boxed_clone(),
),
height_to_supply_in_loss_rel_to_own_supply: LazyVecFrom2::transformed::<PercentageBtcF64>(
&cfg.name("supply_in_loss_rel_to_own_supply"),
cfg.version + v1,
unrealized
.height_to_supply_in_loss_value
.bitcoin
.boxed_clone(),
supply.height_to_supply_value.bitcoin.boxed_clone(),
),
indexes_to_supply_in_profit_rel_to_own_supply:
LazyVecsFrom2FromDateIndex::from_computed::<PercentageSatsF64>(
&cfg.name("supply_in_profit_rel_to_own_supply"),
cfg.version + v1,
&unrealized.indexes_to_supply_in_profit.sats,
&supply.indexes_to_supply.sats,
),
indexes_to_supply_in_loss_rel_to_own_supply: LazyVecsFrom2FromDateIndex::from_computed::<
PercentageSatsF64,
>(
&cfg.name("supply_in_loss_rel_to_own_supply"),
cfg.version + v1,
&unrealized.indexes_to_supply_in_loss.sats,
&supply.indexes_to_supply.sats,
),
// === Supply in Profit/Loss Relative to Circulating Supply (lazy from global supply) ===
height_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all
&& global_supply_btc.is_some())
.then(|| {
LazyVecFrom2::transformed::<PercentageBtcF64>(
&cfg.name("supply_in_profit_rel_to_circulating_supply"),
cfg.version + v1,
unrealized
.height_to_supply_in_profit_value
.bitcoin
.boxed_clone(),
global_supply_btc.unwrap().boxed_clone(),
)
}),
height_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
&& global_supply_btc.is_some())
.then(|| {
LazyVecFrom2::transformed::<PercentageBtcF64>(
&cfg.name("supply_in_loss_rel_to_circulating_supply"),
cfg.version + v1,
unrealized
.height_to_supply_in_loss_value
.bitcoin
.boxed_clone(),
global_supply_btc.unwrap().boxed_clone(),
)
}),
indexes_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all
&& global_supply_sats.is_some())
.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageSatsF64>(
&cfg.name("supply_in_profit_rel_to_circulating_supply"),
cfg.version + v1,
&unrealized.indexes_to_supply_in_profit.sats,
global_supply_sats.unwrap(),
)
}),
indexes_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
&& global_supply_sats.is_some())
.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageSatsF64>(
&cfg.name("supply_in_loss_rel_to_circulating_supply"),
cfg.version + v1,
&unrealized.indexes_to_supply_in_loss.sats,
global_supply_sats.unwrap(),
)
}),
// === Unrealized vs Market Cap (lazy from global market cap) ===
height_to_unrealized_profit_rel_to_market_cap: global_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<PercentageDollarsF32>(
&cfg.name("unrealized_profit_rel_to_market_cap"),
cfg.version + v0,
unrealized.height_to_unrealized_profit.boxed_clone(),
mc.boxed_clone(),
)
}),
height_to_unrealized_loss_rel_to_market_cap: global_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<PercentageDollarsF32>(
&cfg.name("unrealized_loss_rel_to_market_cap"),
cfg.version + v0,
unrealized.height_to_unrealized_loss.boxed_clone(),
mc.boxed_clone(),
)
}),
height_to_neg_unrealized_loss_rel_to_market_cap: global_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<NegPercentageDollarsF32>(
&cfg.name("neg_unrealized_loss_rel_to_market_cap"),
cfg.version + v0,
unrealized.height_to_unrealized_loss.boxed_clone(),
mc.boxed_clone(),
)
}),
height_to_net_unrealized_pnl_rel_to_market_cap: global_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<PercentageDollarsF32>(
&cfg.name("net_unrealized_pnl_rel_to_market_cap"),
cfg.version + v1,
unrealized.height_to_net_unrealized_pnl.boxed_clone(),
mc.boxed_clone(),
)
}),
indexes_to_unrealized_profit_rel_to_market_cap: global_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("unrealized_profit_rel_to_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_unrealized_profit,
mc,
)
}),
indexes_to_unrealized_loss_rel_to_market_cap: global_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("unrealized_loss_rel_to_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_unrealized_loss,
mc,
)
}),
indexes_to_neg_unrealized_loss_rel_to_market_cap: global_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<NegPercentageDollarsF32>(
&cfg.name("neg_unrealized_loss_rel_to_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_unrealized_loss,
mc,
)
}),
indexes_to_net_unrealized_pnl_rel_to_market_cap: global_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("net_unrealized_pnl_rel_to_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_net_unrealized_pnl,
mc,
)
}),
// NUPL is a proxy for net_unrealized_pnl_rel_to_market_cap
indexes_to_nupl: global_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("nupl"),
cfg.version + v2,
&unrealized.indexes_to_net_unrealized_pnl,
mc,
)
}),
// === Unrealized vs Own Market Cap (lazy, optional) ===
height_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<PercentageDollarsF32>(
&cfg.name("unrealized_profit_rel_to_own_market_cap"),
cfg.version + v1,
unrealized.height_to_unrealized_profit.boxed_clone(),
mc.boxed_clone(),
)
})
})
.flatten(),
height_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<PercentageDollarsF32>(
&cfg.name("unrealized_loss_rel_to_own_market_cap"),
cfg.version + v1,
unrealized.height_to_unrealized_loss.boxed_clone(),
mc.boxed_clone(),
)
})
})
.flatten(),
height_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<NegPercentageDollarsF32>(
&cfg.name("neg_unrealized_loss_rel_to_own_market_cap"),
cfg.version + v1,
unrealized.height_to_unrealized_loss.boxed_clone(),
mc.boxed_clone(),
)
})
})
.flatten(),
height_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap_height.map(|mc| {
LazyVecFrom2::transformed::<PercentageDollarsF32>(
&cfg.name("net_unrealized_pnl_rel_to_own_market_cap"),
cfg.version + v2,
unrealized.height_to_net_unrealized_pnl.boxed_clone(),
mc.boxed_clone(),
)
})
})
.flatten(),
indexes_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("unrealized_profit_rel_to_own_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_unrealized_profit,
mc,
)
})
})
.flatten(),
indexes_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("unrealized_loss_rel_to_own_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_unrealized_loss,
mc,
)
})
})
.flatten(),
indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<NegPercentageDollarsF32>(
&cfg.name("neg_unrealized_loss_rel_to_own_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_unrealized_loss,
mc,
)
})
})
.flatten(),
indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
own_market_cap.map(|mc| {
LazyVecsFrom2FromDateIndex::from_computed::<PercentageDollarsF32>(
&cfg.name("net_unrealized_pnl_rel_to_own_market_cap"),
cfg.version + v2,
&unrealized.indexes_to_net_unrealized_pnl,
mc,
)
})
})
.flatten(),
// === Unrealized vs Own Total Unrealized PnL (lazy, optional) ===
height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecFrom2::transformed::<Ratio32>(
&cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"),
cfg.version + v0,
unrealized.height_to_unrealized_profit.boxed_clone(),
unrealized.height_to_total_unrealized_pnl.boxed_clone(),
)
}),
height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecFrom2::transformed::<Ratio32>(
&cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"),
cfg.version + v0,
unrealized.height_to_unrealized_loss.boxed_clone(),
unrealized.height_to_total_unrealized_pnl.boxed_clone(),
)
}),
height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecFrom2::transformed::<NegRatio32>(
&cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"),
cfg.version + v0,
unrealized.height_to_unrealized_loss.boxed_clone(),
unrealized.height_to_total_unrealized_pnl.boxed_clone(),
)
}),
height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecFrom2::transformed::<Ratio32>(
&cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"),
cfg.version + v1,
unrealized.height_to_net_unrealized_pnl.boxed_clone(),
unrealized.height_to_total_unrealized_pnl.boxed_clone(),
)
}),
indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<Ratio32>(
&cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"),
cfg.version + v1,
&unrealized.indexes_to_unrealized_profit,
&unrealized.indexes_to_total_unrealized_pnl,
)
}),
indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<Ratio32>(
&cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"),
cfg.version + v1,
&unrealized.indexes_to_unrealized_loss,
&unrealized.indexes_to_total_unrealized_pnl,
)
}),
indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<NegRatio32>(
&cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"),
cfg.version + v1,
&unrealized.indexes_to_unrealized_loss,
&unrealized.indexes_to_total_unrealized_pnl,
)
}),
indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended.then(|| {
LazyVecsFrom2FromDateIndex::from_computed::<Ratio32>(
&cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"),
cfg.version + v1,
&unrealized.indexes_to_net_unrealized_pnl,
&unrealized.indexes_to_total_unrealized_pnl,
)
}),
})
}
}
@@ -0,0 +1,221 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, Sats, StoredU64, SupplyState, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec,
PcoVec, TypedVecIterator,
};
use crate::{
ComputeIndexes,
internal::{
ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromHeight,
HalfClosePriceTimesSats, HalveDollars, HalveSats, HalveSatsToBitcoin, LazyHeightValueVecs,
LazyValueVecsFromDateIndex, Source, VecBuilderOptions,
},
indexes, price,
};
use super::ImportConfig;
/// Supply and UTXO count metrics for a cohort.
#[derive(Clone, Traversable)]
pub struct SupplyMetrics {
/// Total supply at each height
pub height_to_supply: EagerVec<PcoVec<Height, Sats>>,
/// Supply value in BTC and USD (computed from height_to_supply)
pub height_to_supply_value: ComputedHeightValueVecs,
/// Supply indexed by date
pub indexes_to_supply: ComputedValueVecsFromDateIndex,
/// UTXO count at each height
pub height_to_utxo_count: EagerVec<PcoVec<Height, StoredU64>>,
/// UTXO count indexed by various dimensions
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
/// Half of supply value (used for computing median) - lazy from supply_value
pub height_to_supply_half_value: LazyHeightValueVecs,
/// Half of supply indexed by date - lazy from indexes_to_supply
pub indexes_to_supply_half: LazyValueVecsFromDateIndex,
}
impl SupplyMetrics {
/// Import supply metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let v1 = Version::ONE;
let compute_dollars = cfg.compute_dollars();
let last = VecBuilderOptions::default().add_last();
let height_to_supply: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(cfg.db, &cfg.name("supply"), cfg.version + v0)?;
let price_source = cfg
.price
.map(|p| p.usd.chainindexes_to_price_close.height.boxed_clone());
let height_to_supply_value = ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply"),
Source::Vec(height_to_supply.boxed_clone()),
cfg.version + v0,
price_source.clone(),
)?;
let indexes_to_supply = ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply"),
Source::Compute,
cfg.version + v1,
last,
compute_dollars,
cfg.indexes,
)?;
// Create lazy supply_half from supply sources
let height_to_supply_half_value = LazyHeightValueVecs::from_sources::<
HalveSats,
HalveSatsToBitcoin,
HalfClosePriceTimesSats,
>(
&cfg.name("supply_half"),
height_to_supply.boxed_clone(),
price_source,
cfg.version + v0,
);
let indexes_to_supply_half =
LazyValueVecsFromDateIndex::from_source::<HalveSats, HalveSatsToBitcoin, HalveDollars>(
&cfg.name("supply_half"),
&indexes_to_supply,
cfg.version + v0,
);
let height_to_utxo_count =
EagerVec::forced_import(cfg.db, &cfg.name("utxo_count"), cfg.version + v0)?;
Ok(Self {
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("utxo_count"),
Source::Vec(height_to_utxo_count.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_supply,
height_to_supply_value,
indexes_to_supply,
height_to_utxo_count,
height_to_supply_half_value,
indexes_to_supply_half,
})
}
/// Get minimum length across height-indexed vectors.
pub fn min_len(&self) -> usize {
self.height_to_supply
.len()
.min(self.height_to_utxo_count.len())
}
/// Push supply state values to height-indexed vectors.
pub fn truncate_push(&mut self, height: Height, state: &SupplyState) -> Result<()> {
self.height_to_supply.truncate_push(height, state.value)?;
self.height_to_utxo_count
.truncate_push(height, StoredU64::from(state.utxo_count))?;
Ok(())
}
/// Write height-indexed vectors to disk.
pub fn write(&mut self) -> Result<()> {
self.height_to_supply.write()?;
self.height_to_utxo_count.write()?;
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.height_to_supply as &mut dyn AnyStoredVec,
&mut self.height_to_utxo_count as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_supply.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_supply)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_utxo_count.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_utxo_count)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_supply
.compute_all(price, starting_indexes, exit, |v| {
let mut dateindex_to_height_count_iter =
indexes.time.dateindex_to_height_count.into_iter();
let mut height_to_supply_iter = self.height_to_supply.into_iter();
v.compute_transform(
starting_indexes.dateindex,
&indexes.time.dateindex_to_first_height,
|(i, height, ..)| {
let count = dateindex_to_height_count_iter.get_unwrap(i);
if count == StoredU64::default() {
unreachable!()
}
let supply = height_to_supply_iter.get_unwrap(height + (*count - 1));
(i, supply)
},
exit,
)?;
Ok(())
})?;
self.indexes_to_utxo_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_utxo_count),
)?;
Ok(())
}
}
@@ -0,0 +1,419 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Sats, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec,
LazyVecFrom1, LazyVecFrom2, Negate, PcoVec,
};
use crate::{
ComputeIndexes,
internal::{
ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex,
DollarsMinus, DollarsPlus, LazyVecsFromDateIndex, Source, VecBuilderOptions,
},
distribution::state::UnrealizedState,
};
use super::ImportConfig;
/// Unrealized profit/loss metrics.
#[derive(Clone, Traversable)]
pub struct UnrealizedMetrics {
// === Supply in Profit/Loss ===
pub height_to_supply_in_profit: EagerVec<PcoVec<Height, Sats>>,
pub indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex,
pub height_to_supply_in_loss: EagerVec<PcoVec<Height, Sats>>,
pub indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex,
pub dateindex_to_supply_in_profit: EagerVec<PcoVec<DateIndex, Sats>>,
pub dateindex_to_supply_in_loss: EagerVec<PcoVec<DateIndex, Sats>>,
pub height_to_supply_in_profit_value: ComputedHeightValueVecs,
pub height_to_supply_in_loss_value: ComputedHeightValueVecs,
// === Unrealized Profit/Loss ===
pub height_to_unrealized_profit: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_unrealized_profit: ComputedVecsFromDateIndex<Dollars>,
pub height_to_unrealized_loss: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_unrealized_loss: ComputedVecsFromDateIndex<Dollars>,
pub dateindex_to_unrealized_profit: EagerVec<PcoVec<DateIndex, Dollars>>,
pub dateindex_to_unrealized_loss: EagerVec<PcoVec<DateIndex, Dollars>>,
// === Negated and Net ===
pub height_to_neg_unrealized_loss: LazyVecFrom1<Height, Dollars, Height, Dollars>,
pub indexes_to_neg_unrealized_loss: LazyVecsFromDateIndex<Dollars>,
// net = profit - loss (height is lazy, indexes computed)
pub height_to_net_unrealized_pnl:
LazyVecFrom2<Height, Dollars, Height, Dollars, Height, Dollars>,
pub indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex<Dollars>,
// total = profit + loss (height is lazy, indexes computed)
pub height_to_total_unrealized_pnl:
LazyVecFrom2<Height, Dollars, Height, Dollars, Height, Dollars>,
pub indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex<Dollars>,
}
impl UnrealizedMetrics {
/// Import unrealized metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let compute_dollars = cfg.compute_dollars();
let last = VecBuilderOptions::default().add_last();
let dateindex_to_supply_in_profit =
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_profit"), cfg.version + v0)?;
let dateindex_to_supply_in_loss =
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_loss"), cfg.version + v0)?;
let dateindex_to_unrealized_profit =
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_profit"), cfg.version + v0)?;
let dateindex_to_unrealized_loss =
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_loss"), cfg.version + v0)?;
let height_to_unrealized_loss: EagerVec<PcoVec<Height, Dollars>> =
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_loss"), cfg.version + v0)?;
let height_to_neg_unrealized_loss = LazyVecFrom1::transformed::<Negate>(
&cfg.name("neg_unrealized_loss"),
cfg.version + v0,
height_to_unrealized_loss.boxed_clone(),
);
let indexes_to_unrealized_loss = ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_loss"),
Source::Vec(dateindex_to_unrealized_loss.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?;
let indexes_to_neg_unrealized_loss = LazyVecsFromDateIndex::from_computed::<Negate>(
&cfg.name("neg_unrealized_loss"),
cfg.version + v0,
Some(dateindex_to_unrealized_loss.boxed_clone()),
&indexes_to_unrealized_loss,
);
// Extract profit sources for lazy net/total vecs
let height_to_unrealized_profit: EagerVec<PcoVec<Height, Dollars>> =
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_profit"), cfg.version + v0)?;
let indexes_to_unrealized_profit = ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_profit"),
Source::Vec(dateindex_to_unrealized_profit.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?;
// Create lazy height vecs from profit/loss sources
let height_to_net_unrealized_pnl = LazyVecFrom2::transformed::<DollarsMinus>(
&cfg.name("net_unrealized_pnl"),
cfg.version + v0,
height_to_unrealized_profit.boxed_clone(),
height_to_unrealized_loss.boxed_clone(),
);
let height_to_total_unrealized_pnl = LazyVecFrom2::transformed::<DollarsPlus>(
&cfg.name("total_unrealized_pnl"),
cfg.version + v0,
height_to_unrealized_profit.boxed_clone(),
height_to_unrealized_loss.boxed_clone(),
);
// indexes_to_net/total remain computed (needed by relative.rs)
let indexes_to_net_unrealized_pnl = ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?;
let indexes_to_total_unrealized_pnl = ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("total_unrealized_pnl"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?;
let height_to_supply_in_profit: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_profit"), cfg.version + v0)?;
let height_to_supply_in_loss: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_loss"), cfg.version + v0)?;
let price_source = cfg
.price
.map(|p| p.usd.chainindexes_to_price_close.height.boxed_clone());
let height_to_supply_in_profit_value = ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply_in_profit"),
Source::Vec(height_to_supply_in_profit.boxed_clone()),
cfg.version + v0,
price_source.clone(),
)?;
let height_to_supply_in_loss_value = ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply_in_loss"),
Source::Vec(height_to_supply_in_loss.boxed_clone()),
cfg.version + v0,
price_source,
)?;
Ok(Self {
// === Supply in Profit/Loss ===
height_to_supply_in_profit,
indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_profit"),
Source::Vec(dateindex_to_supply_in_profit.boxed_clone()),
cfg.version + v0,
last,
compute_dollars,
cfg.indexes,
)?,
height_to_supply_in_loss,
indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_loss"),
Source::Vec(dateindex_to_supply_in_loss.boxed_clone()),
cfg.version + v0,
last,
compute_dollars,
cfg.indexes,
)?,
dateindex_to_supply_in_profit,
dateindex_to_supply_in_loss,
height_to_supply_in_profit_value,
height_to_supply_in_loss_value,
// === Unrealized Profit/Loss ===
height_to_unrealized_profit,
indexes_to_unrealized_profit,
height_to_unrealized_loss,
indexes_to_unrealized_loss,
dateindex_to_unrealized_profit,
dateindex_to_unrealized_loss,
height_to_neg_unrealized_loss,
indexes_to_neg_unrealized_loss,
height_to_net_unrealized_pnl,
indexes_to_net_unrealized_pnl,
height_to_total_unrealized_pnl,
indexes_to_total_unrealized_pnl,
})
}
/// Get minimum length across height-indexed vectors written in block loop.
pub fn min_stateful_height_len(&self) -> usize {
self.height_to_supply_in_profit
.len()
.min(self.height_to_supply_in_loss.len())
.min(self.height_to_unrealized_profit.len())
.min(self.height_to_unrealized_loss.len())
}
/// Get minimum length across dateindex-indexed vectors written in block loop.
pub fn min_stateful_dateindex_len(&self) -> usize {
self.dateindex_to_supply_in_profit
.len()
.min(self.dateindex_to_supply_in_loss.len())
.min(self.dateindex_to_unrealized_profit.len())
.min(self.dateindex_to_unrealized_loss.len())
}
/// Push unrealized state values to height-indexed vectors.
pub fn truncate_push(
&mut self,
height: Height,
dateindex: Option<DateIndex>,
height_state: &UnrealizedState,
date_state: Option<&UnrealizedState>,
) -> Result<()> {
self.height_to_supply_in_profit
.truncate_push(height, height_state.supply_in_profit)?;
self.height_to_supply_in_loss
.truncate_push(height, height_state.supply_in_loss)?;
self.height_to_unrealized_profit
.truncate_push(height, height_state.unrealized_profit)?;
self.height_to_unrealized_loss
.truncate_push(height, height_state.unrealized_loss)?;
if let (Some(dateindex), Some(date_state)) = (dateindex, date_state) {
self.dateindex_to_supply_in_profit
.truncate_push(dateindex, date_state.supply_in_profit)?;
self.dateindex_to_supply_in_loss
.truncate_push(dateindex, date_state.supply_in_loss)?;
self.dateindex_to_unrealized_profit
.truncate_push(dateindex, date_state.unrealized_profit)?;
self.dateindex_to_unrealized_loss
.truncate_push(dateindex, date_state.unrealized_loss)?;
}
Ok(())
}
/// Write height-indexed vectors to disk.
pub fn write(&mut self) -> Result<()> {
self.height_to_supply_in_profit.write()?;
self.height_to_supply_in_loss.write()?;
self.height_to_unrealized_profit.write()?;
self.height_to_unrealized_loss.write()?;
self.dateindex_to_supply_in_profit.write()?;
self.dateindex_to_supply_in_loss.write()?;
self.dateindex_to_unrealized_profit.write()?;
self.dateindex_to_unrealized_loss.write()?;
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.height_to_supply_in_profit as &mut dyn AnyStoredVec,
&mut self.height_to_supply_in_loss as &mut dyn AnyStoredVec,
&mut self.height_to_unrealized_profit as &mut dyn AnyStoredVec,
&mut self.height_to_unrealized_loss as &mut dyn AnyStoredVec,
&mut self.dateindex_to_supply_in_profit as &mut dyn AnyStoredVec,
&mut self.dateindex_to_supply_in_loss as &mut dyn AnyStoredVec,
&mut self.dateindex_to_unrealized_profit as &mut dyn AnyStoredVec,
&mut self.dateindex_to_unrealized_loss as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &ComputeIndexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_supply_in_profit.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_supply_in_profit)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_supply_in_loss.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_supply_in_loss)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_unrealized_profit.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_unrealized_profit)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_unrealized_loss.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_unrealized_loss)
.collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_supply_in_profit.compute_sum_of_others(
starting_indexes.dateindex,
&others
.iter()
.map(|v| &v.dateindex_to_supply_in_profit)
.collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_supply_in_loss.compute_sum_of_others(
starting_indexes.dateindex,
&others
.iter()
.map(|v| &v.dateindex_to_supply_in_loss)
.collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_unrealized_profit.compute_sum_of_others(
starting_indexes.dateindex,
&others
.iter()
.map(|v| &v.dateindex_to_unrealized_profit)
.collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_unrealized_loss.compute_sum_of_others(
starting_indexes.dateindex,
&others
.iter()
.map(|v| &v.dateindex_to_unrealized_loss)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
/// First phase of computed metrics.
pub fn compute_rest_part1(
&mut self,
price: Option<&crate::price::Vecs>,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_supply_in_profit.compute_rest(
price,
starting_indexes,
exit,
Some(&self.dateindex_to_supply_in_profit),
)?;
self.indexes_to_supply_in_loss.compute_rest(
price,
starting_indexes,
exit,
Some(&self.dateindex_to_supply_in_loss),
)?;
self.indexes_to_unrealized_profit.compute_rest(
starting_indexes,
exit,
Some(&self.dateindex_to_unrealized_profit),
)?;
self.indexes_to_unrealized_loss.compute_rest(
starting_indexes,
exit,
Some(&self.dateindex_to_unrealized_loss),
)?;
// height_to_net/total are lazy, but indexes still need compute
// total_unrealized_pnl = profit + loss
self.indexes_to_total_unrealized_pnl
.compute_all(starting_indexes, exit, |vec| {
vec.compute_add(
starting_indexes.dateindex,
&self.dateindex_to_unrealized_profit,
&self.dateindex_to_unrealized_loss,
exit,
)?;
Ok(())
})?;
// net_unrealized_pnl = profit - loss
self.indexes_to_net_unrealized_pnl
.compute_all(starting_indexes, exit, |vec| {
vec.compute_subtract(
starting_indexes.dateindex,
&self.dateindex_to_unrealized_profit,
&self.dateindex_to_unrealized_loss,
exit,
)?;
Ok(())
})?;
Ok(())
}
}
@@ -0,0 +1,16 @@
pub mod address;
mod block;
pub mod cohorts;
pub mod compute;
pub mod metrics;
mod range_map;
mod state;
mod vecs;
pub use range_map::RangeMap;
pub use vecs::Vecs;
pub const DB_NAME: &str = "distribution";
pub use address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs};
pub use cohorts::{AddressCohorts, DynCohortVecs, UTXOCohorts};
@@ -0,0 +1,133 @@
use std::marker::PhantomData;
/// Number of ranges to cache. Small enough for O(1) linear scan,
/// large enough to cover the "hot" source blocks in a typical block.
const CACHE_SIZE: usize = 8;
/// Maps ranges of indices to values for efficient reverse lookups.
///
/// Instead of storing a value for every index, stores first_index values
/// in a sorted Vec and uses binary search to find the value for any index.
/// The value is derived from the position in the Vec.
///
/// Includes an LRU cache of recently accessed ranges to avoid binary search
/// when there's locality in access patterns.
#[derive(Debug)]
pub struct RangeMap<I, V> {
/// Sorted vec of first_index values. Position in vec = value.
first_indexes: Vec<I>,
/// LRU cache: (range_low, range_high, value, age). Lower age = more recent.
cache: [(I, I, V, u8); CACHE_SIZE],
cache_len: u8,
_phantom: PhantomData<V>,
}
impl<I: Default + Copy, V: Default + Copy> Default for RangeMap<I, V> {
fn default() -> Self {
Self {
first_indexes: Vec::new(),
cache: [(I::default(), I::default(), V::default(), 0); CACHE_SIZE],
cache_len: 0,
_phantom: PhantomData,
}
}
}
impl<I: Ord + Copy + Default, V: From<usize> + Copy + Default> RangeMap<I, V> {
/// Create with pre-allocated capacity.
pub fn with_capacity(capacity: usize) -> Self {
Self {
first_indexes: Vec::with_capacity(capacity),
cache: [(I::default(), I::default(), V::default(), 0); CACHE_SIZE],
cache_len: 0,
_phantom: PhantomData,
}
}
/// Push a new first_index. Value is implicitly the current length.
/// Must be called in order (first_index must be >= all previous).
#[inline]
pub fn push(&mut self, first_index: I) {
debug_assert!(
self.first_indexes
.last()
.is_none_or(|&last| first_index >= last),
"RangeMap: first_index must be monotonically increasing"
);
self.first_indexes.push(first_index);
}
/// Look up value for an index, checking cache first.
/// Returns the value (position) of the largest first_index <= given index.
#[inline]
pub fn get(&mut self, index: I) -> Option<V> {
if self.first_indexes.is_empty() {
return None;
}
let cache_len = self.cache_len as usize;
// Check cache first (linear scan of small array)
for i in 0..cache_len {
let (low, high, value, _) = self.cache[i];
if index >= low && index < high {
// Cache hit - mark as most recently used
if self.cache[i].3 != 0 {
for j in 0..cache_len {
self.cache[j].3 = self.cache[j].3.saturating_add(1);
}
self.cache[i].3 = 0;
}
return Some(value);
}
}
// Cache miss - binary search
let pos = self.first_indexes.partition_point(|&first| first <= index);
if pos > 0 {
let value = V::from(pos - 1);
let low = self.first_indexes[pos - 1];
// For last range, use low as high (special marker)
// The check `index < high` will fail, but `index >= low` handles it
let high = self.first_indexes.get(pos).copied().unwrap_or(low);
let is_last = pos == self.first_indexes.len();
// Add to cache (skip if last range - unbounded high is tricky)
if !is_last {
self.add_to_cache(low, high, value);
}
Some(value)
} else {
None
}
}
#[inline]
fn add_to_cache(&mut self, low: I, high: I, value: V) {
let cache_len = self.cache_len as usize;
// Age all entries
for i in 0..cache_len {
self.cache[i].3 = self.cache[i].3.saturating_add(1);
}
if cache_len < CACHE_SIZE {
// Not full - append
self.cache[cache_len] = (low, high, value, 0);
self.cache_len += 1;
} else {
// Full - evict oldest (highest age)
let mut oldest_idx = 0;
let mut oldest_age = 0u8;
for i in 0..CACHE_SIZE {
if self.cache[i].3 > oldest_age {
oldest_age = self.cache[i].3;
oldest_idx = i;
}
}
self.cache[oldest_idx] = (low, high, value, 0);
}
}
}
@@ -0,0 +1,34 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_types::{Dollars, SupplyState, Timestamp};
use serde::Serialize;
#[derive(Debug, Clone, Serialize)]
pub struct BlockState {
#[serde(flatten)]
pub supply: SupplyState,
#[serde(skip)]
pub price: Option<Dollars>,
#[serde(skip)]
pub timestamp: Timestamp,
}
impl Add<BlockState> for BlockState {
type Output = Self;
fn add(mut self, rhs: BlockState) -> Self::Output {
self.supply += &rhs.supply;
self
}
}
impl AddAssign<&BlockState> for BlockState {
fn add_assign(&mut self, rhs: &Self) {
self.supply += &rhs.supply;
}
}
impl SubAssign<&BlockState> for BlockState {
fn sub_assign(&mut self, rhs: &Self) {
self.supply -= &rhs.supply;
}
}
@@ -0,0 +1,184 @@
use std::path::Path;
use brk_error::Result;
use brk_types::{Dollars, Height, LoadedAddressData, Sats, SupplyState};
use vecdb::unlikely;
use super::{
super::cost_basis::RealizedState,
base::CohortState,
};
#[derive(Clone)]
pub struct AddressCohortState {
pub addr_count: u64,
pub inner: CohortState,
}
impl AddressCohortState {
pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self {
Self {
addr_count: 0,
inner: CohortState::new(path, name, compute_dollars),
}
}
/// Reset state for fresh start.
pub fn reset(&mut self) {
self.addr_count = 0;
self.inner.supply = SupplyState::default();
self.inner.sent = Sats::ZERO;
self.inner.satblocks_destroyed = Sats::ZERO;
self.inner.satdays_destroyed = Sats::ZERO;
if let Some(realized) = self.inner.realized.as_mut() {
*realized = RealizedState::NAN;
}
}
pub fn reset_price_to_amount_if_needed(&mut self) -> Result<()> {
self.inner.reset_price_to_amount_if_needed()
}
pub fn reset_single_iteration_values(&mut self) {
self.inner.reset_single_iteration_values();
}
#[allow(clippy::too_many_arguments)]
pub fn send(
&mut self,
addressdata: &mut LoadedAddressData,
value: Sats,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
) -> Result<()> {
let compute_price = current_price.is_some();
let prev_realized_price = compute_price.then(|| addressdata.realized_price());
let prev_supply_state = SupplyState {
utxo_count: addressdata.utxo_count() as u64,
value: addressdata.balance(),
};
addressdata.send(value, prev_price)?;
let supply_state = SupplyState {
utxo_count: addressdata.utxo_count() as u64,
value: addressdata.balance(),
};
self.inner.send_(
&SupplyState {
utxo_count: 1,
value,
},
current_price,
prev_price,
blocks_old,
days_old,
older_than_hour,
compute_price.then(|| (addressdata.realized_price(), &supply_state)),
prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)),
);
Ok(())
}
pub fn receive(
&mut self,
address_data: &mut LoadedAddressData,
value: Sats,
price: Option<Dollars>,
) {
self.receive_outputs(address_data, value, price, 1);
}
pub fn receive_outputs(
&mut self,
address_data: &mut LoadedAddressData,
value: Sats,
price: Option<Dollars>,
output_count: u32,
) {
let compute_price = price.is_some();
let prev_realized_price = compute_price.then(|| address_data.realized_price());
let prev_supply_state = SupplyState {
utxo_count: address_data.utxo_count() as u64,
value: address_data.balance(),
};
address_data.receive_outputs(value, price, output_count);
let supply_state = SupplyState {
utxo_count: address_data.utxo_count() as u64,
value: address_data.balance(),
};
self.inner.receive_(
&SupplyState {
utxo_count: output_count as u64,
value,
},
price,
compute_price.then(|| (address_data.realized_price(), &supply_state)),
prev_realized_price.map(|prev_price| (prev_price, &prev_supply_state)),
);
}
pub fn add(&mut self, addressdata: &LoadedAddressData) {
self.addr_count += 1;
self.inner.increment_(
&addressdata.into(),
addressdata.realized_cap,
addressdata.realized_price(),
);
}
pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
let addr_supply: SupplyState = addressdata.into();
let realized_price = addressdata.realized_price();
// Check for potential underflow before it happens
if unlikely(self.inner.supply.utxo_count < addr_supply.utxo_count) {
panic!(
"AddressCohortState::subtract underflow!\n\
Cohort state: addr_count={}, supply={}\n\
Address being subtracted: {}\n\
Address supply: {}\n\
Realized price: {}\n\
This means the address is not properly tracked in this cohort.",
self.addr_count, self.inner.supply, addressdata, addr_supply, realized_price
);
}
if unlikely(self.inner.supply.value < addr_supply.value) {
panic!(
"AddressCohortState::subtract value underflow!\n\
Cohort state: addr_count={}, supply={}\n\
Address being subtracted: {}\n\
Address supply: {}\n\
Realized price: {}\n\
This means the address is not properly tracked in this cohort.",
self.addr_count, self.inner.supply, addressdata, addr_supply, realized_price
);
}
self.addr_count = self.addr_count.checked_sub(1).unwrap_or_else(|| {
panic!(
"AddressCohortState::subtract addr_count underflow! addr_count=0\n\
Address being subtracted: {}\n\
Realized price: {}",
addressdata, realized_price
)
});
self.inner
.decrement_(&addr_supply, addressdata.realized_cap, realized_price);
}
pub fn write(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.inner.write(height, cleanup)
}
}
@@ -0,0 +1,398 @@
use std::path::Path;
use brk_error::Result;
use brk_types::{Dollars, Height, Sats, SupplyState};
use crate::internal::PERCENTILES_LEN;
use super::super::cost_basis::{
CachedUnrealizedState, PriceToAmount, RealizedState, UnrealizedState,
};
/// State tracked for each cohort during computation.
#[derive(Clone)]
pub struct CohortState {
/// Current supply in this cohort
pub supply: SupplyState,
/// Realized cap and profit/loss (requires price data)
pub realized: Option<RealizedState>,
/// Amount sent in current block
pub sent: Sats,
/// Satoshi-blocks destroyed (supply * blocks_old when spent)
pub satblocks_destroyed: Sats,
/// Satoshi-days destroyed (supply * days_old when spent)
pub satdays_destroyed: Sats,
/// Price distribution for percentile calculations (requires price data)
price_to_amount: Option<PriceToAmount>,
/// Cached unrealized state for O(k) incremental updates.
cached_unrealized: Option<CachedUnrealizedState>,
}
impl CohortState {
/// Create new cohort state.
pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self {
Self {
supply: SupplyState::default(),
realized: compute_dollars.then_some(RealizedState::NAN),
sent: Sats::ZERO,
satblocks_destroyed: Sats::ZERO,
satdays_destroyed: Sats::ZERO,
price_to_amount: compute_dollars.then_some(PriceToAmount::create(path, name)),
cached_unrealized: None,
}
}
/// Import state from checkpoint.
pub fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
// Invalidate cache when importing new data
self.cached_unrealized = None;
match self.price_to_amount.as_mut() {
Some(p) => p.import_at_or_before(height),
None => Ok(height),
}
}
/// Reset price_to_amount if needed (for starting fresh).
pub fn reset_price_to_amount_if_needed(&mut self) -> Result<()> {
if let Some(p) = self.price_to_amount.as_mut() {
p.clean()?;
p.init();
}
// Invalidate cache when data is reset
self.cached_unrealized = None;
Ok(())
}
/// Apply pending price_to_amount updates. Must be called before reads.
pub fn apply_pending(&mut self) {
if let Some(p) = self.price_to_amount.as_mut() {
p.apply_pending();
}
}
/// Get first (lowest) price entry in distribution.
pub fn price_to_amount_first_key_value(&self) -> Option<(Dollars, &Sats)> {
self.price_to_amount.as_ref()?.first_key_value()
}
/// Get last (highest) price entry in distribution.
pub fn price_to_amount_last_key_value(&self) -> Option<(Dollars, &Sats)> {
self.price_to_amount.as_ref()?.last_key_value()
}
/// Reset per-block values before processing next block.
pub fn reset_single_iteration_values(&mut self) {
self.sent = Sats::ZERO;
self.satdays_destroyed = Sats::ZERO;
self.satblocks_destroyed = Sats::ZERO;
if let Some(realized) = self.realized.as_mut() {
realized.reset_single_iteration_values();
}
}
/// Add supply to this cohort (e.g., when UTXO ages into cohort).
pub fn increment(&mut self, supply: &SupplyState, price: Option<Dollars>) {
self.supply += supply;
if supply.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
let price = price.unwrap();
realized.increment(supply, price);
self.price_to_amount
.as_mut()
.unwrap()
.increment(price, supply);
// Update cache for added supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_receive(price, supply.value);
}
}
}
/// Add supply with pre-computed realized cap (for address cohorts).
pub fn increment_(
&mut self,
supply: &SupplyState,
realized_cap: Dollars,
realized_price: Dollars,
) {
self.supply += supply;
if supply.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
realized.increment_(realized_cap);
self.price_to_amount
.as_mut()
.unwrap()
.increment(realized_price, supply);
// Update cache for added supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_receive(realized_price, supply.value);
}
}
}
/// Remove supply from this cohort (e.g., when UTXO ages out of cohort).
pub fn decrement(&mut self, supply: &SupplyState, price: Option<Dollars>) {
self.supply -= supply;
if supply.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
let price = price.unwrap();
realized.decrement(supply, price);
self.price_to_amount
.as_mut()
.unwrap()
.decrement(price, supply);
// Update cache for removed supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_send(price, supply.value);
}
}
}
/// Remove supply with pre-computed realized cap (for address cohorts).
pub fn decrement_(
&mut self,
supply: &SupplyState,
realized_cap: Dollars,
realized_price: Dollars,
) {
self.supply -= supply;
if supply.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
realized.decrement_(realized_cap);
self.price_to_amount
.as_mut()
.unwrap()
.decrement(realized_price, supply);
// Update cache for removed supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_send(realized_price, supply.value);
}
}
}
/// Process received output (new UTXO in cohort).
pub fn receive(&mut self, supply: &SupplyState, price: Option<Dollars>) {
self.receive_(supply, price, price.map(|price| (price, supply)), None);
}
/// Process received output with custom price_to_amount updates (for address cohorts).
pub fn receive_(
&mut self,
supply: &SupplyState,
price: Option<Dollars>,
price_to_amount_increment: Option<(Dollars, &SupplyState)>,
price_to_amount_decrement: Option<(Dollars, &SupplyState)>,
) {
self.supply += supply;
if supply.value > Sats::ZERO
&& let Some(realized) = self.realized.as_mut()
{
let price = price.unwrap();
realized.receive(supply, price);
if let Some((price, supply)) = price_to_amount_increment
&& supply.value.is_not_zero()
{
self.price_to_amount
.as_mut()
.unwrap()
.increment(price, supply);
// Update cache for added supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_receive(price, supply.value);
}
}
if let Some((price, supply)) = price_to_amount_decrement
&& supply.value.is_not_zero()
{
self.price_to_amount
.as_mut()
.unwrap()
.decrement(price, supply);
// Update cache for removed supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_send(price, supply.value);
}
}
}
}
/// Process spent input (UTXO leaving cohort).
pub fn send(
&mut self,
supply: &SupplyState,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
) {
self.send_(
supply,
current_price,
prev_price,
blocks_old,
days_old,
older_than_hour,
None,
prev_price.map(|prev_price| (prev_price, supply)),
);
}
/// Process spent input with custom price_to_amount updates (for address cohorts).
#[allow(clippy::too_many_arguments)]
pub fn send_(
&mut self,
supply: &SupplyState,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
price_to_amount_increment: Option<(Dollars, &SupplyState)>,
price_to_amount_decrement: Option<(Dollars, &SupplyState)>,
) {
if supply.utxo_count == 0 {
return;
}
self.supply -= supply;
if supply.value > Sats::ZERO {
self.sent += supply.value;
self.satblocks_destroyed += supply.value * blocks_old;
self.satdays_destroyed +=
Sats::from((u64::from(supply.value) as f64 * days_old).floor() as u64);
if let Some(realized) = self.realized.as_mut() {
let current_price = current_price.unwrap();
let prev_price = prev_price.unwrap();
realized.send(supply, current_price, prev_price, older_than_hour);
if let Some((price, supply)) = price_to_amount_increment
&& supply.value.is_not_zero()
{
self.price_to_amount
.as_mut()
.unwrap()
.increment(price, supply);
// Update cache for added supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_receive(price, supply.value);
}
}
if let Some((price, supply)) = price_to_amount_decrement
&& supply.value.is_not_zero()
{
self.price_to_amount
.as_mut()
.unwrap()
.decrement(price, supply);
// Update cache for removed supply
if let Some(cache) = self.cached_unrealized.as_mut() {
cache.on_send(price, supply.value);
}
}
}
}
}
/// Compute prices at percentile thresholds.
pub fn compute_percentile_prices(&self) -> [Dollars; PERCENTILES_LEN] {
match self.price_to_amount.as_ref() {
Some(p) if !p.is_empty() => p.compute_percentiles(),
_ => [Dollars::NAN; PERCENTILES_LEN],
}
}
/// Compute unrealized profit/loss at current price.
/// Uses O(k) incremental updates for height_price where k = flip range size.
pub fn compute_unrealized_states(
&mut self,
height_price: Dollars,
date_price: Option<Dollars>,
) -> (UnrealizedState, Option<UnrealizedState>) {
let price_to_amount = match self.price_to_amount.as_ref() {
Some(p) if !p.is_empty() => p,
_ => {
return (
UnrealizedState::NAN,
date_price.map(|_| UnrealizedState::NAN),
);
}
};
// Date unrealized: compute from scratch (only at date boundaries, ~144x less frequent)
let date_state = date_price.map(|date_price| {
CachedUnrealizedState::compute_full_standalone(date_price, price_to_amount)
});
// Height unrealized: use incremental cache (O(k) where k = flip range)
let height_state = if let Some(cache) = self.cached_unrealized.as_mut() {
cache.get_at_price(height_price, price_to_amount).clone()
} else {
let cache = CachedUnrealizedState::compute_fresh(height_price, price_to_amount);
let state = cache.state.clone();
self.cached_unrealized = Some(cache);
state
};
(height_state, date_state)
}
/// Flush state to disk at checkpoint.
pub fn write(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(p) = self.price_to_amount.as_mut() {
p.write(height, cleanup)?;
}
Ok(())
}
/// Get first (lowest) price in distribution.
pub fn min_price(&self) -> Option<Dollars> {
self.price_to_amount
.as_ref()?
.first_key_value()
.map(|(k, _)| k)
}
/// Get last (highest) price in distribution.
pub fn max_price(&self) -> Option<Dollars> {
self.price_to_amount
.as_ref()?
.last_key_value()
.map(|(k, _)| k)
}
/// Get iterator over price_to_amount for merged percentile computation.
/// Returns None if price data is not tracked for this cohort.
pub fn price_to_amount_iter(&self) -> Option<impl Iterator<Item = (Dollars, &Sats)>> {
self.price_to_amount.as_ref().map(|p| p.iter())
}
}
@@ -0,0 +1,7 @@
mod address;
mod base;
mod utxo;
pub use address::*;
pub use base::*;
pub use utxo::*;
@@ -0,0 +1,34 @@
use std::path::Path;
use brk_error::Result;
use brk_types::{Sats, SupplyState};
use derive_deref::{Deref, DerefMut};
use super::{
super::cost_basis::RealizedState,
base::CohortState,
};
#[derive(Clone, Deref, DerefMut)]
pub struct UTXOCohortState(CohortState);
impl UTXOCohortState {
pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self {
Self(CohortState::new(path, name, compute_dollars))
}
pub fn reset_price_to_amount_if_needed(&mut self) -> Result<()> {
self.0.reset_price_to_amount_if_needed()
}
/// Reset state for fresh start.
pub fn reset(&mut self) {
self.0.supply = SupplyState::default();
self.0.sent = Sats::ZERO;
self.0.satblocks_destroyed = Sats::ZERO;
self.0.satdays_destroyed = Sats::ZERO;
if let Some(realized) = self.0.realized.as_mut() {
*realized = RealizedState::NAN;
}
}
}
@@ -0,0 +1,7 @@
mod price_to_amount;
mod realized;
mod unrealized;
pub use price_to_amount::*;
pub use realized::*;
pub use unrealized::*;
@@ -0,0 +1,273 @@
use std::{
collections::BTreeMap,
fs,
ops::Bound,
path::{Path, PathBuf},
};
use brk_error::{Error, Result};
use brk_types::{CentsCompact, Dollars, Height, Sats, SupplyState};
use derive_deref::{Deref, DerefMut};
use pco::standalone::{simple_decompress, simpler_compress};
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use vecdb::Bytes;
use crate::{
internal::{PERCENTILES, PERCENTILES_LEN},
utils::OptionExt,
};
#[derive(Clone, Debug)]
pub struct PriceToAmount {
pathbuf: PathBuf,
state: Option<State>,
/// Pending deltas: (total_increment, total_decrement) per price.
/// Flushed to BTreeMap before reads and at end of block.
pending: FxHashMap<CentsCompact, (Sats, Sats)>,
}
const STATE_AT_: &str = "state_at_";
const STATE_TO_KEEP: usize = 10;
impl PriceToAmount {
pub fn create(path: &Path, name: &str) -> Self {
Self {
pathbuf: path.join(format!("{name}_price_to_amount")),
state: None,
pending: FxHashMap::default(),
}
}
pub fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
let files = self.read_dir(None)?;
let (&height, path) = files.range(..=height).next_back().ok_or(Error::NotFound(
"No price state found at or before height".into(),
))?;
self.state = Some(State::deserialize(&fs::read(path)?)?);
self.pending.clear();
Ok(height)
}
fn assert_pending_empty(&self) {
assert!(
self.pending.is_empty(),
"PriceToAmount: pending not empty, call apply_pending first"
);
}
pub fn iter(&self) -> impl Iterator<Item = (Dollars, &Sats)> {
self.assert_pending_empty();
self.state.u().iter().map(|(k, v)| (k.to_dollars(), v))
}
/// Iterate over entries in a price range with explicit bounds.
pub fn range(
&self,
bounds: (Bound<Dollars>, Bound<Dollars>),
) -> impl Iterator<Item = (Dollars, &Sats)> {
self.assert_pending_empty();
let start = match bounds.0 {
Bound::Included(d) => Bound::Included(CentsCompact::from(d)),
Bound::Excluded(d) => Bound::Excluded(CentsCompact::from(d)),
Bound::Unbounded => Bound::Unbounded,
};
let end = match bounds.1 {
Bound::Included(d) => Bound::Included(CentsCompact::from(d)),
Bound::Excluded(d) => Bound::Excluded(CentsCompact::from(d)),
Bound::Unbounded => Bound::Unbounded,
};
self.state
.u()
.range((start, end))
.map(|(k, v)| (k.to_dollars(), v))
}
pub fn is_empty(&self) -> bool {
self.pending.is_empty() && self.state.u().is_empty()
}
pub fn first_key_value(&self) -> Option<(Dollars, &Sats)> {
self.assert_pending_empty();
self.state
.u()
.first_key_value()
.map(|(k, v)| (k.to_dollars(), v))
}
pub fn last_key_value(&self) -> Option<(Dollars, &Sats)> {
self.assert_pending_empty();
self.state
.u()
.last_key_value()
.map(|(k, v)| (k.to_dollars(), v))
}
/// Accumulate increment in pending batch. O(1).
pub fn increment(&mut self, price: Dollars, supply_state: &SupplyState) {
self.pending.entry(CentsCompact::from(price)).or_default().0 += supply_state.value;
}
/// Accumulate decrement in pending batch. O(1).
pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) {
self.pending.entry(CentsCompact::from(price)).or_default().1 += supply_state.value;
}
/// Apply pending deltas to BTreeMap. O(k log n) where k = unique prices in pending.
/// Must be called before any read operations.
pub fn apply_pending(&mut self) {
for (cents, (inc, dec)) in self.pending.drain() {
let entry = self.state.um().entry(cents).or_default();
*entry += inc;
if *entry < dec {
panic!(
"PriceToAmount::apply_pending underflow!\n\
Path: {:?}\n\
Price: {}\n\
Current + increments: {}\n\
Trying to decrement by: {}",
self.pathbuf,
cents.to_dollars(),
entry,
dec
);
}
*entry -= dec;
if *entry == Sats::ZERO {
self.state.um().remove(&cents);
}
}
}
pub fn init(&mut self) {
self.state.replace(State::default());
self.pending.clear();
}
/// Compute percentile prices by iterating the BTreeMap directly.
/// O(n) where n = number of unique prices.
pub fn compute_percentiles(&self) -> [Dollars; PERCENTILES_LEN] {
self.assert_pending_empty();
let state = match self.state.as_ref() {
Some(s) if !s.is_empty() => s,
_ => return [Dollars::NAN; PERCENTILES_LEN],
};
let total: u64 = state.values().map(|&s| u64::from(s)).sum();
if total == 0 {
return [Dollars::NAN; PERCENTILES_LEN];
}
let mut result = [Dollars::NAN; PERCENTILES_LEN];
let mut cumsum = 0u64;
let mut idx = 0;
for (&cents, &amount) in state.iter() {
cumsum += u64::from(amount);
while idx < PERCENTILES_LEN && cumsum >= total * u64::from(PERCENTILES[idx]) / 100 {
result[idx] = cents.to_dollars();
idx += 1;
}
}
result
}
pub fn clean(&mut self) -> Result<()> {
let _ = fs::remove_dir_all(&self.pathbuf);
fs::create_dir_all(&self.pathbuf)?;
Ok(())
}
fn read_dir(&self, keep_only_before: Option<Height>) -> Result<BTreeMap<Height, PathBuf>> {
Ok(fs::read_dir(&self.pathbuf)?
.filter_map(|entry| {
let path = entry.ok()?.path();
let name = path.file_name()?.to_str()?;
let height_str = name.strip_prefix(STATE_AT_).unwrap_or(name);
if let Ok(h) = height_str.parse::<u32>().map(Height::from) {
if keep_only_before.is_none_or(|height| h < height) {
Some((h, path))
} else {
let _ = fs::remove_file(path);
None
}
} else {
None
}
})
.collect::<BTreeMap<Height, PathBuf>>())
}
/// Flush state to disk, optionally cleaning up old state files.
pub fn write(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.apply_pending();
if cleanup {
let files = self.read_dir(Some(height))?;
for (_, path) in files
.iter()
.take(files.len().saturating_sub(STATE_TO_KEEP - 1))
{
fs::remove_file(path)?;
}
}
fs::write(self.path_state(height), self.state.u().serialize()?)?;
Ok(())
}
fn path_state(&self, height: Height) -> PathBuf {
Self::path_state_(&self.pathbuf, height)
}
fn path_state_(path: &Path, height: Height) -> PathBuf {
path.join(u32::from(height).to_string())
}
}
#[derive(Clone, Default, Debug, Deref, DerefMut, Serialize, Deserialize)]
struct State(BTreeMap<CentsCompact, Sats>);
const COMPRESSION_LEVEL: usize = 4;
impl State {
fn serialize(&self) -> vecdb::Result<Vec<u8>> {
let keys: Vec<i32> = self.keys().map(|k| i32::from(*k)).collect();
let values: Vec<u64> = self.values().map(|v| u64::from(*v)).collect();
let compressed_keys = simpler_compress(&keys, COMPRESSION_LEVEL)?;
let compressed_values = simpler_compress(&values, COMPRESSION_LEVEL)?;
let mut buffer = Vec::new();
buffer.extend(keys.len().to_bytes());
buffer.extend(compressed_keys.len().to_bytes());
buffer.extend(compressed_keys);
buffer.extend(compressed_values);
Ok(buffer)
}
fn deserialize(data: &[u8]) -> vecdb::Result<Self> {
let entry_count = usize::from_bytes(&data[0..8])?;
let keys_len = usize::from_bytes(&data[8..16])?;
let keys: Vec<i32> = simple_decompress(&data[16..16 + keys_len])?;
let values: Vec<u64> = simple_decompress(&data[16 + keys_len..])?;
let map: BTreeMap<CentsCompact, Sats> = keys
.into_iter()
.zip(values)
.map(|(k, v)| (CentsCompact::from(k), Sats::from(v)))
.collect();
assert_eq!(map.len(), entry_count);
Ok(Self(map))
}
}
@@ -0,0 +1,102 @@
use std::cmp::Ordering;
use brk_types::{CheckedSub, Dollars, SupplyState};
#[derive(Debug, Default, Clone)]
pub struct RealizedState {
pub cap: Dollars,
pub profit: Dollars,
pub loss: Dollars,
pub value_created: Dollars,
pub adj_value_created: Dollars,
pub value_destroyed: Dollars,
pub adj_value_destroyed: Dollars,
}
impl RealizedState {
pub const NAN: Self = Self {
cap: Dollars::NAN,
profit: Dollars::NAN,
loss: Dollars::NAN,
value_created: Dollars::NAN,
adj_value_created: Dollars::NAN,
value_destroyed: Dollars::NAN,
adj_value_destroyed: Dollars::NAN,
};
pub fn reset_single_iteration_values(&mut self) {
if self.cap != Dollars::NAN {
self.profit = Dollars::ZERO;
self.loss = Dollars::ZERO;
self.value_created = Dollars::ZERO;
self.adj_value_created = Dollars::ZERO;
self.value_destroyed = Dollars::ZERO;
self.adj_value_destroyed = Dollars::ZERO;
}
}
pub fn increment(&mut self, supply_state: &SupplyState, price: Dollars) {
if supply_state.value.is_zero() {
return;
}
self.increment_(price * supply_state.value)
}
pub fn increment_(&mut self, realized_cap: Dollars) {
if self.cap == Dollars::NAN {
self.cap = Dollars::ZERO;
self.profit = Dollars::ZERO;
self.loss = Dollars::ZERO;
self.value_created = Dollars::ZERO;
self.adj_value_created = Dollars::ZERO;
self.value_destroyed = Dollars::ZERO;
self.adj_value_destroyed = Dollars::ZERO;
}
self.cap += realized_cap;
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) {
self.decrement_(price * supply_state.value);
}
pub fn decrement_(&mut self, realized_cap: Dollars) {
self.cap = self.cap.checked_sub(realized_cap).unwrap();
}
pub fn receive(&mut self, supply_state: &SupplyState, current_price: Dollars) {
self.increment(supply_state, current_price);
}
pub fn send(
&mut self,
supply_state: &SupplyState,
current_price: Dollars,
prev_price: Dollars,
older_than_hour: bool,
) {
let current_value = current_price * supply_state.value;
let prev_value = prev_price * supply_state.value;
self.value_created += current_value;
self.value_destroyed += prev_value;
if older_than_hour {
self.adj_value_created += current_value;
self.adj_value_destroyed += prev_value;
}
match current_price.cmp(&prev_price) {
Ordering::Greater => {
self.profit += current_value.checked_sub(prev_value).unwrap();
}
Ordering::Less => {
self.loss += prev_value.checked_sub(current_value).unwrap();
}
Ordering::Equal => {}
}
self.decrement(supply_state, prev_price);
}
}
@@ -0,0 +1,228 @@
use std::ops::Bound;
use brk_types::{Dollars, Sats};
use vecdb::CheckedSub;
use super::price_to_amount::PriceToAmount;
#[derive(Debug, Default, Clone)]
pub struct UnrealizedState {
pub supply_in_profit: Sats,
pub supply_in_loss: Sats,
pub unrealized_profit: Dollars,
pub unrealized_loss: Dollars,
}
impl UnrealizedState {
pub const NAN: Self = Self {
supply_in_profit: Sats::ZERO,
supply_in_loss: Sats::ZERO,
unrealized_profit: Dollars::NAN,
unrealized_loss: Dollars::NAN,
};
pub const ZERO: Self = Self {
supply_in_profit: Sats::ZERO,
supply_in_loss: Sats::ZERO,
unrealized_profit: Dollars::ZERO,
unrealized_loss: Dollars::ZERO,
};
}
/// Cached unrealized state for O(k) incremental updates.
/// k = number of entries in price flip range (typically tiny).
#[derive(Debug, Clone)]
pub struct CachedUnrealizedState {
pub state: UnrealizedState,
at_price: Dollars,
}
impl CachedUnrealizedState {
/// Create new cache by computing from scratch. O(n).
pub fn compute_fresh(price: Dollars, price_to_amount: &PriceToAmount) -> Self {
let state = Self::compute_full_standalone(price, price_to_amount);
Self {
state,
at_price: price,
}
}
/// Get unrealized state at new_price. O(k) where k = flip range size.
pub fn get_at_price(
&mut self,
new_price: Dollars,
price_to_amount: &PriceToAmount,
) -> &UnrealizedState {
if new_price != self.at_price {
self.update_for_price_change(new_price, price_to_amount);
}
&self.state
}
/// Update cached state when a receive happens.
/// Determines profit/loss classification relative to cached price.
pub fn on_receive(&mut self, purchase_price: Dollars, sats: Sats) {
if purchase_price <= self.at_price {
self.state.supply_in_profit += sats;
if purchase_price < self.at_price {
let diff = self.at_price.checked_sub(purchase_price).unwrap();
self.state.unrealized_profit += diff * sats;
}
} else {
self.state.supply_in_loss += sats;
let diff = purchase_price.checked_sub(self.at_price).unwrap();
self.state.unrealized_loss += diff * sats;
}
}
/// Update cached state when a send happens from historical price.
pub fn on_send(&mut self, historical_price: Dollars, sats: Sats) {
if historical_price <= self.at_price {
// Was in profit
self.state.supply_in_profit -= sats;
if historical_price < self.at_price {
let diff = self.at_price.checked_sub(historical_price).unwrap();
let profit_removed = diff * sats;
self.state.unrealized_profit = self
.state
.unrealized_profit
.checked_sub(profit_removed)
.unwrap_or(Dollars::ZERO);
}
} else {
// Was in loss
self.state.supply_in_loss -= sats;
let diff = historical_price.checked_sub(self.at_price).unwrap();
let loss_removed = diff * sats;
self.state.unrealized_loss = self
.state
.unrealized_loss
.checked_sub(loss_removed)
.unwrap_or(Dollars::ZERO);
}
}
/// Incremental update for price change. O(k) where k = entries in flip range.
fn update_for_price_change(&mut self, new_price: Dollars, price_to_amount: &PriceToAmount) {
let old_price = self.at_price;
let delta_f64 = f64::from(new_price) - f64::from(old_price);
// Update profit/loss for entries that DON'T flip
// Profit changes by delta * supply_in_profit
// Loss changes by -delta * supply_in_loss
if delta_f64 > 0.0 {
// Price went up: profits increase, losses decrease
self.state.unrealized_profit += Dollars::from(delta_f64) * self.state.supply_in_profit;
let loss_decrease = Dollars::from(delta_f64) * self.state.supply_in_loss;
self.state.unrealized_loss = self
.state
.unrealized_loss
.checked_sub(loss_decrease)
.unwrap_or(Dollars::ZERO);
} else if delta_f64 < 0.0 {
// Price went down: profits decrease, losses increase
let profit_decrease = Dollars::from(-delta_f64) * self.state.supply_in_profit;
self.state.unrealized_profit = self
.state
.unrealized_profit
.checked_sub(profit_decrease)
.unwrap_or(Dollars::ZERO);
self.state.unrealized_loss += Dollars::from(-delta_f64) * self.state.supply_in_loss;
}
// Handle flipped entries (only iterate the small range between prices)
if new_price > old_price {
// Price went up: entries where old < price <= new flip from loss to profit
for (price, &sats) in
price_to_amount.range((Bound::Excluded(old_price), Bound::Included(new_price)))
{
// Move from loss to profit
self.state.supply_in_loss -= sats;
self.state.supply_in_profit += sats;
// Undo the loss adjustment applied above for this entry
// We decreased loss by delta * sats, but this entry should be removed entirely
// Original loss: (price - old_price) * sats
// After global adjustment: original - delta * sats (negative, wrong)
// Correct: 0 (removed from loss)
// Correction: add back delta * sats, then add original loss
let delta_adj = Dollars::from(delta_f64) * sats;
self.state.unrealized_loss += delta_adj;
if price > old_price {
let original_loss = price.checked_sub(old_price).unwrap() * sats;
self.state.unrealized_loss += original_loss;
}
// Undo the profit adjustment applied above for this entry
// We increased profit by delta * sats, but this entry was not in profit before
// Correct profit: (new_price - price) * sats
// Correction: subtract delta * sats, add correct profit
let profit_adj = Dollars::from(delta_f64) * sats;
self.state.unrealized_profit = self
.state
.unrealized_profit
.checked_sub(profit_adj)
.unwrap_or(Dollars::ZERO);
if new_price > price {
let correct_profit = new_price.checked_sub(price).unwrap() * sats;
self.state.unrealized_profit += correct_profit;
}
}
} else if new_price < old_price {
// Price went down: entries where new < price <= old flip from profit to loss
for (price, &sats) in
price_to_amount.range((Bound::Excluded(new_price), Bound::Included(old_price)))
{
// Move from profit to loss
self.state.supply_in_profit -= sats;
self.state.supply_in_loss += sats;
// Undo the profit adjustment applied above for this entry
let delta_adj = Dollars::from(-delta_f64) * sats;
self.state.unrealized_profit += delta_adj;
if old_price > price {
let original_profit = old_price.checked_sub(price).unwrap() * sats;
self.state.unrealized_profit += original_profit;
}
// Undo the loss adjustment applied above for this entry
let loss_adj = Dollars::from(-delta_f64) * sats;
self.state.unrealized_loss = self
.state
.unrealized_loss
.checked_sub(loss_adj)
.unwrap_or(Dollars::ZERO);
if price > new_price {
let correct_loss = price.checked_sub(new_price).unwrap() * sats;
self.state.unrealized_loss += correct_loss;
}
}
}
self.at_price = new_price;
}
/// Full computation from scratch (no cache). O(n).
pub fn compute_full_standalone(
current_price: Dollars,
price_to_amount: &PriceToAmount,
) -> UnrealizedState {
let mut state = UnrealizedState::ZERO;
for (price, &sats) in price_to_amount.iter() {
if price <= current_price {
state.supply_in_profit += sats;
if price < current_price {
let diff = current_price.checked_sub(price).unwrap();
state.unrealized_profit += diff * sats;
}
} else {
state.supply_in_loss += sats;
let diff = price.checked_sub(current_price).unwrap();
state.unrealized_loss += diff * sats;
}
}
state
}
}
@@ -0,0 +1,9 @@
mod block;
mod cohort;
mod cost_basis;
mod transacted;
pub use block::*;
pub use cohort::*;
pub use cost_basis::*;
pub use transacted::*;
@@ -0,0 +1,50 @@
use std::ops::{Add, AddAssign};
use brk_cohort::{ByAmountRange, GroupedByType};
use brk_types::{OutputType, Sats, SupplyState};
#[derive(Default, Debug)]
pub struct Transacted {
pub spendable_supply: SupplyState,
pub by_type: GroupedByType<SupplyState>,
pub by_size_group: ByAmountRange<SupplyState>,
}
impl Transacted {
#[allow(clippy::inconsistent_digit_grouping)]
pub fn iterate(&mut self, value: Sats, _type: OutputType) {
let supply = SupplyState {
utxo_count: 1,
value,
};
*self.by_type.get_mut(_type) += &supply;
if _type.is_unspendable() {
return;
}
self.spendable_supply += &supply;
*self.by_size_group.get_mut(value) += &supply;
}
}
impl Add for Transacted {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self {
spendable_supply: self.spendable_supply + rhs.spendable_supply,
by_type: self.by_type + rhs.by_type,
by_size_group: self.by_size_group + rhs.by_size_group,
}
}
}
impl AddAssign for Transacted {
fn add_assign(&mut self, rhs: Self) {
self.by_size_group += rhs.by_size_group;
self.spendable_supply += &rhs.spendable_supply;
self.by_type += rhs.by_type;
}
}
@@ -0,0 +1,435 @@
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{
EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, StoredU64,
SupplyState, Version,
};
use log::info;
use vecdb::{
AnyVec, BytesVec, Database, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec,
LazyVecFrom1, PAGE_SIZE, Stamp, TypedVecIterator, VecIndex,
};
use crate::{
ComputeIndexes, blocks,
distribution::{
compute::{StartMode, determine_start_mode, process_blocks, recover_state, reset_state},
state::BlockState,
},
indexes,
inputs,
internal::{ComputedVecsFromHeight, Source, VecBuilderOptions},
outputs, price, transactions,
};
use super::{
AddressCohorts, AddressesDataVecs, AnyAddressIndexesVecs, UTXOCohorts,
address::{AddressTypeToHeightToAddressCount, AddressTypeToIndexesToAddressCount},
compute::aggregates,
};
const VERSION: Version = Version::new(21);
/// Main struct holding all computed vectors and state for stateful computation.
#[derive(Clone, Traversable)]
pub struct Vecs {
#[traversable(skip)]
db: Database,
pub chain_state: BytesVec<Height, SupplyState>,
pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs,
pub utxo_cohorts: UTXOCohorts,
pub address_cohorts: AddressCohorts,
pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount,
pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount,
pub addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount,
pub addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount,
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_empty_addr_count: ComputedVecsFromHeight<StoredU64>,
pub loadedaddressindex_to_loadedaddressindex:
LazyVecFrom1<LoadedAddressIndex, LoadedAddressIndex, LoadedAddressIndex, LoadedAddressData>,
pub emptyaddressindex_to_emptyaddressindex:
LazyVecFrom1<EmptyAddressIndex, EmptyAddressIndex, EmptyAddressIndex, EmptyAddressData>,
}
const SAVED_STAMPED_CHANGES: u16 = 10;
impl Vecs {
pub fn forced_import(
parent: &Path,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let db_path = parent.join(super::DB_NAME);
let states_path = db_path.join("states");
let db = Database::open(&db_path)?;
db.set_min_len(PAGE_SIZE * 20_000_000)?;
db.set_min_regions(50_000)?;
let v0 = version + VERSION + Version::ZERO;
let utxo_cohorts = UTXOCohorts::forced_import(&db, version, indexes, price, &states_path)?;
// Create address cohorts with reference to utxo "all" cohort's supply for global ratios
let address_cohorts = AddressCohorts::forced_import(
&db,
version,
indexes,
price,
&states_path,
Some(&utxo_cohorts.all.metrics.supply),
)?;
// Create address data BytesVecs first so we can also use them for identity mappings
let loadedaddressindex_to_loadedaddressdata = BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "loadedaddressdata", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?;
let emptyaddressindex_to_emptyaddressdata = BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "emptyaddressdata", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?;
// Identity mappings for traversable
let loadedaddressindex_to_loadedaddressindex = LazyVecFrom1::init(
"loadedaddressindex",
v0,
loadedaddressindex_to_loadedaddressdata.boxed_clone(),
|index, _| Some(index),
);
let emptyaddressindex_to_emptyaddressindex = LazyVecFrom1::init(
"emptyaddressindex",
v0,
emptyaddressindex_to_emptyaddressdata.boxed_clone(),
|index, _| Some(index),
);
// Extract address type height vecs before struct literal to use as sources
let addresstype_to_height_to_addr_count =
AddressTypeToHeightToAddressCount::forced_import(&db, "addr_count", v0)?;
let addresstype_to_height_to_empty_addr_count =
AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?;
let this = Self {
chain_state: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
&db,
"addr_count",
Source::Compute,
v0,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_empty_addr_count: ComputedVecsFromHeight::forced_import(
&db,
"empty_addr_count",
Source::Compute,
v0,
indexes,
VecBuilderOptions::default().add_last(),
)?,
addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::forced_import(
&db,
"addr_count",
v0,
indexes,
&addresstype_to_height_to_addr_count,
)?,
addresstype_to_indexes_to_empty_addr_count:
AddressTypeToIndexesToAddressCount::forced_import(
&db,
"empty_addr_count",
v0,
indexes,
&addresstype_to_height_to_empty_addr_count,
)?,
addresstype_to_height_to_addr_count,
addresstype_to_height_to_empty_addr_count,
utxo_cohorts,
address_cohorts,
any_address_indexes: AnyAddressIndexesVecs::forced_import(&db, v0)?,
addresses_data: AddressesDataVecs {
loaded: loadedaddressindex_to_loadedaddressdata,
empty: emptyaddressindex_to_emptyaddressdata,
},
loadedaddressindex_to_loadedaddressindex,
emptyaddressindex_to_emptyaddressindex,
db,
};
this.db.retain_regions(
this.iter_any_exportable()
.flat_map(|v| v.region_names())
.collect(),
)?;
this.db.compact()?;
Ok(this)
}
/// Main computation loop.
///
/// Processes blocks to compute UTXO and address cohort metrics:
/// 1. Recovers state from checkpoints or starts fresh
/// 2. Iterates through blocks, processing outputs/inputs in parallel
/// 3. Flushes checkpoints periodically
/// 4. Computes aggregate cohorts from separate cohorts
/// 5. Computes derived metrics
#[allow(clippy::too_many_arguments)]
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
inputs: &inputs::Vecs,
outputs: &outputs::Vecs,
transactions: &transactions::Vecs,
blocks: &blocks::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &mut ComputeIndexes,
exit: &Exit,
) -> Result<()> {
// 1. Find minimum computed height for recovery
let chain_state_height = Height::from(self.chain_state.len());
let height_based_min = self.min_stateful_height_len();
let dateindex_min = self.min_stateful_dateindex_len();
let stateful_min = adjust_for_dateindex_gap(height_based_min, dateindex_min, indexes)?;
// 2. Determine start mode and recover/reset state
let start_mode = determine_start_mode(stateful_min, chain_state_height);
// Try to resume from checkpoint, fall back to fresh start if needed
let recovered_height = match start_mode {
StartMode::Resume(height) => {
let stamp = Stamp::from(height);
// Rollback BytesVec state and capture results for validation
let chain_state_rollback = self.chain_state.rollback_before(stamp);
// Validate all rollbacks and imports are consistent
let recovered = recover_state(
height,
chain_state_rollback,
&mut self.any_address_indexes,
&mut self.addresses_data,
&mut self.utxo_cohorts,
&mut self.address_cohorts,
)?;
if recovered.starting_height.is_zero() {
info!("State recovery validation failed, falling back to fresh start");
}
recovered.starting_height
}
StartMode::Fresh => Height::ZERO,
};
// Fresh start: reset all state
let (starting_height, mut chain_state) = if recovered_height.is_zero() {
self.chain_state.reset()?;
self.addresstype_to_height_to_addr_count.reset()?;
self.addresstype_to_height_to_empty_addr_count.reset()?;
reset_state(
&mut self.any_address_indexes,
&mut self.addresses_data,
&mut self.utxo_cohorts,
&mut self.address_cohorts,
)?;
info!("State recovery: fresh start");
(Height::ZERO, vec![])
} else {
// Recover chain_state from stored values
let height_to_timestamp = &blocks.time.height_to_timestamp_fixed;
let height_to_price = price.map(|p| &p.usd.chainindexes_to_price_close.height);
let mut height_to_timestamp_iter = height_to_timestamp.into_iter();
let mut height_to_price_iter = height_to_price.map(|v| v.into_iter());
let mut chain_state_iter = self.chain_state.into_iter();
let chain_state = (0..recovered_height.to_usize())
.map(|h| {
let h = Height::from(h);
BlockState {
supply: chain_state_iter.get_unwrap(h),
price: height_to_price_iter.as_mut().map(|v| *v.get_unwrap(h)),
timestamp: height_to_timestamp_iter.get_unwrap(h),
}
})
.collect();
(recovered_height, chain_state)
};
// 2b. Validate computed versions
let base_version = VERSION;
self.utxo_cohorts.validate_computed_versions(base_version)?;
self.address_cohorts
.validate_computed_versions(base_version)?;
// 3. Get last height from indexer
let last_height = Height::from(
indexer
.vecs
.block
.height_to_blockhash
.len()
.saturating_sub(1),
);
// 4. Process blocks
if starting_height <= last_height {
process_blocks(
self,
indexer,
indexes,
inputs,
outputs,
transactions,
blocks,
price,
starting_height,
last_height,
&mut chain_state,
exit,
)?;
}
// 5. Compute aggregates (overlapping cohorts from separate cohorts)
aggregates::compute_overlapping(
&mut self.utxo_cohorts,
&mut self.address_cohorts,
starting_indexes,
exit,
)?;
// 6. Compute rest part1 (dateindex mappings)
aggregates::compute_rest_part1(
&mut self.utxo_cohorts,
&mut self.address_cohorts,
indexes,
price,
starting_indexes,
exit,
)?;
// 7. Compute rest part2 (relative metrics)
let supply_metrics = &self.utxo_cohorts.all.metrics.supply;
let height_to_supply = &supply_metrics.height_to_supply_value.bitcoin.clone();
let height_to_market_cap = supply_metrics
.height_to_supply_value
.dollars
.as_ref()
.cloned();
let dateindex_to_market_cap = supply_metrics
.indexes_to_supply
.dollars
.as_ref()
.and_then(|v| v.dateindex.as_ref().cloned());
let height_to_market_cap_ref = height_to_market_cap.as_ref();
let dateindex_to_market_cap_ref = dateindex_to_market_cap.as_ref();
aggregates::compute_rest_part2(
&mut self.utxo_cohorts,
&mut self.address_cohorts,
indexes,
price,
starting_indexes,
height_to_supply,
height_to_market_cap_ref,
dateindex_to_market_cap_ref,
exit,
)?;
let _lock = exit.lock();
self.db.compact()?;
Ok(())
}
pub fn flush(&self) -> Result<()> {
self.db.flush()?;
Ok(())
}
/// Get minimum length across all height-indexed stateful vectors.
fn min_stateful_height_len(&self) -> Height {
self.utxo_cohorts
.min_separate_stateful_height_len()
.min(self.address_cohorts.min_separate_stateful_height_len())
.min(Height::from(self.chain_state.len()))
.min(self.any_address_indexes.min_stamped_height())
.min(self.addresses_data.min_stamped_height())
.min(Height::from(
self.addresstype_to_height_to_addr_count.min_len(),
))
.min(Height::from(
self.addresstype_to_height_to_empty_addr_count.min_len(),
))
}
/// Get minimum length across all dateindex-indexed stateful vectors.
fn min_stateful_dateindex_len(&self) -> usize {
self.utxo_cohorts
.min_separate_stateful_dateindex_len()
.min(self.utxo_cohorts.min_aggregate_stateful_dateindex_len())
.min(self.address_cohorts.min_separate_stateful_dateindex_len())
}
}
/// Adjust start height if dateindex vecs are behind where they should be.
///
/// To resume at height H (in day D), we need days 0..D-1 complete in dateindex vecs.
/// If dateindex vecs only have length N < D, restart from the first height of day N.
fn adjust_for_dateindex_gap(
height_based_min: Height,
dateindex_min: usize,
indexes: &indexes::Vecs,
) -> Result<Height> {
// Skip check if no dateindex vecs exist or starting from zero
if dateindex_min == usize::MAX || height_based_min.is_zero() {
return Ok(height_based_min);
}
// Skip if height_to_dateindex doesn't cover height_based_min yet
if height_based_min.to_usize() >= indexes.block.height_to_dateindex.len() {
return Ok(height_based_min);
}
// Get the dateindex at the height we want to resume at
let required_dateindex: usize = indexes
.block
.height_to_dateindex
.read_once(height_based_min)?
.into();
// If dateindex vecs are behind, restart from first height of the missing day
if dateindex_min < required_dateindex
&& dateindex_min < indexes.time.dateindex_to_first_height.len()
{
Ok(indexes
.time
.dateindex_to_first_height
.read_once(dateindex_min.into())?)
} else {
Ok(height_based_min)
}
}