diff --git a/Cargo.lock b/Cargo.lock index 420915e2a..1ed36f9a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,7 +631,6 @@ dependencies = [ "log", "pco", "rayon", - "rlimit", "rustc-hash", "serde", "smallvec", @@ -695,6 +694,7 @@ dependencies = [ "log", "mimalloc", "rayon", + "rlimit", "rustc-hash", "vecdb", ] @@ -2594,9 +2594,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -2608,9 +2608,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" @@ -2924,9 +2924,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222" +checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c" dependencies = [ "zlib-rs", ] @@ -3287,9 +3287,9 @@ dependencies = [ [[package]] name = "oxc-browserslist" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f978be538ca5e2a64326d24b7991dc658cc8495132833ae387212ab3b8abd70a" +checksum = "9bd39c45e1d6bd2abfbd4b89cbcaba34bd315cd3cee23aad623fd075acc1ea01" dependencies = [ "bincode", "flate2", @@ -5973,9 +5973,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7" +checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index 2a96d3e5b..b51a32aa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "3" members = ["crates/*"] +default-members = ["crates/brk_cli"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 600aa5a2f..519699d42 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -26,7 +26,6 @@ derive_deref = { workspace = true } log = { workspace = true } pco = "0.4.7" rayon = { workspace = true } -rlimit = "0.10.2" rustc-hash = { workspace = true } serde = { workspace = true } smallvec = "1.15.1" diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 9efda8965..ebc00494b 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -22,6 +22,7 @@ mod market; mod pools; mod price; mod stateful; +mod stateful_new; mod states; mod traits; mod utils; @@ -56,14 +57,6 @@ impl Computer { indexer: &Indexer, fetcher: Option, ) -> Result { - info!("Increasing number of open files..."); - let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?; - rlimit::setrlimit( - rlimit::Resource::NOFILE, - no_file_limit.0.max(10_000), - no_file_limit.1, - )?; - info!("Importing computer..."); let import_start = Instant::now(); diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 9612a53e6..4f082d5f1 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -73,7 +73,7 @@ mod utxo_cohort; mod utxo_cohorts; mod withaddressdatasource; -pub use flushable::{Flushable, HeightFlushable}; +pub use crate::states::{Flushable, HeightFlushable}; use address_indexes::{AddressesDataVecs, AnyAddressIndexesVecs}; use addresstype::*; diff --git a/crates/brk_computer/src/stateful_new/address/address_count.rs b/crates/brk_computer/src/stateful_new/address/address_count.rs new file mode 100644 index 000000000..eda8f02ac --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/address_count.rs @@ -0,0 +1,176 @@ +//! Address count types per address type. + +use brk_error::Result; +use brk_grouper::ByAddressType; +use brk_traversable::Traversable; +use brk_types::{Height, StoredU64, Version}; +use derive_deref::{Deref, DerefMut}; +use vecdb::{Database, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec, TypedVecIterator}; + +use crate::{ + Indexes, + grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, +}; + +/// Address count per address type (runtime state). +#[derive(Debug, Default, Deref, DerefMut)] +pub struct AddressTypeToAddressCount(ByAddressType); + +impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount { + #[inline] + fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self { + if let Some(prev_height) = starting_height.decremented() { + Self(ByAddressType { + p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(), + p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(), + p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(), + p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(), + p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(), + p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(), + p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(), + p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(), + }) + } else { + Default::default() + } + } +} + +/// Address count per address type, indexed by height. +#[derive(Debug, Clone, Deref, DerefMut, Traversable)] +pub struct AddressTypeToHeightToAddressCount(ByAddressType>>); + +impl From>>> + for AddressTypeToHeightToAddressCount +{ + #[inline] + fn from(value: ByAddressType>>) -> Self { + Self(value) + } +} + +impl AddressTypeToHeightToAddressCount { + pub fn forced_import(db: &Database, name: &str, version: Version) -> Result { + Ok(Self::from(ByAddressType::new_with_name(|type_name| { + Ok(EagerVec::forced_import( + db, + &format!("{type_name}_{name}"), + version, + )?) + })?)) + } + + pub fn truncate_push( + &mut self, + height: Height, + addresstype_to_usize: &AddressTypeToAddressCount, + ) -> Result<()> { + self.p2pk65 + .truncate_push(height, addresstype_to_usize.p2pk65.into())?; + self.p2pk33 + .truncate_push(height, addresstype_to_usize.p2pk33.into())?; + self.p2pkh + .truncate_push(height, addresstype_to_usize.p2pkh.into())?; + self.p2sh + .truncate_push(height, addresstype_to_usize.p2sh.into())?; + self.p2wpkh + .truncate_push(height, addresstype_to_usize.p2wpkh.into())?; + self.p2wsh + .truncate_push(height, addresstype_to_usize.p2wsh.into())?; + self.p2tr + .truncate_push(height, addresstype_to_usize.p2tr.into())?; + self.p2a + .truncate_push(height, addresstype_to_usize.p2a.into())?; + Ok(()) + } +} + +/// Address count per address type, indexed by various indexes (dateindex, etc.). +#[derive(Clone, Deref, DerefMut, Traversable)] +pub struct AddressTypeToIndexesToAddressCount(ByAddressType>); + +impl From>> for AddressTypeToIndexesToAddressCount { + #[inline] + fn from(value: ByAddressType>) -> Self { + Self(value) + } +} + +impl AddressTypeToIndexesToAddressCount { + pub fn forced_import( + db: &Database, + name: &str, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + Ok(Self::from(ByAddressType::new_with_name(|type_name| { + ComputedVecsFromHeight::forced_import( + db, + &format!("{type_name}_{name}"), + Source::None, + version, + indexes, + VecBuilderOptions::default().add_last(), + ) + })?)) + } + + pub fn compute( + &mut self, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount, + ) -> Result<()> { + self.p2pk65.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pk65), + )?; + self.p2pk33.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pk33), + )?; + self.p2pkh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pkh), + )?; + self.p2sh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2sh), + )?; + self.p2wpkh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2wpkh), + )?; + self.p2wsh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2wsh), + )?; + self.p2tr.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2tr), + )?; + self.p2a.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2a), + )?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/address/any_address_indexes.rs b/crates/brk_computer/src/stateful_new/address/any_address_indexes.rs new file mode 100644 index 000000000..24535dd78 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/any_address_indexes.rs @@ -0,0 +1,98 @@ +//! Storage for address indexes by type. + +use brk_error::{Error, Result}; +use brk_traversable::Traversable; +use brk_types::{ + AnyAddressIndex, Height, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex, + P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, + TypeIndex, Version, +}; +use vecdb::{ + AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Reader, Stamp, +}; + +const SAVED_STAMPED_CHANGES: u16 = 10; + +/// Macro to define AnyAddressIndexesVecs and its methods. +macro_rules! define_any_address_indexes_vecs { + ($(($field:ident, $variant:ident, $index:ty)),* $(,)?) => { + #[derive(Clone, Traversable)] + pub struct AnyAddressIndexesVecs { + $(pub $field: BytesVec<$index, AnyAddressIndex>,)* + } + + impl AnyAddressIndexesVecs { + /// Import from database. + pub fn forced_import(db: &Database, version: Version) -> Result { + Ok(Self { + $($field: BytesVec::forced_import_with( + ImportOptions::new(db, "anyaddressindex", version) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?,)* + }) + } + + /// Get minimum stamped height across all address types. + pub fn min_stamped_height(&self) -> Height { + [$(Height::from(self.$field.stamp()).incremented()),*] + .into_iter() + .min() + .unwrap_or_default() + } + + /// Rollback all address types to before the given stamp. + pub fn rollback_before(&mut self, stamp: Stamp) -> Result> { + Ok(vec![$(self.$field.rollback_before(stamp)?),*]) + } + + /// Reset all address types. + pub fn reset(&mut self) -> Result<()> { + $(self.$field.reset()?;)* + Ok(()) + } + + /// Get address index for a given type and typeindex. + pub fn get(&self, address_type: OutputType, typeindex: TypeIndex, reader: &Reader) -> AnyAddressIndex { + match address_type { + $(OutputType::$variant => self.$field.get_pushed_or_read_at_unwrap(typeindex.into(), reader),)* + _ => unreachable!("Invalid address type: {:?}", address_type), + } + } + + /// Get address index with single read (no caching). + pub fn get_once(&self, address_type: OutputType, typeindex: TypeIndex) -> Result { + match address_type { + $(OutputType::$variant => self.$field.read_at_once(typeindex.into()).map_err(Into::into),)* + _ => Err(Error::UnsupportedType(address_type.to_string())), + } + } + + /// Update or push address index for a given type. + pub fn update_or_push(&mut self, address_type: OutputType, typeindex: TypeIndex, index: AnyAddressIndex) -> Result<()> { + match address_type { + $(OutputType::$variant => self.$field.update_or_push(typeindex.into(), index)?,)* + _ => unreachable!("Invalid address type: {:?}", address_type), + } + Ok(()) + } + + /// Flush all address types with stamp. + pub fn flush(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> { + $(self.$field.stamped_flush_maybe_with_changes(stamp, with_changes)?;)* + Ok(()) + } + } + }; +} + +// Generate the struct and methods +define_any_address_indexes_vecs!( + (p2a, P2A, P2AAddressIndex), + (p2pk33, P2PK33, P2PK33AddressIndex), + (p2pk65, P2PK65, P2PK65AddressIndex), + (p2pkh, P2PKH, P2PKHAddressIndex), + (p2sh, P2SH, P2SHAddressIndex), + (p2tr, P2TR, P2TRAddressIndex), + (p2wpkh, P2WPKH, P2WPKHAddressIndex), + (p2wsh, P2WSH, P2WSHAddressIndex), +); diff --git a/crates/brk_computer/src/stateful_new/address/data.rs b/crates/brk_computer/src/stateful_new/address/data.rs new file mode 100644 index 000000000..d99bbc68e --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/data.rs @@ -0,0 +1,66 @@ +//! Storage for address data (loaded and empty addresses). + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{ + EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, Version, +}; +use vecdb::{ + AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Stamp, +}; + +const SAVED_STAMPED_CHANGES: u16 = 10; + +/// Storage for both loaded and empty address data. +#[derive(Clone, Traversable)] +pub struct AddressesDataVecs { + pub loaded: BytesVec, + pub empty: BytesVec, +} + +impl AddressesDataVecs { + /// Import from database. + pub fn forced_import(db: &Database, version: Version) -> Result { + Ok(Self { + loaded: BytesVec::forced_import_with( + ImportOptions::new(db, "loadedaddressdata", version) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + empty: BytesVec::forced_import_with( + ImportOptions::new(db, "emptyaddressdata", version) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + }) + } + + /// Get minimum stamped height across loaded and empty data. + pub fn min_stamped_height(&self) -> Height { + Height::from(self.loaded.stamp()) + .incremented() + .min(Height::from(self.empty.stamp()).incremented()) + } + + /// Rollback both loaded and empty data to before the given stamp. + pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> { + Ok([ + self.loaded.rollback_before(stamp)?, + self.empty.rollback_before(stamp)?, + ]) + } + + /// Reset both loaded and empty data. + pub fn reset(&mut self) -> Result<()> { + self.loaded.reset()?; + self.empty.reset()?; + Ok(()) + } + + /// Flush both loaded and empty data with stamp. + pub fn flush(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> { + self.loaded + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.empty + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/address/height_type_vec.rs b/crates/brk_computer/src/stateful_new/address/height_type_vec.rs new file mode 100644 index 000000000..535bcc4d0 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/height_type_vec.rs @@ -0,0 +1,20 @@ +//! Height to AddressTypeToVec hashmap. + +use brk_types::Height; +use derive_deref::{Deref, DerefMut}; +use rustc_hash::FxHashMap; + +use super::type_vec::AddressTypeToVec; + +/// Hashmap from Height to AddressTypeToVec. +#[derive(Debug, Default, Deref, DerefMut)] +pub struct HeightToAddressTypeToVec(FxHashMap>); + +impl HeightToAddressTypeToVec { + /// Merge another map into this one. + pub fn merge_mut(&mut self, other: Self) { + for (height, vec) in other.0 { + self.entry(height).or_default().merge_mut(vec); + } + } +} diff --git a/crates/brk_computer/src/stateful_new/address/mod.rs b/crates/brk_computer/src/stateful_new/address/mod.rs new file mode 100644 index 000000000..f1cb63ff8 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/mod.rs @@ -0,0 +1,28 @@ +//! Address handling with macro-generated code for 8 address types. +//! +//! This module provides: +//! - `AnyAddressIndexesVecs` for storing address indexes by type +//! - `AddressesDataVecs` for storing address data (loaded/empty) +//! - `AddressTypeToTypeIndexMap` for per-type hashmaps +//! - `AddressTypeToVec` for per-type vectors +//! - `HeightToAddressTypeToVec` for height-keyed per-type vectors +//! - `AddressTypeToAddressCount` for runtime address counts +//! - `AddressTypeToHeightToAddressCount` for height-indexed address counts +//! - `AddressTypeToIndexesToAddressCount` for computed address counts + +mod address_count; +mod any_address_indexes; +mod data; +mod height_type_vec; +mod type_index_map; +mod type_vec; + +pub use address_count::{ + AddressTypeToAddressCount, AddressTypeToHeightToAddressCount, + AddressTypeToIndexesToAddressCount, +}; +pub use any_address_indexes::AnyAddressIndexesVecs; +pub use data::AddressesDataVecs; +pub use height_type_vec::HeightToAddressTypeToVec; +pub use type_index_map::AddressTypeToTypeIndexMap; +pub use type_vec::AddressTypeToVec; diff --git a/crates/brk_computer/src/stateful_new/address/type_index_map.rs b/crates/brk_computer/src/stateful_new/address/type_index_map.rs new file mode 100644 index 000000000..9565f7cab --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/type_index_map.rs @@ -0,0 +1,106 @@ +//! Per-address-type hashmap keyed by TypeIndex. + +use std::mem; + +use brk_grouper::ByAddressType; +use brk_types::{OutputType, TypeIndex}; +use derive_deref::{Deref, DerefMut}; +use rustc_hash::FxHashMap; +use smallvec::{Array, SmallVec}; +use std::collections::hash_map::Entry; + +/// A hashmap for each address type, keyed by TypeIndex. +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToTypeIndexMap(ByAddressType>); + +impl Default for AddressTypeToTypeIndexMap { + fn default() -> Self { + Self(ByAddressType { + p2a: FxHashMap::default(), + p2pk33: FxHashMap::default(), + p2pk65: FxHashMap::default(), + p2pkh: FxHashMap::default(), + p2sh: FxHashMap::default(), + p2tr: FxHashMap::default(), + p2wpkh: FxHashMap::default(), + p2wsh: FxHashMap::default(), + }) + } +} + +impl AddressTypeToTypeIndexMap { + /// Merge two maps, consuming other and extending self. + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_single(&mut self.p2a, &mut other.p2a); + Self::merge_single(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_single(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_single(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_single(&mut self.p2sh, &mut other.p2sh); + Self::merge_single(&mut self.p2tr, &mut other.p2tr); + Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_single(&mut self.p2wsh, &mut other.p2wsh); + self + } + + fn merge_single(own: &mut FxHashMap, other: &mut FxHashMap) { + if own.len() < other.len() { + mem::swap(own, other); + } + own.extend(other.drain()); + } + + /// Insert a value for a specific address type and typeindex. + pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) { + self.get_mut(address_type).unwrap().insert(typeindex, value); + } + + /// Remove and return a value for a specific address type and typeindex. + pub fn remove_for_type(&mut self, address_type: OutputType, typeindex: &TypeIndex) -> T { + self.get_mut(address_type) + .unwrap() + .remove(typeindex) + .unwrap() + } + + /// Iterate over sorted entries by address type. + pub fn into_sorted_iter(self) -> impl Iterator)> { + self.0.into_iter().map(|(output_type, map)| { + let mut sorted: Vec<_> = map.into_iter().collect(); + sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex); + (output_type, sorted) + }) + } + + /// Consume and iterate over entries by address type. + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> impl Iterator)> { + self.0.into_iter() + } +} + +impl AddressTypeToTypeIndexMap> +where + T: Array, +{ + /// Merge two maps of SmallVec values, concatenating vectors. + pub fn merge_vec(mut self, other: Self) -> Self { + for (address_type, other_map) in other.0.into_iter() { + let self_map = self.0.get_mut_unwrap(address_type); + for (typeindex, mut other_vec) in other_map { + match self_map.entry(typeindex) { + Entry::Occupied(mut entry) => { + let self_vec = entry.get_mut(); + if other_vec.len() > self_vec.len() { + mem::swap(self_vec, &mut other_vec); + } + self_vec.extend(other_vec); + } + Entry::Vacant(entry) => { + entry.insert(other_vec); + } + } + } + } + self + } +} diff --git a/crates/brk_computer/src/stateful_new/address/type_vec.rs b/crates/brk_computer/src/stateful_new/address/type_vec.rs new file mode 100644 index 000000000..47472ac36 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/address/type_vec.rs @@ -0,0 +1,66 @@ +//! Per-address-type vector. + +use std::mem; + +use brk_grouper::ByAddressType; +use derive_deref::{Deref, DerefMut}; + +/// A vector for each address type. +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToVec(ByAddressType>); + +impl Default for AddressTypeToVec { + fn default() -> Self { + Self(ByAddressType { + p2a: vec![], + p2pk33: vec![], + p2pk65: vec![], + p2pkh: vec![], + p2sh: vec![], + p2tr: vec![], + p2wpkh: vec![], + p2wsh: vec![], + }) + } +} + +impl AddressTypeToVec { + /// Merge two AddressTypeToVec, consuming other. + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_single(&mut self.p2a, &mut other.p2a); + Self::merge_single(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_single(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_single(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_single(&mut self.p2sh, &mut other.p2sh); + Self::merge_single(&mut self.p2tr, &mut other.p2tr); + Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_single(&mut self.p2wsh, &mut other.p2wsh); + self + } + + /// Merge in place. + pub fn merge_mut(&mut self, mut other: Self) { + Self::merge_single(&mut self.p2a, &mut other.p2a); + Self::merge_single(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_single(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_single(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_single(&mut self.p2sh, &mut other.p2sh); + Self::merge_single(&mut self.p2tr, &mut other.p2tr); + Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_single(&mut self.p2wsh, &mut other.p2wsh); + } + + fn merge_single(own: &mut Vec, other: &mut Vec) { + if own.len() >= other.len() { + own.append(other); + } else { + other.append(own); + mem::swap(own, other); + } + } + + /// Unwrap the inner ByAddressType. + pub fn unwrap(self) -> ByAddressType> { + self.0 + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/address.rs b/crates/brk_computer/src/stateful_new/cohorts/address.rs new file mode 100644 index 000000000..0a4a4c362 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/address.rs @@ -0,0 +1,280 @@ +//! Address cohort vectors with metrics and state. + +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{CohortContext, Filter, Filtered}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version}; +use vecdb::{ + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, + PcoVec, +}; + +use crate::{ + Indexes, + grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, price, + states::AddressCohortState, +}; + +use super::super::metrics::{CohortMetrics, ImportConfig}; +use super::traits::{CohortVecs, DynCohortVecs}; + +const VERSION: Version = Version::ZERO; + +/// Address cohort with metrics and optional runtime state. +#[derive(Clone, Traversable)] +pub struct AddressCohortVecs { + /// Starting height when state was imported + starting_height: Option, + + /// Runtime state for block-by-block processing + #[traversable(skip)] + pub state: Option, + + /// Metric vectors + #[traversable(flatten)] + pub metrics: CohortMetrics, + + /// Address count at each height + pub height_to_addr_count: EagerVec>, + + /// Address count indexed by various dimensions + pub indexes_to_addr_count: ComputedVecsFromHeight, +} + +impl AddressCohortVecs { + /// Import address cohort from database. + pub fn forced_import( + db: &Database, + filter: Filter, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: Option<&Path>, + ) -> Result { + let compute_dollars = price.is_some(); + let full_name = filter.to_full_name(CohortContext::Address); + + let cfg = ImportConfig { + db, + filter, + context: CohortContext::Address, + version, + indexes, + price, + }; + + Ok(Self { + starting_height: None, + + state: states_path + .map(|path| AddressCohortState::new(path, &full_name, compute_dollars)), + + metrics: CohortMetrics::forced_import(&cfg)?, + + height_to_addr_count: EagerVec::forced_import( + db, + &cfg.name("addr_count"), + version + VERSION + Version::ZERO, + )?, + + indexes_to_addr_count: ComputedVecsFromHeight::forced_import( + db, + &cfg.name("addr_count"), + Source::None, + version + VERSION + Version::ZERO, + indexes, + VecBuilderOptions::default().add_last(), + )?, + }) + } + + /// Get the starting height when state was imported. + pub fn starting_height(&self) -> Option { + self.starting_height + } + + /// Set the starting height. + pub fn set_starting_height(&mut self, height: Height) { + self.starting_height = Some(height); + } + + /// Reset starting height to zero. + pub fn reset_starting_height(&mut self) { + self.starting_height = Some(Height::ZERO); + } + + /// Get minimum length across height-indexed vectors. + pub fn min_len(&self) -> usize { + self.height_to_addr_count + .len() + .min(self.metrics.supply.min_len()) + .min(self.metrics.activity.min_len()) + } +} + +impl Filtered for AddressCohortVecs { + fn filter(&self) -> &Filter { + &self.metrics.filter + } +} + +impl DynCohortVecs for AddressCohortVecs { + fn min_height_vecs_len(&self) -> usize { + self.min_len() + } + + fn reset_state_starting_height(&mut self) { + self.reset_starting_height(); + } + + fn import_state(&mut self, starting_height: Height) -> Result { + // Import state from runtime state if present + if let Some(state) = self.state.as_mut() { + let imported = state.inner.import_at_or_before(starting_height)?; + self.starting_height = Some(imported); + + // Restore addr_count from last known value + if let Some(prev_height) = imported.decremented() { + use vecdb::TypedVecIterator; + state.addr_count = *self + .height_to_addr_count + .into_iter() + .get_unwrap(prev_height); + } + + Ok(imported) + } else { + self.starting_height = Some(starting_height); + Ok(starting_height) + } + } + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + use vecdb::GenericStoredVec; + self.height_to_addr_count + .validate_computed_version_or_reset( + base_version + self.height_to_addr_count.inner_version(), + )?; + self.metrics.validate_computed_versions(base_version)?; + Ok(()) + } + + fn truncate_push(&mut self, height: Height) -> Result<()> { + if self.starting_height.map_or(false, |h| h > height) { + return Ok(()); + } + + // Push addr_count from state + if let Some(state) = self.state.as_ref() { + self.height_to_addr_count + .truncate_push(height, state.addr_count.into())?; + self.metrics.truncate_push(height, &state.inner)?; + } + + Ok(()) + } + + fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + ) -> Result<()> { + if let Some(state) = self.state.as_ref() { + self.metrics.compute_then_truncate_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + &state.inner, + )?; + } + Ok(()) + } + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.height_to_addr_count.safe_write(exit)?; + self.metrics.safe_flush(exit)?; + + if let Some(state) = self.state.as_mut() { + state.inner.commit(height)?; + } + + Ok(()) + } + + fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.indexes_to_addr_count.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_addr_count), + )?; + self.metrics + .compute_rest_part1(indexes, price, starting_indexes, exit)?; + Ok(()) + } +} + +impl CohortVecs for AddressCohortVecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_addr_count.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_addr_count) + .collect::>() + .as_slice(), + exit, + )?; + self.metrics.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.metrics).collect::>(), + exit, + )?; + Ok(()) + } + + fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.metrics.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + )?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/address_cohorts.rs b/crates/brk_computer/src/stateful_new/cohorts/address_cohorts.rs new file mode 100644 index 000000000..a05a2e14a --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/address_cohorts.rs @@ -0,0 +1,230 @@ +//! Container for all Address cohorts organized by filter type. + +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{ + AddressGroups, AmountFilter, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, Filter, + Filtered, +}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version}; +use derive_deref::{Deref, DerefMut}; +use rayon::prelude::*; +use vecdb::{Database, Exit, IterableVec}; + +use crate::{Indexes, indexes, price, stateful_new::DynCohortVecs}; + +use super::{AddressCohortVecs, CohortVecs}; + +const VERSION: Version = Version::new(0); + +/// All Address cohorts organized by filter type. +#[derive(Clone, Deref, DerefMut, Traversable)] +pub struct AddressCohorts(AddressGroups); + +impl AddressCohorts { + /// Import all Address cohorts from database. + pub fn forced_import( + db: &Database, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: &Path, + ) -> Result { + let v = version + VERSION + Version::ZERO; + + // Helper to create a cohort - only amount_range cohorts have state + let create = |filter: Filter, has_state: bool| -> Result { + let states_path = if has_state { Some(states_path) } else { None }; + AddressCohortVecs::forced_import(db, filter, v, indexes, price, states_path) + }; + + let full = |f: Filter| create(f, true); + let none = |f: Filter| create(f, false); + + Ok(Self(AddressGroups { + amount_range: ByAmountRange { + _0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?, + _1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?, + _10sats_to_100sats: full(Filter::Amount(AmountFilter::Range( + Sats::_10..Sats::_100, + )))?, + _100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_100..Sats::_1K, + )))?, + _1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_1K..Sats::_10K, + )))?, + _10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_10K..Sats::_100K, + )))?, + _100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_100K..Sats::_1M, + )))?, + _1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_1M..Sats::_10M, + )))?, + _10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10M..Sats::_1BTC, + )))?, + _1btc_to_10btc: full(Filter::Amount(AmountFilter::Range( + Sats::_1BTC..Sats::_10BTC, + )))?, + _10btc_to_100btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10BTC..Sats::_100BTC, + )))?, + _100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_100BTC..Sats::_1K_BTC, + )))?, + _1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_1K_BTC..Sats::_10K_BTC, + )))?, + _10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10K_BTC..Sats::_100K_BTC, + )))?, + _100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual( + Sats::_100K_BTC, + )))?, + }, + + lt_amount: ByLowerThanAmount { + _10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?, + _100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?, + _1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?, + _10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?, + _100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?, + _1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?, + _10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?, + _1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?, + _10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?, + _100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?, + _1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?, + _10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?, + _100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?, + }, + + ge_amount: ByGreatEqualAmount { + _1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?, + _10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?, + _100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?, + _1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?, + _10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?, + _100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?, + _1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?, + _10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?, + _1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?, + _10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?, + _100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?, + _1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?, + _10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?, + }, + })) + } + + /// Compute overlapping cohorts from component amount_range cohorts. + /// + /// For example, ">=1 BTC" cohort is computed from sum of amount_range cohorts that match. + pub fn compute_overlapping_vecs( + &mut self, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + let by_amount_range = &self.0.amount_range; + + // ge_amount cohorts computed from matching amount_range cohorts + [ + self.0 + .ge_amount + .par_iter_mut() + .map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_amount_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + }) + .collect::>(), + // lt_amount cohorts computed from matching amount_range cohorts + self.0 + .lt_amount + .par_iter_mut() + .map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_amount_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + }) + .collect::>(), + ] + .into_iter() + .flatten() + .try_for_each(|(vecs, components)| { + vecs.compute_from_stateful(starting_indexes, &components, exit) + }) + } + + /// First phase of post-processing: compute index transforms. + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.par_iter_mut() + .try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit)) + } + + /// Second phase of post-processing: compute relative metrics. + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &S, + dateindex_to_supply: &D, + height_to_market_cap: Option<&HM>, + dateindex_to_market_cap: Option<&DM>, + height_to_realized_cap: Option<&HR>, + dateindex_to_realized_cap: Option<&DR>, + exit: &Exit, + ) -> Result<()> + where + S: IterableVec + Sync, + D: IterableVec + Sync, + HM: IterableVec + Sync, + DM: IterableVec + Sync, + HR: IterableVec + Sync, + DR: IterableVec + Sync, + { + self.0.par_iter_mut().try_for_each(|v| { + v.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + }) + } + + /// Flush stateful vectors for separate cohorts. + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.par_iter_separate_mut() + .try_for_each(|v| v.safe_flush_stateful_vecs(height, exit)) + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/mod.rs b/crates/brk_computer/src/stateful_new/cohorts/mod.rs new file mode 100644 index 000000000..bbf2c7b5c --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/mod.rs @@ -0,0 +1,22 @@ +//! Cohort management for UTXO and address groupings. +//! +//! Cohorts are groups of UTXOs or addresses filtered by criteria like: +//! - Age (0-1d, 1-7d, etc.) +//! - Amount (< 1 BTC, 1-10 BTC, etc.) +//! - Type (P2PKH, P2SH, etc.) +//! - Term (short-term holder, long-term holder) + +mod address; +mod address_cohorts; +mod state; +mod traits; +mod utxo; +mod utxo_cohorts; + +pub use address::AddressCohortVecs; +pub use address_cohorts::AddressCohorts; +pub use crate::states::{Flushable, HeightFlushable}; +pub use state::CohortState; +pub use traits::{CohortVecs, DynCohortVecs}; +pub use utxo::UTXOCohortVecs; +pub use utxo_cohorts::UTXOCohorts; diff --git a/crates/brk_computer/src/stateful_new/cohorts/state.rs b/crates/brk_computer/src/stateful_new/cohorts/state.rs new file mode 100644 index 000000000..25abaa7d7 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/state.rs @@ -0,0 +1,243 @@ +//! Cohort state tracking during computation. +//! +//! This state is maintained in memory during block processing and periodically flushed. + +use std::cmp::Ordering; +use std::path::Path; + +use brk_error::Result; +use brk_types::{CheckedSub, Dollars, Height, Sats}; + +use crate::{ + PriceToAmount, RealizedState, SupplyState, UnrealizedState, + grouped::{PERCENTILES, PERCENTILES_LEN}, + utils::OptionExt, +}; + +/// State tracked for each cohort during computation. +#[derive(Clone)] +pub struct CohortState { + /// Current supply in this cohort + pub supply: SupplyState, + + /// Realized cap and profit/loss (requires price data) + pub realized: Option, + + /// Amount sent in current block + pub sent: Sats, + + /// Satoshi-blocks destroyed (supply * blocks_old when spent) + pub satblocks_destroyed: Sats, + + /// Satoshi-days destroyed (supply * days_old when spent) + pub satdays_destroyed: Sats, + + /// Price distribution for percentile calculations (requires price data) + price_to_amount: Option, +} + +impl CohortState { + /// Create new cohort state. + pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self { + Self { + supply: SupplyState::default(), + realized: compute_dollars.then_some(RealizedState::NAN), + sent: Sats::ZERO, + satblocks_destroyed: Sats::ZERO, + satdays_destroyed: Sats::ZERO, + price_to_amount: compute_dollars.then_some(PriceToAmount::create(path, name)), + } + } + + /// Import state from checkpoint. + pub fn import_at_or_before(&mut self, height: Height) -> Result { + match self.price_to_amount.as_mut() { + Some(p) => p.import_at_or_before(height), + None => Ok(height), + } + } + + /// Reset price_to_amount if needed (for starting fresh). + pub fn reset_price_to_amount(&mut self) -> Result<()> { + if let Some(p) = self.price_to_amount.as_mut() { + p.clean()?; + p.init(); + } + Ok(()) + } + + /// Reset per-block values before processing next block. + pub fn reset_block_values(&mut self) { + self.sent = Sats::ZERO; + self.satdays_destroyed = Sats::ZERO; + self.satblocks_destroyed = Sats::ZERO; + if let Some(realized) = self.realized.as_mut() { + realized.reset_single_iteration_values(); + } + } + + /// Add supply to this cohort (e.g., when UTXO ages into cohort). + pub fn increment(&mut self, supply: &SupplyState, price: Option) { + self.supply += supply; + + if supply.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + let price = price.unwrap(); + realized.increment(supply, price); + self.price_to_amount.as_mut().unwrap().increment(price, supply); + } + } + } + + /// Remove supply from this cohort (e.g., when UTXO ages out of cohort). + pub fn decrement(&mut self, supply: &SupplyState, price: Option) { + self.supply -= supply; + + if supply.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + let price = price.unwrap(); + realized.decrement(supply, price); + self.price_to_amount.as_mut().unwrap().decrement(price, supply); + } + } + } + + /// Process received output (new UTXO in cohort). + pub fn receive(&mut self, supply: &SupplyState, price: Option) { + self.supply += supply; + + if supply.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + let price = price.unwrap(); + realized.receive(supply, price); + self.price_to_amount.as_mut().unwrap().increment(price, supply); + } + } + } + + /// Process spent input (UTXO leaving cohort). + pub fn send( + &mut self, + supply: &SupplyState, + current_price: Option, + prev_price: Option, + blocks_old: usize, + days_old: f64, + older_than_hour: bool, + ) { + if supply.utxo_count == 0 { + return; + } + + self.supply -= supply; + + if supply.value > Sats::ZERO { + self.sent += supply.value; + self.satblocks_destroyed += supply.value * blocks_old; + self.satdays_destroyed += + Sats::from((u64::from(supply.value) as f64 * days_old).floor() as u64); + + if let Some(realized) = self.realized.as_mut() { + let current_price = current_price.unwrap(); + let prev_price = prev_price.unwrap(); + realized.send(supply, current_price, prev_price, older_than_hour); + self.price_to_amount.as_mut().unwrap().decrement(prev_price, supply); + } + } + } + + /// Compute prices at percentile thresholds. + pub fn compute_percentile_prices(&self) -> [Dollars; PERCENTILES_LEN] { + let mut result = [Dollars::NAN; PERCENTILES_LEN]; + + let price_to_amount = match self.price_to_amount.as_ref() { + Some(p) => p, + None => return result, + }; + + if price_to_amount.is_empty() || self.supply.value == Sats::ZERO { + return result; + } + + let total = u64::from(self.supply.value); + let targets = PERCENTILES.map(|p| total * u64::from(p) / 100); + + let mut accumulated = 0u64; + let mut pct_idx = 0; + + for (&price, &sats) in price_to_amount.iter() { + accumulated += u64::from(sats); + + while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] { + result[pct_idx] = price; + pct_idx += 1; + } + + if pct_idx >= PERCENTILES_LEN { + break; + } + } + + result + } + + /// Compute unrealized profit/loss at current price. + pub fn compute_unrealized( + &self, + height_price: Dollars, + date_price: Option, + ) -> (UnrealizedState, Option) { + let price_to_amount = match self.price_to_amount.as_ref() { + Some(p) if !p.is_empty() => p, + _ => return (UnrealizedState::NAN, date_price.map(|_| UnrealizedState::NAN)), + }; + + let mut height_state = UnrealizedState::ZERO; + let mut date_state = date_price.map(|_| UnrealizedState::ZERO); + + for (&price, &sats) in price_to_amount.iter() { + Self::update_unrealized(price, height_price, sats, &mut height_state); + + if let Some(date_price) = date_price { + Self::update_unrealized(price, date_price, sats, date_state.um()); + } + } + + (height_state, date_state) + } + + fn update_unrealized(price: Dollars, current: Dollars, sats: Sats, state: &mut UnrealizedState) { + match price.cmp(¤t) { + Ordering::Less | Ordering::Equal => { + state.supply_in_profit += sats; + if price < current && price > Dollars::ZERO && current > Dollars::ZERO { + state.unrealized_profit += current.checked_sub(price).unwrap() * sats; + } + } + Ordering::Greater => { + state.supply_in_loss += sats; + if price > Dollars::ZERO && current > Dollars::ZERO { + state.unrealized_loss += price.checked_sub(current).unwrap() * sats; + } + } + } + } + + /// Flush state to disk at checkpoint. + pub fn commit(&mut self, height: Height) -> Result<()> { + if let Some(p) = self.price_to_amount.as_mut() { + p.flush(height)?; + } + Ok(()) + } + + /// Get first (lowest) price in distribution. + pub fn min_price(&self) -> Option<&Dollars> { + self.price_to_amount.as_ref()?.first_key_value().map(|(k, _)| k) + } + + /// Get last (highest) price in distribution. + pub fn max_price(&self) -> Option<&Dollars> { + self.price_to_amount.as_ref()?.last_key_value().map(|(k, _)| k) + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/traits.rs b/crates/brk_computer/src/stateful_new/cohorts/traits.rs new file mode 100644 index 000000000..d92978010 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/traits.rs @@ -0,0 +1,76 @@ +//! Traits for cohort vector operations. + +use brk_error::Result; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; +use vecdb::{Exit, IterableVec}; + +use crate::{indexes, price, Indexes}; + +/// Dynamic dispatch trait for cohort vectors. +/// +/// This trait enables heterogeneous cohort processing via trait objects. +pub trait DynCohortVecs: Send + Sync { + /// Get minimum length across height-indexed vectors. + fn min_height_vecs_len(&self) -> usize; + + /// Reset the starting height for state tracking. + fn reset_state_starting_height(&mut self); + + /// Import state from checkpoint at or before the given height. + fn import_state(&mut self, starting_height: Height) -> Result; + + /// Validate that computed vectors have correct versions. + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>; + + /// Push state to height-indexed vectors (truncating if needed). + fn truncate_push(&mut self, height: Height) -> Result<()>; + + /// Compute and push unrealized profit/loss states. + fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + ) -> Result<()>; + + /// Flush stateful vectors to disk. + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>; + + /// First phase of post-processing computations. + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()>; +} + +/// Static dispatch trait for cohort vectors with additional methods. +pub trait CohortVecs: DynCohortVecs { + /// Compute aggregate cohort from component cohorts. + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()>; + + /// Second phase of post-processing computations. + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()>; +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/utxo.rs b/crates/brk_computer/src/stateful_new/cohorts/utxo.rs new file mode 100644 index 000000000..7b28d7af8 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/utxo.rs @@ -0,0 +1,224 @@ +//! UTXO cohort vectors with metrics and state. + +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{CohortContext, Filter, Filtered, StateLevel}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; +use vecdb::{Database, Exit, IterableVec}; + +use crate::{ + Indexes, PriceToAmount, UTXOCohortState, indexes, price, + stateful_new::{CohortVecs, DynCohortVecs}, +}; + +use super::super::metrics::{CohortMetrics, ImportConfig}; + +/// UTXO cohort with metrics and optional runtime state. +#[derive(Clone, Traversable)] +pub struct UTXOCohortVecs { + /// Starting height when state was imported + state_starting_height: Option, + + /// Runtime state for block-by-block processing + #[traversable(skip)] + pub state: Option, + + /// For aggregate cohorts that only need price_to_amount for percentiles + #[traversable(skip)] + pub price_to_amount: Option, + + /// Metric vectors + #[traversable(flatten)] + pub metrics: CohortMetrics, +} + +impl UTXOCohortVecs { + /// Import UTXO cohort from database. + pub fn forced_import( + db: &Database, + filter: Filter, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: &Path, + state_level: StateLevel, + ) -> Result { + let compute_dollars = price.is_some(); + let full_name = filter.to_full_name(CohortContext::Utxo); + + let cfg = ImportConfig { + db, + filter, + context: CohortContext::Utxo, + version, + indexes, + price, + }; + + Ok(Self { + state_starting_height: None, + + state: if state_level.is_full() { + Some(UTXOCohortState::new( + states_path, + &full_name, + compute_dollars, + )) + } else { + None + }, + + price_to_amount: if state_level.is_price_only() && compute_dollars { + Some(PriceToAmount::create(states_path, &full_name)) + } else { + None + }, + + metrics: CohortMetrics::forced_import(&cfg)?, + }) + } + + /// Get the starting height when state was imported. + pub fn state_starting_height(&self) -> Option { + self.state_starting_height + } + + /// Set the state starting height. + pub fn set_state_starting_height(&mut self, height: Height) { + self.state_starting_height = Some(height); + } + + /// Reset state starting height to zero. + pub fn reset_state_starting_height(&mut self) { + self.state_starting_height = Some(Height::ZERO); + } +} + +impl Filtered for UTXOCohortVecs { + fn filter(&self) -> &Filter { + &self.metrics.filter + } +} + +impl DynCohortVecs for UTXOCohortVecs { + fn min_height_vecs_len(&self) -> usize { + self.metrics.min_len() + } + + fn reset_state_starting_height(&mut self) { + self.state_starting_height = Some(Height::ZERO); + } + + fn import_state(&mut self, starting_height: Height) -> Result { + // Import state from runtime state if present + if let Some(state) = self.state.as_mut() { + let imported = state.import_at_or_before(starting_height)?; + self.state_starting_height = Some(imported); + Ok(imported) + } else { + self.state_starting_height = Some(starting_height); + Ok(starting_height) + } + } + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.metrics.validate_computed_versions(base_version) + } + + fn truncate_push(&mut self, height: Height) -> Result<()> { + if self.state_starting_height.map_or(false, |h| h > height) { + return Ok(()); + } + + // Push from state to metrics + if let Some(state) = self.state.as_ref() { + self.metrics.truncate_push(height, &state)?; + } + + Ok(()) + } + + fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + ) -> Result<()> { + if let Some(state) = self.state.as_ref() { + self.metrics.compute_then_truncate_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + state, + )?; + } + Ok(()) + } + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.metrics.safe_flush(exit)?; + + if let Some(state) = self.state.as_mut() { + state.commit(height)?; + } + + Ok(()) + } + + fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.metrics + .compute_rest_part1(indexes, price, starting_indexes, exit) + } +} + +impl CohortVecs for UTXOCohortVecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.metrics.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.metrics).collect::>(), + exit, + ) + } + + fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.metrics.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/mod.rs b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/mod.rs new file mode 100644 index 000000000..426372bdc --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/mod.rs @@ -0,0 +1,376 @@ +//! Container for all UTXO cohorts organized by filter type. + +mod receive; +mod send; +mod tick_tock; + +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{ + AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, + ByMaxAge, ByMinAge, BySpendableType, ByTerm, Filter, Filtered, StateLevel, Term, TimeFilter, + UTXOGroups, +}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, HalvingEpoch, Height, OutputType, Sats, Version}; +use derive_deref::{Deref, DerefMut}; +use rayon::prelude::*; +use vecdb::{Database, Exit, IterableVec}; + +use crate::{Indexes, indexes, price, stateful_new::DynCohortVecs}; + +use super::{CohortVecs, HeightFlushable, UTXOCohortVecs}; + +const VERSION: Version = Version::new(0); + +/// All UTXO cohorts organized by filter type. +#[derive(Clone, Deref, DerefMut, Traversable)] +pub struct UTXOCohorts(pub(crate) UTXOGroups); + +impl UTXOCohorts { + /// Import all UTXO cohorts from database. + pub fn forced_import( + db: &Database, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: &Path, + ) -> Result { + let v = version + VERSION + Version::ZERO; + + let create = |filter: Filter, state_level: StateLevel| -> Result { + UTXOCohortVecs::forced_import(db, filter, v, indexes, price, states_path, state_level) + }; + + let full = |f: Filter| create(f, StateLevel::Full); + let none = |f: Filter| create(f, StateLevel::None); + + Ok(Self(UTXOGroups { + all: UTXOCohortVecs::forced_import( + db, + Filter::All, + version + VERSION + Version::ONE, + indexes, + price, + states_path, + StateLevel::PriceOnly, + )?, + + term: ByTerm { + short: create(Filter::Term(Term::Sth), StateLevel::PriceOnly)?, + long: create(Filter::Term(Term::Lth), StateLevel::PriceOnly)?, + }, + + epoch: ByEpoch { + _0: full(Filter::Epoch(HalvingEpoch::new(0)))?, + _1: full(Filter::Epoch(HalvingEpoch::new(1)))?, + _2: full(Filter::Epoch(HalvingEpoch::new(2)))?, + _3: full(Filter::Epoch(HalvingEpoch::new(3)))?, + _4: full(Filter::Epoch(HalvingEpoch::new(4)))?, + }, + + type_: BySpendableType { + p2pk65: full(Filter::Type(OutputType::P2PK65))?, + p2pk33: full(Filter::Type(OutputType::P2PK33))?, + p2pkh: full(Filter::Type(OutputType::P2PKH))?, + p2sh: full(Filter::Type(OutputType::P2SH))?, + p2wpkh: full(Filter::Type(OutputType::P2WPKH))?, + p2wsh: full(Filter::Type(OutputType::P2WSH))?, + p2tr: full(Filter::Type(OutputType::P2TR))?, + p2a: full(Filter::Type(OutputType::P2A))?, + p2ms: full(Filter::Type(OutputType::P2MS))?, + empty: full(Filter::Type(OutputType::Empty))?, + unknown: full(Filter::Type(OutputType::Unknown))?, + }, + + max_age: ByMaxAge { + _1w: none(Filter::Time(TimeFilter::LowerThan(7)))?, + _1m: none(Filter::Time(TimeFilter::LowerThan(30)))?, + _2m: none(Filter::Time(TimeFilter::LowerThan(2 * 30)))?, + _3m: none(Filter::Time(TimeFilter::LowerThan(3 * 30)))?, + _4m: none(Filter::Time(TimeFilter::LowerThan(4 * 30)))?, + _5m: none(Filter::Time(TimeFilter::LowerThan(5 * 30)))?, + _6m: none(Filter::Time(TimeFilter::LowerThan(6 * 30)))?, + _1y: none(Filter::Time(TimeFilter::LowerThan(365)))?, + _2y: none(Filter::Time(TimeFilter::LowerThan(2 * 365)))?, + _3y: none(Filter::Time(TimeFilter::LowerThan(3 * 365)))?, + _4y: none(Filter::Time(TimeFilter::LowerThan(4 * 365)))?, + _5y: none(Filter::Time(TimeFilter::LowerThan(5 * 365)))?, + _6y: none(Filter::Time(TimeFilter::LowerThan(6 * 365)))?, + _7y: none(Filter::Time(TimeFilter::LowerThan(7 * 365)))?, + _8y: none(Filter::Time(TimeFilter::LowerThan(8 * 365)))?, + _10y: none(Filter::Time(TimeFilter::LowerThan(10 * 365)))?, + _12y: none(Filter::Time(TimeFilter::LowerThan(12 * 365)))?, + _15y: none(Filter::Time(TimeFilter::LowerThan(15 * 365)))?, + }, + + min_age: ByMinAge { + _1d: none(Filter::Time(TimeFilter::GreaterOrEqual(1)))?, + _1w: none(Filter::Time(TimeFilter::GreaterOrEqual(7)))?, + _1m: none(Filter::Time(TimeFilter::GreaterOrEqual(30)))?, + _2m: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 30)))?, + _3m: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 30)))?, + _4m: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 30)))?, + _5m: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 30)))?, + _6m: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 30)))?, + _1y: none(Filter::Time(TimeFilter::GreaterOrEqual(365)))?, + _2y: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 365)))?, + _3y: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 365)))?, + _4y: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 365)))?, + _5y: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 365)))?, + _6y: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 365)))?, + _7y: none(Filter::Time(TimeFilter::GreaterOrEqual(7 * 365)))?, + _8y: none(Filter::Time(TimeFilter::GreaterOrEqual(8 * 365)))?, + _10y: none(Filter::Time(TimeFilter::GreaterOrEqual(10 * 365)))?, + _12y: none(Filter::Time(TimeFilter::GreaterOrEqual(12 * 365)))?, + }, + + age_range: ByAgeRange { + up_to_1d: full(Filter::Time(TimeFilter::Range(0..1)))?, + _1d_to_1w: full(Filter::Time(TimeFilter::Range(1..7)))?, + _1w_to_1m: full(Filter::Time(TimeFilter::Range(7..30)))?, + _1m_to_2m: full(Filter::Time(TimeFilter::Range(30..2 * 30)))?, + _2m_to_3m: full(Filter::Time(TimeFilter::Range(2 * 30..3 * 30)))?, + _3m_to_4m: full(Filter::Time(TimeFilter::Range(3 * 30..4 * 30)))?, + _4m_to_5m: full(Filter::Time(TimeFilter::Range(4 * 30..5 * 30)))?, + _5m_to_6m: full(Filter::Time(TimeFilter::Range(5 * 30..6 * 30)))?, + _6m_to_1y: full(Filter::Time(TimeFilter::Range(6 * 30..365)))?, + _1y_to_2y: full(Filter::Time(TimeFilter::Range(365..2 * 365)))?, + _2y_to_3y: full(Filter::Time(TimeFilter::Range(2 * 365..3 * 365)))?, + _3y_to_4y: full(Filter::Time(TimeFilter::Range(3 * 365..4 * 365)))?, + _4y_to_5y: full(Filter::Time(TimeFilter::Range(4 * 365..5 * 365)))?, + _5y_to_6y: full(Filter::Time(TimeFilter::Range(5 * 365..6 * 365)))?, + _6y_to_7y: full(Filter::Time(TimeFilter::Range(6 * 365..7 * 365)))?, + _7y_to_8y: full(Filter::Time(TimeFilter::Range(7 * 365..8 * 365)))?, + _8y_to_10y: full(Filter::Time(TimeFilter::Range(8 * 365..10 * 365)))?, + _10y_to_12y: full(Filter::Time(TimeFilter::Range(10 * 365..12 * 365)))?, + _12y_to_15y: full(Filter::Time(TimeFilter::Range(12 * 365..15 * 365)))?, + from_15y: full(Filter::Time(TimeFilter::GreaterOrEqual(15 * 365)))?, + }, + + amount_range: ByAmountRange { + _0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?, + _1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?, + _10sats_to_100sats: full(Filter::Amount(AmountFilter::Range( + Sats::_10..Sats::_100, + )))?, + _100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_100..Sats::_1K, + )))?, + _1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_1K..Sats::_10K, + )))?, + _10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_10K..Sats::_100K, + )))?, + _100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_100K..Sats::_1M, + )))?, + _1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_1M..Sats::_10M, + )))?, + _10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10M..Sats::_1BTC, + )))?, + _1btc_to_10btc: full(Filter::Amount(AmountFilter::Range( + Sats::_1BTC..Sats::_10BTC, + )))?, + _10btc_to_100btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10BTC..Sats::_100BTC, + )))?, + _100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_100BTC..Sats::_1K_BTC, + )))?, + _1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_1K_BTC..Sats::_10K_BTC, + )))?, + _10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10K_BTC..Sats::_100K_BTC, + )))?, + _100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual( + Sats::_100K_BTC, + )))?, + }, + + lt_amount: ByLowerThanAmount { + _10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?, + _100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?, + _1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?, + _10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?, + _100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?, + _1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?, + _10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?, + _1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?, + _10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?, + _100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?, + _1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?, + _10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?, + _100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?, + }, + + ge_amount: ByGreatEqualAmount { + _1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?, + _10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?, + _100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?, + _1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?, + _10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?, + _100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?, + _1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?, + _10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?, + _1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?, + _10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?, + _100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?, + _1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?, + _10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?, + }, + })) + } + + /// Compute overlapping cohorts from component age/amount range cohorts. + pub fn compute_overlapping_vecs( + &mut self, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + let by_age_range = &self.0.age_range; + let by_amount_range = &self.0.amount_range; + + [(&mut self.0.all, by_age_range.iter().collect::>())] + .into_par_iter() + .chain(self.0.min_age.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_age_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.max_age.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_age_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.term.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_age_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.ge_amount.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_amount_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.lt_amount.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_amount_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .try_for_each(|(vecs, components)| { + vecs.compute_from_stateful(starting_indexes, &components, exit) + }) + } + + /// First phase of post-processing: compute index transforms. + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.par_iter_mut() + .try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit)) + } + + /// Second phase of post-processing: compute relative metrics. + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &S, + dateindex_to_supply: &D, + height_to_market_cap: Option<&HM>, + dateindex_to_market_cap: Option<&DM>, + height_to_realized_cap: Option<&HR>, + dateindex_to_realized_cap: Option<&DR>, + exit: &Exit, + ) -> Result<()> + where + S: IterableVec + Sync, + D: IterableVec + Sync, + HM: IterableVec + Sync, + DM: IterableVec + Sync, + HR: IterableVec + Sync, + DR: IterableVec + Sync, + { + self.par_iter_mut().try_for_each(|v| { + v.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + }) + } + + /// Flush stateful vectors for separate cohorts. + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.par_iter_separate_mut() + .try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))?; + + self.0 + .par_iter_aggregate_mut() + .try_for_each(|v| v.price_to_amount.flush_at_height(height, exit)) + } + + /// Reset aggregate cohorts' price_to_amount for fresh start. + pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> { + self.0 + .iter_aggregate_mut() + .try_for_each(|v| v.price_to_amount.reset()) + } + + /// Import aggregate cohorts' price_to_amount when resuming from checkpoint. + pub fn import_aggregate_price_to_amount(&mut self, height: Height) -> Result { + let Some(mut prev_height) = height.decremented() else { + return Ok(Height::ZERO); + }; + + for v in self.0.iter_aggregate_mut() { + prev_height = prev_height.min(v.price_to_amount.import_at_or_before(prev_height)?); + } + + Ok(prev_height.incremented()) + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/receive.rs b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/receive.rs new file mode 100644 index 000000000..89854c3ae --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/receive.rs @@ -0,0 +1,72 @@ +//! Processing received outputs (new UTXOs). + +use brk_grouper::{Filter, Filtered}; +use brk_types::{Dollars, Height}; + +use crate::states::Transacted; + +use super::UTXOCohorts; + +impl UTXOCohorts { + /// Process received outputs for this block. + /// + /// New UTXOs are added to: + /// - The "up_to_1d" age cohort (all new UTXOs start at 0 days old) + /// - The appropriate epoch cohort based on block height + /// - The appropriate output type cohort (P2PKH, P2SH, etc.) + /// - The appropriate amount range cohort based on value + pub fn receive(&mut self, received: Transacted, height: Height, price: Option) { + let supply_state = received.spendable_supply; + + // New UTXOs go into up_to_1d and current epoch + [ + &mut self.0.age_range.up_to_1d, + self.0.epoch.mut_vec_from_height(height), + ] + .into_iter() + .for_each(|v| { + v.state.as_mut().unwrap().receive(&supply_state, price); + }); + + // Update aggregate cohorts' price_to_amount + // New UTXOs have days_old = 0, so check if filter includes day 0 + if let Some(price) = price + && supply_state.value.is_not_zero() + { + self.0 + .iter_aggregate_mut() + .filter(|v| v.filter().contains_time(0)) + .for_each(|v| { + v.price_to_amount + .as_mut() + .unwrap() + .increment(price, &supply_state); + }); + } + + // Update output type cohorts + self.type_.iter_mut().for_each(|vecs| { + let output_type = match vecs.filter() { + Filter::Type(output_type) => *output_type, + _ => unreachable!(), + }; + vecs.state + .as_mut() + .unwrap() + .receive(received.by_type.get(output_type), price) + }); + + // Update amount range cohorts + received + .by_size_group + .iter_typed() + .for_each(|(group, supply_state)| { + self.amount_range + .get_mut(group) + .state + .as_mut() + .unwrap() + .receive(supply_state, price); + }); + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/send.rs b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/send.rs new file mode 100644 index 000000000..bc3479c86 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/send.rs @@ -0,0 +1,135 @@ +//! Processing spent inputs (UTXOs being spent). + +use brk_grouper::{Filter, Filtered, TimeFilter, UTXOGroups}; +use brk_types::{CheckedSub, HalvingEpoch, Height}; +use rustc_hash::FxHashMap; +use vecdb::VecIndex; + +use crate::{states::{BlockState, Transacted}, utils::OptionExt, PriceToAmount}; + +use super::UTXOCohorts; + +impl UTXOCohorts { + /// Process spent inputs for this block. + /// + /// Each input references a UTXO created at some previous height. + /// We need to update the cohort states based on when that UTXO was created. + pub fn send( + &mut self, + height_to_sent: FxHashMap, + chain_state: &mut [BlockState], + ) { + let UTXOGroups { + all, + term, + age_range, + epoch, + type_, + amount_range, + .. + } = &mut self.0; + + // Time-based cohorts: age_range + epoch + let mut time_cohorts: Vec<_> = age_range + .iter_mut() + .chain(epoch.iter_mut()) + .collect(); + + // Aggregate cohorts' price_to_amount + let mut aggregate_p2a: Vec<(Filter, Option<&mut PriceToAmount>)> = vec![ + (all.filter().clone(), all.price_to_amount.as_mut()), + ( + term.short.filter().clone(), + term.short.price_to_amount.as_mut(), + ), + ( + term.long.filter().clone(), + term.long.price_to_amount.as_mut(), + ), + ]; + + let last_block = chain_state.last().unwrap(); + let last_timestamp = last_block.timestamp; + let current_price = last_block.price; + let chain_len = chain_state.len(); + + for (height, sent) in height_to_sent { + // Update chain_state to reflect spent supply + chain_state[height.to_usize()].supply -= &sent.spendable_supply; + + let block_state = &chain_state[height.to_usize()]; + let prev_price = block_state.price; + let blocks_old = chain_len - 1 - height.to_usize(); + let days_old = last_timestamp.difference_in_days_between(block_state.timestamp); + let days_old_float = + last_timestamp.difference_in_days_between_float(block_state.timestamp); + let older_than_hour = last_timestamp + .checked_sub(block_state.timestamp) + .unwrap() + .is_more_than_hour(); + + // Update time-based cohorts + time_cohorts + .iter_mut() + .filter(|v| match v.filter() { + Filter::Time(TimeFilter::GreaterOrEqual(from)) => *from <= days_old, + Filter::Time(TimeFilter::LowerThan(to)) => *to > days_old, + Filter::Time(TimeFilter::Range(range)) => range.contains(&days_old), + Filter::Epoch(e) => *e == HalvingEpoch::from(height), + _ => unreachable!(), + }) + .for_each(|vecs| { + vecs.state.um().send( + &sent.spendable_supply, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); + }); + + // Update output type cohorts + sent.by_type + .spendable + .iter_typed() + .for_each(|(output_type, supply_state)| { + type_.get_mut(output_type).state.um().send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ) + }); + + // Update amount range cohorts + sent.by_size_group + .iter_typed() + .for_each(|(group, supply_state)| { + amount_range.get_mut(group).state.um().send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); + }); + + // Update aggregate cohorts' price_to_amount + if let Some(prev_price) = prev_price { + let supply_state = &sent.spendable_supply; + if supply_state.value.is_not_zero() { + aggregate_p2a + .iter_mut() + .filter(|(f, _)| f.contains_time(days_old)) + .for_each(|(_, p2a)| { + p2a.um().decrement(prev_price, supply_state); + }); + } + } + } + } +} diff --git a/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/tick_tock.rs b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/tick_tock.rs new file mode 100644 index 000000000..bee1018d8 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/cohorts/utxo_cohorts/tick_tock.rs @@ -0,0 +1,107 @@ +//! Age-based state transitions for UTXO cohorts. +//! +//! When a new block arrives, UTXOs age. Some cross day boundaries +//! and need to move between age-based cohorts. + +use brk_grouper::{Filter, Filtered, UTXOGroups}; +use brk_types::{ONE_DAY_IN_SEC, Sats, Timestamp}; + +use crate::{states::BlockState, utils::OptionExt, PriceToAmount}; + +use super::UTXOCohorts; + +impl UTXOCohorts { + /// Handle age transitions when processing a new block. + /// + /// UTXOs age with each block. When they cross day boundaries, + /// they move between age-based cohorts (e.g., from "0-1d" to "1-7d"). + pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) { + if chain_state.is_empty() { + return; + } + + let prev_timestamp = chain_state.last().unwrap().timestamp; + + // Optimization: Only blocks whose age % ONE_DAY >= threshold can cross a day boundary. + // Saves computation vs checking days_old for every block. + let elapsed = (*timestamp).saturating_sub(*prev_timestamp); + let threshold = ONE_DAY_IN_SEC.saturating_sub(elapsed); + + // Extract mutable references to avoid borrow checker issues + let UTXOGroups { + all, + term, + age_range, + .. + } = &mut self.0; + + // Collect age_range cohorts with their filters and states + let mut age_cohorts: Vec<(Filter, &mut Option<_>)> = age_range + .iter_mut() + .map(|v| (v.filter().clone(), &mut v.state)) + .collect(); + + // Collect aggregate cohorts' price_to_amount for age transitions + let mut aggregate_p2a: Vec<(Filter, Option<&mut PriceToAmount>)> = vec![ + (all.filter().clone(), all.price_to_amount.as_mut()), + ( + term.short.filter().clone(), + term.short.price_to_amount.as_mut(), + ), + ( + term.long.filter().clone(), + term.long.price_to_amount.as_mut(), + ), + ]; + + // Process blocks that might cross a day boundary + chain_state + .iter() + .filter(|block_state| { + let age = (*prev_timestamp).saturating_sub(*block_state.timestamp); + age % ONE_DAY_IN_SEC >= threshold + }) + .for_each(|block_state| { + let prev_days = prev_timestamp.difference_in_days_between(block_state.timestamp); + let curr_days = timestamp.difference_in_days_between(block_state.timestamp); + + if prev_days == curr_days { + return; + } + + // Update age_range cohort states + age_cohorts.iter_mut().for_each(|(filter, state)| { + let is_now = filter.contains_time(curr_days); + let was_before = filter.contains_time(prev_days); + + if is_now && !was_before { + state + .as_mut() + .unwrap() + .increment(&block_state.supply, block_state.price); + } else if was_before && !is_now { + state + .as_mut() + .unwrap() + .decrement(&block_state.supply, block_state.price); + } + }); + + // Update aggregate cohorts' price_to_amount + if let Some(price) = block_state.price + && block_state.supply.value > Sats::ZERO + { + aggregate_p2a.iter_mut().for_each(|(filter, p2a)| { + let is_now = filter.contains_time(curr_days); + let was_before = filter.contains_time(prev_days); + + if is_now && !was_before { + p2a.um().increment(price, &block_state.supply); + } else if was_before && !is_now { + p2a.um().decrement(price, &block_state.supply); + } + }); + } + }); + } +} diff --git a/crates/brk_computer/src/stateful_new/compute/aggregates.rs b/crates/brk_computer/src/stateful_new/compute/aggregates.rs new file mode 100644 index 000000000..1807bf458 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/aggregates.rs @@ -0,0 +1,110 @@ +//! Aggregate cohort computation. +//! +//! After block processing, compute derived metrics: +//! 1. Overlapping cohorts (e.g., ">=1d" from sum of age_range cohorts) +//! 2. Index-based transforms (height -> dateindex, etc.) +//! 3. Relative metrics (supply ratios, market cap ratios) + +use brk_error::Result; +use brk_types::{Bitcoin, DateIndex, Dollars, Height}; +use log::info; +use vecdb::{Exit, IterableVec}; + +use crate::{Indexes, indexes, price}; + +use super::super::cohorts::{AddressCohorts, UTXOCohorts}; + +/// Compute overlapping cohorts from component cohorts. +/// +/// For example: +/// - ">=1d" UTXO cohort is computed from sum of age_range cohorts that match +/// - ">=1 BTC" address cohort is computed from sum of amount_range cohorts that match +pub fn compute_overlapping( + utxo_cohorts: &mut UTXOCohorts, + address_cohorts: &mut AddressCohorts, + starting_indexes: &Indexes, + exit: &Exit, +) -> Result<()> { + info!("Computing overlapping cohorts..."); + + utxo_cohorts.compute_overlapping_vecs(starting_indexes, exit)?; + address_cohorts.compute_overlapping_vecs(starting_indexes, exit)?; + + Ok(()) +} + +/// First phase of post-processing: compute index transforms. +/// +/// Converts height-indexed data to dateindex-indexed data and other transforms. +pub fn compute_rest_part1( + utxo_cohorts: &mut UTXOCohorts, + address_cohorts: &mut AddressCohorts, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, +) -> Result<()> { + info!("Computing rest part 1..."); + + utxo_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?; + address_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?; + + Ok(()) +} + +/// Second phase of post-processing: compute relative metrics. +/// +/// Computes supply ratios, market cap ratios, etc. using total references. +#[allow(clippy::too_many_arguments)] +pub fn compute_rest_part2( + utxo_cohorts: &mut UTXOCohorts, + address_cohorts: &mut AddressCohorts, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &S, + dateindex_to_supply: &D, + height_to_market_cap: Option<&HM>, + dateindex_to_market_cap: Option<&DM>, + height_to_realized_cap: Option<&HR>, + dateindex_to_realized_cap: Option<&DR>, + exit: &Exit, +) -> Result<()> +where + S: IterableVec + Sync, + D: IterableVec + Sync, + HM: IterableVec + Sync, + DM: IterableVec + Sync, + HR: IterableVec + Sync, + DR: IterableVec + Sync, +{ + info!("Computing rest part 2..."); + + utxo_cohorts.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + )?; + + address_cohorts.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + )?; + + Ok(()) +} diff --git a/crates/brk_computer/src/stateful_new/compute/block_loop.rs b/crates/brk_computer/src/stateful_new/compute/block_loop.rs new file mode 100644 index 000000000..5708d76c8 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/block_loop.rs @@ -0,0 +1,323 @@ +//! Main block processing loop. +//! +//! Iterates through blocks and processes each one: +//! 1. Reset per-block state values +//! 2. Tick-tock age transitions +//! 3. Process outputs (receive) in parallel +//! 4. Process inputs (send) in parallel +//! 5. Push to height-indexed vectors +//! 6. Periodically flush checkpoints + +use std::thread; + +use brk_error::Result; +use brk_grouper::ByAddressType; +use brk_indexer::Indexer; +use brk_types::{DateIndex, Height, OutputType, Sats}; +use log::info; +use vecdb::{Exit, GenericStoredVec, IterableVec, VecIndex}; + +use crate::states::{BlockState, Transacted}; +use crate::{chain, indexes, price}; + +use super::super::cohorts::{AddressCohorts, DynCohortVecs, UTXOCohorts}; +use super::super::vecs::Vecs; +use super::{ + FLUSH_INTERVAL, IndexerReaders, build_txinindex_to_txindex, build_txoutindex_to_height_map, + build_txoutindex_to_txindex, process_inputs, process_outputs, +}; + +/// BIP30 duplicate coinbase heights - must handle specially. +const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842; +const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880; +const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812; +const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722; + +/// Process all blocks from starting_height to last_height. +#[allow(clippy::too_many_arguments)] +pub fn process_blocks( + vecs: &mut Vecs, + indexer: &Indexer, + indexes: &indexes::Vecs, + chain: &chain::Vecs, + price: Option<&price::Vecs>, + starting_height: Height, + last_height: Height, + chain_state: &mut Vec, + exit: &Exit, +) -> Result<()> { + if starting_height > last_height { + return Ok(()); + } + + info!( + "Processing blocks {} to {}...", + starting_height, last_height + ); + + // Pre-compute iterators for fast access + let mut height_to_first_txindex = indexes.height_to_first_txindex.boxed_iter(); + let mut height_to_tx_count = chain.height_to_tx_count.boxed_iter(); + let mut height_to_first_txoutindex = indexes.height_to_first_txoutindex.boxed_iter(); + let mut height_to_output_count = chain.height_to_output_count.boxed_iter(); + let mut height_to_first_txinindex = indexes.height_to_first_txinindex.boxed_iter(); + let mut height_to_input_count = chain.height_to_input_count.boxed_iter(); + let mut height_to_timestamp = chain.height_to_timestamp.boxed_iter(); + let mut height_to_unclaimed_rewards = chain.height_to_unclaimed_reward.boxed_iter(); + let mut height_to_date = indexes.height_to_date.boxed_iter(); + let mut dateindex_to_first_height = indexes.dateindex_to_first_height.boxed_iter(); + let mut dateindex_to_height_count = indexes.dateindex_to_height_count.boxed_iter(); + let mut txindex_to_output_count = chain.txindex_to_output_count.boxed_iter(); + let mut txindex_to_input_count = chain.txindex_to_input_count.boxed_iter(); + + let mut height_to_price = price.map(|p| p.height_to_close.boxed_iter()); + let mut dateindex_to_price = price.map(|p| p.dateindex_to_close.boxed_iter()); + + // Build txoutindex -> height map for input processing + let txoutindex_to_height = build_txoutindex_to_height_map(&indexes.height_to_first_txoutindex); + + // Create readers for parallel data access + let ir = IndexerReaders::new(indexer); + + // Track running totals + let mut unspendable_supply = Sats::ZERO; + let mut opreturn_supply = Sats::ZERO; + let mut addresstype_to_addr_count = ByAddressType::::default(); + let mut addresstype_to_empty_addr_count = ByAddressType::::default(); + + // Recover initial values if resuming + if starting_height > Height::ZERO { + let prev_height = starting_height.decremented().unwrap(); + unspendable_supply = vecs + .height_to_unspendable_supply + .get(prev_height) + .unwrap_or_default(); + opreturn_supply = vecs + .height_to_opreturn_supply + .get(prev_height) + .unwrap_or_default(); + } + + // Main block iteration + for height in starting_height.to_usize()..=last_height.to_usize() { + let height = Height::from(height); + + if height.to_usize() % 10000 == 0 { + info!("Processing chain at {}...", height); + } + + // Get block metadata + let first_txindex = height_to_first_txindex.get_unwrap(height); + let tx_count = u64::from(height_to_tx_count.get_unwrap(height)); + let first_txoutindex = height_to_first_txoutindex.get_unwrap(height).to_usize(); + let output_count = u64::from(height_to_output_count.get_unwrap(height)) as usize; + let first_txinindex = height_to_first_txinindex.get_unwrap(height).to_usize(); + let input_count = u64::from(height_to_input_count.get_unwrap(height)) as usize; + let timestamp = height_to_timestamp.get_unwrap(height); + let block_price = height_to_price.as_mut().map(|v| v.get_unwrap(height)); + + // Build txindex mappings for this block + let txoutindex_to_txindex = + build_txoutindex_to_txindex(first_txindex, tx_count, &mut txindex_to_output_count); + let txinindex_to_txindex = + build_txinindex_to_txindex(first_txindex, tx_count, &mut txindex_to_input_count); + + // Reset per-block values for all separate cohorts + reset_block_values(&mut vecs.utxo_cohorts, &mut vecs.address_cohorts); + + // Process outputs and inputs in parallel with tick-tock + let (outputs_result, inputs_result) = thread::scope(|scope| { + // Tick-tock age transitions in background + scope.spawn(|| { + vecs.utxo_cohorts + .tick_tock_next_block(chain_state, timestamp); + }); + + // Process outputs (receive) + let outputs_result = process_outputs( + first_txoutindex, + output_count, + &txoutindex_to_txindex, + &indexer.vecs.txoutindex_to_value, + &indexer.vecs.txoutindex_to_outputtype, + &indexer.vecs.txoutindex_to_typeindex, + &ir, + ); + + // Process inputs (send) - skip coinbase input + let inputs_result = if input_count > 1 { + process_inputs( + first_txinindex + 1, // Skip coinbase + input_count - 1, + &txinindex_to_txindex[1..], // Skip coinbase + &indexer.vecs.txinindex_to_outpoint, + &indexer.vecs.txindex_to_first_txoutindex, + &indexer.vecs.txoutindex_to_value, + &indexer.vecs.txoutindex_to_outputtype, + &indexer.vecs.txoutindex_to_typeindex, + &txoutindex_to_height, + &ir, + ) + } else { + super::InputsResult { + height_to_sent: Default::default(), + sent_data: Default::default(), + } + }; + + (outputs_result, inputs_result) + }); + + let mut transacted = outputs_result.transacted; + let mut height_to_sent = inputs_result.height_to_sent; + + // Update supply tracking + unspendable_supply += transacted.by_type.unspendable.opreturn.value + + height_to_unclaimed_rewards.get_unwrap(height); + opreturn_supply += transacted.by_type.unspendable.opreturn.value; + + // Handle special cases + if height == Height::ZERO { + // Genesis block - reset transacted, add 50 BTC to unspendable + transacted = Transacted::default(); + unspendable_supply += Sats::FIFTY_BTC; + } else if height == Height::new(BIP30_DUPLICATE_HEIGHT_1) + || height == Height::new(BIP30_DUPLICATE_HEIGHT_2) + { + // BIP30: Add 50 BTC to spent from original height + let original_height = if height == Height::new(BIP30_DUPLICATE_HEIGHT_1) { + Height::new(BIP30_ORIGINAL_HEIGHT_1) + } else { + Height::new(BIP30_ORIGINAL_HEIGHT_2) + }; + height_to_sent + .entry(original_height) + .or_default() + .iterate(Sats::FIFTY_BTC, OutputType::P2PK65); + } + + // Push current block state before processing cohort updates + chain_state.push(BlockState { + supply: transacted.spendable_supply.clone(), + price: block_price, + timestamp, + }); + + // Update UTXO cohorts + vecs.utxo_cohorts.receive(transacted, height, block_price); + vecs.utxo_cohorts.send(height_to_sent, chain_state); + + // Push to height-indexed vectors + vecs.height_to_unspendable_supply + .truncate_push(height, unspendable_supply)?; + vecs.height_to_opreturn_supply + .truncate_push(height, opreturn_supply)?; + vecs.addresstype_to_height_to_addr_count + .truncate_push(height, &addresstype_to_addr_count)?; + vecs.addresstype_to_height_to_empty_addr_count + .truncate_push(height, &addresstype_to_empty_addr_count)?; + + // Get date info for unrealized state computation + let date = height_to_date.get_unwrap(height); + let dateindex = DateIndex::try_from(date).unwrap(); + let date_first_height = dateindex_to_first_height.get_unwrap(dateindex); + let date_height_count = dateindex_to_height_count.get_unwrap(dateindex); + let is_date_last_height = + date_first_height + Height::from(date_height_count).decremented().unwrap() == height; + let date_price = dateindex_to_price + .as_mut() + .map(|v| is_date_last_height.then(|| v.get_unwrap(dateindex))); + let dateindex_opt = is_date_last_height.then_some(dateindex); + + // Push cohort states and compute unrealized + push_cohort_states( + &mut vecs.utxo_cohorts, + &mut vecs.address_cohorts, + height, + block_price, + dateindex_opt, + date_price, + )?; + + // Periodic checkpoint flush + if height != last_height + && height != Height::ZERO + && height.to_usize() % FLUSH_INTERVAL == 0 + { + let _lock = exit.lock(); + flush_checkpoint(vecs, height, exit)?; + } + } + + // Final flush + let _lock = exit.lock(); + flush_checkpoint(vecs, last_height, exit)?; + + Ok(()) +} + +/// Reset per-block values for all separate cohorts. +fn reset_block_values(utxo_cohorts: &mut UTXOCohorts, address_cohorts: &mut AddressCohorts) { + utxo_cohorts.par_iter_separate_mut().for_each(|v| { + if let Some(state) = v.state.as_mut() { + state.reset_single_iteration_values(); + } + }); + + address_cohorts.par_iter_separate_mut().for_each(|v| { + if let Some(state) = v.state.as_mut() { + state.inner.reset_single_iteration_values(); + } + }); +} + +/// Push cohort states to height-indexed vectors. +fn push_cohort_states( + utxo_cohorts: &mut UTXOCohorts, + address_cohorts: &mut AddressCohorts, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, +) -> Result<()> { + utxo_cohorts + .par_iter_separate_mut() + .map(|v| v as &mut dyn DynCohortVecs) + .chain( + address_cohorts + .par_iter_separate_mut() + .map(|v| v as &mut dyn DynCohortVecs), + ) + .try_for_each(|v| { + v.truncate_push(height)?; + v.compute_then_truncate_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + ) + })?; + + Ok(()) +} + +/// Flush checkpoint to disk. +fn flush_checkpoint(vecs: &mut Vecs, height: Height, exit: &Exit) -> Result<()> { + info!("Flushing checkpoint at height {}...", height); + + // Flush cohort states + vecs.utxo_cohorts.safe_flush_stateful_vecs(height, exit)?; + vecs.address_cohorts.safe_flush_stateful_vecs(height, exit)?; + + // Flush height-indexed vectors + vecs.height_to_unspendable_supply.safe_write(exit)?; + vecs.height_to_opreturn_supply.safe_write(exit)?; + vecs.addresstype_to_height_to_addr_count.safe_flush(exit)?; + vecs.addresstype_to_height_to_empty_addr_count + .safe_flush(exit)?; + + // Flush chain state with stamp + vecs.chain_state.safe_write_with_stamp(height.into(), exit)?; + + Ok(()) +} diff --git a/crates/brk_computer/src/stateful_new/compute/context.rs b/crates/brk_computer/src/stateful_new/compute/context.rs new file mode 100644 index 000000000..433e5797f --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/context.rs @@ -0,0 +1,39 @@ +//! Computation context holding shared state during block processing. + +use brk_types::{Dollars, Height, Timestamp}; +use vecdb::VecIndex; + +use crate::price; + +/// Context shared across block processing. +pub struct ComputeContext<'a> { + /// Starting height for this computation run + pub starting_height: Height, + + /// Last height to process + pub last_height: Height, + + /// Whether price data is available + pub compute_dollars: bool, + + /// Price data (optional) + pub price: Option<&'a price::Vecs>, + + /// Pre-computed height -> timestamp mapping + pub height_to_timestamp: Vec, + + /// Pre-computed height -> price mapping (if available) + pub height_to_price: Option>, +} + +impl<'a> ComputeContext<'a> { + /// Get price at height (None if no price data or height out of range). + pub fn price_at(&self, height: Height) -> Option { + self.height_to_price.as_ref()?.get(height.to_usize()).copied() + } + + /// Get timestamp at height. + pub fn timestamp_at(&self, height: Height) -> Timestamp { + self.height_to_timestamp[height.to_usize()] + } +} diff --git a/crates/brk_computer/src/stateful_new/compute/flush.rs b/crates/brk_computer/src/stateful_new/compute/flush.rs new file mode 100644 index 000000000..0815bb2fa --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/flush.rs @@ -0,0 +1,87 @@ +//! State flushing logic for checkpoints. +//! +//! Handles periodic flushing of all stateful data to disk, +//! including cohort states, address data, and chain state. + +use brk_error::Result; +use brk_types::{ + AddressDataSource, AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, Height, + LoadedAddressData, LoadedAddressIndex, +}; +use log::info; +use vecdb::{Exit, Stamp}; + +use crate::stateful_new::process::{process_empty_addresses, process_loaded_addresses}; + +use super::super::address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs}; +use super::super::cohorts::DynCohortVecs; + +/// Flush all cohort stateful vectors. +pub fn flush_cohort_states( + height: Height, + utxo_vecs: &mut [&mut dyn DynCohortVecs], + address_vecs: &mut [&mut dyn DynCohortVecs], + exit: &Exit, +) -> Result<()> { + for v in utxo_vecs.iter_mut() { + v.safe_flush_stateful_vecs(height, exit)?; + } + for v in address_vecs.iter_mut() { + v.safe_flush_stateful_vecs(height, exit)?; + } + Ok(()) +} + +/// Apply address index updates to the index storage. +fn apply_address_index_updates( + address_indexes: &mut AnyAddressIndexesVecs, + updates: AddressTypeToTypeIndexMap, +) -> Result<()> { + for (address_type, sorted) in updates.into_sorted_iter() { + for (typeindex, any_index) in sorted { + address_indexes.update_or_push(address_type, typeindex, any_index)?; + } + } + Ok(()) +} + +/// Full state flush at a checkpoint. +/// +/// This is the main entry point for checkpoint flushing: +/// 1. Flush cohort stateful vectors +/// 2. Process address data updates (empty and loaded) +/// 3. Update address indexes +/// 4. Stamped flush address indexes and data +/// 5. Flush chain state +#[allow(clippy::too_many_arguments)] +pub fn flush_checkpoint( + height: Height, + utxo_vecs: &mut [&mut dyn DynCohortVecs], + address_vecs: &mut [&mut dyn DynCohortVecs], + address_indexes: &mut AnyAddressIndexesVecs, + addresses_data: &mut AddressesDataVecs, + empty_updates: AddressTypeToTypeIndexMap>, + loaded_updates: AddressTypeToTypeIndexMap>, + with_changes: bool, + exit: &Exit, +) -> Result<()> { + info!("Flushing at height {}...", height); + + // 1. Flush cohort states + flush_cohort_states(height, utxo_vecs, address_vecs, exit)?; + + // 2. Process address updates - empty first, then loaded + let empty_result = process_empty_addresses(addresses_data, empty_updates)?; + let loaded_result = process_loaded_addresses(addresses_data, loaded_updates)?; + let all_updates = empty_result.merge(loaded_result); + + // 3. Apply index updates + apply_address_index_updates(address_indexes, all_updates)?; + + // 4. Stamped flush + let stamp = Stamp::from(height); + address_indexes.flush(stamp, with_changes)?; + addresses_data.flush(stamp, with_changes)?; + + Ok(()) +} diff --git a/crates/brk_computer/src/stateful_new/compute/mod.rs b/crates/brk_computer/src/stateful_new/compute/mod.rs new file mode 100644 index 000000000..f99046d2e --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/mod.rs @@ -0,0 +1,36 @@ +//! Block processing pipeline. +//! +//! This module handles the main computation loop that processes blocks: +//! 1. Recover state from checkpoint or start fresh +//! 2. Process each block's outputs and inputs +//! 3. Update cohort states +//! 4. Periodically flush to disk +//! 5. Compute aggregate cohorts from separate cohorts + +mod aggregates; +// mod block_loop; +mod context; +mod flush; +mod readers; +mod recover; + +pub use aggregates::{compute_overlapping, compute_rest_part1, compute_rest_part2}; +// pub use block_loop::process_blocks; +pub use context::ComputeContext; +pub use flush::{flush_checkpoint, flush_cohort_states}; +pub use readers::{ + IndexerReaders, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex, +}; +pub use recover::{ + RecoveredState, StartMode, determine_start_mode, find_min_height, + import_aggregate_price_to_amount, import_cohort_states, reset_all_state, rollback_states, +}; + +/// Flush checkpoint interval (every N blocks). +pub const FLUSH_INTERVAL: usize = 10_000; + +// BIP30 duplicate coinbase heights (special case handling) +pub const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842; +pub const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880; +pub const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812; +pub const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722; diff --git a/crates/brk_computer/src/stateful_new/compute/readers.rs b/crates/brk_computer/src/stateful_new/compute/readers.rs new file mode 100644 index 000000000..0746dcd0f --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/readers.rs @@ -0,0 +1,119 @@ +//! Cached readers for efficient data access during computation. +//! +//! Readers provide mmap-based access to indexed data without repeated syscalls. + +use brk_grouper::{ByAddressType, ByAnyAddress}; +use brk_indexer::Indexer; +use brk_types::{OutputType, StoredU64, TxIndex}; +use vecdb::{BoxedVecIterator, GenericStoredVec, Reader, VecIndex}; + +use crate::stateful_new::address::{AddressesDataVecs, AnyAddressIndexesVecs}; + +/// Cached readers for indexer vectors. +pub struct IndexerReaders { + pub txinindex_to_outpoint: Reader, + pub txindex_to_first_txoutindex: Reader, + pub txoutindex_to_value: Reader, + pub txoutindex_to_outputtype: Reader, + pub txoutindex_to_typeindex: Reader, +} + +impl IndexerReaders { + pub fn new(indexer: &Indexer) -> Self { + Self { + txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(), + txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(), + txoutindex_to_value: indexer.vecs.txoutindex_to_value.create_reader(), + txoutindex_to_outputtype: indexer.vecs.txoutindex_to_outputtype.create_reader(), + txoutindex_to_typeindex: indexer.vecs.txoutindex_to_typeindex.create_reader(), + } + } +} + +/// Cached readers for stateful vectors. +pub struct VecsReaders { + pub addresstypeindex_to_anyaddressindex: ByAddressType, + pub anyaddressindex_to_anyaddressdata: ByAnyAddress, +} + +impl VecsReaders { + pub fn new( + any_address_indexes: &AnyAddressIndexesVecs, + addresses_data: &AddressesDataVecs, + ) -> Self { + Self { + addresstypeindex_to_anyaddressindex: ByAddressType { + p2a: any_address_indexes.p2a.create_reader(), + p2pk33: any_address_indexes.p2pk33.create_reader(), + p2pk65: any_address_indexes.p2pk65.create_reader(), + p2pkh: any_address_indexes.p2pkh.create_reader(), + p2sh: any_address_indexes.p2sh.create_reader(), + p2tr: any_address_indexes.p2tr.create_reader(), + p2wpkh: any_address_indexes.p2wpkh.create_reader(), + p2wsh: any_address_indexes.p2wsh.create_reader(), + }, + anyaddressindex_to_anyaddressdata: ByAnyAddress { + loaded: addresses_data.loaded.create_reader(), + empty: addresses_data.empty.create_reader(), + }, + } + } + + /// Get reader for specific address type. + pub fn address_reader(&self, address_type: OutputType) -> &Reader { + self.addresstypeindex_to_anyaddressindex + .get_unwrap(address_type) + } +} + +/// Build txoutindex -> txindex mapping for a block. +pub fn build_txoutindex_to_txindex<'a>( + block_first_txindex: TxIndex, + block_tx_count: u64, + txindex_to_output_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>, +) -> Vec { + let first = block_first_txindex.to_usize(); + + let counts: Vec = (0..block_tx_count as usize) + .map(|offset| { + let txindex = TxIndex::from(first + offset); + u64::from(txindex_to_output_count.get_unwrap(txindex)) + }) + .collect(); + + let total: u64 = counts.iter().sum(); + let mut result = Vec::with_capacity(total as usize); + + for (offset, &count) in counts.iter().enumerate() { + let txindex = TxIndex::from(first + offset); + result.extend(std::iter::repeat(txindex).take(count as usize)); + } + + result +} + +/// Build txinindex -> txindex mapping for a block. +pub fn build_txinindex_to_txindex<'a>( + block_first_txindex: TxIndex, + block_tx_count: u64, + txindex_to_input_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>, +) -> Vec { + let first = block_first_txindex.to_usize(); + + let counts: Vec = (0..block_tx_count as usize) + .map(|offset| { + let txindex = TxIndex::from(first + offset); + u64::from(txindex_to_input_count.get_unwrap(txindex)) + }) + .collect(); + + let total: u64 = counts.iter().sum(); + let mut result = Vec::with_capacity(total as usize); + + for (offset, &count) in counts.iter().enumerate() { + let txindex = TxIndex::from(first + offset); + result.extend(std::iter::repeat_n(txindex, count as usize)); + } + + result +} diff --git a/crates/brk_computer/src/stateful_new/compute/recover.rs b/crates/brk_computer/src/stateful_new/compute/recover.rs new file mode 100644 index 000000000..e204c3ece --- /dev/null +++ b/crates/brk_computer/src/stateful_new/compute/recover.rs @@ -0,0 +1,163 @@ +//! State recovery logic for checkpoint/resume. +//! +//! Determines starting height and imports saved state from checkpoints. + +use std::cmp::Ordering; +use std::collections::BTreeSet; + +use brk_error::Result; +use brk_types::Height; +use vecdb::{AnyVec, Stamp}; + +use super::super::address::AnyAddressIndexesVecs; +use super::super::cohorts::{DynCohortVecs, UTXOCohorts}; + +/// Result of state recovery. +pub struct RecoveredState { + /// Height to start processing from. + pub starting_height: Height, + /// Whether state was successfully restored (vs starting fresh). + pub restored: bool, +} + +/// Determine starting height from vector lengths. +pub fn find_min_height( + utxo_vecs: &[&mut dyn DynCohortVecs], + address_vecs: &[&mut dyn DynCohortVecs], + chain_state_len: usize, + address_indexes_min_height: Height, + address_data_min_height: Height, + other_vec_lens: &[usize], +) -> Height { + let utxo_min = utxo_vecs + .iter() + .map(|v| Height::from(v.min_height_vecs_len())) + .min() + .unwrap_or_default(); + + let address_min = address_vecs + .iter() + .map(|v| Height::from(v.min_height_vecs_len())) + .min() + .unwrap_or_default(); + + let other_min = other_vec_lens + .iter() + .map(|&len| Height::from(len)) + .min() + .unwrap_or_default(); + + utxo_min + .min(address_min) + .min(Height::from(chain_state_len)) + .min(address_indexes_min_height) + .min(address_data_min_height) + .min(other_min) +} + +/// Check if we can resume from a checkpoint or need to start fresh. +pub fn determine_start_mode(computed_min: Height, chain_state_height: Height) -> StartMode { + match computed_min.cmp(&chain_state_height) { + Ordering::Greater => unreachable!("min height > chain state height"), + Ordering::Equal => StartMode::Resume(chain_state_height), + Ordering::Less => StartMode::Fresh, + } +} + +/// Whether to resume from checkpoint or start fresh. +pub enum StartMode { + /// Resume from the given height. + Resume(Height), + /// Start from height 0. + Fresh, +} + +/// Rollback state vectors to before a given stamp. +/// +/// Returns the consistent starting height if all vectors agree, +/// otherwise returns Height::ZERO (need fresh start). +pub fn rollback_states( + stamp: Stamp, + chain_state_rollback: Result, + address_indexes_rollbacks: Vec>, + address_data_rollbacks: Vec>, +) -> Height { + let mut heights: BTreeSet = [chain_state_rollback] + .into_iter() + .chain(address_indexes_rollbacks) + .chain(address_data_rollbacks) + .filter_map(|r| r.ok()) + .map(Height::from) + .map(Height::incremented) + .collect(); + + if heights.len() == 1 { + heights.pop_first().unwrap() + } else { + Height::ZERO + } +} + +/// Import state for all separate cohorts. +/// +/// Returns the starting height if all imports succeed with the same height, +/// otherwise returns Height::ZERO. +pub fn import_cohort_states( + starting_height: Height, + cohorts: &mut [&mut dyn DynCohortVecs], +) -> Height { + if starting_height.is_zero() { + return Height::ZERO; + } + + let all_match = cohorts + .iter_mut() + .map(|v| v.import_state(starting_height).unwrap_or_default()) + .all(|h| h == starting_height); + + if all_match { + starting_height + } else { + Height::ZERO + } +} + +/// Import aggregate price_to_amount for UTXO cohorts. +pub fn import_aggregate_price_to_amount( + starting_height: Height, + utxo_cohorts: &mut UTXOCohorts, +) -> Result { + if starting_height.is_zero() { + return Ok(Height::ZERO); + } + + let imported = utxo_cohorts.import_aggregate_price_to_amount(starting_height)?; + + Ok(if imported == starting_height { + starting_height + } else { + Height::ZERO + }) +} + +/// Reset all state for fresh start. +pub fn reset_all_state( + address_indexes: &mut AnyAddressIndexesVecs, + utxo_vecs: &mut [&mut dyn DynCohortVecs], + address_vecs: &mut [&mut dyn DynCohortVecs], + utxo_cohorts: &mut UTXOCohorts, +) -> Result<()> { + address_indexes.reset()?; + + for v in utxo_vecs.iter_mut() { + v.reset_state_starting_height(); + } + + for v in address_vecs.iter_mut() { + v.reset_state_starting_height(); + } + + utxo_cohorts.reset_aggregate_price_to_amount()?; + + Ok(()) +} diff --git a/crates/brk_computer/src/stateful_new/metrics/activity.rs b/crates/brk_computer/src/stateful_new/metrics/activity.rs new file mode 100644 index 000000000..47f25ba30 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/activity.rs @@ -0,0 +1,201 @@ +//! Transaction activity metrics. +//! +//! These metrics track amounts sent and destruction of satoshi-days/blocks. + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, Height, Sats, StoredF64, Version}; +use vecdb::{AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec}; + +use crate::{ + Indexes, + grouped::{ComputedValueVecsFromHeight, ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, price, +}; + +use super::ImportConfig; + +/// Activity metrics for a cohort. +#[derive(Clone, Traversable)] +pub struct ActivityMetrics { + /// Total satoshis sent at each height + pub height_to_sent: EagerVec>, + + /// Sent amounts indexed by various dimensions + pub indexes_to_sent: ComputedValueVecsFromHeight, + + /// Satoshi-blocks destroyed (supply * blocks_old when spent) + pub height_to_satblocks_destroyed: EagerVec>, + + /// Satoshi-days destroyed (supply * days_old when spent) + pub height_to_satdays_destroyed: EagerVec>, + + /// Coin-blocks destroyed (in BTC rather than sats) + pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight, + + /// Coin-days destroyed (in BTC rather than sats) + pub indexes_to_coindays_destroyed: ComputedVecsFromHeight, +} + +impl ActivityMetrics { + /// Import activity metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let v0 = Version::ZERO; + let compute_dollars = cfg.compute_dollars(); + let sum = VecBuilderOptions::default().add_sum(); + + Ok(Self { + height_to_sent: EagerVec::forced_import(cfg.db, &cfg.name("sent"), cfg.version + v0)?, + + indexes_to_sent: ComputedValueVecsFromHeight::forced_import( + cfg.db, + &cfg.name("sent"), + Source::None, + cfg.version + v0, + sum, + compute_dollars, + cfg.indexes, + )?, + + height_to_satblocks_destroyed: EagerVec::forced_import( + cfg.db, + &cfg.name("satblocks_destroyed"), + cfg.version + v0, + )?, + + height_to_satdays_destroyed: EagerVec::forced_import( + cfg.db, + &cfg.name("satdays_destroyed"), + cfg.version + v0, + )?, + + indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("coinblocks_destroyed"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + sum.clone(), + )?, + + indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("coindays_destroyed"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + sum, + )?, + }) + } + + /// Get minimum length across height-indexed vectors. + pub fn min_len(&self) -> usize { + self.height_to_sent + .len() + .min(self.height_to_satblocks_destroyed.len()) + .min(self.height_to_satdays_destroyed.len()) + } + + /// Push activity state values to height-indexed vectors. + pub fn truncate_push( + &mut self, + height: Height, + sent: Sats, + satblocks_destroyed: Sats, + satdays_destroyed: Sats, + ) -> Result<()> { + self.height_to_sent.truncate_push(height, sent)?; + self.height_to_satblocks_destroyed + .truncate_push(height, satblocks_destroyed)?; + self.height_to_satdays_destroyed + .truncate_push(height, satdays_destroyed)?; + Ok(()) + } + + /// Flush height-indexed vectors to disk. + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + self.height_to_sent.safe_write(exit)?; + self.height_to_satblocks_destroyed.safe_write(exit)?; + self.height_to_satdays_destroyed.safe_write(exit)?; + Ok(()) + } + + /// Validate computed versions against base version. + pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> { + // Validation logic for computed vecs + Ok(()) + } + + /// Compute aggregate values from separate cohorts. + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_sent.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_sent).collect::>(), + exit, + )?; + self.height_to_satblocks_destroyed.compute_sum_of_others( + starting_indexes.height, + &others + .iter() + .map(|v| &v.height_to_satblocks_destroyed) + .collect::>(), + exit, + )?; + self.height_to_satdays_destroyed.compute_sum_of_others( + starting_indexes.height, + &others + .iter() + .map(|v| &v.height_to_satdays_destroyed) + .collect::>(), + exit, + )?; + Ok(()) + } + + /// First phase of computed metrics (indexes from height). + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.indexes_to_sent.compute_rest( + indexes, + price, + starting_indexes, + exit, + Some(&self.height_to_sent), + )?; + + self.indexes_to_coinblocks_destroyed + .compute_all(indexes, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.height, + &self.height_to_satblocks_destroyed, + |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_coindays_destroyed + .compute_all(indexes, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.height, + &self.height_to_satdays_destroyed, + |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), + exit, + )?; + Ok(()) + })?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/config.rs b/crates/brk_computer/src/stateful_new/metrics/config.rs new file mode 100644 index 000000000..8cfdacf0b --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/config.rs @@ -0,0 +1,49 @@ +//! Configuration for metric imports. + +use brk_grouper::{CohortContext, Filter}; +use brk_types::Version; +use vecdb::Database; + +use crate::{indexes, price}; + +/// Configuration for importing metrics. +pub struct ImportConfig<'a> { + pub db: &'a Database, + pub filter: Filter, + pub context: CohortContext, + pub version: Version, + pub indexes: &'a indexes::Vecs, + pub price: Option<&'a price::Vecs>, +} + +impl<'a> ImportConfig<'a> { + /// Whether price data is available (enables realized/unrealized metrics). + pub fn compute_dollars(&self) -> bool { + self.price.is_some() + } + + /// Whether this is an extended cohort (more relative metrics). + pub fn extended(&self) -> bool { + self.filter.is_extended(self.context) + } + + /// Whether to compute relative-to-all metrics. + pub fn compute_rel_to_all(&self) -> bool { + self.filter.compute_rel_to_all() + } + + /// Whether to compute adjusted metrics (SOPR, etc.). + pub fn compute_adjusted(&self) -> bool { + self.filter.compute_adjusted(self.context) + } + + /// Get full metric name with filter prefix. + pub fn name(&self, suffix: &str) -> String { + let prefix = self.filter.to_full_name(self.context); + if prefix.is_empty() { + suffix.to_string() + } else { + format!("{prefix}_{suffix}") + } + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/mod.rs b/crates/brk_computer/src/stateful_new/metrics/mod.rs new file mode 100644 index 000000000..6c6a978b0 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/mod.rs @@ -0,0 +1,293 @@ +//! Metric vectors organized by category. +//! +//! Instead of a single 80+ field struct, metrics are grouped into logical categories: +//! - `supply`: Supply and UTXO count metrics (always computed) +//! - `activity`: Transaction activity metrics (always computed) +//! - `realized`: Realized cap, profit/loss, SOPR (requires price) +//! - `unrealized`: Unrealized profit/loss (requires price) +//! - `price`: Price paid metrics and percentiles (requires price) +//! - `relative`: Ratios relative to market cap, etc. (requires price) + +mod activity; +mod config; +mod price_paid; +mod realized; +mod relative; +mod supply; +mod unrealized; + +pub use activity::ActivityMetrics; +pub use config::ImportConfig; +pub use price_paid::PricePaidMetrics; +pub use realized::RealizedMetrics; +pub use relative::RelativeMetrics; +pub use supply::SupplyMetrics; +pub use unrealized::UnrealizedMetrics; + +use brk_error::Result; +use brk_grouper::Filter; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; +use vecdb::{Exit, IterableVec}; + +use crate::{Indexes, indexes, price, states::CohortState}; + +/// All metrics for a cohort, organized by category. +#[derive(Clone, Traversable)] +pub struct CohortMetrics { + #[traversable(skip)] + pub filter: Filter, + + /// Supply and UTXO count (always computed) + pub supply: SupplyMetrics, + + /// Transaction activity (always computed) + pub activity: ActivityMetrics, + + /// Realized cap and profit/loss (requires price data) + pub realized: Option, + + /// Unrealized profit/loss (requires price data) + pub unrealized: Option, + + /// Price paid metrics (requires price data) + pub price_paid: Option, + + /// Relative metrics (requires price data) + pub relative: Option, +} + +impl CohortMetrics { + /// Import all metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let compute_dollars = cfg.compute_dollars(); + + Ok(Self { + filter: cfg.filter.clone(), + supply: SupplyMetrics::forced_import(cfg)?, + activity: ActivityMetrics::forced_import(cfg)?, + realized: compute_dollars + .then(|| RealizedMetrics::forced_import(cfg)) + .transpose()?, + unrealized: compute_dollars + .then(|| UnrealizedMetrics::forced_import(cfg)) + .transpose()?, + price_paid: compute_dollars + .then(|| PricePaidMetrics::forced_import(cfg)) + .transpose()?, + relative: compute_dollars + .then(|| RelativeMetrics::forced_import(cfg)) + .transpose()?, + }) + } + + /// Get minimum length across height-indexed vectors. + pub fn min_len(&self) -> usize { + self.supply.min_len().min(self.activity.min_len()) + } + + /// Push state values to height-indexed vectors. + pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> { + self.supply.truncate_push(height, &state.supply)?; + self.activity.truncate_push( + height, + state.sent, + state.satblocks_destroyed, + state.satdays_destroyed, + )?; + + if let (Some(realized), Some(realized_state)) = + (self.realized.as_mut(), state.realized.as_ref()) + { + realized.truncate_push(height, realized_state)?; + } + + Ok(()) + } + + /// Flush height-indexed vectors to disk. + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + self.supply.safe_flush(exit)?; + self.activity.safe_flush(exit)?; + + if let Some(realized) = self.realized.as_mut() { + realized.safe_flush(exit)?; + } + + if let Some(unrealized) = self.unrealized.as_mut() { + unrealized.safe_flush(exit)?; + } + + if let Some(price_paid) = self.price_paid.as_mut() { + price_paid.safe_flush(exit)?; + } + + Ok(()) + } + + /// Validate computed versions against base version. + pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.supply.validate_computed_versions(base_version)?; + self.activity.validate_computed_versions(base_version)?; + + if let Some(realized) = self.realized.as_mut() { + realized.validate_computed_versions(base_version)?; + } + + Ok(()) + } + + /// Compute and push unrealized states. + pub fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + state: &CohortState, + ) -> Result<()> { + if let (Some(unrealized), Some(price_paid), Some(height_price)) = ( + self.unrealized.as_mut(), + self.price_paid.as_mut(), + height_price, + ) { + // Push price paid min/max + price_paid.truncate_push_minmax(height, state)?; + + // Compute unrealized states from price_to_amount + let (height_unrealized_state, date_unrealized_state) = + state.compute_unrealized_states(height_price, date_price.unwrap()); + + unrealized.truncate_push( + height, + dateindex, + &height_unrealized_state, + date_unrealized_state.as_ref(), + )?; + + // Compute and push price percentiles + price_paid.truncate_push_percentiles(height, state)?; + } + + Ok(()) + } + + /// Compute aggregate cohort values from separate cohorts. + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.supply.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.supply).collect::>(), + exit, + )?; + self.activity.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.activity).collect::>(), + exit, + )?; + + if let Some(realized) = self.realized.as_mut() { + realized.compute_from_stateful( + starting_indexes, + &others + .iter() + .filter_map(|v| v.realized.as_ref()) + .collect::>(), + exit, + )?; + } + + if let Some(unrealized) = self.unrealized.as_mut() { + unrealized.compute_from_stateful( + starting_indexes, + &others + .iter() + .filter_map(|v| v.unrealized.as_ref()) + .collect::>(), + exit, + )?; + } + + if let Some(price_paid) = self.price_paid.as_mut() { + price_paid.compute_from_stateful( + starting_indexes, + &others + .iter() + .filter_map(|v| v.price_paid.as_ref()) + .collect::>(), + exit, + )?; + } + + Ok(()) + } + + /// First phase of computed metrics (indexes from height). + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.supply + .compute_rest_part1(indexes, price, starting_indexes, exit)?; + self.activity + .compute_rest_part1(indexes, price, starting_indexes, exit)?; + + if let Some(realized) = self.realized.as_mut() { + realized.compute_rest_part1(indexes, price, starting_indexes, exit)?; + } + + Ok(()) + } + + /// Second phase of computed metrics (ratios, relative values). + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.supply.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + exit, + )?; + + if let Some(relative) = self.relative.as_mut() { + relative.compute_rest_part2( + indexes, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + &self.supply, + self.realized.as_ref(), + exit, + )?; + } + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/price_paid.rs b/crates/brk_computer/src/stateful_new/metrics/price_paid.rs new file mode 100644 index 000000000..cc3006670 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/price_paid.rs @@ -0,0 +1,144 @@ +//! Price paid metrics and percentiles. +//! +//! Tracks min/max price paid for UTXOs and price distribution percentiles. + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Dollars, Height, Version}; +use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec}; + +use crate::{ + Indexes, + grouped::{ComputedVecsFromHeight, PricePercentiles, Source, VecBuilderOptions}, + states::{CohortState, Flushable}, +}; + +use super::ImportConfig; + +/// Price paid metrics. +#[derive(Clone, Traversable)] +pub struct PricePaidMetrics { + /// Minimum price paid for any UTXO at this height + pub height_to_min_price_paid: EagerVec>, + pub indexes_to_min_price_paid: ComputedVecsFromHeight, + + /// Maximum price paid for any UTXO at this height + pub height_to_max_price_paid: EagerVec>, + pub indexes_to_max_price_paid: ComputedVecsFromHeight, + + /// Price distribution percentiles (median, quartiles, etc.) + pub price_percentiles: Option, +} + +impl PricePaidMetrics { + /// Import price paid metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let v0 = Version::ZERO; + let extended = cfg.extended(); + let last = VecBuilderOptions::default().add_last(); + + Ok(Self { + height_to_min_price_paid: EagerVec::forced_import( + cfg.db, + &cfg.name("min_price_paid"), + cfg.version + v0, + )?, + indexes_to_min_price_paid: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("min_price_paid"), + Source::None, + cfg.version + v0, + cfg.indexes, + last, + )?, + height_to_max_price_paid: EagerVec::forced_import( + cfg.db, + &cfg.name("max_price_paid"), + cfg.version + v0, + )?, + indexes_to_max_price_paid: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("max_price_paid"), + Source::None, + cfg.version + v0, + cfg.indexes, + last, + )?, + price_percentiles: extended + .then(|| { + PricePercentiles::forced_import( + cfg.db, + &cfg.name(""), + cfg.version + v0, + cfg.indexes, + true, + ) + }) + .transpose()?, + }) + } + + /// Push min/max price paid from state. + pub fn truncate_push_minmax(&mut self, height: Height, state: &CohortState) -> Result<()> { + self.height_to_min_price_paid.truncate_push( + height, + state + .price_to_amount_first_key_value() + .map(|(&dollars, _)| dollars) + .unwrap_or(Dollars::NAN), + )?; + self.height_to_max_price_paid.truncate_push( + height, + state + .price_to_amount_last_key_value() + .map(|(&dollars, _)| dollars) + .unwrap_or(Dollars::NAN), + )?; + Ok(()) + } + + /// Push price percentiles from state. + pub fn truncate_push_percentiles(&mut self, height: Height, state: &CohortState) -> Result<()> { + if let Some(price_percentiles) = self.price_percentiles.as_mut() { + let percentile_prices = state.compute_percentile_prices(); + price_percentiles.truncate_push(height, &percentile_prices)?; + } + Ok(()) + } + + /// Flush height-indexed vectors to disk. + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + self.height_to_min_price_paid.safe_write(exit)?; + self.height_to_max_price_paid.safe_write(exit)?; + if let Some(price_percentiles) = self.price_percentiles.as_mut() { + price_percentiles.safe_flush(exit)?; + } + Ok(()) + } + + /// Compute aggregate values from separate cohorts. + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_min_price_paid.compute_min_of_others( + starting_indexes.height, + &others + .iter() + .map(|v| &v.height_to_min_price_paid) + .collect::>(), + exit, + )?; + self.height_to_max_price_paid.compute_max_of_others( + starting_indexes.height, + &others + .iter() + .map(|v| &v.height_to_max_price_paid) + .collect::>(), + exit, + )?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/realized.rs b/crates/brk_computer/src/stateful_new/metrics/realized.rs new file mode 100644 index 000000000..491356ec5 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/realized.rs @@ -0,0 +1,556 @@ +//! Realized cap and profit/loss metrics. +//! +//! These metrics require price data and track realized value based on acquisition price. + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{DateIndex, Dollars, Height, StoredF32, StoredF64, Version}; +use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec}; + +use crate::{ + Indexes, + grouped::{ + ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, + VecBuilderOptions, + }, + indexes, price, + states::RealizedState, + utils::OptionExt, +}; + +use super::ImportConfig; + +/// Realized cap and related metrics. +#[derive(Clone, Traversable)] +pub struct RealizedMetrics { + // === Realized Cap === + pub height_to_realized_cap: EagerVec>, + pub indexes_to_realized_cap: ComputedVecsFromHeight, + pub indexes_to_realized_price: ComputedVecsFromHeight, + pub indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex, + pub indexes_to_realized_cap_rel_to_own_market_cap: Option>, + pub indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex, + + // === Realized Profit/Loss === + pub height_to_realized_profit: EagerVec>, + pub indexes_to_realized_profit: ComputedVecsFromHeight, + pub height_to_realized_loss: EagerVec>, + pub indexes_to_realized_loss: ComputedVecsFromHeight, + pub indexes_to_neg_realized_loss: ComputedVecsFromHeight, + pub indexes_to_net_realized_pnl: ComputedVecsFromHeight, + pub indexes_to_realized_value: ComputedVecsFromHeight, + + // === Realized vs Realized Cap Ratios === + pub indexes_to_realized_profit_rel_to_realized_cap: ComputedVecsFromHeight, + pub indexes_to_realized_loss_rel_to_realized_cap: ComputedVecsFromHeight, + pub indexes_to_net_realized_pnl_rel_to_realized_cap: ComputedVecsFromHeight, + + // === Total Realized PnL === + pub height_to_total_realized_pnl: EagerVec>, + pub indexes_to_total_realized_pnl: ComputedVecsFromDateIndex, + pub dateindex_to_realized_profit_to_loss_ratio: Option>>, + + // === Value Created/Destroyed === + pub height_to_value_created: EagerVec>, + pub indexes_to_value_created: ComputedVecsFromHeight, + pub height_to_value_destroyed: EagerVec>, + pub indexes_to_value_destroyed: ComputedVecsFromHeight, + + // === Adjusted Value (optional) === + pub height_to_adjusted_value_created: Option>>, + pub indexes_to_adjusted_value_created: Option>, + pub height_to_adjusted_value_destroyed: Option>>, + pub indexes_to_adjusted_value_destroyed: Option>, + + // === SOPR (Spent Output Profit Ratio) === + pub dateindex_to_sopr: EagerVec>, + pub dateindex_to_sopr_7d_ema: EagerVec>, + pub dateindex_to_sopr_30d_ema: EagerVec>, + pub dateindex_to_adjusted_sopr: Option>>, + pub dateindex_to_adjusted_sopr_7d_ema: Option>>, + pub dateindex_to_adjusted_sopr_30d_ema: Option>>, + + // === Sell Side Risk === + pub dateindex_to_sell_side_risk_ratio: EagerVec>, + pub dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec>, + pub dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec>, + + // === Net Realized PnL Deltas === + pub indexes_to_net_realized_pnl_cumulative_30d_delta: ComputedVecsFromDateIndex, + pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap: + ComputedVecsFromDateIndex, + pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: + ComputedVecsFromDateIndex, +} + +impl RealizedMetrics { + /// Import realized metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let v0 = Version::ZERO; + let v1 = Version::ONE; + let v3 = Version::new(3); + let extended = cfg.extended(); + let compute_adjusted = cfg.compute_adjusted(); + let last = VecBuilderOptions::default().add_last(); + let sum = VecBuilderOptions::default().add_sum(); + let sum_cum = VecBuilderOptions::default().add_sum().add_cumulative(); + + Ok(Self { + // === Realized Cap === + height_to_realized_cap: EagerVec::forced_import( + cfg.db, + &cfg.name("realized_cap"), + cfg.version + v0, + )?, + indexes_to_realized_cap: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_cap"), + Source::None, + cfg.version + v0, + cfg.indexes, + last, + )?, + indexes_to_realized_price: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_price"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + )?, + indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("realized_price"), + Source::None, + cfg.version + v0, + cfg.indexes, + extended, + )?, + indexes_to_realized_cap_rel_to_own_market_cap: extended + .then(|| { + ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_cap_rel_to_own_market_cap"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("realized_cap_30d_delta"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + )?, + + // === Realized Profit/Loss === + height_to_realized_profit: EagerVec::forced_import( + cfg.db, + &cfg.name("realized_profit"), + cfg.version + v0, + )?, + indexes_to_realized_profit: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_profit"), + Source::None, + cfg.version + v0, + cfg.indexes, + sum_cum, + )?, + height_to_realized_loss: EagerVec::forced_import( + cfg.db, + &cfg.name("realized_loss"), + cfg.version + v0, + )?, + indexes_to_realized_loss: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_loss"), + Source::None, + cfg.version + v0, + cfg.indexes, + sum_cum, + )?, + indexes_to_neg_realized_loss: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("neg_realized_loss"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + sum_cum, + )?, + indexes_to_net_realized_pnl: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("net_realized_pnl"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + sum_cum, + )?, + indexes_to_realized_value: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_value"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + sum, + )?, + + // === Realized vs Realized Cap Ratios === + indexes_to_realized_profit_rel_to_realized_cap: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_profit_rel_to_realized_cap"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + sum, + )?, + indexes_to_realized_loss_rel_to_realized_cap: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_loss_rel_to_realized_cap"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + sum, + )?, + indexes_to_net_realized_pnl_rel_to_realized_cap: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("net_realized_pnl_rel_to_realized_cap"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + sum, + )?, + + // === Total Realized PnL === + height_to_total_realized_pnl: EagerVec::forced_import( + cfg.db, + &cfg.name("total_realized_pnl"), + cfg.version + v0, + )?, + indexes_to_total_realized_pnl: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("total_realized_pnl"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + sum, + )?, + dateindex_to_realized_profit_to_loss_ratio: extended + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("realized_profit_to_loss_ratio"), + cfg.version + v1, + ) + }) + .transpose()?, + + // === Value Created/Destroyed === + height_to_value_created: EagerVec::forced_import( + cfg.db, + &cfg.name("value_created"), + cfg.version + v0, + )?, + indexes_to_value_created: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("value_created"), + Source::None, + cfg.version + v0, + cfg.indexes, + sum, + )?, + height_to_value_destroyed: EagerVec::forced_import( + cfg.db, + &cfg.name("value_destroyed"), + cfg.version + v0, + )?, + indexes_to_value_destroyed: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("value_destroyed"), + Source::None, + cfg.version + v0, + cfg.indexes, + sum, + )?, + + // === Adjusted Value (optional) === + height_to_adjusted_value_created: compute_adjusted + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("adjusted_value_created"), + cfg.version + v0, + ) + }) + .transpose()?, + indexes_to_adjusted_value_created: compute_adjusted + .then(|| { + ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("adjusted_value_created"), + Source::None, + cfg.version + v0, + cfg.indexes, + sum, + ) + }) + .transpose()?, + height_to_adjusted_value_destroyed: compute_adjusted + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("adjusted_value_destroyed"), + cfg.version + v0, + ) + }) + .transpose()?, + indexes_to_adjusted_value_destroyed: compute_adjusted + .then(|| { + ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("adjusted_value_destroyed"), + Source::None, + cfg.version + v0, + cfg.indexes, + sum, + ) + }) + .transpose()?, + + // === SOPR === + dateindex_to_sopr: EagerVec::forced_import( + cfg.db, + &cfg.name("sopr"), + cfg.version + v1, + )?, + dateindex_to_sopr_7d_ema: EagerVec::forced_import( + cfg.db, + &cfg.name("sopr_7d_ema"), + cfg.version + v1, + )?, + dateindex_to_sopr_30d_ema: EagerVec::forced_import( + cfg.db, + &cfg.name("sopr_30d_ema"), + cfg.version + v1, + )?, + dateindex_to_adjusted_sopr: compute_adjusted + .then(|| { + EagerVec::forced_import(cfg.db, &cfg.name("adjusted_sopr"), cfg.version + v1) + }) + .transpose()?, + dateindex_to_adjusted_sopr_7d_ema: compute_adjusted + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("adjusted_sopr_7d_ema"), + cfg.version + v1, + ) + }) + .transpose()?, + dateindex_to_adjusted_sopr_30d_ema: compute_adjusted + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("adjusted_sopr_30d_ema"), + cfg.version + v1, + ) + }) + .transpose()?, + + // === Sell Side Risk === + dateindex_to_sell_side_risk_ratio: EagerVec::forced_import( + cfg.db, + &cfg.name("sell_side_risk_ratio"), + cfg.version + v1, + )?, + dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec::forced_import( + cfg.db, + &cfg.name("sell_side_risk_ratio_7d_ema"), + cfg.version + v1, + )?, + dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec::forced_import( + cfg.db, + &cfg.name("sell_side_risk_ratio_30d_ema"), + cfg.version + v1, + )?, + + // === Net Realized PnL Deltas === + indexes_to_net_realized_pnl_cumulative_30d_delta: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_realized_pnl_cumulative_30d_delta"), + Source::Compute, + cfg.version + v3, + cfg.indexes, + last, + )?, + indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap"), + Source::Compute, + cfg.version + v3, + cfg.indexes, + last, + )?, + indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_market_cap"), + Source::Compute, + cfg.version + v3, + cfg.indexes, + last, + )?, + }) + } + + /// Push realized state values to height-indexed vectors. + pub fn truncate_push(&mut self, height: Height, state: &RealizedState) -> Result<()> { + self.height_to_realized_cap.truncate_push(height, state.cap)?; + self.height_to_realized_profit.truncate_push(height, state.profit)?; + self.height_to_realized_loss.truncate_push(height, state.loss)?; + self.height_to_value_created.truncate_push(height, state.value_created)?; + self.height_to_value_destroyed.truncate_push(height, state.value_destroyed)?; + + if let Some(v) = self.height_to_adjusted_value_created.as_mut() { + v.truncate_push(height, state.adj_value_created)?; + } + if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() { + v.truncate_push(height, state.adj_value_destroyed)?; + } + + Ok(()) + } + + /// Flush height-indexed vectors to disk. + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + self.height_to_realized_cap.safe_write(exit)?; + self.height_to_realized_profit.safe_write(exit)?; + self.height_to_realized_loss.safe_write(exit)?; + self.height_to_value_created.safe_write(exit)?; + self.height_to_value_destroyed.safe_write(exit)?; + self.height_to_adjusted_value_created.um().safe_write(exit)?; + self.height_to_adjusted_value_destroyed.um().safe_write(exit)?; + Ok(()) + } + + /// Validate computed versions against base version. + pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> { + // Validation logic for computed vecs + Ok(()) + } + + /// Compute aggregate values from separate cohorts. + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_realized_cap.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_realized_cap).collect::>(), + exit, + )?; + self.height_to_realized_profit.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_realized_profit).collect::>(), + exit, + )?; + self.height_to_realized_loss.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_realized_loss).collect::>(), + exit, + )?; + self.height_to_value_created.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_value_created).collect::>(), + exit, + )?; + self.height_to_value_destroyed.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_value_destroyed).collect::>(), + exit, + )?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created.um().compute_sum_of_others( + starting_indexes.height, + &others + .iter() + .map(|v| { + v.height_to_adjusted_value_created + .as_ref() + .unwrap_or(&v.height_to_value_created) + }) + .collect::>(), + exit, + )?; + self.height_to_adjusted_value_destroyed.um().compute_sum_of_others( + starting_indexes.height, + &others + .iter() + .map(|v| { + v.height_to_adjusted_value_destroyed + .as_ref() + .unwrap_or(&v.height_to_value_destroyed) + }) + .collect::>(), + exit, + )?; + } + + Ok(()) + } + + /// First phase of computed metrics (indexes from height). + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + _price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.indexes_to_realized_cap.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_realized_cap), + )?; + + self.indexes_to_realized_profit.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_realized_profit), + )?; + + self.indexes_to_realized_loss.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_realized_loss), + )?; + + self.indexes_to_value_created.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_value_created), + )?; + + self.indexes_to_value_destroyed.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_value_destroyed), + )?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/relative.rs b/crates/brk_computer/src/stateful_new/metrics/relative.rs new file mode 100644 index 000000000..359074453 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/relative.rs @@ -0,0 +1,450 @@ +//! Relative metrics (ratios to market cap, realized cap, supply, etc.) +//! +//! These are computed ratios comparing cohort metrics to global metrics. + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version}; +use vecdb::{EagerVec, Exit, ImportableVec, IterableVec, PcoVec}; + +use crate::{ + Indexes, + grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, +}; + +use super::{ImportConfig, RealizedMetrics, SupplyMetrics}; + +/// Relative metrics comparing cohort values to global values. +#[derive(Clone, Traversable)] +pub struct RelativeMetrics { + // === Supply Relative to Circulating Supply === + pub indexes_to_supply_rel_to_circulating_supply: Option>, + + // === Supply in Profit/Loss Relative to Own Supply === + pub height_to_supply_in_profit_rel_to_own_supply: EagerVec>, + pub height_to_supply_in_loss_rel_to_own_supply: EagerVec>, + pub indexes_to_supply_in_profit_rel_to_own_supply: ComputedVecsFromDateIndex, + pub indexes_to_supply_in_loss_rel_to_own_supply: ComputedVecsFromDateIndex, + + // === Supply in Profit/Loss Relative to Circulating Supply === + pub height_to_supply_in_profit_rel_to_circulating_supply: + Option>>, + pub height_to_supply_in_loss_rel_to_circulating_supply: + Option>>, + pub indexes_to_supply_in_profit_rel_to_circulating_supply: + Option>, + pub indexes_to_supply_in_loss_rel_to_circulating_supply: + Option>, + + // === Unrealized vs Market Cap === + pub height_to_unrealized_profit_rel_to_market_cap: EagerVec>, + pub height_to_unrealized_loss_rel_to_market_cap: EagerVec>, + pub height_to_neg_unrealized_loss_rel_to_market_cap: EagerVec>, + pub height_to_net_unrealized_pnl_rel_to_market_cap: EagerVec>, + pub indexes_to_unrealized_profit_rel_to_market_cap: ComputedVecsFromDateIndex, + pub indexes_to_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex, + pub indexes_to_neg_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex, + pub indexes_to_net_unrealized_pnl_rel_to_market_cap: ComputedVecsFromDateIndex, + + // === Unrealized vs Own Market Cap (optional) === + pub height_to_unrealized_profit_rel_to_own_market_cap: + Option>>, + pub height_to_unrealized_loss_rel_to_own_market_cap: + Option>>, + pub height_to_neg_unrealized_loss_rel_to_own_market_cap: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_own_market_cap: + Option>>, + pub indexes_to_unrealized_profit_rel_to_own_market_cap: + Option>, + pub indexes_to_unrealized_loss_rel_to_own_market_cap: + Option>, + pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap: + Option>, + pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap: + Option>, + + // === Unrealized vs Own Total Unrealized PnL (optional) === + pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: + Option>>, + pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>>, + pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: + Option>>, + pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: + Option>, + pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>, + pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>, + pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: + Option>, +} + +impl RelativeMetrics { + /// Import relative metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let v0 = Version::ZERO; + let v1 = Version::ONE; + let v2 = Version::new(2); + let extended = cfg.extended(); + let compute_rel_to_all = cfg.compute_rel_to_all(); + let last = VecBuilderOptions::default().add_last(); + + Ok(Self { + // === Supply Relative to Circulating Supply === + indexes_to_supply_rel_to_circulating_supply: compute_rel_to_all + .then(|| { + ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("supply_rel_to_circulating_supply"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + + // === Supply in Profit/Loss Relative to Own Supply === + height_to_supply_in_profit_rel_to_own_supply: EagerVec::forced_import( + cfg.db, + &cfg.name("supply_in_profit_rel_to_own_supply"), + cfg.version + v1, + )?, + height_to_supply_in_loss_rel_to_own_supply: EagerVec::forced_import( + cfg.db, + &cfg.name("supply_in_loss_rel_to_own_supply"), + cfg.version + v1, + )?, + indexes_to_supply_in_profit_rel_to_own_supply: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_in_profit_rel_to_own_supply"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + )?, + indexes_to_supply_in_loss_rel_to_own_supply: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_in_loss_rel_to_own_supply"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + )?, + + // === Supply in Profit/Loss Relative to Circulating Supply === + height_to_supply_in_profit_rel_to_circulating_supply: compute_rel_to_all + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("supply_in_profit_rel_to_circulating_supply"), + cfg.version + v1, + ) + }) + .transpose()?, + height_to_supply_in_loss_rel_to_circulating_supply: compute_rel_to_all + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("supply_in_loss_rel_to_circulating_supply"), + cfg.version + v1, + ) + }) + .transpose()?, + indexes_to_supply_in_profit_rel_to_circulating_supply: compute_rel_to_all + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_in_profit_rel_to_circulating_supply"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_supply_in_loss_rel_to_circulating_supply: compute_rel_to_all + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_in_loss_rel_to_circulating_supply"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + + // === Unrealized vs Market Cap === + height_to_unrealized_profit_rel_to_market_cap: EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_profit_rel_to_market_cap"), + cfg.version + v0, + )?, + height_to_unrealized_loss_rel_to_market_cap: EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_loss_rel_to_market_cap"), + cfg.version + v0, + )?, + height_to_neg_unrealized_loss_rel_to_market_cap: EagerVec::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss_rel_to_market_cap"), + cfg.version + v0, + )?, + height_to_net_unrealized_pnl_rel_to_market_cap: EagerVec::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl_rel_to_market_cap"), + cfg.version + v1, + )?, + indexes_to_unrealized_profit_rel_to_market_cap: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_profit_rel_to_market_cap"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + )?, + indexes_to_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_loss_rel_to_market_cap"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + )?, + indexes_to_neg_unrealized_loss_rel_to_market_cap: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss_rel_to_market_cap"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + )?, + indexes_to_net_unrealized_pnl_rel_to_market_cap: + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl_rel_to_market_cap"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + )?, + + // === Unrealized vs Own Market Cap (optional) === + height_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_profit_rel_to_own_market_cap"), + cfg.version + v1, + ) + }) + .transpose()?, + height_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_loss_rel_to_own_market_cap"), + cfg.version + v1, + ) + }) + .transpose()?, + height_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss_rel_to_own_market_cap"), + cfg.version + v1, + ) + }) + .transpose()?, + height_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl_rel_to_own_market_cap"), + cfg.version + v2, + ) + }) + .transpose()?, + indexes_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_profit_rel_to_own_market_cap"), + Source::Compute, + cfg.version + v2, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_loss_rel_to_own_market_cap"), + Source::Compute, + cfg.version + v2, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss_rel_to_own_market_cap"), + Source::Compute, + cfg.version + v2, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all) + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl_rel_to_own_market_cap"), + Source::Compute, + cfg.version + v2, + cfg.indexes, + last, + ) + }) + .transpose()?, + + // === Unrealized vs Own Total Unrealized PnL (optional) === + height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"), + cfg.version + v0, + ) + }) + .transpose()?, + height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"), + cfg.version + v0, + ) + }) + .transpose()?, + height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"), + cfg.version + v0, + ) + }) + .transpose()?, + height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"), + cfg.version + v1, + ) + }) + .transpose()?, + indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended + .then(|| { + ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"), + Source::Compute, + cfg.version + v1, + cfg.indexes, + last, + ) + }) + .transpose()?, + }) + } + + /// Second phase of computed metrics (ratios, relative values). + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + _height_to_realized_cap: Option<&impl IterableVec>, + _dateindex_to_realized_cap: Option<&impl IterableVec>, + supply: &SupplyMetrics, + _realized: Option<&RealizedMetrics>, + exit: &Exit, + ) -> Result<()> { + // Supply relative to circulating supply + if let Some(v) = self.indexes_to_supply_rel_to_circulating_supply.as_mut() { + v.compute_all(indexes, starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.height, + &supply.height_to_supply_value.bitcoin, + height_to_supply, + exit, + )?; + Ok(()) + })?; + } + + let _ = (dateindex_to_supply, height_to_market_cap, dateindex_to_market_cap); + + // Additional relative metrics computed here + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/supply.rs b/crates/brk_computer/src/stateful_new/metrics/supply.rs new file mode 100644 index 000000000..05197aefe --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/supply.rs @@ -0,0 +1,252 @@ +//! Supply and UTXO count metrics. +//! +//! These metrics are always computed regardless of price data availability. + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, StoredF64, StoredU64, Version}; +use vecdb::{ + AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, PcoVec, + TypedVecIterator, +}; + +use crate::{ + Indexes, + grouped::{ + ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromHeight, Source, + VecBuilderOptions, + }, + indexes, price, + states::SupplyState, +}; + +use super::ImportConfig; + +/// Supply and UTXO count metrics for a cohort. +#[derive(Clone, Traversable)] +pub struct SupplyMetrics { + /// Total supply at each height + pub height_to_supply: EagerVec>, + + /// Supply value in BTC and USD (computed from height_to_supply) + pub height_to_supply_value: ComputedHeightValueVecs, + + /// Supply indexed by date + pub indexes_to_supply: ComputedValueVecsFromDateIndex, + + /// UTXO count at each height + pub height_to_utxo_count: EagerVec>, + + /// UTXO count indexed by various dimensions + pub indexes_to_utxo_count: ComputedVecsFromHeight, + + /// Half of supply value (used for computing median) + pub height_to_supply_half_value: ComputedHeightValueVecs, + + /// Half of supply indexed by date + pub indexes_to_supply_half: ComputedValueVecsFromDateIndex, +} + +impl SupplyMetrics { + /// Import supply metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let v0 = Version::ZERO; + let v1 = Version::ONE; + let compute_dollars = cfg.compute_dollars(); + let last = VecBuilderOptions::default().add_last(); + + Ok(Self { + height_to_supply: EagerVec::forced_import( + cfg.db, + &cfg.name("supply"), + cfg.version + v0, + )?, + + height_to_supply_value: ComputedHeightValueVecs::forced_import( + cfg.db, + &cfg.name("supply"), + Source::None, + cfg.version + v0, + compute_dollars, + )?, + + indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply"), + Source::Compute, + cfg.version + v1, + last, + compute_dollars, + cfg.indexes, + )?, + + height_to_utxo_count: EagerVec::forced_import( + cfg.db, + &cfg.name("utxo_count"), + cfg.version + v0, + )?, + + indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("utxo_count"), + Source::None, + cfg.version + v0, + cfg.indexes, + last, + )?, + + height_to_supply_half_value: ComputedHeightValueVecs::forced_import( + cfg.db, + &cfg.name("supply_half"), + Source::Compute, + cfg.version + v0, + compute_dollars, + )?, + + indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_half"), + Source::Compute, + cfg.version + v0, + last, + compute_dollars, + cfg.indexes, + )?, + }) + } + + /// Get minimum length across height-indexed vectors. + pub fn min_len(&self) -> usize { + self.height_to_supply + .len() + .min(self.height_to_utxo_count.len()) + } + + /// Push supply state values to height-indexed vectors. + pub fn truncate_push(&mut self, height: Height, state: &SupplyState) -> Result<()> { + self.height_to_supply.truncate_push(height, state.value)?; + self.height_to_utxo_count + .truncate_push(height, StoredU64::from(state.utxo_count))?; + Ok(()) + } + + /// Flush height-indexed vectors to disk. + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + self.height_to_supply.safe_write(exit)?; + self.height_to_utxo_count.safe_write(exit)?; + Ok(()) + } + + /// Validate computed versions against base version. + pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> { + // Validation logic for computed vecs + Ok(()) + } + + /// Compute aggregate values from separate cohorts. + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_supply.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_supply).collect::>(), + exit, + )?; + self.height_to_utxo_count.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_utxo_count).collect::>(), + exit, + )?; + Ok(()) + } + + /// First phase of computed metrics (indexes from height). + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.height_to_supply_value.compute_rest( + price, + starting_indexes, + exit, + Some(&self.height_to_supply), + )?; + + self.indexes_to_supply + .compute_all(price, starting_indexes, exit, |v| { + let mut dateindex_to_height_count_iter = + indexes.dateindex_to_height_count.into_iter(); + let mut height_to_supply_iter = self.height_to_supply.into_iter(); + v.compute_transform( + starting_indexes.dateindex, + &indexes.dateindex_to_first_height, + |(i, height, ..)| { + let count = dateindex_to_height_count_iter.get_unwrap(i); + if count == StoredU64::default() { + unreachable!() + } + let supply = height_to_supply_iter.get_unwrap(height + (*count - 1)); + (i, supply) + }, + exit, + )?; + Ok(()) + })?; + + self.indexes_to_utxo_count.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_utxo_count), + )?; + + self.height_to_supply_half_value + .compute_all(price, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.height, + &self.height_to_supply, + |(h, v, ..)| (h, v / 2), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_supply_half + .compute_all(price, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + self.indexes_to_supply.sats.dateindex.as_ref().unwrap(), + |(i, sats, ..)| (i, sats / 2), + exit, + )?; + Ok(()) + })?; + + Ok(()) + } + + /// Second phase of computed metrics (ratios, relative values). + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + _dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + let _ = (indexes, price, height_to_supply, height_to_market_cap, dateindex_to_market_cap); + + // Supply relative metrics computed here if needed + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/metrics/unrealized.rs b/crates/brk_computer/src/stateful_new/metrics/unrealized.rs new file mode 100644 index 000000000..c5f1a96ff --- /dev/null +++ b/crates/brk_computer/src/stateful_new/metrics/unrealized.rs @@ -0,0 +1,281 @@ +//! Unrealized profit/loss metrics. +//! +//! These metrics track paper gains/losses based on current vs acquisition price. + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{DateIndex, Dollars, Height, Sats, Version}; +use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec}; + +use crate::{ + Indexes, + grouped::{ + ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, Source, + VecBuilderOptions, + }, + states::UnrealizedState, +}; + +use super::ImportConfig; + +/// Unrealized profit/loss metrics. +#[derive(Clone, Traversable)] +pub struct UnrealizedMetrics { + // === Supply in Profit/Loss === + pub height_to_supply_in_profit: EagerVec>, + pub indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex, + pub height_to_supply_in_loss: EagerVec>, + pub indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex, + pub dateindex_to_supply_in_profit: EagerVec>, + pub dateindex_to_supply_in_loss: EagerVec>, + pub height_to_supply_in_profit_value: ComputedHeightValueVecs, + pub height_to_supply_in_loss_value: ComputedHeightValueVecs, + + // === Unrealized Profit/Loss === + pub height_to_unrealized_profit: EagerVec>, + pub indexes_to_unrealized_profit: ComputedVecsFromDateIndex, + pub height_to_unrealized_loss: EagerVec>, + pub indexes_to_unrealized_loss: ComputedVecsFromDateIndex, + pub dateindex_to_unrealized_profit: EagerVec>, + pub dateindex_to_unrealized_loss: EagerVec>, + + // === Negated and Net === + pub height_to_neg_unrealized_loss: EagerVec>, + pub indexes_to_neg_unrealized_loss: ComputedVecsFromDateIndex, + pub height_to_net_unrealized_pnl: EagerVec>, + pub indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex, + pub height_to_total_unrealized_pnl: EagerVec>, + pub indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex, +} + +impl UnrealizedMetrics { + /// Import unrealized metrics from database. + pub fn forced_import(cfg: &ImportConfig) -> Result { + let v0 = Version::ZERO; + let compute_dollars = cfg.compute_dollars(); + let last = VecBuilderOptions::default().add_last(); + + // Pre-import the dateindex vecs that are used as sources + let dateindex_to_supply_in_profit = + EagerVec::forced_import(cfg.db, &cfg.name("supply_in_profit"), cfg.version + v0)?; + let dateindex_to_supply_in_loss = + EagerVec::forced_import(cfg.db, &cfg.name("supply_in_loss"), cfg.version + v0)?; + let dateindex_to_unrealized_profit = + EagerVec::forced_import(cfg.db, &cfg.name("unrealized_profit"), cfg.version + v0)?; + let dateindex_to_unrealized_loss = + EagerVec::forced_import(cfg.db, &cfg.name("unrealized_loss"), cfg.version + v0)?; + + Ok(Self { + // === Supply in Profit/Loss === + height_to_supply_in_profit: EagerVec::forced_import( + cfg.db, + &cfg.name("supply_in_profit"), + cfg.version + v0, + )?, + indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_in_profit"), + Source::Vec(dateindex_to_supply_in_profit.boxed_clone()), + cfg.version + v0, + last, + compute_dollars, + cfg.indexes, + )?, + height_to_supply_in_loss: EagerVec::forced_import( + cfg.db, + &cfg.name("supply_in_loss"), + cfg.version + v0, + )?, + indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply_in_loss"), + Source::Vec(dateindex_to_supply_in_loss.boxed_clone()), + cfg.version + v0, + last, + compute_dollars, + cfg.indexes, + )?, + dateindex_to_supply_in_profit, + dateindex_to_supply_in_loss, + height_to_supply_in_profit_value: ComputedHeightValueVecs::forced_import( + cfg.db, + &cfg.name("supply_in_profit"), + Source::None, + cfg.version + v0, + compute_dollars, + )?, + height_to_supply_in_loss_value: ComputedHeightValueVecs::forced_import( + cfg.db, + &cfg.name("supply_in_loss"), + Source::None, + cfg.version + v0, + compute_dollars, + )?, + + // === Unrealized Profit/Loss === + height_to_unrealized_profit: EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_profit"), + cfg.version + v0, + )?, + indexes_to_unrealized_profit: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_profit"), + Source::Vec(dateindex_to_unrealized_profit.boxed_clone()), + cfg.version + v0, + cfg.indexes, + last, + )?, + height_to_unrealized_loss: EagerVec::forced_import( + cfg.db, + &cfg.name("unrealized_loss"), + cfg.version + v0, + )?, + indexes_to_unrealized_loss: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("unrealized_loss"), + Source::Vec(dateindex_to_unrealized_loss.boxed_clone()), + cfg.version + v0, + cfg.indexes, + last, + )?, + dateindex_to_unrealized_profit, + dateindex_to_unrealized_loss, + + // === Negated and Net === + height_to_neg_unrealized_loss: EagerVec::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss"), + cfg.version + v0, + )?, + indexes_to_neg_unrealized_loss: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("neg_unrealized_loss"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + )?, + height_to_net_unrealized_pnl: EagerVec::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl"), + cfg.version + v0, + )?, + indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("net_unrealized_pnl"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + )?, + height_to_total_unrealized_pnl: EagerVec::forced_import( + cfg.db, + &cfg.name("total_unrealized_pnl"), + cfg.version + v0, + )?, + indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("total_unrealized_pnl"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + )?, + }) + } + + /// Push unrealized state values to height-indexed vectors. + pub fn truncate_push( + &mut self, + height: Height, + dateindex: Option, + height_state: &UnrealizedState, + date_state: Option<&UnrealizedState>, + ) -> Result<()> { + self.height_to_supply_in_profit + .truncate_push(height, height_state.supply_in_profit)?; + self.height_to_supply_in_loss + .truncate_push(height, height_state.supply_in_loss)?; + self.height_to_unrealized_profit + .truncate_push(height, height_state.unrealized_profit)?; + self.height_to_unrealized_loss + .truncate_push(height, height_state.unrealized_loss)?; + + if let (Some(dateindex), Some(date_state)) = (dateindex, date_state) { + self.dateindex_to_supply_in_profit + .truncate_push(dateindex, date_state.supply_in_profit)?; + self.dateindex_to_supply_in_loss + .truncate_push(dateindex, date_state.supply_in_loss)?; + self.dateindex_to_unrealized_profit + .truncate_push(dateindex, date_state.unrealized_profit)?; + self.dateindex_to_unrealized_loss + .truncate_push(dateindex, date_state.unrealized_loss)?; + } + + Ok(()) + } + + /// Flush height-indexed vectors to disk. + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + self.height_to_supply_in_profit.safe_write(exit)?; + self.height_to_supply_in_loss.safe_write(exit)?; + self.height_to_unrealized_profit.safe_write(exit)?; + self.height_to_unrealized_loss.safe_write(exit)?; + self.dateindex_to_supply_in_profit.safe_write(exit)?; + self.dateindex_to_supply_in_loss.safe_write(exit)?; + self.dateindex_to_unrealized_profit.safe_write(exit)?; + self.dateindex_to_unrealized_loss.safe_write(exit)?; + Ok(()) + } + + /// Compute aggregate values from separate cohorts. + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_supply_in_profit.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_supply_in_profit).collect::>(), + exit, + )?; + self.height_to_supply_in_loss.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_supply_in_loss).collect::>(), + exit, + )?; + self.height_to_unrealized_profit.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_unrealized_profit).collect::>(), + exit, + )?; + self.height_to_unrealized_loss.compute_sum_of_others( + starting_indexes.height, + &others.iter().map(|v| &v.height_to_unrealized_loss).collect::>(), + exit, + )?; + self.dateindex_to_supply_in_profit.compute_sum_of_others( + starting_indexes.dateindex, + &others.iter().map(|v| &v.dateindex_to_supply_in_profit).collect::>(), + exit, + )?; + self.dateindex_to_supply_in_loss.compute_sum_of_others( + starting_indexes.dateindex, + &others.iter().map(|v| &v.dateindex_to_supply_in_loss).collect::>(), + exit, + )?; + self.dateindex_to_unrealized_profit.compute_sum_of_others( + starting_indexes.dateindex, + &others.iter().map(|v| &v.dateindex_to_unrealized_profit).collect::>(), + exit, + )?; + self.dateindex_to_unrealized_loss.compute_sum_of_others( + starting_indexes.dateindex, + &others.iter().map(|v| &v.dateindex_to_unrealized_loss).collect::>(), + exit, + )?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_new/mod.rs b/crates/brk_computer/src/stateful_new/mod.rs new file mode 100644 index 000000000..f162ba9a7 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/mod.rs @@ -0,0 +1,57 @@ +//! Stateful computation for Bitcoin UTXO and address cohort metrics. +//! +//! This module processes blockchain data to compute metrics for various cohorts +//! (groups of UTXOs or addresses filtered by age, amount, type, etc.). +//! +//! ## Module Structure +//! +//! ```text +//! stateful/ +//! ├── address/ # Address type handling (indexes, data storage) +//! ├── cohorts/ # Cohort traits and state management +//! ├── compute/ # Block processing pipeline +//! └── metrics/ # Metric vectors organized by category +//! ``` +//! +//! ## Data Flow +//! +//! 1. **Import**: Load from checkpoint or start fresh +//! 2. **Process blocks**: For each block, process outputs/inputs in parallel +//! 3. **Update cohorts**: Track supply, realized/unrealized P&L per cohort +//! 4. **Flush**: Periodically checkpoint state to disk +//! 5. **Compute aggregates**: Derive aggregate cohorts from separate cohorts + +pub mod address; +pub mod cohorts; +pub mod compute; +pub mod metrics; +mod process; +mod vecs; + +use process::*; + +pub use vecs::Vecs; + +// Address re-exports +pub use address::{ + AddressTypeToTypeIndexMap, AddressTypeToVec, AddressesDataVecs, AnyAddressIndexesVecs, + HeightToAddressTypeToVec, +}; + +// Cohort re-exports +pub use cohorts::{ + AddressCohortVecs, AddressCohorts, CohortState, CohortVecs, DynCohortVecs, Flushable, + HeightFlushable, UTXOCohortVecs, UTXOCohorts, +}; + +// Compute re-exports +pub use compute::{ + BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1, + BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexerReaders, VecsReaders, +}; + +// Metrics re-exports +pub use metrics::{ + ActivityMetrics, CohortMetrics, ImportConfig, PricePaidMetrics, RealizedMetrics, + RelativeMetrics, SupplyMetrics, UnrealizedMetrics, +}; diff --git a/crates/brk_computer/src/stateful_new/process/empty_addresses.rs b/crates/brk_computer/src/stateful_new/process/empty_addresses.rs new file mode 100644 index 000000000..ce7e77c5e --- /dev/null +++ b/crates/brk_computer/src/stateful_new/process/empty_addresses.rs @@ -0,0 +1,44 @@ +use brk_error::Result; +use brk_types::{AddressDataSource, AnyAddressIndex, EmptyAddressData}; + +use crate::stateful_new::{AddressTypeToTypeIndexMap, AddressesDataVecs}; + +/// Process empty address data updates. +/// +/// Handles three cases: +/// - New empty address: push to empty storage +/// - Updated empty address (was empty): update in place +/// - Transition loaded -> empty: delete from loaded, push to empty +pub fn process_empty_addresses( + addresses_data: &mut AddressesDataVecs, + empty_updates: AddressTypeToTypeIndexMap>, +) -> Result> { + let mut result = AddressTypeToTypeIndexMap::default(); + + for (address_type, sorted) in empty_updates.into_sorted_iter() { + for (typeindex, source) in sorted { + match source { + AddressDataSource::New(data) => { + let index = addresses_data.empty.fill_first_hole_or_push(data)?; + result + .get_mut(address_type) + .unwrap() + .insert(typeindex, AnyAddressIndex::from(index)); + } + AddressDataSource::FromEmpty((index, data)) => { + addresses_data.empty.update(index, data)?; + } + AddressDataSource::FromLoaded((loaded_index, data)) => { + addresses_data.loaded.delete(loaded_index); + let empty_index = addresses_data.empty.fill_first_hole_or_push(data)?; + result + .get_mut(address_type) + .unwrap() + .insert(typeindex, AnyAddressIndex::from(empty_index)); + } + } + } + } + + Ok(result) +} diff --git a/crates/brk_computer/src/stateful_new/process/inputs.rs b/crates/brk_computer/src/stateful_new/process/inputs.rs new file mode 100644 index 000000000..78cfaf87b --- /dev/null +++ b/crates/brk_computer/src/stateful_new/process/inputs.rs @@ -0,0 +1,128 @@ +//! Parallel input processing. +//! +//! Processes a block's inputs (spent UTXOs) in parallel, building: +//! - height_to_sent: map from creation height -> Transacted for sends +//! - Address data for address cohort tracking (optional) + +use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TxOutIndex, TypeIndex}; +use rayon::prelude::*; +use rustc_hash::FxHashMap; +use vecdb::{BytesVec, GenericStoredVec, PcoVec}; + +use crate::{ + stateful_new::{IndexerReaders, process::RangeMap}, + states::Transacted, +}; + +use super::super::address::HeightToAddressTypeToVec; + +/// Result of processing inputs for a block. +pub struct InputsResult { + /// Map from UTXO creation height -> aggregated sent supply. + pub height_to_sent: FxHashMap, + /// Per-height, per-address-type sent data: (typeindex, value) for each address. + pub sent_data: HeightToAddressTypeToVec<(TypeIndex, Sats)>, +} + +/// Process inputs (spent UTXOs) for a block in parallel. +/// +/// For each input: +/// 1. Read outpoint, resolve to txoutindex +/// 2. Get the creation height from txoutindex_to_height map +/// 3. Read value and type from the referenced output +/// 4. Accumulate into height_to_sent map +/// 5. Track address-specific data if input references an address type +#[allow(clippy::too_many_arguments)] +pub fn process_inputs( + first_txinindex: usize, + input_count: usize, + txinindex_to_txindex: &[TxIndex], + txinindex_to_outpoint: &PcoVec, + txindex_to_first_txoutindex: &BytesVec, + txoutindex_to_value: &BytesVec, + txoutindex_to_outputtype: &BytesVec, + txoutindex_to_typeindex: &BytesVec, + txoutindex_to_height: &RangeMap, + ir: &IndexerReaders, +) -> InputsResult { + let (height_to_sent, sent_data) = (first_txinindex..first_txinindex + input_count) + .into_par_iter() + .map(|i| { + let txinindex = TxInIndex::from(i); + let local_idx = i - first_txinindex; + let _txindex = txinindex_to_txindex[local_idx]; + + // Get outpoint and resolve to txoutindex + let outpoint = txinindex_to_outpoint.read_unwrap(txinindex, &ir.txinindex_to_outpoint); + let first_txoutindex = txindex_to_first_txoutindex + .read_unwrap(outpoint.txindex(), &ir.txindex_to_first_txoutindex); + let txoutindex = first_txoutindex + outpoint.vout(); + + // Get creation height + let prev_height = *txoutindex_to_height.get(txoutindex).unwrap(); + + // Get value and type from the output being spent + let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value); + let input_type = + txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype); + + // Non-address inputs don't need typeindex + if input_type.is_not_address() { + return (prev_height, value, input_type, None); + } + + let typeindex = + txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex); + + (prev_height, value, input_type, Some((typeindex, value))) + }) + .fold( + || { + ( + FxHashMap::::default(), + HeightToAddressTypeToVec::default(), + ) + }, + |(mut height_to_sent, mut sent_data), (prev_height, value, output_type, addr_data)| { + height_to_sent + .entry(prev_height) + .or_default() + .iterate(value, output_type); + + if let Some((typeindex, value)) = addr_data { + sent_data + .entry(prev_height) + .or_default() + .get_mut(output_type) + .unwrap() + .push((typeindex, value)); + } + + (height_to_sent, sent_data) + }, + ) + .reduce( + || { + ( + FxHashMap::::default(), + HeightToAddressTypeToVec::default(), + ) + }, + |(mut h1, mut s1), (h2, s2)| { + // Merge height_to_sent maps + for (k, v) in h2 { + *h1.entry(k).or_default() += v; + } + + // Merge sent_data maps + s1.merge_mut(s2); + + (h1, s1) + }, + ); + + InputsResult { + height_to_sent, + sent_data, + } +} diff --git a/crates/brk_computer/src/stateful_new/process/loaded_addresses.rs b/crates/brk_computer/src/stateful_new/process/loaded_addresses.rs new file mode 100644 index 000000000..3580f7bc5 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/process/loaded_addresses.rs @@ -0,0 +1,44 @@ +use brk_error::Result; +use brk_types::{AddressDataSource, AnyAddressIndex, LoadedAddressData}; + +use crate::stateful_new::{AddressTypeToTypeIndexMap, AddressesDataVecs}; + +/// Process loaded address data updates. +/// +/// Handles three cases: +/// - New loaded address: push to loaded storage +/// - Updated loaded address (was loaded): update in place +/// - Transition empty -> loaded: delete from empty, push to loaded +pub fn process_loaded_addresses( + addresses_data: &mut AddressesDataVecs, + loaded_updates: AddressTypeToTypeIndexMap>, +) -> Result> { + let mut result = AddressTypeToTypeIndexMap::default(); + + for (address_type, sorted) in loaded_updates.into_sorted_iter() { + for (typeindex, source) in sorted { + match source { + AddressDataSource::New(data) => { + let index = addresses_data.loaded.fill_first_hole_or_push(data)?; + result + .get_mut(address_type) + .unwrap() + .insert(typeindex, AnyAddressIndex::from(index)); + } + AddressDataSource::FromLoaded((index, data)) => { + addresses_data.loaded.update(index, data)?; + } + AddressDataSource::FromEmpty((empty_index, data)) => { + addresses_data.empty.delete(empty_index); + let loaded_index = addresses_data.loaded.fill_first_hole_or_push(data)?; + result + .get_mut(address_type) + .unwrap() + .insert(typeindex, AnyAddressIndex::from(loaded_index)); + } + } + } + } + + Ok(result) +} diff --git a/crates/brk_computer/src/stateful_new/process/mod.rs b/crates/brk_computer/src/stateful_new/process/mod.rs new file mode 100644 index 000000000..7ee37a162 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/process/mod.rs @@ -0,0 +1,11 @@ +mod empty_addresses; +mod inputs; +mod loaded_addresses; +mod outputs; +mod range_map; + +pub use empty_addresses::*; +pub use inputs::*; +pub use loaded_addresses::*; +pub use outputs::*; +pub use range_map::*; diff --git a/crates/brk_computer/src/stateful_new/process/outputs.rs b/crates/brk_computer/src/stateful_new/process/outputs.rs new file mode 100644 index 000000000..c471ac936 --- /dev/null +++ b/crates/brk_computer/src/stateful_new/process/outputs.rs @@ -0,0 +1,83 @@ +//! Parallel output processing. +//! +//! Processes a block's outputs (new UTXOs) in parallel, building: +//! - Transacted: aggregated supply by output type and amount range +//! - Address data for address cohort tracking (optional) + +use brk_types::{OutputType, Sats, TxIndex, TxOutIndex, TypeIndex}; +use rayon::prelude::*; +use vecdb::{BytesVec, GenericStoredVec}; + +use crate::{stateful_new::IndexerReaders, states::Transacted}; + +use super::super::address::AddressTypeToVec; + +/// Result of processing outputs for a block. +pub struct OutputsResult { + /// Aggregated supply transacted in this block. + pub transacted: Transacted, + /// Per-address-type received data: (typeindex, value) for each address. + pub received_data: AddressTypeToVec<(TypeIndex, Sats)>, +} + +/// Process outputs (new UTXOs) for a block in parallel. +/// +/// For each output: +/// 1. Read value and output type from indexer +/// 2. Accumulate into Transacted by type and amount +/// 3. Track address-specific data if output is an address type +pub fn process_outputs( + first_txoutindex: usize, + output_count: usize, + txoutindex_to_txindex: &[TxIndex], + txoutindex_to_value: &BytesVec, + txoutindex_to_outputtype: &BytesVec, + txoutindex_to_typeindex: &BytesVec, + ir: &IndexerReaders, +) -> OutputsResult { + let (transacted, received_data) = (first_txoutindex..first_txoutindex + output_count) + .into_par_iter() + .map(|i| { + let txoutindex = TxOutIndex::from(i); + let local_idx = i - first_txoutindex; + let _txindex = txoutindex_to_txindex[local_idx]; + + let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value); + let output_type = + txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype); + + // Non-address outputs don't need typeindex + if output_type.is_not_address() { + return (value, output_type, None); + } + + let typeindex = + txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex); + + (value, output_type, Some((typeindex, value))) + }) + .fold( + || (Transacted::default(), AddressTypeToVec::default()), + |(mut transacted, mut received_data), (value, output_type, addr_data)| { + transacted.iterate(value, output_type); + + if let Some((typeindex, value)) = addr_data { + received_data + .get_mut(output_type) + .unwrap() + .push((typeindex, value)); + } + + (transacted, received_data) + }, + ) + .reduce( + || (Transacted::default(), AddressTypeToVec::default()), + |(t1, r1), (t2, r2)| (t1 + t2, r1.merge(r2)), + ); + + OutputsResult { + transacted, + received_data, + } +} diff --git a/crates/brk_computer/src/stateful_new/process/range_map.rs b/crates/brk_computer/src/stateful_new/process/range_map.rs new file mode 100644 index 000000000..88a44ce7c --- /dev/null +++ b/crates/brk_computer/src/stateful_new/process/range_map.rs @@ -0,0 +1,65 @@ +//! Main block processing loop. +//! +//! Iterates through blocks, processing outputs (receive) and inputs (send) in parallel. + +use std::collections::BTreeMap; + +use brk_types::{Height, TxOutIndex}; +use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex}; + +/// Maps ranges of indices to their corresponding height. +/// Used to efficiently look up which block a txoutindex belongs to. +#[derive(Debug)] +pub struct RangeMap(BTreeMap); + +impl RangeMap +where + I: VecIndex, + T: VecIndex, +{ + /// Look up value for a key using range search. + /// Returns the value associated with the largest key <= given key. + #[inline] + pub fn get(&self, key: I) -> Option<&T> { + self.0.range(..=key).next_back().map(|(_, value)| value) + } +} + +impl From<&BytesVec> for RangeMap +where + I: VecIndex, + T: VecIndex + BytesVecValue, +{ + #[inline] + fn from(vec: &BytesVec) -> Self { + Self( + vec.into_iter() + .enumerate() + .map(|(i, v)| (v, I::from(i))) + .collect(), + ) + } +} + +impl From<&PcoVec> for RangeMap +where + I: VecIndex, + T: VecIndex + PcoVecValue, +{ + #[inline] + fn from(vec: &PcoVec) -> Self { + Self( + vec.into_iter() + .enumerate() + .map(|(i, v)| (v, I::from(i))) + .collect(), + ) + } +} + +/// Creates a RangeMap from height_to_first_txoutindex for fast txoutindex -> height lookups. +pub fn build_txoutindex_to_height_map( + height_to_first_txoutindex: &PcoVec, +) -> RangeMap { + RangeMap::from(height_to_first_txoutindex) +} diff --git a/crates/brk_computer/src/stateful_new/vecs.rs b/crates/brk_computer/src/stateful_new/vecs.rs new file mode 100644 index 000000000..16d42e67f --- /dev/null +++ b/crates/brk_computer/src/stateful_new/vecs.rs @@ -0,0 +1,231 @@ +//! Main Vecs struct for stateful computation. + +use std::path::Path; + +use brk_error::Result; +use brk_indexer::Indexer; +use brk_traversable::Traversable; +use brk_types::{Dollars, Height, Sats, StoredU64, Version}; +use vecdb::{ + BytesVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, LazyVecFrom1, + PAGE_SIZE, PcoVec, +}; + +use crate::{ + Indexes, SupplyState, chain, + grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, price, +}; + +use super::{ + AddressCohorts, AddressesDataVecs, AnyAddressIndexesVecs, UTXOCohorts, + address::{AddressTypeToHeightToAddressCount, AddressTypeToIndexesToAddressCount}, +}; + +const VERSION: Version = Version::new(21); + +/// Main struct holding all computed vectors and state for stateful computation. +#[derive(Clone, Traversable)] +pub struct Vecs { + #[traversable(skip)] + db: Database, + + // --- + // States + // --- + pub chain_state: BytesVec, + pub any_address_indexes: AnyAddressIndexesVecs, + pub addresses_data: AddressesDataVecs, + pub utxo_cohorts: UTXOCohorts, + pub address_cohorts: AddressCohorts, + + pub height_to_unspendable_supply: EagerVec>, + pub height_to_opreturn_supply: EagerVec>, + pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount, + pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount, + + // --- + // Computed + // --- + pub addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount, + pub addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount, + pub indexes_to_addr_count: ComputedVecsFromHeight, + pub indexes_to_empty_addr_count: ComputedVecsFromHeight, + pub height_to_market_cap: Option>, + pub indexes_to_market_cap: Option>, +} + +const SAVED_STAMPED_CHANGES: u16 = 10; + +impl Vecs { + pub fn forced_import( + parent: &Path, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + ) -> Result { + let db_path = parent.join("stateful"); + let states_path = db_path.join("states"); + + let db = Database::open(&db_path)?; + db.set_min_len(PAGE_SIZE * 20_000_000)?; + db.set_min_regions(50_000)?; + + let compute_dollars = price.is_some(); + let v0 = version + VERSION + Version::ZERO; + let v2 = version + VERSION + Version::TWO; + + let utxo_cohorts = UTXOCohorts::forced_import(&db, version, indexes, price, &states_path)?; + + Ok(Self { + chain_state: BytesVec::forced_import_with( + vecdb::ImportOptions::new(&db, "chain", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + + height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?, + height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?, + + indexes_to_addr_count: ComputedVecsFromHeight::forced_import( + &db, + "addr_count", + Source::Compute, + v0, + indexes, + VecBuilderOptions::default().add_last(), + )?, + indexes_to_empty_addr_count: ComputedVecsFromHeight::forced_import( + &db, + "empty_addr_count", + Source::Compute, + v0, + indexes, + VecBuilderOptions::default().add_last(), + )?, + + height_to_market_cap: compute_dollars.then(|| { + LazyVecFrom1::init( + "market_cap", + v0, + utxo_cohorts + .all + .metrics + .supply + .height_to_supply_value + .dollars + .as_ref() + .unwrap() + .boxed_clone(), + |height: Height, iter| iter.get(height), + ) + }), + + indexes_to_market_cap: compute_dollars.then(|| { + ComputedVecsFromDateIndex::forced_import( + &db, + "market_cap", + Source::Compute, + v2, + indexes, + VecBuilderOptions::default().add_last(), + ) + .unwrap() + }), + + addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::forced_import( + &db, + "addr_count", + v0, + )?, + addresstype_to_height_to_empty_addr_count: + AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?, + addresstype_to_indexes_to_addr_count: + AddressTypeToIndexesToAddressCount::forced_import(&db, "addr_count", v0, indexes)?, + addresstype_to_indexes_to_empty_addr_count: + AddressTypeToIndexesToAddressCount::forced_import( + &db, + "empty_addr_count", + v0, + indexes, + )?, + + utxo_cohorts, + + address_cohorts: AddressCohorts::forced_import( + &db, + version, + indexes, + price, + &states_path, + )?, + + any_address_indexes: AnyAddressIndexesVecs::forced_import(&db, v0)?, + addresses_data: AddressesDataVecs::forced_import(&db, v0)?, + + db, + }) + } + + /// Main computation loop. + /// + /// Processes blocks to compute UTXO and address cohort metrics: + /// 1. Recovers state from checkpoints or starts fresh + /// 2. Iterates through blocks, processing outputs/inputs in parallel + /// 3. Flushes checkpoints periodically + /// 4. Computes aggregate cohorts from separate cohorts + /// 5. Computes derived metrics + /// + /// NOTE: This is a placeholder. The full implementation needs to be ported + /// from stateful/mod.rs once all the supporting methods on UTXOCohorts, + /// AddressCohorts, and state types are implemented. + #[allow(clippy::too_many_arguments)] + pub fn compute( + &mut self, + _indexer: &Indexer, + _indexes: &indexes::Vecs, + _chain: &chain::Vecs, + _price: Option<&price::Vecs>, + _starting_indexes: &mut Indexes, + _exit: &Exit, + ) -> Result<()> { + // The full compute implementation requires these methods to be implemented: + // + // On UTXOCohorts: + // - tick_tock_next_block(&chain_state, timestamp) + // - receive(transacted, height, price) + // - send(height_to_sent, &mut chain_state) + // - truncate_push_aggregate_percentiles(height) + // - import_aggregate_price_to_amount(height) + // - reset_aggregate_price_to_amount() + // + // On UTXOCohortState: + // - reset_block_values() + // - reset_price_to_amount() + // + // On AddressCohortState: + // - inner.reset_block_values() + // - inner.reset_price_to_amount() + // + // On AddressTypeToHeightToAddressCount: + // - safe_flush(exit) + // - truncate_push(height, &count) + // + // See stateful/mod.rs:368-1397 for the full implementation. + // + // The basic structure is: + // 1. Validate computed versions against base version + // 2. Find min stateful height and recover state + // 3. For each block: + // a. Reset per-block values + // b. Process outputs in parallel (receive) + // c. Process inputs in parallel (send) + // d. Push to height-indexed vectors + // e. Flush checkpoint every 10,000 blocks + // 4. Compute aggregate cohorts from separate cohorts + // 5. Compute rest_part1 (dateindex mappings) + // 6. Compute rest_part2 (ratios and relative metrics) + + self.db.compact()?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/address_cohort.rs b/crates/brk_computer/src/stateful_old/address_cohort.rs new file mode 100644 index 000000000..5eb4349bd --- /dev/null +++ b/crates/brk_computer/src/stateful_old/address_cohort.rs @@ -0,0 +1,239 @@ +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{CohortContext, Filter, Filtered}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version}; +use vecdb::{ + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, + PcoVec, TypedVecIterator, +}; + +use crate::{ + Indexes, + grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, price, + stateful::{ + common, + r#trait::{CohortVecs, DynCohortVecs}, + }, + states::AddressCohortState, + utils::OptionExt, +}; + +const VERSION: Version = Version::ZERO; + +#[derive(Clone, Traversable)] +pub struct Vecs { + starting_height: Option, + + #[traversable(skip)] + pub state: Option, + + #[traversable(flatten)] + pub inner: common::Vecs, + + pub height_to_addr_count: EagerVec>, + pub indexes_to_addr_count: ComputedVecsFromHeight, +} + +impl Vecs { + pub fn forced_import( + db: &Database, + filter: Filter, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: Option<&Path>, + ) -> Result { + let compute_dollars = price.is_some(); + + let full_name = filter.to_full_name(CohortContext::Address); + let suffix = |s: &str| { + if full_name.is_empty() { + s.to_string() + } else { + format!("{full_name}_{s}") + } + }; + + Ok(Self { + starting_height: None, + state: states_path.map(|states_path| { + AddressCohortState::new(states_path, &full_name, compute_dollars) + }), + height_to_addr_count: EagerVec::forced_import( + db, + &suffix("addr_count"), + version + VERSION + Version::ZERO, + )?, + indexes_to_addr_count: ComputedVecsFromHeight::forced_import( + db, + &suffix("addr_count"), + Source::None, + version + VERSION + Version::ZERO, + indexes, + VecBuilderOptions::default().add_last(), + )?, + inner: common::Vecs::forced_import( + db, + filter, + CohortContext::Address, + version, + indexes, + price, + )?, + }) + } +} + +impl DynCohortVecs for Vecs { + fn min_height_vecs_len(&self) -> usize { + std::cmp::min( + self.height_to_addr_count.len(), + self.inner.min_height_vecs_len(), + ) + } + + fn reset_state_starting_height(&mut self) { + self.starting_height = Some(Height::ZERO); + } + + fn import_state(&mut self, starting_height: Height) -> Result { + let starting_height = self + .inner + .import_state(starting_height, &mut self.state.um().inner)?; + + self.starting_height = Some(starting_height); + + if let Some(prev_height) = starting_height.decremented() { + self.state.um().addr_count = *self + .height_to_addr_count + .into_iter() + .get_unwrap(prev_height); + } + + Ok(starting_height) + } + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.height_to_addr_count + .validate_computed_version_or_reset( + base_version + self.height_to_addr_count.inner_version(), + )?; + + self.inner.validate_computed_versions(base_version) + } + + fn truncate_push(&mut self, height: Height) -> Result<()> { + if self.starting_height.unwrap() > height { + return Ok(()); + } + + self.height_to_addr_count + .truncate_push(height, self.state.u().addr_count.into())?; + + self.inner + .truncate_push(height, &self.state.u().inner) + } + + fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + ) -> Result<()> { + self.inner.compute_then_truncate_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + &self.state.u().inner, + ) + } + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.height_to_addr_count.safe_write(exit)?; + + self.inner + .safe_flush_stateful_vecs(height, exit, &mut self.state.um().inner) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.indexes_to_addr_count.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_addr_count), + )?; + + self.inner + .compute_rest_part1(indexes, price, starting_indexes, exit) + } +} + +impl CohortVecs for Vecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_addr_count.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_addr_count) + .collect::>() + .as_slice(), + exit, + )?; + self.inner.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.inner).collect::>(), + exit, + ) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.inner.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + } +} + +impl Filtered for Vecs { + fn filter(&self) -> &Filter { + &self.inner.filter + } +} diff --git a/crates/brk_computer/src/stateful_old/address_cohorts.rs b/crates/brk_computer/src/stateful_old/address_cohorts.rs new file mode 100644 index 000000000..e6e37fa09 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/address_cohorts.rs @@ -0,0 +1,139 @@ +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{AddressGroups, AmountFilter, Filter, Filtered}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; +use derive_deref::{Deref, DerefMut}; +use rayon::prelude::*; +use vecdb::{Database, Exit, IterableVec}; + +use crate::{ + Indexes, indexes, price, + stateful::{ + address_cohort, + r#trait::{CohortVecs, DynCohortVecs}, + }, +}; + +const VERSION: Version = Version::new(0); + +#[derive(Clone, Deref, DerefMut, Traversable)] +pub struct Vecs(AddressGroups); + +impl Vecs { + pub fn forced_import( + db: &Database, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: &Path, + ) -> Result { + Ok(Self(AddressGroups::new(|filter| { + let states_path = match &filter { + Filter::Amount(AmountFilter::Range(_)) => Some(states_path), + _ => None, + }; + + address_cohort::Vecs::forced_import( + db, + filter, + version + VERSION + Version::ZERO, + indexes, + price, + states_path, + ) + .unwrap() + }))) + } + + pub fn compute_overlapping_vecs( + &mut self, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + let by_size_range = &self.0.amount_range; + + [ + self.0 + .ge_amount + .par_iter_mut() + .map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_size_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + }) + .collect::>(), + self.0 + .lt_amount + .par_iter_mut() + .map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_size_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + }) + .collect::>(), + ] + .into_iter() + .flatten() + .try_for_each(|(vecs, stateful)| { + vecs.compute_from_stateful(starting_indexes, &stateful, exit) + }) + } + + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.par_iter_mut() + .try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit)) + } + + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.0.par_iter_mut().try_for_each(|v| { + v.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + }) + } + + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.par_iter_separate_mut() + .try_for_each(|v| v.safe_flush_stateful_vecs(height, exit)) + } +} diff --git a/crates/brk_computer/src/stateful_old/address_indexes.rs b/crates/brk_computer/src/stateful_old/address_indexes.rs new file mode 100644 index 000000000..fbc6b1cc1 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/address_indexes.rs @@ -0,0 +1,226 @@ +use brk_error::{Error, Result}; +use brk_traversable::Traversable; +use brk_types::{ + AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, + LoadedAddressIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex, + P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, + TypeIndex, +}; +use vecdb::{AnyStoredVec, BytesVec, GenericStoredVec, Reader, Stamp}; + +#[derive(Clone, Traversable)] +pub struct AnyAddressIndexesVecs { + pub p2pk33: BytesVec, + pub p2pk65: BytesVec, + pub p2pkh: BytesVec, + pub p2sh: BytesVec, + pub p2tr: BytesVec, + pub p2wpkh: BytesVec, + pub p2wsh: BytesVec, + pub p2a: BytesVec, +} + +impl AnyAddressIndexesVecs { + pub fn min_stamped_height(&self) -> Height { + Height::from(self.p2pk33.stamp()) + .incremented() + .min(Height::from(self.p2pk65.stamp()).incremented()) + .min(Height::from(self.p2pkh.stamp()).incremented()) + .min(Height::from(self.p2sh.stamp()).incremented()) + .min(Height::from(self.p2tr.stamp()).incremented()) + .min(Height::from(self.p2wpkh.stamp()).incremented()) + .min(Height::from(self.p2wsh.stamp()).incremented()) + .min(Height::from(self.p2a.stamp()).incremented()) + } + + pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 8]> { + Ok([ + self.p2pk33.rollback_before(stamp)?, + self.p2pk65.rollback_before(stamp)?, + self.p2pkh.rollback_before(stamp)?, + self.p2sh.rollback_before(stamp)?, + self.p2tr.rollback_before(stamp)?, + self.p2wpkh.rollback_before(stamp)?, + self.p2wsh.rollback_before(stamp)?, + self.p2a.rollback_before(stamp)?, + ]) + } + + pub fn reset(&mut self) -> Result<()> { + self.p2pk33.reset()?; + self.p2pk65.reset()?; + self.p2pkh.reset()?; + self.p2sh.reset()?; + self.p2tr.reset()?; + self.p2wpkh.reset()?; + self.p2wsh.reset()?; + self.p2a.reset()?; + Ok(()) + } + + pub fn get_anyaddressindex( + &self, + address_type: OutputType, + typeindex: TypeIndex, + reader: &Reader, + ) -> AnyAddressIndex { + match address_type { + OutputType::P2PK33 => self + .p2pk33 + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2PK65 => self + .p2pk65 + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2PKH => self + .p2pkh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2SH => self + .p2sh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2TR => self + .p2tr + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2WPKH => self + .p2wpkh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2WSH => self + .p2wsh + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + OutputType::P2A => self + .p2a + .get_pushed_or_read_at_unwrap(typeindex.into(), reader), + _ => unreachable!(), + } + } + + pub fn get_anyaddressindex_once( + &self, + address_type: OutputType, + typeindex: TypeIndex, + ) -> Result { + match address_type { + OutputType::P2PK33 => self + .p2pk33 + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2PK65 => self + .p2pk65 + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2PKH => self + .p2pkh + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2SH => self + .p2sh + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2TR => self + .p2tr + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2WPKH => self + .p2wpkh + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2WSH => self + .p2wsh + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + OutputType::P2A => self + .p2a + .read_at_once(typeindex.into()) + .map_err(|e| e.into()), + _ => Err(Error::UnsupportedType(address_type.to_string())), + } + } + + pub fn update_or_push( + &mut self, + address_type: OutputType, + typeindex: TypeIndex, + anyaddressindex: AnyAddressIndex, + ) -> Result<()> { + (match address_type { + OutputType::P2PK33 => self + .p2pk33 + .update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2PK65 => self + .p2pk65 + .update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2PKH => self.p2pkh.update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2SH => self.p2sh.update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2TR => self.p2tr.update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2WPKH => self + .p2wpkh + .update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2WSH => self.p2wsh.update_or_push(typeindex.into(), anyaddressindex), + OutputType::P2A => self.p2a.update_or_push(typeindex.into(), anyaddressindex), + _ => unreachable!(), + })?; + Ok(()) + } + + pub fn stamped_flush_maybe_with_changes( + &mut self, + stamp: Stamp, + with_changes: bool, + ) -> Result<()> { + self.p2pk33 + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2pk65 + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2pkh + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2sh + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2tr + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2wpkh + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2wsh + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.p2a + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + Ok(()) + } +} + +#[derive(Clone, Traversable)] +pub struct AddressesDataVecs { + pub loaded: BytesVec, + pub empty: BytesVec, +} + +impl AddressesDataVecs { + pub fn min_stamped_height(&self) -> Height { + Height::from(self.loaded.stamp()) + .incremented() + .min(Height::from(self.empty.stamp()).incremented()) + } + + pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> { + Ok([ + self.loaded.rollback_before(stamp)?, + self.empty.rollback_before(stamp)?, + ]) + } + + pub fn reset(&mut self) -> Result<()> { + self.loaded.reset()?; + self.empty.reset()?; + Ok(()) + } + + pub fn stamped_flush_maybe_with_changes( + &mut self, + stamp: Stamp, + with_changes: bool, + ) -> Result<()> { + self.loaded + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.empty + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/addresstype/addresscount.rs b/crates/brk_computer/src/stateful_old/addresstype/addresscount.rs new file mode 100644 index 000000000..1cdd92eb5 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/addresscount.rs @@ -0,0 +1,29 @@ +use brk_grouper::ByAddressType; +use brk_types::Height; +use derive_deref::{Deref, DerefMut}; +use vecdb::TypedVecIterator; + +use super::AddressTypeToHeightToAddressCount; + +#[derive(Debug, Default, Deref, DerefMut)] +pub struct AddressTypeToAddressCount(ByAddressType); + +impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount { + #[inline] + fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self { + if let Some(prev_height) = starting_height.decremented() { + Self(ByAddressType { + p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(), + p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(), + p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(), + p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(), + p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(), + p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(), + p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(), + p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(), + }) + } else { + Default::default() + } + } +} diff --git a/crates/brk_computer/src/stateful_old/addresstype/height_to_addresscount.rs b/crates/brk_computer/src/stateful_old/addresstype/height_to_addresscount.rs new file mode 100644 index 000000000..b9e753ea2 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/height_to_addresscount.rs @@ -0,0 +1,45 @@ +use brk_error::Result; +use brk_grouper::ByAddressType; +use brk_traversable::Traversable; +use brk_types::{Height, StoredU64}; +use derive_deref::{Deref, DerefMut}; +use vecdb::{PcoVec, EagerVec, GenericStoredVec}; + +use super::AddressTypeToAddressCount; + +#[derive(Debug, Clone, Deref, DerefMut, Traversable)] +pub struct AddressTypeToHeightToAddressCount(ByAddressType>>); + +impl From>>> for AddressTypeToHeightToAddressCount { + #[inline] + fn from(value: ByAddressType>>) -> Self { + Self(value) + } +} + +impl AddressTypeToHeightToAddressCount { + pub fn truncate_push( + &mut self, + height: Height, + addresstype_to_usize: &AddressTypeToAddressCount, + ) -> Result<()> { + self.p2pk65 + .truncate_push(height, addresstype_to_usize.p2pk65.into())?; + self.p2pk33 + .truncate_push(height, addresstype_to_usize.p2pk33.into())?; + self.p2pkh + .truncate_push(height, addresstype_to_usize.p2pkh.into())?; + self.p2sh + .truncate_push(height, addresstype_to_usize.p2sh.into())?; + self.p2wpkh + .truncate_push(height, addresstype_to_usize.p2wpkh.into())?; + self.p2wsh + .truncate_push(height, addresstype_to_usize.p2wsh.into())?; + self.p2tr + .truncate_push(height, addresstype_to_usize.p2tr.into())?; + self.p2a + .truncate_push(height, addresstype_to_usize.p2a.into())?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/addresstype/height_to_vec.rs b/crates/brk_computer/src/stateful_old/addresstype/height_to_vec.rs new file mode 100644 index 000000000..db471d658 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/height_to_vec.rs @@ -0,0 +1,9 @@ +use std::collections::BTreeMap; + +use brk_types::Height; +use derive_deref::{Deref, DerefMut}; + +use crate::stateful::AddressTypeToVec; + +#[derive(Debug, Default, Deref, DerefMut)] +pub struct HeightToAddressTypeToVec(pub BTreeMap>); diff --git a/crates/brk_computer/src/stateful_old/addresstype/indexes_to_addresscount.rs b/crates/brk_computer/src/stateful_old/addresstype/indexes_to_addresscount.rs new file mode 100644 index 000000000..ae8e32af5 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/indexes_to_addresscount.rs @@ -0,0 +1,80 @@ +use brk_error::Result; +use brk_grouper::ByAddressType; +use brk_traversable::Traversable; +use brk_types::StoredU64; +use derive_deref::{Deref, DerefMut}; +use vecdb::Exit; + +use crate::{Indexes, grouped::ComputedVecsFromHeight, indexes}; + +use super::AddressTypeToHeightToAddressCount; + +#[derive(Clone, Deref, DerefMut, Traversable)] +pub struct AddressTypeToIndexesToAddressCount(ByAddressType>); + +impl From>> for AddressTypeToIndexesToAddressCount { + #[inline] + fn from(value: ByAddressType>) -> Self { + Self(value) + } +} + +impl AddressTypeToIndexesToAddressCount { + pub fn compute( + &mut self, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount, + ) -> Result<()> { + self.p2pk65.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pk65), + )?; + self.p2pk33.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pk33), + )?; + self.p2pkh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2pkh), + )?; + self.p2sh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2sh), + )?; + self.p2wpkh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2wpkh), + )?; + self.p2wsh.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2wsh), + )?; + self.p2tr.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2tr), + )?; + self.p2a.compute_rest( + indexes, + starting_indexes, + exit, + Some(&addresstype_to_height_to_addresscount.p2a), + )?; + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/addresstype/mod.rs b/crates/brk_computer/src/stateful_old/addresstype/mod.rs new file mode 100644 index 000000000..8b2fc4845 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/mod.rs @@ -0,0 +1,13 @@ +mod addresscount; +mod height_to_addresscount; +mod height_to_vec; +mod indexes_to_addresscount; +mod typeindex_map; +mod vec; + +pub use addresscount::*; +pub use height_to_addresscount::*; +pub use height_to_vec::*; +pub use indexes_to_addresscount::*; +pub use typeindex_map::*; +pub use vec::*; diff --git a/crates/brk_computer/src/stateful_old/addresstype/typeindex_map.rs b/crates/brk_computer/src/stateful_old/addresstype/typeindex_map.rs new file mode 100644 index 000000000..a6a9cce6c --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/typeindex_map.rs @@ -0,0 +1,100 @@ +use std::{collections::hash_map::Entry, mem}; + +use brk_grouper::ByAddressType; +use brk_types::{OutputType, TypeIndex}; +use derive_deref::{Deref, DerefMut}; +use rustc_hash::FxHashMap; +use smallvec::{Array, SmallVec}; + +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToTypeIndexMap(ByAddressType>); + +impl AddressTypeToTypeIndexMap { + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + self + } + + fn merge_(own: &mut FxHashMap, other: &mut FxHashMap) { + if own.len() < other.len() { + mem::swap(own, other); + } + own.extend(other.drain()); + } + + // pub fn get_for_type(&self, address_type: OutputType, typeindex: &TypeIndex) -> Option<&T> { + // self.get(address_type).unwrap().get(typeindex) + // } + + pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) { + self.get_mut(address_type).unwrap().insert(typeindex, value); + } + + pub fn remove_for_type(&mut self, address_type: OutputType, typeindex: &TypeIndex) -> T { + self.get_mut(address_type) + .unwrap() + .remove(typeindex) + .unwrap() + } + + pub fn into_sorted_iter(self) -> impl Iterator)> { + self.0.into_iter().map(|(output_type, map)| { + let mut sorted: Vec<_> = map.into_iter().collect(); + sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex); + (output_type, sorted) + }) + } + + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> impl Iterator)> { + self.0.into_iter() + } +} + +impl Default for AddressTypeToTypeIndexMap { + fn default() -> Self { + Self(ByAddressType { + p2pk65: FxHashMap::default(), + p2pk33: FxHashMap::default(), + p2pkh: FxHashMap::default(), + p2sh: FxHashMap::default(), + p2wpkh: FxHashMap::default(), + p2wsh: FxHashMap::default(), + p2tr: FxHashMap::default(), + p2a: FxHashMap::default(), + }) + } +} + +impl AddressTypeToTypeIndexMap> +where + T: Array, +{ + pub fn merge_vec(mut self, other: Self) -> Self { + for (address_type, other_map) in other.0.into_iter() { + let self_map = self.0.get_mut_unwrap(address_type); + for (typeindex, mut other_vec) in other_map { + match self_map.entry(typeindex) { + Entry::Occupied(mut entry) => { + let self_vec = entry.get_mut(); + if other_vec.len() > self_vec.len() { + mem::swap(self_vec, &mut other_vec); + } + self_vec.extend(other_vec); + } + Entry::Vacant(entry) => { + entry.insert(other_vec); + } + } + } + } + self + } +} diff --git a/crates/brk_computer/src/stateful_old/addresstype/vec.rs b/crates/brk_computer/src/stateful_old/addresstype/vec.rs new file mode 100644 index 000000000..8ba575292 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/addresstype/vec.rs @@ -0,0 +1,60 @@ +use std::mem; + +use brk_grouper::ByAddressType; +use derive_deref::{Deref, DerefMut}; + +#[derive(Debug, Deref, DerefMut)] +pub struct AddressTypeToVec(ByAddressType>); + +impl AddressTypeToVec { + pub fn merge(mut self, mut other: Self) -> Self { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + self + } + + pub fn merge_mut(&mut self, mut other: Self) { + Self::merge_(&mut self.p2pk65, &mut other.p2pk65); + Self::merge_(&mut self.p2pk33, &mut other.p2pk33); + Self::merge_(&mut self.p2pkh, &mut other.p2pkh); + Self::merge_(&mut self.p2sh, &mut other.p2sh); + Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh); + Self::merge_(&mut self.p2wsh, &mut other.p2wsh); + Self::merge_(&mut self.p2tr, &mut other.p2tr); + Self::merge_(&mut self.p2a, &mut other.p2a); + } + + fn merge_(own: &mut Vec, other: &mut Vec) { + if own.len() >= other.len() { + own.append(other); + } else { + other.append(own); + mem::swap(own, other); + } + } + + pub fn unwrap(self) -> ByAddressType> { + self.0 + } +} + +impl Default for AddressTypeToVec { + fn default() -> Self { + Self(ByAddressType { + p2pk65: vec![], + p2pk33: vec![], + p2pkh: vec![], + p2sh: vec![], + p2wpkh: vec![], + p2wsh: vec![], + p2tr: vec![], + p2a: vec![], + }) + } +} diff --git a/crates/brk_computer/src/stateful_old/common/compute.rs b/crates/brk_computer/src/stateful_old/common/compute.rs new file mode 100644 index 000000000..6df334f19 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/common/compute.rs @@ -0,0 +1,1241 @@ +//! Compute methods for Vecs. +//! +//! This module contains methods for post-processing computations: +//! - `compute_from_stateful`: Compute aggregate cohort values from separate cohorts +//! - `compute_rest_part1`: First phase of computed metrics +//! - `compute_rest_part2`: Second phase of computed metrics + +use brk_error::Result; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF64, StoredU64}; +use vecdb::{Exit, IterableVec, TypedVecIterator}; + +use crate::{Indexes, indexes, price, utils::OptionExt}; + +use super::Vecs; + +impl Vecs { + pub fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.height_to_supply.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_supply) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_utxo_count.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_utxo_count) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_sent.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_sent) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_satblocks_destroyed.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_satblocks_destroyed) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_satdays_destroyed.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| &v.height_to_satdays_destroyed) + .collect::>() + .as_slice(), + exit, + )?; + + if let Some(height_to_realized_cap) = &mut self.height_to_realized_cap { + height_to_realized_cap.compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_realized_cap.u()) + .collect::>() + .as_slice(), + exit, + )?; + + self.height_to_min_price_paid.um().compute_min_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_min_price_paid.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_max_price_paid.um().compute_max_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_max_price_paid.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_realized_profit.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_realized_profit.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_realized_loss.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_realized_loss.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_value_created.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_value_created.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_value_destroyed.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_value_destroyed.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_supply_in_profit.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_supply_in_profit.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_supply_in_loss.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_supply_in_loss.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_unrealized_profit + .um() + .compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_unrealized_profit.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_unrealized_loss.um().compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_unrealized_loss.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.dateindex_to_supply_in_profit + .um() + .compute_sum_of_others( + starting_indexes.dateindex, + others + .iter() + .map(|v| v.dateindex_to_supply_in_profit.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.dateindex_to_supply_in_loss + .um() + .compute_sum_of_others( + starting_indexes.dateindex, + others + .iter() + .map(|v| v.dateindex_to_supply_in_loss.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.dateindex_to_unrealized_profit + .um() + .compute_sum_of_others( + starting_indexes.dateindex, + others + .iter() + .map(|v| v.dateindex_to_unrealized_profit.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.dateindex_to_unrealized_loss + .um() + .compute_sum_of_others( + starting_indexes.dateindex, + others + .iter() + .map(|v| v.dateindex_to_unrealized_loss.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_min_price_paid.um().compute_min_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_min_price_paid.u()) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_max_price_paid.um().compute_max_of_others( + starting_indexes.height, + others + .iter() + .map(|v| v.height_to_max_price_paid.u()) + .collect::>() + .as_slice(), + exit, + )?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created + .um() + .compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| { + v.height_to_adjusted_value_created + .as_ref() + .unwrap_or(v.height_to_value_created.u()) + }) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_adjusted_value_destroyed + .um() + .compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| { + v.height_to_adjusted_value_destroyed + .as_ref() + .unwrap_or(v.height_to_value_destroyed.u()) + }) + .collect::>() + .as_slice(), + exit, + )?; + } + } + + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.height_to_supply_value.compute_rest( + price, + starting_indexes, + exit, + Some(&self.height_to_supply), + )?; + + self.indexes_to_supply + .compute_all(price, starting_indexes, exit, |v| { + let mut dateindex_to_height_count_iter = + indexes.dateindex_to_height_count.into_iter(); + let mut height_to_supply_iter = self.height_to_supply.into_iter(); + v.compute_transform( + starting_indexes.dateindex, + &indexes.dateindex_to_first_height, + |(i, height, ..)| { + let count = dateindex_to_height_count_iter.get_unwrap(i); + if count == StoredU64::default() { + unreachable!() + } + let supply = height_to_supply_iter.get_unwrap(height + (*count - 1)); + (i, supply) + }, + exit, + )?; + Ok(()) + })?; + + self.indexes_to_utxo_count.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_utxo_count), + )?; + + self.height_to_supply_half_value + .compute_all(price, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.height, + &self.height_to_supply, + |(h, v, ..)| (h, v / 2), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_supply_half + .compute_all(price, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + self.indexes_to_supply.sats.dateindex.u(), + |(i, sats, ..)| (i, sats / 2), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_sent.compute_rest( + indexes, + price, + starting_indexes, + exit, + Some(&self.height_to_sent), + )?; + + self.indexes_to_coinblocks_destroyed + .compute_all(indexes, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.height, + &self.height_to_satblocks_destroyed, + |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_coindays_destroyed + .compute_all(indexes, starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.height, + &self.height_to_satdays_destroyed, + |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), + exit, + )?; + Ok(()) + })?; + + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + if let Some(v) = self.indexes_to_supply_rel_to_circulating_supply.as_mut() { + v.compute_all(indexes, starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.height, + &self.height_to_supply_value.bitcoin, + height_to_supply, + exit, + )?; + Ok(()) + })?; + } + + if let Some(indexes_to_realized_cap) = self.indexes_to_realized_cap.as_mut() { + let height_to_market_cap = height_to_market_cap.unwrap(); + let dateindex_to_market_cap = dateindex_to_market_cap.unwrap(); + + indexes_to_realized_cap.compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_realized_cap.u()), + )?; + + self.indexes_to_realized_price.um().compute_all( + indexes, + starting_indexes, + exit, + |vec| { + vec.compute_divide( + starting_indexes.height, + self.height_to_realized_cap.u(), + &self.height_to_supply_value.bitcoin, + exit, + )?; + Ok(()) + }, + )?; + + self.indexes_to_realized_price_extra.um().compute_rest( + price.u(), + starting_indexes, + exit, + Some(self.indexes_to_realized_price.u().dateindex.unwrap_last()), + )?; + + self.indexes_to_realized_profit.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_realized_profit.u()), + )?; + + self.indexes_to_realized_loss.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_realized_loss.u()), + )?; + + self.indexes_to_neg_realized_loss.um().compute_all( + indexes, + starting_indexes, + exit, + |vec| { + vec.compute_transform( + starting_indexes.height, + self.height_to_realized_loss.u(), + |(i, v, ..)| (i, v * -1_i64), + exit, + )?; + Ok(()) + }, + )?; + + self.indexes_to_value_created.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_value_created.u()), + )?; + + self.indexes_to_value_destroyed.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_value_destroyed.u()), + )?; + + self.indexes_to_realized_cap_30d_delta.um().compute_all( + starting_indexes, + exit, + |vec| { + vec.compute_change( + starting_indexes.dateindex, + self.indexes_to_realized_cap.u().dateindex.unwrap_last(), + 30, + exit, + )?; + Ok(()) + }, + )?; + + self.indexes_to_net_realized_pnl.um().compute_all( + indexes, + starting_indexes, + exit, + |vec| { + vec.compute_subtract( + starting_indexes.height, + self.height_to_realized_profit.u(), + self.height_to_realized_loss.u(), + exit, + )?; + Ok(()) + }, + )?; + + self.indexes_to_realized_value.um().compute_all( + indexes, + starting_indexes, + exit, + |vec| { + vec.compute_add( + starting_indexes.height, + self.height_to_realized_profit.u(), + self.height_to_realized_loss.u(), + exit, + )?; + Ok(()) + }, + )?; + + self.dateindex_to_sopr.um().compute_divide( + starting_indexes.dateindex, + self.indexes_to_value_created.u().dateindex.unwrap_sum(), + self.indexes_to_value_destroyed.u().dateindex.unwrap_sum(), + exit, + )?; + + self.dateindex_to_sopr_7d_ema.um().compute_ema( + starting_indexes.dateindex, + self.dateindex_to_sopr.u(), + 7, + exit, + )?; + + self.dateindex_to_sopr_30d_ema.um().compute_ema( + starting_indexes.dateindex, + self.dateindex_to_sopr.u(), + 30, + exit, + )?; + + self.dateindex_to_sell_side_risk_ratio + .um() + .compute_percentage( + starting_indexes.dateindex, + self.indexes_to_realized_value.u().dateindex.unwrap_sum(), + self.indexes_to_realized_cap.u().dateindex.unwrap_last(), + exit, + )?; + + self.dateindex_to_sell_side_risk_ratio_7d_ema + .um() + .compute_ema( + starting_indexes.dateindex, + self.dateindex_to_sell_side_risk_ratio.u(), + 7, + exit, + )?; + + self.dateindex_to_sell_side_risk_ratio_30d_ema + .um() + .compute_ema( + starting_indexes.dateindex, + self.dateindex_to_sell_side_risk_ratio.u(), + 30, + exit, + )?; + + self.indexes_to_supply_in_profit.um().compute_rest( + price, + starting_indexes, + exit, + Some(self.dateindex_to_supply_in_profit.u()), + )?; + self.indexes_to_supply_in_loss.um().compute_rest( + price, + starting_indexes, + exit, + Some(self.dateindex_to_supply_in_loss.u()), + )?; + self.indexes_to_unrealized_profit.um().compute_rest( + starting_indexes, + exit, + Some(self.dateindex_to_unrealized_profit.u()), + )?; + self.indexes_to_unrealized_loss.um().compute_rest( + starting_indexes, + exit, + Some(self.dateindex_to_unrealized_loss.u()), + )?; + self.height_to_total_unrealized_pnl.um().compute_add( + starting_indexes.height, + self.height_to_unrealized_profit.u(), + self.height_to_unrealized_loss.u(), + exit, + )?; + self.indexes_to_total_unrealized_pnl.um().compute_all( + starting_indexes, + exit, + |vec| { + vec.compute_add( + starting_indexes.dateindex, + self.dateindex_to_unrealized_profit.u(), + self.dateindex_to_unrealized_loss.u(), + exit, + )?; + Ok(()) + }, + )?; + self.height_to_total_realized_pnl.um().compute_add( + starting_indexes.height, + self.height_to_realized_profit.u(), + self.height_to_realized_loss.u(), + exit, + )?; + self.indexes_to_total_realized_pnl + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_add( + starting_indexes.dateindex, + self.indexes_to_realized_profit.u().dateindex.unwrap_sum(), + self.indexes_to_realized_loss.u().dateindex.unwrap_sum(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_min_price_paid.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_min_price_paid.u()), + )?; + self.indexes_to_max_price_paid.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_max_price_paid.u()), + )?; + + self.height_to_neg_unrealized_loss.um().compute_transform( + starting_indexes.height, + self.height_to_unrealized_loss.u(), + |(h, v, ..)| (h, v * -1_i64), + exit, + )?; + self.indexes_to_neg_unrealized_loss + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + self.dateindex_to_unrealized_loss.u(), + |(h, v, ..)| (h, v * -1_i64), + exit, + )?; + Ok(()) + })?; + self.height_to_net_unrealized_pnl.um().compute_subtract( + starting_indexes.height, + self.height_to_unrealized_profit.u(), + self.height_to_unrealized_loss.u(), + exit, + )?; + + self.indexes_to_net_unrealized_pnl + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_subtract( + starting_indexes.dateindex, + self.dateindex_to_unrealized_profit.u(), + self.dateindex_to_unrealized_loss.u(), + exit, + )?; + Ok(()) + })?; + self.height_to_unrealized_profit_rel_to_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_unrealized_profit.u(), + height_to_market_cap, + exit, + )?; + self.height_to_unrealized_loss_rel_to_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_unrealized_loss.u(), + height_to_market_cap, + exit, + )?; + self.height_to_neg_unrealized_loss_rel_to_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_neg_unrealized_loss.u(), + height_to_market_cap, + exit, + )?; + self.height_to_net_unrealized_pnl_rel_to_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_net_unrealized_pnl.u(), + height_to_market_cap, + exit, + )?; + self.indexes_to_unrealized_profit_rel_to_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.dateindex_to_unrealized_profit.u(), + dateindex_to_market_cap, + exit, + )?; + Ok(()) + })?; + self.indexes_to_unrealized_loss_rel_to_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.dateindex_to_unrealized_loss.u(), + dateindex_to_market_cap, + exit, + )?; + Ok(()) + })?; + self.indexes_to_neg_unrealized_loss_rel_to_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_neg_unrealized_loss.u().dateindex.u(), + dateindex_to_market_cap, + exit, + )?; + Ok(()) + })?; + self.indexes_to_net_unrealized_pnl_rel_to_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_net_unrealized_pnl.u().dateindex.u(), + dateindex_to_market_cap, + exit, + )?; + Ok(()) + })?; + + if self + .height_to_unrealized_profit_rel_to_own_market_cap + .is_some() + { + self.height_to_unrealized_profit_rel_to_own_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_unrealized_profit.u(), + self.height_to_supply_value.dollars.u(), + exit, + )?; + self.height_to_unrealized_loss_rel_to_own_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_unrealized_loss.u(), + self.height_to_supply_value.dollars.u(), + exit, + )?; + self.height_to_neg_unrealized_loss_rel_to_own_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_neg_unrealized_loss.u(), + self.height_to_supply_value.dollars.u(), + exit, + )?; + self.height_to_net_unrealized_pnl_rel_to_own_market_cap + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_net_unrealized_pnl.u(), + self.height_to_supply_value.dollars.u(), + exit, + )?; + self.indexes_to_unrealized_profit_rel_to_own_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.dateindex_to_unrealized_profit.u(), + self.indexes_to_supply + .dollars + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_unrealized_loss_rel_to_own_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.dateindex_to_unrealized_loss.u(), + self.indexes_to_supply + .dollars + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_neg_unrealized_loss_rel_to_own_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_neg_unrealized_loss + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + self.indexes_to_supply + .dollars + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_net_unrealized_pnl_rel_to_own_market_cap + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_net_unrealized_pnl + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + self.indexes_to_supply + .dollars + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + } + + if self + .height_to_unrealized_profit_rel_to_own_total_unrealized_pnl + .is_some() + { + self.height_to_unrealized_profit_rel_to_own_total_unrealized_pnl + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_unrealized_profit.u(), + self.height_to_total_unrealized_pnl.u(), + exit, + )?; + self.height_to_unrealized_loss_rel_to_own_total_unrealized_pnl + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_unrealized_loss.u(), + self.height_to_total_unrealized_pnl.u(), + exit, + )?; + self.height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_neg_unrealized_loss.u(), + self.height_to_total_unrealized_pnl.u(), + exit, + )?; + self.height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl + .um() + .compute_percentage( + starting_indexes.height, + self.height_to_net_unrealized_pnl.u(), + self.height_to_total_unrealized_pnl.u(), + exit, + )?; + self.indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.dateindex_to_unrealized_profit.u(), + self.indexes_to_total_unrealized_pnl + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.dateindex_to_unrealized_loss.u(), + self.indexes_to_total_unrealized_pnl + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_neg_unrealized_loss + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + self.indexes_to_total_unrealized_pnl + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl + .um() + .compute_all(starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_net_unrealized_pnl + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + self.indexes_to_total_unrealized_pnl + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + exit, + )?; + Ok(()) + })?; + } + + self.indexes_to_realized_profit_rel_to_realized_cap + .um() + .compute_all(indexes, starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.height, + self.height_to_realized_profit.u(), + *height_to_realized_cap.u(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_realized_loss_rel_to_realized_cap + .um() + .compute_all(indexes, starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.height, + self.height_to_realized_loss.u(), + *height_to_realized_cap.u(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_net_realized_pnl_rel_to_realized_cap + .um() + .compute_all(indexes, starting_indexes, exit, |vec| { + vec.compute_percentage( + starting_indexes.height, + self.indexes_to_net_realized_pnl.u().height.u(), + *height_to_realized_cap.u(), + exit, + )?; + Ok(()) + })?; + + self.height_to_supply_in_loss_value.um().compute_rest( + price, + starting_indexes, + exit, + Some(self.height_to_supply_in_loss.u()), + )?; + self.height_to_supply_in_profit_value.um().compute_rest( + price, + starting_indexes, + exit, + Some(self.height_to_supply_in_profit.u()), + )?; + self.height_to_supply_in_loss_rel_to_own_supply + .um() + .compute_percentage( + starting_indexes.height, + &self.height_to_supply_in_loss_value.u().bitcoin, + &self.height_to_supply_value.bitcoin, + exit, + )?; + self.height_to_supply_in_profit_rel_to_own_supply + .um() + .compute_percentage( + starting_indexes.height, + &self.height_to_supply_in_profit_value.u().bitcoin, + &self.height_to_supply_value.bitcoin, + exit, + )?; + self.indexes_to_supply_in_loss_rel_to_own_supply + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_supply_in_loss.u().bitcoin.dateindex.u(), + self.indexes_to_supply.bitcoin.dateindex.u(), + exit, + )?; + Ok(()) + })?; + self.indexes_to_supply_in_profit_rel_to_own_supply + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_supply_in_profit.u().bitcoin.dateindex.u(), + self.indexes_to_supply.bitcoin.dateindex.u(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_net_realized_pnl_cumulative_30d_delta + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_change( + starting_indexes.dateindex, + self.indexes_to_net_realized_pnl + .u() + .dateindex + .unwrap_cumulative(), + 30, + exit, + )?; + Ok(()) + })?; + + self.indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_net_realized_pnl_cumulative_30d_delta + .u() + .dateindex + .u(), + *dateindex_to_realized_cap.u(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_net_realized_pnl_cumulative_30d_delta + .u() + .dateindex + .u(), + dateindex_to_market_cap, + exit, + )?; + Ok(()) + })?; + + if self + .height_to_supply_in_profit_rel_to_circulating_supply + .as_mut() + .is_some() + { + self.height_to_supply_in_loss_rel_to_circulating_supply + .um() + .compute_percentage( + starting_indexes.height, + &self.height_to_supply_in_loss_value.u().bitcoin, + height_to_supply, + exit, + )?; + self.height_to_supply_in_profit_rel_to_circulating_supply + .um() + .compute_percentage( + starting_indexes.height, + &self.height_to_supply_in_profit_value.u().bitcoin, + height_to_supply, + exit, + )?; + self.indexes_to_supply_in_loss_rel_to_circulating_supply + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_supply_in_loss + .as_ref() + .unwrap() + .bitcoin + .dateindex + .as_ref() + .unwrap(), + dateindex_to_supply, + exit, + )?; + Ok(()) + })?; + self.indexes_to_supply_in_profit_rel_to_circulating_supply + .um() + .compute_all(starting_indexes, exit, |v| { + v.compute_percentage( + starting_indexes.dateindex, + self.indexes_to_supply_in_profit + .as_ref() + .unwrap() + .bitcoin + .dateindex + .as_ref() + .unwrap(), + dateindex_to_supply, + exit, + )?; + Ok(()) + })?; + } + + if self.indexes_to_adjusted_value_created.is_some() { + self.indexes_to_adjusted_value_created.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_adjusted_value_created.u()), + )?; + + self.indexes_to_adjusted_value_destroyed.um().compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_adjusted_value_destroyed.u()), + )?; + + self.dateindex_to_adjusted_sopr.um().compute_divide( + starting_indexes.dateindex, + self.indexes_to_adjusted_value_created + .u() + .dateindex + .unwrap_sum(), + self.indexes_to_adjusted_value_destroyed + .u() + .dateindex + .unwrap_sum(), + exit, + )?; + + self.dateindex_to_adjusted_sopr_7d_ema.um().compute_ema( + starting_indexes.dateindex, + self.dateindex_to_adjusted_sopr.u(), + 7, + exit, + )?; + + self.dateindex_to_adjusted_sopr_30d_ema.um().compute_ema( + starting_indexes.dateindex, + self.dateindex_to_adjusted_sopr.u(), + 30, + exit, + )?; + } + + if let Some(indexes_to_realized_cap_rel_to_own_market_cap) = + self.indexes_to_realized_cap_rel_to_own_market_cap.as_mut() + { + indexes_to_realized_cap_rel_to_own_market_cap.compute_all( + indexes, + starting_indexes, + exit, + |v| { + v.compute_percentage( + starting_indexes.height, + self.height_to_realized_cap.u(), + self.height_to_supply_value.dollars.u(), + exit, + )?; + Ok(()) + }, + )?; + } + } + + if let Some(dateindex_to_realized_profit_to_loss_ratio) = + self.dateindex_to_realized_profit_to_loss_ratio.as_mut() + { + dateindex_to_realized_profit_to_loss_ratio.compute_divide( + starting_indexes.dateindex, + self.indexes_to_realized_profit.u().dateindex.unwrap_sum(), + self.indexes_to_realized_loss.u().dateindex.unwrap_sum(), + exit, + )?; + } + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/common/import.rs b/crates/brk_computer/src/stateful_old/common/import.rs new file mode 100644 index 000000000..33fc6d90c --- /dev/null +++ b/crates/brk_computer/src/stateful_old/common/import.rs @@ -0,0 +1,914 @@ +//! Import and validation methods for Vecs. +//! +//! This module contains methods for: +//! - `forced_import`: Creating a new Vecs instance from database +//! - `import_state`: Importing state when resuming from checkpoint +//! - `validate_computed_versions`: Version validation +//! - `min_height_vecs_len`: Finding minimum vector length + +use brk_error::{Error, Result}; +use brk_grouper::{CohortContext, Filter}; +use brk_types::{DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, Version}; +use vecdb::{ + AnyVec, Database, EagerVec, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec, + StoredVec, TypedVecIterator, +}; + +use crate::{ + grouped::{ + ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex, + ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, + PricePercentiles, Source, VecBuilderOptions, + }, + indexes, price, + states::CohortState, + utils::OptionExt, +}; + +use super::Vecs; + +impl Vecs { + #[allow(clippy::too_many_arguments)] + pub fn forced_import( + db: &Database, + filter: Filter, + context: CohortContext, + parent_version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + ) -> Result { + let compute_dollars = price.is_some(); + let extended = filter.is_extended(context); + let compute_rel_to_all = filter.compute_rel_to_all(); + let compute_adjusted = filter.compute_adjusted(context); + + let version = parent_version + Version::ZERO; + + let name_prefix = filter.to_full_name(context); + let suffix = |s: &str| { + if name_prefix.is_empty() { + s.to_string() + } else { + format!("{name_prefix}_{s}") + } + }; + + // Helper macros for imports + macro_rules! eager { + ($idx:ty, $val:ty, $name:expr, $v:expr) => { + EagerVec::>::forced_import(db, &suffix($name), version + $v) + .unwrap() + }; + } + macro_rules! computed_h { + ($name:expr, $source:expr, $v:expr, $opts:expr $(,)?) => { + ComputedVecsFromHeight::forced_import( + db, + &suffix($name), + $source, + version + $v, + indexes, + $opts, + ) + .unwrap() + }; + } + macro_rules! computed_di { + ($name:expr, $source:expr, $v:expr, $opts:expr $(,)?) => { + ComputedVecsFromDateIndex::forced_import( + db, + &suffix($name), + $source, + version + $v, + indexes, + $opts, + ) + .unwrap() + }; + } + + // Common version patterns + let v0 = Version::ZERO; + let v1 = Version::ONE; + let v2 = Version::TWO; + let v3 = Version::new(3); + let last = || VecBuilderOptions::default().add_last(); + let sum = || VecBuilderOptions::default().add_sum(); + let sum_cum = || VecBuilderOptions::default().add_sum().add_cumulative(); + + // Pre-create dateindex vecs that are used in computed vecs + let dateindex_to_supply_in_profit = + compute_dollars.then(|| eager!(DateIndex, Sats, "supply_in_profit", v0)); + let dateindex_to_supply_in_loss = + compute_dollars.then(|| eager!(DateIndex, Sats, "supply_in_loss", v0)); + let dateindex_to_unrealized_profit = + compute_dollars.then(|| eager!(DateIndex, Dollars, "unrealized_profit", v0)); + let dateindex_to_unrealized_loss = + compute_dollars.then(|| eager!(DateIndex, Dollars, "unrealized_loss", v0)); + + Ok(Self { + filter, + + // ==================== SUPPLY & UTXO COUNT ==================== + height_to_supply: EagerVec::forced_import(db, &suffix("supply"), version + v0)?, + height_to_supply_value: ComputedHeightValueVecs::forced_import( + db, + &suffix("supply"), + Source::None, + version + v0, + compute_dollars, + )?, + indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( + db, + &suffix("supply"), + Source::Compute, + version + v1, + last(), + compute_dollars, + indexes, + )?, + height_to_utxo_count: EagerVec::forced_import(db, &suffix("utxo_count"), version + v0)?, + indexes_to_utxo_count: computed_h!("utxo_count", Source::None, v0, last()), + height_to_supply_half_value: ComputedHeightValueVecs::forced_import( + db, + &suffix("supply_half"), + Source::Compute, + version + v0, + compute_dollars, + )?, + indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import( + db, + &suffix("supply_half"), + Source::Compute, + version + v0, + last(), + compute_dollars, + indexes, + )?, + + // ==================== ACTIVITY ==================== + height_to_sent: EagerVec::forced_import(db, &suffix("sent"), version + v0)?, + indexes_to_sent: ComputedValueVecsFromHeight::forced_import( + db, + &suffix("sent"), + Source::None, + version + v0, + sum(), + compute_dollars, + indexes, + )?, + height_to_satblocks_destroyed: EagerVec::forced_import( + db, + &suffix("satblocks_destroyed"), + version + v0, + )?, + height_to_satdays_destroyed: EagerVec::forced_import( + db, + &suffix("satdays_destroyed"), + version + v0, + )?, + indexes_to_coinblocks_destroyed: computed_h!( + "coinblocks_destroyed", + Source::Compute, + v2, + sum_cum(), + ), + indexes_to_coindays_destroyed: computed_h!( + "coindays_destroyed", + Source::Compute, + v2, + sum_cum(), + ), + + // ==================== REALIZED CAP & PRICE ==================== + height_to_realized_cap: compute_dollars + .then(|| eager!(Height, Dollars, "realized_cap", v0)), + indexes_to_realized_cap: compute_dollars + .then(|| computed_h!("realized_cap", Source::None, v0, last())), + indexes_to_realized_price: compute_dollars + .then(|| computed_h!("realized_price", Source::Compute, v0, last())), + indexes_to_realized_price_extra: compute_dollars.then(|| { + ComputedRatioVecsFromDateIndex::forced_import( + db, + &suffix("realized_price"), + Source::None, + version + v0, + indexes, + extended, + ) + .unwrap() + }), + indexes_to_realized_cap_rel_to_own_market_cap: (compute_dollars && extended).then( + || { + computed_h!( + "realized_cap_rel_to_own_market_cap", + Source::Compute, + v0, + last() + ) + }, + ), + indexes_to_realized_cap_30d_delta: compute_dollars + .then(|| computed_di!("realized_cap_30d_delta", Source::Compute, v0, last())), + + // ==================== REALIZED PROFIT & LOSS ==================== + height_to_realized_profit: compute_dollars + .then(|| eager!(Height, Dollars, "realized_profit", v0)), + indexes_to_realized_profit: compute_dollars + .then(|| computed_h!("realized_profit", Source::None, v0, sum_cum())), + height_to_realized_loss: compute_dollars + .then(|| eager!(Height, Dollars, "realized_loss", v0)), + indexes_to_realized_loss: compute_dollars + .then(|| computed_h!("realized_loss", Source::None, v0, sum_cum())), + indexes_to_neg_realized_loss: compute_dollars + .then(|| computed_h!("neg_realized_loss", Source::Compute, v1, sum_cum())), + indexes_to_net_realized_pnl: compute_dollars + .then(|| computed_h!("net_realized_pnl", Source::Compute, v0, sum_cum())), + indexes_to_realized_value: compute_dollars + .then(|| computed_h!("realized_value", Source::Compute, v0, sum())), + indexes_to_realized_profit_rel_to_realized_cap: compute_dollars.then(|| { + computed_h!( + "realized_profit_rel_to_realized_cap", + Source::Compute, + v0, + sum() + ) + }), + indexes_to_realized_loss_rel_to_realized_cap: compute_dollars.then(|| { + computed_h!( + "realized_loss_rel_to_realized_cap", + Source::Compute, + v0, + sum() + ) + }), + indexes_to_net_realized_pnl_rel_to_realized_cap: compute_dollars.then(|| { + computed_h!( + "net_realized_pnl_rel_to_realized_cap", + Source::Compute, + v1, + sum() + ) + }), + height_to_total_realized_pnl: compute_dollars + .then(|| eager!(Height, Dollars, "total_realized_pnl", v0)), + indexes_to_total_realized_pnl: compute_dollars + .then(|| computed_di!("total_realized_pnl", Source::Compute, v1, sum())), + dateindex_to_realized_profit_to_loss_ratio: (compute_dollars && extended) + .then(|| eager!(DateIndex, StoredF64, "realized_profit_to_loss_ratio", v1)), + + // ==================== VALUE CREATED & DESTROYED ==================== + height_to_value_created: compute_dollars + .then(|| eager!(Height, Dollars, "value_created", v0)), + indexes_to_value_created: compute_dollars + .then(|| computed_h!("value_created", Source::None, v0, sum())), + height_to_value_destroyed: compute_dollars + .then(|| eager!(Height, Dollars, "value_destroyed", v0)), + indexes_to_value_destroyed: compute_dollars + .then(|| computed_h!("value_destroyed", Source::None, v0, sum())), + height_to_adjusted_value_created: (compute_dollars && compute_adjusted) + .then(|| eager!(Height, Dollars, "adjusted_value_created", v0)), + indexes_to_adjusted_value_created: (compute_dollars && compute_adjusted) + .then(|| computed_h!("adjusted_value_created", Source::None, v0, sum())), + height_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted) + .then(|| eager!(Height, Dollars, "adjusted_value_destroyed", v0)), + indexes_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted) + .then(|| computed_h!("adjusted_value_destroyed", Source::None, v0, sum())), + + // ==================== SOPR ==================== + dateindex_to_sopr: compute_dollars.then(|| eager!(DateIndex, StoredF64, "sopr", v1)), + dateindex_to_sopr_7d_ema: compute_dollars + .then(|| eager!(DateIndex, StoredF64, "sopr_7d_ema", v1)), + dateindex_to_sopr_30d_ema: compute_dollars + .then(|| eager!(DateIndex, StoredF64, "sopr_30d_ema", v1)), + dateindex_to_adjusted_sopr: (compute_dollars && compute_adjusted) + .then(|| eager!(DateIndex, StoredF64, "adjusted_sopr", v1)), + dateindex_to_adjusted_sopr_7d_ema: (compute_dollars && compute_adjusted) + .then(|| eager!(DateIndex, StoredF64, "adjusted_sopr_7d_ema", v1)), + dateindex_to_adjusted_sopr_30d_ema: (compute_dollars && compute_adjusted) + .then(|| eager!(DateIndex, StoredF64, "adjusted_sopr_30d_ema", v1)), + + // ==================== SELL SIDE RISK ==================== + dateindex_to_sell_side_risk_ratio: compute_dollars + .then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio", v1)), + dateindex_to_sell_side_risk_ratio_7d_ema: compute_dollars + .then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio_7d_ema", v1)), + dateindex_to_sell_side_risk_ratio_30d_ema: compute_dollars + .then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio_30d_ema", v1)), + + // ==================== SUPPLY IN PROFIT/LOSS ==================== + height_to_supply_in_profit: compute_dollars + .then(|| eager!(Height, Sats, "supply_in_profit", v0)), + indexes_to_supply_in_profit: compute_dollars.then(|| { + ComputedValueVecsFromDateIndex::forced_import( + db, + &suffix("supply_in_profit"), + dateindex_to_supply_in_profit + .as_ref() + .map(|v| v.boxed_clone()) + .into(), + version + v0, + last(), + compute_dollars, + indexes, + ) + .unwrap() + }), + height_to_supply_in_loss: compute_dollars + .then(|| eager!(Height, Sats, "supply_in_loss", v0)), + indexes_to_supply_in_loss: compute_dollars.then(|| { + ComputedValueVecsFromDateIndex::forced_import( + db, + &suffix("supply_in_loss"), + dateindex_to_supply_in_loss + .as_ref() + .map(|v| v.boxed_clone()) + .into(), + version + v0, + last(), + compute_dollars, + indexes, + ) + .unwrap() + }), + dateindex_to_supply_in_profit, + dateindex_to_supply_in_loss, + height_to_supply_in_profit_value: compute_dollars.then(|| { + ComputedHeightValueVecs::forced_import( + db, + &suffix("supply_in_profit"), + Source::None, + version + v0, + compute_dollars, + ) + .unwrap() + }), + height_to_supply_in_loss_value: compute_dollars.then(|| { + ComputedHeightValueVecs::forced_import( + db, + &suffix("supply_in_loss"), + Source::None, + version + v0, + compute_dollars, + ) + .unwrap() + }), + + // ==================== UNREALIZED PROFIT & LOSS ==================== + height_to_unrealized_profit: compute_dollars + .then(|| eager!(Height, Dollars, "unrealized_profit", v0)), + indexes_to_unrealized_profit: compute_dollars.then(|| { + ComputedVecsFromDateIndex::forced_import( + db, + &suffix("unrealized_profit"), + dateindex_to_unrealized_profit + .as_ref() + .map(|v| v.boxed_clone()) + .into(), + version + v0, + indexes, + last(), + ) + .unwrap() + }), + height_to_unrealized_loss: compute_dollars + .then(|| eager!(Height, Dollars, "unrealized_loss", v0)), + indexes_to_unrealized_loss: compute_dollars.then(|| { + ComputedVecsFromDateIndex::forced_import( + db, + &suffix("unrealized_loss"), + dateindex_to_unrealized_loss + .as_ref() + .map(|v| v.boxed_clone()) + .into(), + version + v0, + indexes, + last(), + ) + .unwrap() + }), + dateindex_to_unrealized_profit, + dateindex_to_unrealized_loss, + height_to_neg_unrealized_loss: compute_dollars + .then(|| eager!(Height, Dollars, "neg_unrealized_loss", v0)), + indexes_to_neg_unrealized_loss: compute_dollars + .then(|| computed_di!("neg_unrealized_loss", Source::Compute, v0, last())), + height_to_net_unrealized_pnl: compute_dollars + .then(|| eager!(Height, Dollars, "net_unrealized_pnl", v0)), + indexes_to_net_unrealized_pnl: compute_dollars + .then(|| computed_di!("net_unrealized_pnl", Source::Compute, v0, last())), + height_to_total_unrealized_pnl: compute_dollars + .then(|| eager!(Height, Dollars, "total_unrealized_pnl", v0)), + indexes_to_total_unrealized_pnl: compute_dollars + .then(|| computed_di!("total_unrealized_pnl", Source::Compute, v0, last())), + + // ==================== PRICE PAID ==================== + height_to_min_price_paid: compute_dollars + .then(|| eager!(Height, Dollars, "min_price_paid", v0)), + indexes_to_min_price_paid: compute_dollars + .then(|| computed_h!("min_price_paid", Source::None, v0, last())), + height_to_max_price_paid: compute_dollars + .then(|| eager!(Height, Dollars, "max_price_paid", v0)), + indexes_to_max_price_paid: compute_dollars + .then(|| computed_h!("max_price_paid", Source::None, v0, last())), + price_percentiles: (compute_dollars && extended).then(|| { + PricePercentiles::forced_import(db, &suffix(""), version + v0, indexes, true) + .unwrap() + }), + + // ==================== RELATIVE METRICS: UNREALIZED vs MARKET CAP ==================== + height_to_unrealized_profit_rel_to_market_cap: compute_dollars + .then(|| eager!(Height, StoredF32, "unrealized_profit_rel_to_market_cap", v0)), + height_to_unrealized_loss_rel_to_market_cap: compute_dollars + .then(|| eager!(Height, StoredF32, "unrealized_loss_rel_to_market_cap", v0)), + height_to_neg_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| { + eager!( + Height, + StoredF32, + "neg_unrealized_loss_rel_to_market_cap", + v0 + ) + }), + height_to_net_unrealized_pnl_rel_to_market_cap: compute_dollars.then(|| { + eager!( + Height, + StoredF32, + "net_unrealized_pnl_rel_to_market_cap", + v1 + ) + }), + indexes_to_unrealized_profit_rel_to_market_cap: compute_dollars.then(|| { + computed_di!( + "unrealized_profit_rel_to_market_cap", + Source::Compute, + v1, + last() + ) + }), + indexes_to_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| { + computed_di!( + "unrealized_loss_rel_to_market_cap", + Source::Compute, + v1, + last() + ) + }), + indexes_to_neg_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| { + computed_di!( + "neg_unrealized_loss_rel_to_market_cap", + Source::Compute, + v1, + last() + ) + }), + indexes_to_net_unrealized_pnl_rel_to_market_cap: compute_dollars.then(|| { + computed_di!( + "net_unrealized_pnl_rel_to_market_cap", + Source::Compute, + v1, + last() + ) + }), + + // ==================== RELATIVE METRICS: UNREALIZED vs OWN MARKET CAP ==================== + height_to_unrealized_profit_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + eager!( + Height, + StoredF32, + "unrealized_profit_rel_to_own_market_cap", + v1 + ) + }), + height_to_unrealized_loss_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + eager!( + Height, + StoredF32, + "unrealized_loss_rel_to_own_market_cap", + v1 + ) + }), + height_to_neg_unrealized_loss_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + eager!( + Height, + StoredF32, + "neg_unrealized_loss_rel_to_own_market_cap", + v1 + ) + }), + height_to_net_unrealized_pnl_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + eager!( + Height, + StoredF32, + "net_unrealized_pnl_rel_to_own_market_cap", + v2 + ) + }), + indexes_to_unrealized_profit_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + computed_di!( + "unrealized_profit_rel_to_own_market_cap", + Source::Compute, + v2, + last() + ) + }), + indexes_to_unrealized_loss_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + computed_di!( + "unrealized_loss_rel_to_own_market_cap", + Source::Compute, + v2, + last() + ) + }), + indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + computed_di!( + "neg_unrealized_loss_rel_to_own_market_cap", + Source::Compute, + v2, + last() + ) + }), + indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (compute_dollars + && extended + && compute_rel_to_all) + .then(|| { + computed_di!( + "net_unrealized_pnl_rel_to_own_market_cap", + Source::Compute, + v2, + last() + ) + }), + + // ==================== RELATIVE METRICS: UNREALIZED vs OWN TOTAL UNREALIZED ==================== + height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + eager!( + Height, + StoredF32, + "unrealized_profit_rel_to_own_total_unrealized_pnl", + v0 + ) + }), + height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + eager!( + Height, + StoredF32, + "unrealized_loss_rel_to_own_total_unrealized_pnl", + v0 + ) + }), + height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + eager!( + Height, + StoredF32, + "neg_unrealized_loss_rel_to_own_total_unrealized_pnl", + v0 + ) + }), + height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + eager!( + Height, + StoredF32, + "net_unrealized_pnl_rel_to_own_total_unrealized_pnl", + v1 + ) + }), + indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + computed_di!( + "unrealized_profit_rel_to_own_total_unrealized_pnl", + Source::Compute, + v1, + last() + ) + }), + indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + computed_di!( + "unrealized_loss_rel_to_own_total_unrealized_pnl", + Source::Compute, + v1, + last() + ) + }), + indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + computed_di!( + "neg_unrealized_loss_rel_to_own_total_unrealized_pnl", + Source::Compute, + v1, + last() + ) + }), + indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: (compute_dollars + && extended) + .then(|| { + computed_di!( + "net_unrealized_pnl_rel_to_own_total_unrealized_pnl", + Source::Compute, + v1, + last() + ) + }), + + // ==================== RELATIVE METRICS: SUPPLY vs CIRCULATING/OWN ==================== + indexes_to_supply_rel_to_circulating_supply: compute_rel_to_all.then(|| { + computed_h!( + "supply_rel_to_circulating_supply", + Source::Compute, + v1, + last() + ) + }), + height_to_supply_in_profit_rel_to_own_supply: compute_dollars + .then(|| eager!(Height, StoredF64, "supply_in_profit_rel_to_own_supply", v1)), + height_to_supply_in_loss_rel_to_own_supply: compute_dollars + .then(|| eager!(Height, StoredF64, "supply_in_loss_rel_to_own_supply", v1)), + indexes_to_supply_in_profit_rel_to_own_supply: compute_dollars.then(|| { + computed_di!( + "supply_in_profit_rel_to_own_supply", + Source::Compute, + v1, + last() + ) + }), + indexes_to_supply_in_loss_rel_to_own_supply: compute_dollars.then(|| { + computed_di!( + "supply_in_loss_rel_to_own_supply", + Source::Compute, + v1, + last() + ) + }), + height_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all + && compute_dollars) + .then(|| { + eager!( + Height, + StoredF64, + "supply_in_profit_rel_to_circulating_supply", + v1 + ) + }), + height_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all + && compute_dollars) + .then(|| { + eager!( + Height, + StoredF64, + "supply_in_loss_rel_to_circulating_supply", + v1 + ) + }), + indexes_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all + && compute_dollars) + .then(|| { + computed_di!( + "supply_in_profit_rel_to_circulating_supply", + Source::Compute, + v1, + last() + ) + }), + indexes_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all + && compute_dollars) + .then(|| { + computed_di!( + "supply_in_loss_rel_to_circulating_supply", + Source::Compute, + v1, + last() + ) + }), + + // ==================== NET REALIZED PNL DELTAS ==================== + indexes_to_net_realized_pnl_cumulative_30d_delta: compute_dollars.then(|| { + computed_di!( + "net_realized_pnl_cumulative_30d_delta", + Source::Compute, + v3, + last() + ) + }), + indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap: compute_dollars + .then(|| { + computed_di!( + "net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap", + Source::Compute, + v3, + last() + ) + }), + indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: compute_dollars + .then(|| { + computed_di!( + "net_realized_pnl_cumulative_30d_delta_rel_to_market_cap", + Source::Compute, + v3, + last() + ) + }), + }) + } + + /// Returns the minimum length of all height-indexed vectors. + /// Used to determine the starting point for processing. + pub fn min_height_vecs_len(&self) -> usize { + [ + self.height_to_supply.len(), + self.height_to_utxo_count.len(), + self.height_to_realized_cap + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_realized_profit + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_realized_loss + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_value_created + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_adjusted_value_created + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_value_destroyed + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_adjusted_value_destroyed + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_supply_in_profit + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_supply_in_loss + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_unrealized_profit + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_unrealized_loss + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_min_price_paid + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_max_price_paid + .as_ref() + .map_or(usize::MAX, |v| v.len()), + self.height_to_sent.len(), + self.height_to_satdays_destroyed.len(), + self.height_to_satblocks_destroyed.len(), + ] + .into_iter() + .min() + .unwrap() + } + + /// Import state from a checkpoint when resuming processing. + /// Returns the next height to process from. + pub fn import_state( + &mut self, + starting_height: Height, + state: &mut CohortState, + ) -> Result { + if let Some(mut prev_height) = starting_height.decremented() { + if self.height_to_realized_cap.as_mut().is_some() { + prev_height = state.import_at_or_before(prev_height)?; + } + + state.supply.value = self.height_to_supply.into_iter().get_unwrap(prev_height); + state.supply.utxo_count = *self + .height_to_utxo_count + .into_iter() + .get_unwrap(prev_height); + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { + state.realized.um().cap = + height_to_realized_cap.into_iter().get_unwrap(prev_height); + } + + Ok(prev_height.incremented()) + } else { + Err(Error::Str("Unset")) + } + } + + /// Validate that all computed versions match expected values, resetting if needed. + pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + // Always-present vecs + self.height_to_supply.validate_computed_version_or_reset( + base_version + self.height_to_supply.inner_version(), + )?; + self.height_to_utxo_count + .validate_computed_version_or_reset( + base_version + self.height_to_utxo_count.inner_version(), + )?; + self.height_to_sent.validate_computed_version_or_reset( + base_version + self.height_to_sent.inner_version(), + )?; + self.height_to_satblocks_destroyed + .validate_computed_version_or_reset( + base_version + self.height_to_satblocks_destroyed.inner_version(), + )?; + self.height_to_satdays_destroyed + .validate_computed_version_or_reset( + base_version + self.height_to_satdays_destroyed.inner_version(), + )?; + + // Dollar-dependent vecs + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() { + height_to_realized_cap.validate_computed_version_or_reset( + base_version + height_to_realized_cap.inner_version(), + )?; + + Self::validate_optional_vec_version(&mut self.height_to_realized_profit, base_version)?; + Self::validate_optional_vec_version(&mut self.height_to_realized_loss, base_version)?; + Self::validate_optional_vec_version(&mut self.height_to_value_created, base_version)?; + Self::validate_optional_vec_version(&mut self.height_to_value_destroyed, base_version)?; + Self::validate_optional_vec_version( + &mut self.height_to_supply_in_profit, + base_version, + )?; + Self::validate_optional_vec_version(&mut self.height_to_supply_in_loss, base_version)?; + Self::validate_optional_vec_version( + &mut self.height_to_unrealized_profit, + base_version, + )?; + Self::validate_optional_vec_version(&mut self.height_to_unrealized_loss, base_version)?; + Self::validate_optional_vec_version( + &mut self.dateindex_to_supply_in_profit, + base_version, + )?; + Self::validate_optional_vec_version( + &mut self.dateindex_to_supply_in_loss, + base_version, + )?; + Self::validate_optional_vec_version( + &mut self.dateindex_to_unrealized_profit, + base_version, + )?; + Self::validate_optional_vec_version( + &mut self.dateindex_to_unrealized_loss, + base_version, + )?; + Self::validate_optional_vec_version(&mut self.height_to_min_price_paid, base_version)?; + Self::validate_optional_vec_version(&mut self.height_to_max_price_paid, base_version)?; + + if self.height_to_adjusted_value_created.is_some() { + Self::validate_optional_vec_version( + &mut self.height_to_adjusted_value_created, + base_version, + )?; + Self::validate_optional_vec_version( + &mut self.height_to_adjusted_value_destroyed, + base_version, + )?; + } + } + + Ok(()) + } + + /// Helper to validate an optional vec's version. + fn validate_optional_vec_version( + vec: &mut Option>, + base_version: Version, + ) -> Result<()> { + if let Some(v) = vec.as_mut() { + v.validate_computed_version_or_reset(base_version + v.inner_version())?; + } + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/common/mod.rs b/crates/brk_computer/src/stateful_old/common/mod.rs new file mode 100644 index 000000000..fc02e4666 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/common/mod.rs @@ -0,0 +1,19 @@ +//! Common vector structs and logic shared between UTXO and Address cohorts. +//! +//! This module contains the `Vecs` struct which holds all the computed vectors +//! for a single cohort, along with methods for importing, flushing, and computing. +//! +//! ## Module Organization +//! +//! The implementation is split across multiple files for maintainability: +//! - `vecs.rs`: Struct definition with field documentation +//! - `import.rs`: Import, validation, and initialization methods +//! - `push.rs`: Per-block push and flush methods +//! - `compute.rs`: Post-processing computation methods + +mod compute; +mod import; +mod push; +mod vecs; + +pub use vecs::Vecs; diff --git a/crates/brk_computer/src/stateful_old/common/push.rs b/crates/brk_computer/src/stateful_old/common/push.rs new file mode 100644 index 000000000..b28b9e3df --- /dev/null +++ b/crates/brk_computer/src/stateful_old/common/push.rs @@ -0,0 +1,178 @@ +//! Push and flush methods for Vecs. +//! +//! This module contains methods for: +//! - `truncate_push`: Push state values to height-indexed vectors +//! - `compute_then_truncate_push_unrealized_states`: Compute and push unrealized states +//! - `safe_flush_stateful_vecs`: Safely flush all stateful vectors + +use brk_error::Result; +use brk_types::{DateIndex, Dollars, Height, StoredU64}; +use vecdb::{AnyStoredVec, Exit, GenericStoredVec}; + +use crate::{stateful::Flushable, states::CohortState, utils::OptionExt}; + +use super::Vecs; + +impl Vecs { + pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> { + self.height_to_supply + .truncate_push(height, state.supply.value)?; + + self.height_to_utxo_count + .truncate_push(height, StoredU64::from(state.supply.utxo_count))?; + + self.height_to_sent.truncate_push(height, state.sent)?; + + self.height_to_satblocks_destroyed + .truncate_push(height, state.satblocks_destroyed)?; + + self.height_to_satdays_destroyed + .truncate_push(height, state.satdays_destroyed)?; + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { + let realized = state.realized.as_ref().unwrap_or_else(|| { + dbg!((&state.realized, &state.supply)); + panic!(); + }); + + height_to_realized_cap.truncate_push(height, realized.cap)?; + + self.height_to_realized_profit + .um() + .truncate_push(height, realized.profit)?; + self.height_to_realized_loss + .um() + .truncate_push(height, realized.loss)?; + self.height_to_value_created + .um() + .truncate_push(height, realized.value_created)?; + self.height_to_value_destroyed + .um() + .truncate_push(height, realized.value_destroyed)?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created + .um() + .truncate_push(height, realized.adj_value_created)?; + self.height_to_adjusted_value_destroyed + .um() + .truncate_push(height, realized.adj_value_destroyed)?; + } + } + Ok(()) + } + + pub fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + state: &CohortState, + ) -> Result<()> { + if let Some(height_price) = height_price { + self.height_to_min_price_paid.um().truncate_push( + height, + state + .price_to_amount_first_key_value() + .map(|(&dollars, _)| dollars) + .unwrap_or(Dollars::NAN), + )?; + self.height_to_max_price_paid.um().truncate_push( + height, + state + .price_to_amount_last_key_value() + .map(|(&dollars, _)| dollars) + .unwrap_or(Dollars::NAN), + )?; + + let (height_unrealized_state, date_unrealized_state) = + state.compute_unrealized_states(height_price, date_price.unwrap()); + + self.height_to_supply_in_profit + .um() + .truncate_push(height, height_unrealized_state.supply_in_profit)?; + self.height_to_supply_in_loss + .um() + .truncate_push(height, height_unrealized_state.supply_in_loss)?; + self.height_to_unrealized_profit + .um() + .truncate_push(height, height_unrealized_state.unrealized_profit)?; + self.height_to_unrealized_loss + .um() + .truncate_push(height, height_unrealized_state.unrealized_loss)?; + + if let Some(date_unrealized_state) = date_unrealized_state { + let dateindex = dateindex.unwrap(); + + self.dateindex_to_supply_in_profit + .um() + .truncate_push(dateindex, date_unrealized_state.supply_in_profit)?; + self.dateindex_to_supply_in_loss + .um() + .truncate_push(dateindex, date_unrealized_state.supply_in_loss)?; + self.dateindex_to_unrealized_profit + .um() + .truncate_push(dateindex, date_unrealized_state.unrealized_profit)?; + self.dateindex_to_unrealized_loss + .um() + .truncate_push(dateindex, date_unrealized_state.unrealized_loss)?; + } + + // Compute and push price percentiles + if let Some(price_percentiles) = self.price_percentiles.as_mut() { + let percentile_prices = state.compute_percentile_prices(); + price_percentiles.truncate_push(height, &percentile_prices)?; + } + } + + Ok(()) + } + + pub fn safe_flush_stateful_vecs( + &mut self, + height: Height, + exit: &Exit, + state: &mut CohortState, + ) -> Result<()> { + self.height_to_supply.safe_write(exit)?; + self.height_to_utxo_count.safe_write(exit)?; + self.height_to_sent.safe_write(exit)?; + self.height_to_satdays_destroyed.safe_write(exit)?; + self.height_to_satblocks_destroyed.safe_write(exit)?; + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { + height_to_realized_cap.safe_write(exit)?; + self.height_to_realized_profit.um().safe_write(exit)?; + self.height_to_realized_loss.um().safe_write(exit)?; + self.height_to_value_created.um().safe_write(exit)?; + self.height_to_value_destroyed.um().safe_write(exit)?; + self.height_to_supply_in_profit.um().safe_write(exit)?; + self.height_to_supply_in_loss.um().safe_write(exit)?; + self.height_to_unrealized_profit.um().safe_write(exit)?; + self.height_to_unrealized_loss.um().safe_write(exit)?; + self.dateindex_to_supply_in_profit.um().safe_write(exit)?; + self.dateindex_to_supply_in_loss.um().safe_write(exit)?; + self.dateindex_to_unrealized_profit.um().safe_write(exit)?; + self.dateindex_to_unrealized_loss.um().safe_write(exit)?; + self.height_to_min_price_paid.um().safe_write(exit)?; + self.height_to_max_price_paid.um().safe_write(exit)?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created + .um() + .safe_write(exit)?; + self.height_to_adjusted_value_destroyed + .um() + .safe_write(exit)?; + } + + // Uses Flushable trait - Option impl handles None case + self.price_percentiles.safe_write(exit)?; + } + + state.commit(height)?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/common/vecs.rs b/crates/brk_computer/src/stateful_old/common/vecs.rs new file mode 100644 index 000000000..1ea0d06f9 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/common/vecs.rs @@ -0,0 +1,210 @@ +use brk_grouper::Filter; +use brk_traversable::Traversable; +use brk_types::{DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64}; +use vecdb::{EagerVec, PcoVec}; + +use crate::grouped::{ + ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex, + ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, + PricePercentiles, +}; + +/// Common vectors shared between UTXO and Address cohorts. +/// +/// This struct contains all the computed vectors for a single cohort. The fields are +/// organized into logical groups matching the initialization order in `forced_import`. +/// +/// ## Field Groups +/// - **Supply & UTXO count**: Basic supply metrics (always computed) +/// - **Activity**: Sent amounts, satblocks/satdays destroyed +/// - **Realized**: Realized cap, profit/loss, value created/destroyed, SOPR +/// - **Unrealized**: Unrealized profit/loss, supply in profit/loss +/// - **Price**: Min/max price paid, price percentiles +/// - **Relative metrics**: Ratios relative to market cap, realized cap, etc. +#[derive(Clone, Traversable)] +pub struct Vecs { + #[traversable(skip)] + pub filter: Filter, + + // ==================== SUPPLY & UTXO COUNT ==================== + // Always computed - core supply metrics + pub height_to_supply: EagerVec>, + pub height_to_supply_value: ComputedHeightValueVecs, + pub indexes_to_supply: ComputedValueVecsFromDateIndex, + pub height_to_utxo_count: EagerVec>, + pub indexes_to_utxo_count: ComputedVecsFromHeight, + pub height_to_supply_half_value: ComputedHeightValueVecs, + pub indexes_to_supply_half: ComputedValueVecsFromDateIndex, + + // ==================== ACTIVITY ==================== + // Always computed - transaction activity metrics + pub height_to_sent: EagerVec>, + pub indexes_to_sent: ComputedValueVecsFromHeight, + pub height_to_satblocks_destroyed: EagerVec>, + pub height_to_satdays_destroyed: EagerVec>, + pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight, + pub indexes_to_coindays_destroyed: ComputedVecsFromHeight, + + // ==================== REALIZED CAP & PRICE ==================== + // Conditional on compute_dollars + pub height_to_realized_cap: Option>>, + pub indexes_to_realized_cap: Option>, + pub indexes_to_realized_price: Option>, + pub indexes_to_realized_price_extra: Option, + pub indexes_to_realized_cap_rel_to_own_market_cap: Option>, + pub indexes_to_realized_cap_30d_delta: Option>, + + // ==================== REALIZED PROFIT & LOSS ==================== + // Conditional on compute_dollars + pub height_to_realized_profit: Option>>, + pub indexes_to_realized_profit: Option>, + pub height_to_realized_loss: Option>>, + pub indexes_to_realized_loss: Option>, + pub indexes_to_neg_realized_loss: Option>, + pub indexes_to_net_realized_pnl: Option>, + pub indexes_to_realized_value: Option>, + pub indexes_to_realized_profit_rel_to_realized_cap: Option>, + pub indexes_to_realized_loss_rel_to_realized_cap: Option>, + pub indexes_to_net_realized_pnl_rel_to_realized_cap: Option>, + pub height_to_total_realized_pnl: Option>>, + pub indexes_to_total_realized_pnl: Option>, + pub dateindex_to_realized_profit_to_loss_ratio: Option>>, + + // ==================== VALUE CREATED & DESTROYED ==================== + // Conditional on compute_dollars + pub height_to_value_created: Option>>, + pub indexes_to_value_created: Option>, + pub height_to_value_destroyed: Option>>, + pub indexes_to_value_destroyed: Option>, + pub height_to_adjusted_value_created: Option>>, + pub indexes_to_adjusted_value_created: Option>, + pub height_to_adjusted_value_destroyed: Option>>, + pub indexes_to_adjusted_value_destroyed: Option>, + + // ==================== SOPR ==================== + // Spent Output Profit Ratio - conditional on compute_dollars + pub dateindex_to_sopr: Option>>, + pub dateindex_to_sopr_7d_ema: Option>>, + pub dateindex_to_sopr_30d_ema: Option>>, + pub dateindex_to_adjusted_sopr: Option>>, + pub dateindex_to_adjusted_sopr_7d_ema: Option>>, + pub dateindex_to_adjusted_sopr_30d_ema: Option>>, + + // ==================== SELL SIDE RISK ==================== + // Conditional on compute_dollars + pub dateindex_to_sell_side_risk_ratio: Option>>, + pub dateindex_to_sell_side_risk_ratio_7d_ema: Option>>, + pub dateindex_to_sell_side_risk_ratio_30d_ema: Option>>, + + // ==================== SUPPLY IN PROFIT/LOSS ==================== + // Conditional on compute_dollars + pub height_to_supply_in_profit: Option>>, + pub indexes_to_supply_in_profit: Option, + pub height_to_supply_in_loss: Option>>, + pub indexes_to_supply_in_loss: Option, + pub dateindex_to_supply_in_profit: Option>>, + pub dateindex_to_supply_in_loss: Option>>, + pub height_to_supply_in_profit_value: Option, + pub height_to_supply_in_loss_value: Option, + + // ==================== UNREALIZED PROFIT & LOSS ==================== + // Conditional on compute_dollars + pub height_to_unrealized_profit: Option>>, + pub indexes_to_unrealized_profit: Option>, + pub height_to_unrealized_loss: Option>>, + pub indexes_to_unrealized_loss: Option>, + pub dateindex_to_unrealized_profit: Option>>, + pub dateindex_to_unrealized_loss: Option>>, + pub height_to_neg_unrealized_loss: Option>>, + pub indexes_to_neg_unrealized_loss: Option>, + pub height_to_net_unrealized_pnl: Option>>, + pub indexes_to_net_unrealized_pnl: Option>, + pub height_to_total_unrealized_pnl: Option>>, + pub indexes_to_total_unrealized_pnl: Option>, + + // ==================== PRICE PAID ==================== + // Conditional on compute_dollars + pub height_to_min_price_paid: Option>>, + pub indexes_to_min_price_paid: Option>, + pub height_to_max_price_paid: Option>>, + pub indexes_to_max_price_paid: Option>, + pub price_percentiles: Option, + + // ==================== RELATIVE METRICS: UNREALIZED vs MARKET CAP ==================== + // Conditional on compute_dollars + pub height_to_unrealized_profit_rel_to_market_cap: Option>>, + pub height_to_unrealized_loss_rel_to_market_cap: Option>>, + pub height_to_neg_unrealized_loss_rel_to_market_cap: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_market_cap: Option>>, + pub indexes_to_unrealized_profit_rel_to_market_cap: + Option>, + pub indexes_to_unrealized_loss_rel_to_market_cap: Option>, + pub indexes_to_neg_unrealized_loss_rel_to_market_cap: + Option>, + pub indexes_to_net_unrealized_pnl_rel_to_market_cap: + Option>, + + // ==================== RELATIVE METRICS: UNREALIZED vs OWN MARKET CAP ==================== + // Conditional on compute_dollars && extended && compute_rel_to_all + pub height_to_unrealized_profit_rel_to_own_market_cap: + Option>>, + pub height_to_unrealized_loss_rel_to_own_market_cap: + Option>>, + pub height_to_neg_unrealized_loss_rel_to_own_market_cap: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_own_market_cap: + Option>>, + pub indexes_to_unrealized_profit_rel_to_own_market_cap: + Option>, + pub indexes_to_unrealized_loss_rel_to_own_market_cap: + Option>, + pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap: + Option>, + pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap: + Option>, + + // ==================== RELATIVE METRICS: UNREALIZED vs OWN TOTAL UNREALIZED ==================== + // Conditional on compute_dollars && extended + pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: + Option>>, + pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>>, + pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>>, + pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: + Option>>, + pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: + Option>, + pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>, + pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: + Option>, + pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: + Option>, + + // ==================== RELATIVE METRICS: SUPPLY vs CIRCULATING/OWN ==================== + // Conditional on compute_dollars + pub indexes_to_supply_rel_to_circulating_supply: Option>, + pub height_to_supply_in_profit_rel_to_own_supply: Option>>, + pub height_to_supply_in_loss_rel_to_own_supply: Option>>, + pub indexes_to_supply_in_profit_rel_to_own_supply: Option>, + pub indexes_to_supply_in_loss_rel_to_own_supply: Option>, + pub height_to_supply_in_profit_rel_to_circulating_supply: + Option>>, + pub height_to_supply_in_loss_rel_to_circulating_supply: + Option>>, + pub indexes_to_supply_in_profit_rel_to_circulating_supply: + Option>, + pub indexes_to_supply_in_loss_rel_to_circulating_supply: + Option>, + + // ==================== NET REALIZED PNL DELTAS ==================== + // Conditional on compute_dollars + pub indexes_to_net_realized_pnl_cumulative_30d_delta: + Option>, + pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap: + Option>, + pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: + Option>, +} diff --git a/crates/brk_computer/src/stateful_old/flushable.rs b/crates/brk_computer/src/stateful_old/flushable.rs new file mode 100644 index 000000000..449e93f39 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/flushable.rs @@ -0,0 +1,80 @@ +//! Traits for consistent state flushing and importing. +//! +//! These traits ensure all stateful components follow the same patterns +//! for checkpoint/resume operations, preventing bugs where new fields +//! are forgotten during flush operations. + +use brk_error::Result; +use brk_types::Height; +use vecdb::Exit; + +/// Trait for components that can be flushed to disk. +/// +/// This is for simple flush operations that don't require height tracking. +pub trait Flushable { + /// Safely flush data to disk. + fn safe_flush(&mut self, exit: &Exit) -> Result<()>; + + /// Write to mmap without fsync. Data visible to readers immediately but not durable. + fn safe_write(&mut self, exit: &Exit) -> Result<()>; +} + +/// Trait for stateful components that track data indexed by height. +/// +/// This ensures consistent patterns for: +/// - Flushing state at checkpoints +/// - Importing state when resuming from a checkpoint +/// - Resetting state when starting from scratch +pub trait HeightFlushable { + /// Flush state to disk at the given height checkpoint. + fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()>; + + /// Import state from the most recent checkpoint at or before the given height. + /// Returns the actual height that was imported. + fn import_at_or_before(&mut self, height: Height) -> Result; + + /// Reset state for starting from scratch. + fn reset(&mut self) -> Result<()>; +} + +/// Blanket implementation for Option where T: Flushable +impl Flushable for Option { + fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.safe_flush(exit)?; + } + Ok(()) + } + + fn safe_write(&mut self, exit: &Exit) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.safe_write(exit)?; + } + Ok(()) + } +} + +/// Blanket implementation for Option where T: HeightFlushable +impl HeightFlushable for Option { + fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.flush_at_height(height, exit)?; + } + Ok(()) + } + + fn import_at_or_before(&mut self, height: Height) -> Result { + if let Some(inner) = self.as_mut() { + inner.import_at_or_before(height) + } else { + Ok(height) + } + } + + fn reset(&mut self) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.reset()?; + } + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/mod.rs b/crates/brk_computer/src/stateful_old/mod.rs new file mode 100644 index 000000000..9612a53e6 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/mod.rs @@ -0,0 +1,1616 @@ +//! Stateful computation module for Bitcoin UTXO and address cohort analysis. +//! +//! This module contains the main computation loop that processes blocks and computes +//! various metrics for UTXO cohorts (grouped by age, amount, etc.) and address cohorts. +//! +//! ## Architecture +//! +//! The module is organized as follows: +//! +//! - **`Vecs`**: Main struct holding all computed vectors and state +//! - **Cohort Types**: +//! - **Separate cohorts**: Have full state tracking (e.g., UTXOs 1-2 years old) +//! - **Aggregate cohorts**: Computed from separate cohorts (e.g., all, sth, lth) +//! +//! ## Checkpoint/Resume +//! +//! The computation supports checkpointing via `flush_states()` which saves: +//! - All separate cohorts' state (via `safe_flush_stateful_vecs`) +//! - Aggregate cohorts' `price_to_amount` (via `HeightFlushable` trait) +//! - Aggregate cohorts' `price_percentiles` (via `Flushable` trait) +//! +//! Resume is handled by: +//! - `import_state()` for separate cohorts +//! - `import_aggregate_price_to_amount()` for aggregate cohorts +//! +//! ## Key Traits +//! +//! - `Flushable`: Simple flush operations (no height tracking) +//! - `HeightFlushable`: Height-indexed state (flush, import, reset) + +use std::{cmp::Ordering, collections::BTreeSet, mem, path::Path, thread}; + +use brk_error::Result; +use brk_grouper::ByAddressType; +use brk_indexer::Indexer; +use brk_traversable::Traversable; +use brk_types::{ + AnyAddressDataIndexEnum, AnyAddressIndex, DateIndex, Dollars, EmptyAddressData, + EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, OutputType, Sats, StoredU64, + TxInIndex, TxIndex, TxOutIndex, TypeIndex, Version, +}; +use log::info; +use rayon::prelude::*; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; +use vecdb::{ + AnyStoredVec, AnyVec, BytesVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, + ImportOptions, ImportableVec, IterableCloneableVec, IterableVec, LazyVecFrom1, PAGE_SIZE, + PcoVec, Stamp, TypedVecIterator, VecIndex, +}; + +use crate::{ + BlockState, Indexes, SupplyState, Transacted, chain, + grouped::{ + ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, + VecBuilderOptions, + }, + indexes, price, + utils::OptionExt, +}; + +mod address_cohort; +mod address_cohorts; +mod address_indexes; +mod addresstype; +mod common; +mod flushable; +mod range_map; +mod readers; +mod r#trait; +mod transaction_processing; +mod utxo_cohort; +mod utxo_cohorts; +mod withaddressdatasource; + +pub use flushable::{Flushable, HeightFlushable}; + +use address_indexes::{AddressesDataVecs, AnyAddressIndexesVecs}; +use addresstype::*; +use range_map::*; +use readers::{ + IndexerReaders, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex, +}; +use r#trait::*; +use withaddressdatasource::*; + +type TxIndexVec = SmallVec<[TxIndex; 4]>; + +const VERSION: Version = Version::new(21); + +const BIP30_DUPLICATE_COINBASE_HEIGHT_1: u32 = 91_842; +const BIP30_DUPLICATE_COINBASE_HEIGHT_2: u32 = 91_880; +const BIP30_ORIGINAL_COINBASE_HEIGHT_1: u32 = 91_812; +const BIP30_ORIGINAL_COINBASE_HEIGHT_2: u32 = 91_722; +const FLUSH_INTERVAL: usize = 10_000; + +#[derive(Clone, Traversable)] +pub struct Vecs { + db: Database, + + // --- + // States + // --- + pub chain_state: BytesVec, + pub any_address_indexes: AnyAddressIndexesVecs, + pub addresses_data: AddressesDataVecs, + pub utxo_cohorts: utxo_cohorts::Vecs, + pub address_cohorts: address_cohorts::Vecs, + + pub height_to_unspendable_supply: EagerVec>, + pub height_to_opreturn_supply: EagerVec>, + pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount, + pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount, + + // --- + // Computed + // --- + pub addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount, + pub addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount, + pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, + pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight, + pub indexes_to_addr_count: ComputedVecsFromHeight, + pub indexes_to_empty_addr_count: ComputedVecsFromHeight, + pub height_to_market_cap: Option>, + pub indexes_to_market_cap: Option>, + pub loadedaddressindex_to_loadedaddressindex: + LazyVecFrom1, + pub emptyaddressindex_to_emptyaddressindex: + LazyVecFrom1, +} + +const SAVED_STAMPED_CHANGES: u16 = 10; + +impl Vecs { + pub fn forced_import( + parent: &Path, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + ) -> Result { + let db_path = parent.join("stateful"); + let states_path = db_path.join("states"); + + let db = Database::open(&db_path)?; + db.set_min_len(PAGE_SIZE * 20_000_000)?; + db.set_min_regions(50_000)?; + + let compute_dollars = price.is_some(); + let v0 = version + VERSION + Version::ZERO; + let v1 = version + VERSION + Version::ONE; + let v2 = version + VERSION + Version::TWO; + + let utxo_cohorts = + utxo_cohorts::Vecs::forced_import(&db, version, indexes, price, &states_path)?; + + let loadedaddressindex_to_loadedaddressdata = BytesVec::forced_import_with( + ImportOptions::new(&db, "loadedaddressdata", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?; + let emptyaddressindex_to_emptyaddressdata = BytesVec::forced_import_with( + ImportOptions::new(&db, "emptyaddressdata", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?; + let loadedaddressindex_to_loadedaddressindex = LazyVecFrom1::init( + "loadedaddressindex", + v0, + loadedaddressindex_to_loadedaddressdata.boxed_clone(), + |index, _| Some(index), + ); + let emptyaddressindex_to_emptyaddressindex = LazyVecFrom1::init( + "emptyaddressindex", + v0, + emptyaddressindex_to_emptyaddressdata.boxed_clone(), + |index, _| Some(index), + ); + + let this = Self { + chain_state: BytesVec::forced_import_with( + ImportOptions::new(&db, "chain", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + + height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?, + indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( + &db, + "unspendable_supply", + Source::None, + v0, + VecBuilderOptions::default().add_last(), + compute_dollars, + indexes, + )?, + height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?, + indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( + &db, + "opreturn_supply", + Source::None, + v0, + VecBuilderOptions::default().add_last(), + compute_dollars, + indexes, + )?, + indexes_to_addr_count: ComputedVecsFromHeight::forced_import( + &db, + "addr_count", + Source::Compute, + v0, + indexes, + VecBuilderOptions::default().add_last(), + )?, + indexes_to_empty_addr_count: ComputedVecsFromHeight::forced_import( + &db, + "empty_addr_count", + Source::Compute, + v0, + indexes, + VecBuilderOptions::default().add_last(), + )?, + height_to_market_cap: compute_dollars.then(|| { + LazyVecFrom1::init( + "market_cap", + v1, + utxo_cohorts + .all + .inner + .height_to_supply_value + .dollars + .as_ref() + .unwrap() + .boxed_clone(), + |height: Height, iter| iter.get(height), + ) + }), + indexes_to_market_cap: compute_dollars.then(|| { + ComputedVecsFromDateIndex::forced_import( + &db, + "market_cap", + Source::Compute, + v2, + indexes, + VecBuilderOptions::default().add_last(), + ) + .unwrap() + }), + addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::from( + ByAddressType::new_with_name(|name| { + Ok(EagerVec::forced_import( + &db, + &format!("{name}_addr_count"), + v0, + )?) + })?, + ), + addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount::from( + ByAddressType::new_with_name(|name| { + Ok(EagerVec::forced_import( + &db, + &format!("{name}_empty_addr_count"), + v0, + )?) + })?, + ), + addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::from( + ByAddressType::new_with_name(|name| { + ComputedVecsFromHeight::forced_import( + &db, + &format!("{name}_addr_count"), + Source::None, + v0, + indexes, + VecBuilderOptions::default().add_last(), + ) + })?, + ), + addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount::from( + ByAddressType::new_with_name(|name| { + ComputedVecsFromHeight::forced_import( + &db, + &format!("{name}_empty_addr_count"), + Source::None, + v0, + indexes, + VecBuilderOptions::default().add_last(), + ) + })?, + ), + utxo_cohorts, + address_cohorts: address_cohorts::Vecs::forced_import( + &db, + version, + indexes, + price, + &states_path, + )?, + + any_address_indexes: AnyAddressIndexesVecs { + p2a: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2pk33: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2pk65: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2pkh: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2sh: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2tr: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2wpkh: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + p2wsh: BytesVec::forced_import_with( + ImportOptions::new(&db, "anyaddressindex", v0) + .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), + )?, + }, + addresses_data: AddressesDataVecs { + loaded: loadedaddressindex_to_loadedaddressdata, + empty: emptyaddressindex_to_emptyaddressdata, + }, + loadedaddressindex_to_loadedaddressindex, + emptyaddressindex_to_emptyaddressindex, + + db, + }; + + this.db.retain_regions( + this.iter_any_exportable() + .flat_map(|v| v.region_names()) + .collect(), + )?; + + this.db.compact()?; + + Ok(this) + } + + #[allow(clippy::too_many_arguments)] + pub fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + chain: &chain::Vecs, + price: Option<&price::Vecs>, + // Must take ownership as its indexes will be updated for this specific function + starting_indexes: &mut Indexes, + exit: &Exit, + ) -> Result<()> { + self.compute_(indexer, indexes, chain, price, starting_indexes, exit)?; + self.db.compact()?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn compute_( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + chain: &chain::Vecs, + price: Option<&price::Vecs>, + // Must take ownership as its indexes will be updated for this specific function + starting_indexes: &mut Indexes, + exit: &Exit, + ) -> Result<()> { + let dateindex_to_first_height = &indexes.dateindex_to_first_height; + let dateindex_to_height_count = &indexes.dateindex_to_height_count; + let dateindex_to_price_close = price + .as_ref() + .map(|price| price.timeindexes_to_price_close.dateindex.u()); + let height_to_date_fixed = &indexes.height_to_date_fixed; + let height_to_first_p2aaddressindex = &indexer.vecs.height_to_first_p2aaddressindex; + let height_to_first_p2pk33addressindex = &indexer.vecs.height_to_first_p2pk33addressindex; + let height_to_first_p2pk65addressindex = &indexer.vecs.height_to_first_p2pk65addressindex; + let height_to_first_p2pkhaddressindex = &indexer.vecs.height_to_first_p2pkhaddressindex; + let height_to_first_p2shaddressindex = &indexer.vecs.height_to_first_p2shaddressindex; + let height_to_first_p2traddressindex = &indexer.vecs.height_to_first_p2traddressindex; + let height_to_first_p2wpkhaddressindex = &indexer.vecs.height_to_first_p2wpkhaddressindex; + let height_to_first_p2wshaddressindex = &indexer.vecs.height_to_first_p2wshaddressindex; + let height_to_first_txindex = &indexer.vecs.height_to_first_txindex; + let height_to_txindex_count = chain.indexes_to_tx_count.height.u(); + let height_to_first_txinindex = &indexer.vecs.height_to_first_txinindex; + let height_to_first_txoutindex = &indexer.vecs.height_to_first_txoutindex; + let height_to_input_count = chain.indexes_to_input_count.height.unwrap_sum(); + let height_to_output_count = chain.indexes_to_output_count.height.unwrap_sum(); + let height_to_price_close = price + .as_ref() + .map(|price| &price.chainindexes_to_price_close.height); + let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed; + let height_to_tx_count = chain.indexes_to_tx_count.height.u(); + let height_to_unclaimed_rewards = chain + .indexes_to_unclaimed_rewards + .sats + .height + .as_ref() + .unwrap(); + let txindex_to_first_txoutindex = &indexer.vecs.txindex_to_first_txoutindex; + let txindex_to_height = &indexer.vecs.txindex_to_height; + let txindex_to_input_count = &indexes.txindex_to_input_count; + let txindex_to_output_count = &indexes.txindex_to_output_count; + let txinindex_to_outpoint = &indexer.vecs.txinindex_to_outpoint; + let txoutindex_to_outputtype = &indexer.vecs.txoutindex_to_outputtype; + let txoutindex_to_txindex = &indexer.vecs.txoutindex_to_txindex; + let txoutindex_to_typeindex = &indexer.vecs.txoutindex_to_typeindex; + let txoutindex_to_value = &indexer.vecs.txoutindex_to_value; + + let mut height_to_price_close_iter = height_to_price_close.as_ref().map(|v| v.into_iter()); + let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter(); + + let base_version = Version::ZERO + + dateindex_to_first_height.version() + + dateindex_to_height_count.version() + + dateindex_to_price_close + .as_ref() + .map_or(Version::ZERO, |v| v.version()) + + height_to_date_fixed.version() + + height_to_first_p2aaddressindex.version() + + height_to_first_p2pk33addressindex.version() + + height_to_first_p2pk65addressindex.version() + + height_to_first_p2pkhaddressindex.version() + + height_to_first_p2shaddressindex.version() + + height_to_first_p2traddressindex.version() + + height_to_first_p2wpkhaddressindex.version() + + height_to_first_p2wshaddressindex.version() + + height_to_first_txindex.version() + + height_to_txindex_count.version() + + height_to_first_txinindex.version() + + height_to_first_txoutindex.version() + + height_to_input_count.version() + + height_to_output_count.version() + + height_to_price_close + .as_ref() + .map_or(Version::ZERO, |v| v.version()) + + height_to_timestamp_fixed.version() + + height_to_tx_count.version() + + height_to_unclaimed_rewards.version() + + txindex_to_first_txoutindex.version() + + txindex_to_height.version() + + txindex_to_input_count.version() + + txindex_to_output_count.version() + + txinindex_to_outpoint.version() + + txoutindex_to_outputtype.version() + + txoutindex_to_txindex.version() + + txoutindex_to_typeindex.version() + + txoutindex_to_value.version(); + + let mut separate_utxo_vecs = self.utxo_cohorts.iter_separate_mut().collect::>(); + let mut separate_address_vecs = + self.address_cohorts.iter_separate_mut().collect::>(); + + separate_utxo_vecs + .par_iter_mut() + .try_for_each(|v| v.validate_computed_versions(base_version))?; + separate_address_vecs + .par_iter_mut() + .try_for_each(|v| v.validate_computed_versions(base_version))?; + self.height_to_unspendable_supply + .validate_computed_version_or_reset( + base_version + self.height_to_unspendable_supply.inner_version(), + )?; + self.height_to_opreturn_supply + .validate_computed_version_or_reset( + base_version + self.height_to_opreturn_supply.inner_version(), + )?; + + let mut chain_state_starting_height = Height::from(self.chain_state.len()); + let stateful_starting_height = match separate_utxo_vecs + .par_iter_mut() + .map(|v| Height::from(v.min_height_vecs_len())) + .min() + .unwrap_or_default() + .min( + separate_address_vecs + .par_iter_mut() + .map(|v| Height::from(v.min_height_vecs_len())) + .min() + .unwrap_or_default(), + ) + .min(chain_state_starting_height) + .min(self.any_address_indexes.min_stamped_height()) + .min(self.addresses_data.min_stamped_height()) + .min(Height::from(self.height_to_unspendable_supply.len())) + .min(Height::from(self.height_to_opreturn_supply.len())) + .cmp(&chain_state_starting_height) + { + Ordering::Greater => unreachable!(), + Ordering::Equal => chain_state_starting_height, + Ordering::Less => Height::ZERO, + }; + + let starting_height = starting_indexes.height.min(stateful_starting_height); + let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp()); + if starting_height <= last_height { + let stamp = starting_height.into(); + let starting_height = if starting_height.is_not_zero() { + let mut set = [self.chain_state.rollback_before(stamp)?] + .into_iter() + .chain(self.any_address_indexes.rollback_before(stamp)?) + .chain(self.addresses_data.rollback_before(stamp)?) + .map(Height::from) + .map(Height::incremented) + .collect::>(); + + if set.len() == 1 { + set.pop_first().unwrap() + } else { + Height::ZERO + } + } else { + Height::ZERO + }; + + let starting_height = if starting_height.is_not_zero() + && separate_utxo_vecs + .iter_mut() + .map(|v| v.import_state(starting_height).unwrap_or_default()) + .all(|h| h == starting_height) + { + starting_height + } else { + Height::ZERO + }; + + let starting_height = if starting_height.is_not_zero() + && separate_address_vecs + .iter_mut() + .map(|v| v.import_state(starting_height).unwrap_or_default()) + .all(|h| h == starting_height) + { + starting_height + } else { + Height::ZERO + }; + + // Import aggregate cohorts' price_to_amount. + // Need to temporarily release the separate vecs borrows since iter_aggregate_mut + // borrows the whole UTXOGroups struct, even though it accesses non-overlapping fields. + let starting_height = { + drop(separate_utxo_vecs); + drop(separate_address_vecs); + let imported_height = self + .utxo_cohorts + .import_aggregate_price_to_amount(starting_height)?; + let result = if starting_height.is_not_zero() && imported_height == starting_height + { + starting_height + } else { + Height::ZERO + }; + separate_utxo_vecs = self.utxo_cohorts.iter_separate_mut().collect(); + separate_address_vecs = self.address_cohorts.iter_separate_mut().collect(); + result + }; + + let mut chain_state: Vec; + if starting_height.is_not_zero() { + chain_state = self + .chain_state + .collect_range(None, None) + .into_iter() + .enumerate() + .map(|(height, supply)| { + let height = Height::from(height); + let timestamp = height_to_timestamp_fixed_iter.get_unwrap(height); + let price = height_to_price_close_iter + .as_mut() + .map(|i| *i.get_unwrap(height)); + BlockState { + timestamp, + price, + supply, + } + }) + .collect::>(); + } else { + info!("Starting processing utxos from the start"); + + chain_state = vec![]; + + self.any_address_indexes.reset()?; + self.addresses_data.reset()?; + + separate_utxo_vecs.par_iter_mut().try_for_each(|v| { + v.reset_state_starting_height(); + v.state.um().reset_price_to_amount_if_needed() + })?; + + // Reset aggregate cohorts' price_to_amount + self.utxo_cohorts.reset_aggregate_price_to_amount()?; + + separate_address_vecs.par_iter_mut().try_for_each(|v| { + v.reset_state_starting_height(); + v.state.um().reset_price_to_amount_if_needed() + })?; + } + + chain_state_starting_height = starting_height; + + starting_indexes.update_from_height(starting_height, indexes); + + let ir = IndexerReaders::new(indexer); + + let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter(); + let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter(); + let mut dateindex_to_price_close_iter = + dateindex_to_price_close.as_ref().map(|v| v.into_iter()); + let mut height_to_date_fixed_iter = height_to_date_fixed.into_iter(); + let mut height_to_first_p2aaddressindex_iter = + height_to_first_p2aaddressindex.into_iter(); + let mut height_to_first_p2pk33addressindex_iter = + height_to_first_p2pk33addressindex.into_iter(); + let mut height_to_first_p2pk65addressindex_iter = + height_to_first_p2pk65addressindex.into_iter(); + let mut height_to_first_p2pkhaddressindex_iter = + height_to_first_p2pkhaddressindex.into_iter(); + let mut height_to_first_p2shaddressindex_iter = + height_to_first_p2shaddressindex.into_iter(); + let mut height_to_first_p2traddressindex_iter = + height_to_first_p2traddressindex.into_iter(); + let mut height_to_first_p2wpkhaddressindex_iter = + height_to_first_p2wpkhaddressindex.into_iter(); + let mut height_to_first_p2wshaddressindex_iter = + height_to_first_p2wshaddressindex.into_iter(); + let mut height_to_first_txindex_iter = height_to_first_txindex.into_iter(); + let mut height_to_first_txinindex_iter = height_to_first_txinindex.into_iter(); + let mut height_to_first_txoutindex_iter = height_to_first_txoutindex.into_iter(); + let mut height_to_input_count_iter = height_to_input_count.into_iter(); + let mut height_to_output_count_iter = height_to_output_count.into_iter(); + let mut height_to_tx_count_iter = height_to_tx_count.into_iter(); + let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter(); + let mut txindex_to_input_count_iter = txindex_to_input_count.iter(); + let mut txindex_to_output_count_iter = txindex_to_output_count.iter(); + + let height_to_price_close_vec = + height_to_price_close.map(|height_to_price_close| height_to_price_close.collect()); + + let height_to_timestamp_fixed_vec = height_to_timestamp_fixed.collect(); + let txoutindex_range_to_height = RangeMap::from(height_to_first_txoutindex); + + let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() { + self.height_to_unspendable_supply + .into_iter() + .get_unwrap(prev_height) + } else { + Sats::ZERO + }; + let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() { + self.height_to_opreturn_supply + .into_iter() + .get_unwrap(prev_height) + } else { + Sats::ZERO + }; + let mut addresstype_to_addr_count = AddressTypeToAddressCount::from(( + &self.addresstype_to_height_to_addr_count, + starting_height, + )); + let mut addresstype_to_empty_addr_count = AddressTypeToAddressCount::from(( + &self.addresstype_to_height_to_empty_addr_count, + starting_height, + )); + + let mut height = starting_height; + + let mut addresstype_to_typeindex_to_loadedaddressdata = + AddressTypeToTypeIndexMap::>::default(); + let mut addresstype_to_typeindex_to_emptyaddressdata = + AddressTypeToTypeIndexMap::>::default(); + + let mut vr = VecsReaders::new(self); + + let last_height = Height::from( + height_to_date_fixed + .len() + .checked_sub(1) + .unwrap_or_default(), + ); + + for _height in (height.to_usize()..height_to_date_fixed.len()).map(Height::from) { + height = _height; + + info!("Processing chain at {height}..."); + + self.utxo_cohorts + .iter_separate_mut() + .for_each(|v| v.state.um().reset_single_iteration_values()); + + self.address_cohorts + .iter_separate_mut() + .for_each(|v| v.state.um().reset_single_iteration_values()); + + let timestamp = height_to_timestamp_fixed_iter.get_unwrap(height); + let price = height_to_price_close_iter + .as_mut() + .map(|i| *i.get_unwrap(height)); + let first_txindex = height_to_first_txindex_iter.get_unwrap(height); + let first_txoutindex = height_to_first_txoutindex_iter + .get_unwrap(height) + .to_usize(); + let first_txinindex = height_to_first_txinindex_iter.get_unwrap(height).to_usize(); + let tx_count = height_to_tx_count_iter.get_unwrap(height); + let output_count = height_to_output_count_iter.get_unwrap(height); + let input_count = height_to_input_count_iter.get_unwrap(height); + + let txoutindex_to_txindex = build_txoutindex_to_txindex( + first_txindex, + u64::from(tx_count), + &mut txindex_to_output_count_iter, + ); + + let txinindex_to_txindex = build_txinindex_to_txindex( + first_txindex, + u64::from(tx_count), + &mut txindex_to_input_count_iter, + ); + + let first_addressindexes: ByAddressType = ByAddressType { + p2a: height_to_first_p2aaddressindex_iter + .get_unwrap(height) + .into(), + p2pk33: height_to_first_p2pk33addressindex_iter + .get_unwrap(height) + .into(), + p2pk65: height_to_first_p2pk65addressindex_iter + .get_unwrap(height) + .into(), + p2pkh: height_to_first_p2pkhaddressindex_iter + .get_unwrap(height) + .into(), + p2sh: height_to_first_p2shaddressindex_iter + .get_unwrap(height) + .into(), + p2tr: height_to_first_p2traddressindex_iter + .get_unwrap(height) + .into(), + p2wpkh: height_to_first_p2wpkhaddressindex_iter + .get_unwrap(height) + .into(), + p2wsh: height_to_first_p2wshaddressindex_iter + .get_unwrap(height) + .into(), + }; + + let ( + mut transacted, + addresstype_to_typedindex_to_received_data, + mut height_to_sent, + addresstype_to_typedindex_to_sent_data, + mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource, + mut combined_txindex_vecs, + ) = thread::scope(|scope| { + scope.spawn(|| { + self.utxo_cohorts + .tick_tock_next_block(&chain_state, timestamp); + }); + + let (transacted, addresstype_to_typedindex_to_received_data, receiving_addresstype_to_typeindex_to_addressdatawithsource, output_txindex_vecs) = (first_txoutindex..first_txoutindex + usize::from(output_count)) + .into_par_iter() + .map(|i| { + let txoutindex = TxOutIndex::from(i); + + let local_idx = i - first_txoutindex; + let txindex = txoutindex_to_txindex[local_idx]; + + let value = txoutindex_to_value + .read_unwrap(txoutindex, &ir.txoutindex_to_value); + + let output_type = txoutindex_to_outputtype + .read_unwrap(txoutindex, &ir.txoutindex_to_outputtype); + + if output_type.is_not_address() { + return (txindex, value, output_type, None); + } + + let typeindex = txoutindex_to_typeindex + .read_unwrap(txoutindex, &ir.txoutindex_to_typeindex); + + let addressdata_opt = Self::get_addressdatawithsource( + output_type, + typeindex, + &first_addressindexes, + &addresstype_to_typeindex_to_loadedaddressdata, + &addresstype_to_typeindex_to_emptyaddressdata, + &vr, + &self.any_address_indexes, + &self.addresses_data, + ); + + (txindex, value, output_type, Some(( typeindex, addressdata_opt))) + }).fold( + || { + ( + Transacted::default(), + AddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexMap::default(), + AddressTypeToTypeIndexMap::::default(), + ) + }, + |(mut transacted, mut addresstype_to_typedindex_to_data, mut addresstype_to_typeindex_to_addressdatawithsource, mut txindex_vecs), + ( + txindex, + value, + output_type, + typeindex_with_addressdata_opt, + )| { + transacted.iterate(value, output_type); + + if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt { + if let Some(addressdata) = addressdata_opt + { + addresstype_to_typeindex_to_addressdatawithsource + .insert_for_type(output_type, typeindex, addressdata); + } + + let addr_type = output_type; + + addresstype_to_typedindex_to_data + .get_mut(addr_type) + .unwrap() + .push((typeindex, value)); + + txindex_vecs + .get_mut(addr_type) + .unwrap() + .entry(typeindex) + .or_insert_with(TxIndexVec::new) + .push(txindex); + } + + (transacted, addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource, txindex_vecs) + }).reduce( + || { + ( + Transacted::default(), + AddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexMap::default(), + AddressTypeToTypeIndexMap::::default(), + ) + }, + |(transacted, addresstype_to_typedindex_to_data, addresstype_to_typeindex_to_addressdatawithsource, txindex_vecs), (transacted2, addresstype_to_typedindex_to_data2, addresstype_to_typeindex_to_addressdatawithsource2, txindex_vecs2)| { + (transacted + transacted2, addresstype_to_typedindex_to_data.merge(addresstype_to_typedindex_to_data2), addresstype_to_typeindex_to_addressdatawithsource.merge(addresstype_to_typeindex_to_addressdatawithsource2), txindex_vecs.merge_vec(txindex_vecs2)) + }, + ); + + // Skip coinbase + let ( + height_to_sent, + addresstype_to_typedindex_to_sent_data, + sending_addresstype_to_typeindex_to_addressdatawithsource, + input_txindex_vecs, + ) = (first_txinindex + 1..first_txinindex + usize::from(input_count)) + .into_par_iter() + .map(|i| { + let txinindex = TxInIndex::from(i); + + let local_idx = i - first_txinindex; + let txindex = txinindex_to_txindex[local_idx]; + + let outpoint = txinindex_to_outpoint + .read_unwrap(txinindex, &ir.txinindex_to_outpoint); + + let txoutindex = txindex_to_first_txoutindex + .read_unwrap(outpoint.txindex(), &ir.txindex_to_first_txoutindex) + + outpoint.vout(); + + let value = txoutindex_to_value + .read_unwrap(txoutindex, &ir.txoutindex_to_value); + + let input_type = txoutindex_to_outputtype + .read_unwrap(txoutindex, &ir.txoutindex_to_outputtype); + + let prev_height = *txoutindex_range_to_height.get(txoutindex).unwrap(); + + if input_type.is_not_address() { + return (txindex, prev_height, value, input_type, None); + } + + let typeindex = txoutindex_to_typeindex + .read_unwrap(txoutindex, &ir.txoutindex_to_typeindex); + + let addressdata_opt = Self::get_addressdatawithsource( + input_type, + typeindex, + &first_addressindexes, + &addresstype_to_typeindex_to_loadedaddressdata, + &addresstype_to_typeindex_to_emptyaddressdata, + &vr, + &self.any_address_indexes, + &self.addresses_data, + ); + + ( + txindex, + prev_height, + value, + input_type, + Some((typeindex, addressdata_opt)), + ) + }) + .fold( + || { + ( + FxHashMap::::default(), + HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexMap::default(), + AddressTypeToTypeIndexMap::::default(), + ) + }, + |( + mut height_to_transacted, + mut height_to_addresstype_to_typedindex_to_data, + mut addresstype_to_typeindex_to_addressdatawithsource, + mut txindex_vecs, + ), + ( + txindex, + prev_height, + value, + output_type, + typeindex_with_addressdata_opt, + )| { + height_to_transacted + .entry(prev_height) + .or_default() + .iterate(value, output_type); + + if let Some((typeindex, addressdata_opt)) = + typeindex_with_addressdata_opt + { + if let Some(addressdata) = addressdata_opt { + addresstype_to_typeindex_to_addressdatawithsource + .insert_for_type(output_type, typeindex, addressdata); + } + + let addr_type = output_type; + + height_to_addresstype_to_typedindex_to_data + .entry(prev_height) + .or_default() + .get_mut(addr_type) + .unwrap() + .push((typeindex, value)); + + txindex_vecs + .get_mut(addr_type) + .unwrap() + .entry(typeindex) + .or_insert_with(TxIndexVec::new) + .push(txindex); + } + + ( + height_to_transacted, + height_to_addresstype_to_typedindex_to_data, + addresstype_to_typeindex_to_addressdatawithsource, + txindex_vecs, + ) + }, + ) + .reduce( + || { + ( + FxHashMap::::default(), + HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(), + AddressTypeToTypeIndexMap::default(), + AddressTypeToTypeIndexMap::::default(), + ) + }, + |( + height_to_transacted, + addresstype_to_typedindex_to_data, + addresstype_to_typeindex_to_addressdatawithsource, + txindex_vecs, + ), + ( + height_to_transacted2, + addresstype_to_typedindex_to_data2, + addresstype_to_typeindex_to_addressdatawithsource2, + txindex_vecs2, + )| { + let (mut height_to_transacted, height_to_transacted_consumed) = + if height_to_transacted.len() > height_to_transacted2.len() { + (height_to_transacted, height_to_transacted2) + } else { + (height_to_transacted2, height_to_transacted) + }; + height_to_transacted_consumed + .into_iter() + .for_each(|(k, v)| { + *height_to_transacted.entry(k).or_default() += v; + }); + + let ( + mut addresstype_to_typedindex_to_data, + addresstype_to_typedindex_to_data_consumed, + ) = if addresstype_to_typedindex_to_data.len() + > addresstype_to_typedindex_to_data2.len() + { + ( + addresstype_to_typedindex_to_data, + addresstype_to_typedindex_to_data2, + ) + } else { + ( + addresstype_to_typedindex_to_data2, + addresstype_to_typedindex_to_data, + ) + }; + addresstype_to_typedindex_to_data_consumed + .0 + .into_iter() + .for_each(|(k, v)| { + addresstype_to_typedindex_to_data + .entry(k) + .or_default() + .merge_mut(v); + }); + + ( + height_to_transacted, + addresstype_to_typedindex_to_data, + addresstype_to_typeindex_to_addressdatawithsource + .merge(addresstype_to_typeindex_to_addressdatawithsource2), + txindex_vecs.merge_vec(txindex_vecs2), + ) + }, + ); + + let addresstype_to_typeindex_to_addressdatawithsource = + receiving_addresstype_to_typeindex_to_addressdatawithsource + .merge(sending_addresstype_to_typeindex_to_addressdatawithsource); + + let combined_txindex_vecs = output_txindex_vecs.merge_vec(input_txindex_vecs); + + ( + transacted, + addresstype_to_typedindex_to_received_data, + height_to_sent, + addresstype_to_typedindex_to_sent_data, + addresstype_to_typeindex_to_addressdatawithsource, + combined_txindex_vecs, + ) + }); + + combined_txindex_vecs + .par_values_mut() + .flat_map(|typeindex_to_txindexes| typeindex_to_txindexes.par_iter_mut()) + .map(|(_, v)| v) + .filter(|txindex_vec| txindex_vec.len() > 1) + .for_each(|txindex_vec| { + txindex_vec.sort_unstable(); + txindex_vec.dedup(); + }); + + for (address_type, typeindex, txindex_vec) in combined_txindex_vecs + .into_iter() + .flat_map(|(t, m)| m.into_iter().map(move |(i, v)| (t, i, v))) + { + let tx_count = txindex_vec.len() as u32; + + if let Some(addressdata) = addresstype_to_typeindex_to_loadedaddressdata + .get_mut_unwrap(address_type) + .get_mut(&typeindex) + { + addressdata.deref_mut().tx_count += tx_count; + } else if let Some(addressdata) = addresstype_to_typeindex_to_emptyaddressdata + .get_mut_unwrap(address_type) + .get_mut(&typeindex) + { + addressdata.deref_mut().tx_count += tx_count; + } else if let Some(addressdata) = + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource + .get_mut_unwrap(address_type) + .get_mut(&typeindex) + { + addressdata.deref_mut().tx_count += tx_count; + } + } + + thread::scope(|scope| { + scope.spawn(|| { + addresstype_to_typedindex_to_received_data.process_received( + &mut self.address_cohorts, + &mut addresstype_to_typeindex_to_loadedaddressdata, + &mut addresstype_to_typeindex_to_emptyaddressdata, + price, + &mut addresstype_to_addr_count, + &mut addresstype_to_empty_addr_count, + &mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource, + ); + + addresstype_to_typedindex_to_sent_data + .process_sent( + &mut self.address_cohorts, + &mut addresstype_to_typeindex_to_loadedaddressdata, + &mut addresstype_to_typeindex_to_emptyaddressdata, + price, + &mut addresstype_to_addr_count, + &mut addresstype_to_empty_addr_count, + height_to_price_close_vec.as_ref(), + &height_to_timestamp_fixed_vec, + height, + timestamp, + &mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource, + ) + .unwrap(); + }); + + debug_assert!( + chain_state_starting_height <= height, + "chain_state_starting_height ({chain_state_starting_height}) > height ({height})" + ); + + // NOTE: If ByUnspendableType gains more fields, change to .as_vec().into_iter().map(|s| s.value).sum() + unspendable_supply += transacted.by_type.unspendable.opreturn.value + + height_to_unclaimed_rewards_iter.get_unwrap(height); + + opreturn_supply += transacted.by_type.unspendable.opreturn.value; + + if height == Height::ZERO { + transacted = Transacted::default(); + unspendable_supply += Sats::FIFTY_BTC; + } else if height == Height::new(BIP30_DUPLICATE_COINBASE_HEIGHT_1) + || height == Height::new(BIP30_DUPLICATE_COINBASE_HEIGHT_2) + { + if height == Height::new(BIP30_DUPLICATE_COINBASE_HEIGHT_1) { + height_to_sent + .entry(Height::new(BIP30_ORIGINAL_COINBASE_HEIGHT_1)) + .or_default() + } else { + height_to_sent + .entry(Height::new(BIP30_ORIGINAL_COINBASE_HEIGHT_2)) + .or_default() + } + .iterate(Sats::FIFTY_BTC, OutputType::P2PK65); + } + + // Push current block state before processing sends and receives + chain_state.push(BlockState { + supply: transacted.spendable_supply.clone(), + price, + timestamp, + }); + + self.utxo_cohorts.receive(transacted, height, price); + + self.utxo_cohorts.send(height_to_sent, &mut chain_state); + }); + + self.height_to_unspendable_supply + .truncate_push(height, unspendable_supply)?; + + self.height_to_opreturn_supply + .truncate_push(height, opreturn_supply)?; + + self.addresstype_to_height_to_addr_count + .truncate_push(height, &addresstype_to_addr_count)?; + + self.addresstype_to_height_to_empty_addr_count + .truncate_push(height, &addresstype_to_empty_addr_count)?; + + let date = height_to_date_fixed_iter.get_unwrap(height); + let dateindex = DateIndex::try_from(date).unwrap(); + let date_first_height = dateindex_to_first_height_iter.get_unwrap(dateindex); + let date_height_count = dateindex_to_height_count_iter.get_unwrap(dateindex); + let is_date_last_height = date_first_height + + Height::from(date_height_count).decremented().unwrap() + == height; + let date_price = dateindex_to_price_close_iter + .as_mut() + .map(|v| is_date_last_height.then(|| *v.get_unwrap(dateindex))); + + let dateindex = is_date_last_height.then_some(dateindex); + + self.utxo_cohorts + .par_iter_separate_mut() + .map(|v| v as &mut dyn DynCohortVecs) + .chain( + self.address_cohorts + .par_iter_separate_mut() + .map(|v| v as &mut dyn DynCohortVecs), + ) + .try_for_each(|v| { + v.truncate_push(height)?; + v.compute_then_truncate_push_unrealized_states( + height, price, dateindex, date_price, + ) + })?; + + // Compute and push percentiles for aggregate cohorts (all, sth, lth) + self.utxo_cohorts + .truncate_push_aggregate_percentiles(height)?; + + if height != last_height + && height != Height::ZERO + && height.to_usize() % FLUSH_INTERVAL == 0 + { + let _lock = exit.lock(); + + drop(vr); + + self.flush_states( + height, + &chain_state, + mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata), + mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), + false, + exit, + )?; + + vr = VecsReaders::new(self); + } + } + + drop(vr); + + let _lock = exit.lock(); + self.flush_states( + height, + &chain_state, + mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata), + mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), + true, + exit, + )?; + } + + info!("Computing overlapping..."); + + self.utxo_cohorts + .compute_overlapping_vecs(starting_indexes, exit)?; + + self.address_cohorts + .compute_overlapping_vecs(starting_indexes, exit)?; + + info!("Computing rest part 1..."); + + self.indexes_to_addr_count + .compute_all(indexes, starting_indexes, exit, |v| { + v.compute_sum_of_others( + starting_indexes.height, + &self + .addresstype_to_height_to_addr_count + .iter() + .map(|(_, v)| v) + .collect::>(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_empty_addr_count + .compute_all(indexes, starting_indexes, exit, |v| { + v.compute_sum_of_others( + starting_indexes.height, + &self + .addresstype_to_height_to_empty_addr_count + .iter() + .map(|(_, v)| v) + .collect::>(), + exit, + )?; + Ok(()) + })?; + + self.indexes_to_unspendable_supply.compute_rest( + indexes, + price, + starting_indexes, + exit, + Some(&self.height_to_unspendable_supply), + )?; + self.indexes_to_opreturn_supply.compute_rest( + indexes, + price, + starting_indexes, + exit, + Some(&self.height_to_opreturn_supply), + )?; + + self.addresstype_to_indexes_to_addr_count.compute( + indexes, + starting_indexes, + exit, + &self.addresstype_to_height_to_addr_count, + )?; + + self.addresstype_to_indexes_to_empty_addr_count.compute( + indexes, + starting_indexes, + exit, + &self.addresstype_to_height_to_empty_addr_count, + )?; + + self.utxo_cohorts + .compute_rest_part1(indexes, price, starting_indexes, exit)?; + + self.address_cohorts + .compute_rest_part1(indexes, price, starting_indexes, exit)?; + + if let Some(indexes_to_market_cap) = self.indexes_to_market_cap.as_mut() { + indexes_to_market_cap.compute_all(starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + self.utxo_cohorts + .all + .inner + .indexes_to_supply + .dollars + .as_ref() + .unwrap() + .dateindex + .as_ref() + .unwrap(), + |(i, v, ..)| (i, v), + exit, + )?; + Ok(()) + })?; + } + + info!("Computing rest part 2..."); + + let height_to_supply = &self + .utxo_cohorts + .all + .inner + .height_to_supply_value + .bitcoin + .clone(); + let dateindex_to_supply = self + .utxo_cohorts + .all + .inner + .indexes_to_supply + .bitcoin + .dateindex + .clone(); + let height_to_market_cap = self.height_to_market_cap.clone(); + let dateindex_to_market_cap = self + .indexes_to_market_cap + .as_ref() + .map(|v| v.dateindex.u().clone()); + let height_to_realized_cap = self.utxo_cohorts.all.inner.height_to_realized_cap.clone(); + let dateindex_to_realized_cap = self + .utxo_cohorts + .all + .inner + .indexes_to_realized_cap + .as_ref() + .map(|v| v.dateindex.unwrap_last().clone()); + let dateindex_to_supply_ref = dateindex_to_supply.u(); + let height_to_market_cap_ref = height_to_market_cap.as_ref(); + let dateindex_to_market_cap_ref = dateindex_to_market_cap.as_ref(); + let height_to_realized_cap_ref = height_to_realized_cap.as_ref(); + let dateindex_to_realized_cap_ref = dateindex_to_realized_cap.as_ref(); + + self.utxo_cohorts.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply_ref, + height_to_market_cap_ref, + dateindex_to_market_cap_ref, + height_to_realized_cap_ref, + dateindex_to_realized_cap_ref, + exit, + )?; + + self.address_cohorts.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply_ref, + height_to_market_cap_ref, + dateindex_to_market_cap_ref, + height_to_realized_cap_ref, + dateindex_to_realized_cap_ref, + exit, + )?; + + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn get_addressdatawithsource( + address_type: OutputType, + typeindex: TypeIndex, + first_addressindexes: &ByAddressType, + addresstype_to_typeindex_to_loadedaddressdata: &AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: &AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + vr: &VecsReaders, + any_address_indexes: &AnyAddressIndexesVecs, + addresses_data: &AddressesDataVecs, + ) -> Option> { + let first = *first_addressindexes.get(address_type).unwrap(); + if first <= typeindex { + return Some(WithAddressDataSource::New(LoadedAddressData::default())); + } + + if addresstype_to_typeindex_to_loadedaddressdata + .get(address_type) + .unwrap() + .contains_key(&typeindex) + || addresstype_to_typeindex_to_emptyaddressdata + .get(address_type) + .unwrap() + .contains_key(&typeindex) + { + return None; + } + + let reader = vr.get_anyaddressindex_reader(address_type); + + let anyaddressindex = + any_address_indexes.get_anyaddressindex(address_type, typeindex, reader); + + Some(match anyaddressindex.to_enum() { + AnyAddressDataIndexEnum::Loaded(loadedaddressindex) => { + let reader = &vr.anyaddressindex_to_anyaddressdata.loaded; + + let loadedaddressdata = addresses_data + .loaded + .get_pushed_or_read_unwrap(loadedaddressindex, reader); + + WithAddressDataSource::FromLoadedAddressDataVec(( + loadedaddressindex, + loadedaddressdata, + )) + } + AnyAddressDataIndexEnum::Empty(emtpyaddressindex) => { + let reader = &vr.anyaddressindex_to_anyaddressdata.empty; + + let emptyaddressdata = addresses_data + .empty + .get_pushed_or_read_unwrap(emtpyaddressindex, reader); + + WithAddressDataSource::FromEmptyAddressDataVec(( + emtpyaddressindex, + emptyaddressdata.into(), + )) + } + }) + } + + fn flush_states( + &mut self, + height: Height, + chain_state: &[BlockState], + addresstype_to_typeindex_to_loadedaddressdata: AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + with_changes: bool, + exit: &Exit, + ) -> Result<()> { + info!("Flushing..."); + + self.utxo_cohorts.safe_flush_stateful_vecs(height, exit)?; + self.address_cohorts + .safe_flush_stateful_vecs(height, exit)?; + self.height_to_unspendable_supply.safe_write(exit)?; + self.height_to_opreturn_supply.safe_write(exit)?; + self.addresstype_to_height_to_addr_count + .values_mut() + .try_for_each(|v| v.safe_flush(exit))?; + self.addresstype_to_height_to_empty_addr_count + .values_mut() + .try_for_each(|v| v.safe_flush(exit))?; + + let mut addresstype_to_typeindex_to_new_or_updated_anyaddressindex = + AddressTypeToTypeIndexMap::default(); + + for (address_type, sorted) in + addresstype_to_typeindex_to_emptyaddressdata.into_sorted_iter() + { + for (typeindex, emptyaddressdata_with_source) in sorted.into_iter() { + match emptyaddressdata_with_source { + WithAddressDataSource::New(emptyaddressdata) => { + let emptyaddressindex = self + .addresses_data + .empty + .fill_first_hole_or_push(emptyaddressdata)?; + + let anyaddressindex = AnyAddressIndex::from(emptyaddressindex); + + addresstype_to_typeindex_to_new_or_updated_anyaddressindex + .get_mut(address_type) + .unwrap() + .insert(typeindex, anyaddressindex); + } + WithAddressDataSource::FromEmptyAddressDataVec(( + emptyaddressindex, + emptyaddressdata, + )) => self + .addresses_data + .empty + .update(emptyaddressindex, emptyaddressdata)?, + WithAddressDataSource::FromLoadedAddressDataVec(( + loadedaddressindex, + emptyaddressdata, + )) => { + self.addresses_data.loaded.delete(loadedaddressindex); + + let emptyaddressindex = self + .addresses_data + .empty + .fill_first_hole_or_push(emptyaddressdata)?; + + let anyaddressindex = emptyaddressindex.into(); + + addresstype_to_typeindex_to_new_or_updated_anyaddressindex + .get_mut(address_type) + .unwrap() + .insert(typeindex, anyaddressindex); + } + } + } + } + + for (address_type, sorted) in + addresstype_to_typeindex_to_loadedaddressdata.into_sorted_iter() + { + for (typeindex, loadedaddressdata_with_source) in sorted.into_iter() { + match loadedaddressdata_with_source { + WithAddressDataSource::New(loadedaddressdata) => { + let loadedaddressindex = self + .addresses_data + .loaded + .fill_first_hole_or_push(loadedaddressdata)?; + + let anyaddressindex = AnyAddressIndex::from(loadedaddressindex); + + addresstype_to_typeindex_to_new_or_updated_anyaddressindex + .get_mut(address_type) + .unwrap() + .insert(typeindex, anyaddressindex); + } + WithAddressDataSource::FromLoadedAddressDataVec(( + loadedaddressindex, + loadedaddressdata, + )) => self + .addresses_data + .loaded + .update(loadedaddressindex, loadedaddressdata)?, + WithAddressDataSource::FromEmptyAddressDataVec(( + emptyaddressindex, + loadedaddressdata, + )) => { + self.addresses_data.empty.delete(emptyaddressindex); + + let loadedaddressindex = self + .addresses_data + .loaded + .fill_first_hole_or_push(loadedaddressdata)?; + + let anyaddressindex = loadedaddressindex.into(); + + addresstype_to_typeindex_to_new_or_updated_anyaddressindex + .get_mut(address_type) + .unwrap() + .insert(typeindex, anyaddressindex); + } + } + } + } + + for (address_type, sorted) in + addresstype_to_typeindex_to_new_or_updated_anyaddressindex.into_sorted_iter() + { + for (typeindex, anyaddressindex) in sorted { + self.any_address_indexes.update_or_push( + address_type, + typeindex, + anyaddressindex, + )?; + } + } + + let stamp = Stamp::from(height); + + self.any_address_indexes + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + self.addresses_data + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + + self.chain_state.truncate_if_needed(Height::ZERO)?; + chain_state.iter().for_each(|block_state| { + self.chain_state.push(block_state.supply.clone()); + }); + self.chain_state + .stamped_flush_maybe_with_changes(stamp, with_changes)?; + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/range_map.rs b/crates/brk_computer/src/stateful_old/range_map.rs new file mode 100644 index 000000000..87076dc32 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/range_map.rs @@ -0,0 +1,53 @@ +use std::collections::BTreeMap; + +use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex}; + +#[derive(Debug)] +pub struct RangeMap(BTreeMap); + +impl RangeMap +where + I: VecIndex, + T: VecIndex, +{ + pub fn get(&self, key: I) -> Option<&T> { + self.0.range(..=key).next_back().map(|(&min, value)| { + if min > key { + unreachable!() + } + value + }) + } +} + +impl From<&BytesVec> for RangeMap +where + I: VecIndex, + T: VecIndex + BytesVecValue, +{ + #[inline] + fn from(vec: &BytesVec) -> Self { + Self( + vec.into_iter() + .enumerate() + .map(|(i, v)| (v, I::from(i))) + .collect::>(), + ) + } +} + +impl From<&PcoVec> for RangeMap +where + I: VecIndex, + T: VecIndex + PcoVecValue, +{ + #[inline] + fn from(vec: &PcoVec) -> Self { + Self( + vec.into_iter() + .enumerate() + .map(|(i, v)| (v, I::from(i))) + .collect::>(), + ) + } +} diff --git a/crates/brk_computer/src/stateful_old/readers.rs b/crates/brk_computer/src/stateful_old/readers.rs new file mode 100644 index 000000000..5597fac56 --- /dev/null +++ b/crates/brk_computer/src/stateful_old/readers.rs @@ -0,0 +1,111 @@ +use brk_grouper::{ByAddressType, ByAnyAddress}; +use brk_indexer::Indexer; +use brk_types::{OutputType, StoredU64, TxIndex}; +use vecdb::{BoxedVecIterator, GenericStoredVec, Reader, VecIndex}; + +use super::Vecs; + +pub struct IndexerReaders { + pub txinindex_to_outpoint: Reader, + pub txindex_to_first_txoutindex: Reader, + pub txoutindex_to_value: Reader, + pub txoutindex_to_outputtype: Reader, + pub txoutindex_to_typeindex: Reader, +} + +impl IndexerReaders { + pub fn new(indexer: &Indexer) -> Self { + Self { + txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(), + txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(), + txoutindex_to_value: indexer.vecs.txoutindex_to_value.create_reader(), + txoutindex_to_outputtype: indexer.vecs.txoutindex_to_outputtype.create_reader(), + txoutindex_to_typeindex: indexer.vecs.txoutindex_to_typeindex.create_reader(), + } + } +} + +pub struct VecsReaders { + pub addresstypeindex_to_anyaddressindex: ByAddressType, + pub anyaddressindex_to_anyaddressdata: ByAnyAddress, +} + +impl VecsReaders { + pub fn new(vecs: &Vecs) -> Self { + Self { + addresstypeindex_to_anyaddressindex: ByAddressType { + p2pk33: vecs.any_address_indexes.p2pk33.create_reader(), + p2pk65: vecs.any_address_indexes.p2pk65.create_reader(), + p2pkh: vecs.any_address_indexes.p2pkh.create_reader(), + p2sh: vecs.any_address_indexes.p2sh.create_reader(), + p2tr: vecs.any_address_indexes.p2tr.create_reader(), + p2wpkh: vecs.any_address_indexes.p2wpkh.create_reader(), + p2wsh: vecs.any_address_indexes.p2wsh.create_reader(), + p2a: vecs.any_address_indexes.p2a.create_reader(), + }, + anyaddressindex_to_anyaddressdata: ByAnyAddress { + loaded: vecs.addresses_data.loaded.create_reader(), + empty: vecs.addresses_data.empty.create_reader(), + }, + } + } + + pub fn get_anyaddressindex_reader(&self, address_type: OutputType) -> &Reader { + self.addresstypeindex_to_anyaddressindex + .get_unwrap(address_type) + } +} + +pub fn build_txoutindex_to_txindex<'a>( + block_first_txindex: TxIndex, + block_tx_count: u64, + txindex_to_output_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>, +) -> Vec { + let block_first_txindex = block_first_txindex.to_usize(); + + let counts: Vec<_> = (0..block_tx_count as usize) + .map(|tx_offset| { + let txindex = TxIndex::from(block_first_txindex + tx_offset); + u64::from(txindex_to_output_count.get_unwrap(txindex)) + }) + .collect(); + + let total: u64 = counts.iter().sum(); + let mut vec = Vec::with_capacity(total as usize); + + for (tx_offset, &output_count) in counts.iter().enumerate() { + let txindex = TxIndex::from(block_first_txindex + tx_offset); + for _ in 0..output_count { + vec.push(txindex); + } + } + + vec +} + +pub fn build_txinindex_to_txindex<'a>( + block_first_txindex: TxIndex, + block_tx_count: u64, + txindex_to_input_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>, +) -> Vec { + let block_first_txindex = block_first_txindex.to_usize(); + + let counts: Vec<_> = (0..block_tx_count as usize) + .map(|tx_offset| { + let txindex = TxIndex::from(block_first_txindex + tx_offset); + u64::from(txindex_to_input_count.get_unwrap(txindex)) + }) + .collect(); + + let total: u64 = counts.iter().sum(); + let mut vec = Vec::with_capacity(total as usize); + + for (tx_offset, &input_count) in counts.iter().enumerate() { + let txindex = TxIndex::from(block_first_txindex + tx_offset); + for _ in 0..input_count { + vec.push(txindex); + } + } + + vec +} diff --git a/crates/brk_computer/src/stateful_old/trait.rs b/crates/brk_computer/src/stateful_old/trait.rs new file mode 100644 index 000000000..64005bbae --- /dev/null +++ b/crates/brk_computer/src/stateful_old/trait.rs @@ -0,0 +1,59 @@ +use brk_error::Result; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version}; +use vecdb::{Exit, IterableVec}; + +use crate::{Indexes, indexes, price}; + +pub trait DynCohortVecs: Send + Sync { + fn min_height_vecs_len(&self) -> usize; + fn reset_state_starting_height(&mut self); + + fn import_state(&mut self, starting_height: Height) -> Result; + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>; + + fn truncate_push(&mut self, height: Height) -> Result<()>; + + fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + ) -> Result<()>; + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>; + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()>; +} + +pub trait CohortVecs: DynCohortVecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()>; + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()>; +} diff --git a/crates/brk_computer/src/stateful_old/transaction_processing.rs b/crates/brk_computer/src/stateful_old/transaction_processing.rs new file mode 100644 index 000000000..c425c0d0d --- /dev/null +++ b/crates/brk_computer/src/stateful_old/transaction_processing.rs @@ -0,0 +1,217 @@ +use brk_error::Result; +use brk_grouper::{ByAddressType, Filtered}; +use brk_types::{ + CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex, +}; +use vecdb::VecIndex; + +use crate::utils::OptionExt; + +use super::{ + address_cohorts, + addresstype::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec}, + withaddressdatasource::WithAddressDataSource, +}; + +impl AddressTypeToVec<(TypeIndex, Sats)> { + #[allow(clippy::too_many_arguments)] + pub fn process_received( + self, + vecs: &mut address_cohorts::Vecs, + addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + price: Option, + addresstype_to_addr_count: &mut ByAddressType, + addresstype_to_empty_addr_count: &mut ByAddressType, + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + ) { + self.unwrap().into_iter().for_each(|(_type, vec)| { + vec.into_iter().for_each(|(type_index, value)| { + let mut is_new = false; + let mut from_any_empty = false; + + let addressdata_withsource = addresstype_to_typeindex_to_loadedaddressdata + .get_mut(_type) + .unwrap() + .entry(type_index) + .or_insert_with(|| { + addresstype_to_typeindex_to_emptyaddressdata + .get_mut(_type) + .unwrap() + .remove(&type_index) + .map(|ad| { + from_any_empty = true; + ad.into() + }) + .unwrap_or_else(|| { + let addressdata = + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource + .remove_for_type(_type, &type_index); + is_new = addressdata.is_new(); + from_any_empty = addressdata.is_from_emptyaddressdata(); + addressdata + }) + }); + + if is_new || from_any_empty { + (*addresstype_to_addr_count.get_mut(_type).unwrap()) += 1; + if from_any_empty { + (*addresstype_to_empty_addr_count.get_mut(_type).unwrap()) -= 1; + } + } + + let addressdata = addressdata_withsource.deref_mut(); + + let prev_amount = addressdata.balance(); + + let amount = prev_amount + value; + + let filters_differ = vecs.amount_range.get(amount).filter() + != vecs.amount_range.get(prev_amount).filter(); + + if is_new || from_any_empty || filters_differ { + if !is_new && !from_any_empty { + vecs.amount_range + .get_mut(prev_amount) + .state + .um() + .subtract(addressdata); + } + + addressdata.receive(value, price); + + vecs.amount_range + .get_mut(amount) + .state + .um() + .add(addressdata); + } else { + vecs.amount_range + .get_mut(amount) + .state + .um() + .receive(addressdata, value, price); + } + }); + }); + } +} + +impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { + #[allow(clippy::too_many_arguments)] + pub fn process_sent( + self, + vecs: &mut address_cohorts::Vecs, + addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + price: Option, + addresstype_to_addr_count: &mut ByAddressType, + addresstype_to_empty_addr_count: &mut ByAddressType, + height_to_price_close_vec: Option<&Vec>>, + height_to_timestamp_fixed_vec: &[Timestamp], + height: Height, + timestamp: Timestamp, + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexMap< + WithAddressDataSource, + >, + ) -> Result<()> { + self.0.into_iter().try_for_each(|(prev_height, v)| { + let prev_price = height_to_price_close_vec + .as_ref() + .map(|v| **v.get(prev_height.to_usize()).unwrap()); + + let prev_timestamp = *height_to_timestamp_fixed_vec + .get(prev_height.to_usize()) + .unwrap(); + + let blocks_old = height.to_usize() - prev_height.to_usize(); + + let days_old = timestamp.difference_in_days_between_float(prev_timestamp); + + let older_than_hour = timestamp + .checked_sub(prev_timestamp) + .unwrap() + .is_more_than_hour(); + + v.unwrap().into_iter().try_for_each(|(_type, vec)| { + vec.into_iter().try_for_each(|(type_index, value)| { + let typeindex_to_loadedaddressdata = + addresstype_to_typeindex_to_loadedaddressdata.get_mut_unwrap(_type); + + let addressdata_withsource = typeindex_to_loadedaddressdata + .entry(type_index) + .or_insert_with(|| { + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource + .remove_for_type(_type, &type_index) + }); + + let addressdata = addressdata_withsource.deref_mut(); + + let prev_amount = addressdata.balance(); + + let amount = prev_amount.checked_sub(value).unwrap(); + + let will_be_empty = addressdata.has_1_utxos(); + + let filters_differ = vecs.amount_range.get(amount).filter() + != vecs.amount_range.get(prev_amount).filter(); + + if will_be_empty || filters_differ { + vecs.amount_range + .get_mut(prev_amount) + .state + .um() + .subtract(addressdata); + + addressdata.send(value, prev_price)?; + + if will_be_empty { + if amount.is_not_zero() { + unreachable!() + } + + (*addresstype_to_addr_count.get_mut(_type).unwrap()) -= 1; + (*addresstype_to_empty_addr_count.get_mut(_type).unwrap()) += 1; + + let addressdata = + typeindex_to_loadedaddressdata.remove(&type_index).unwrap(); + + addresstype_to_typeindex_to_emptyaddressdata + .get_mut(_type) + .unwrap() + .insert(type_index, addressdata.into()); + } else { + vecs.amount_range + .get_mut(amount) + .state + .um() + .add(addressdata); + } + } else { + vecs.amount_range.get_mut(amount).state.um().send( + addressdata, + value, + price, + prev_price, + blocks_old, + days_old, + older_than_hour, + )?; + } + + Ok(()) + }) + }) + }) + } +} diff --git a/crates/brk_computer/src/stateful_old/utxo_cohort.rs b/crates/brk_computer/src/stateful_old/utxo_cohort.rs new file mode 100644 index 000000000..aebe57d9b --- /dev/null +++ b/crates/brk_computer/src/stateful_old/utxo_cohort.rs @@ -0,0 +1,241 @@ +use std::{ops::Deref, path::Path}; + +use brk_error::Result; +use brk_grouper::{CohortContext, Filter, Filtered, StateLevel}; +use brk_traversable::Traversable; +use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version}; +use vecdb::{Database, Exit, IterableVec}; + +use crate::{ + Indexes, PriceToAmount, UTXOCohortState, + grouped::{PERCENTILES, PERCENTILES_LEN}, + indexes, price, + stateful::{ + common, + r#trait::{CohortVecs, DynCohortVecs}, + }, + utils::OptionExt, +}; + +#[derive(Clone, Traversable)] +pub struct Vecs { + state_starting_height: Option, + + #[traversable(skip)] + pub state: Option, + + /// For aggregate cohorts (all, sth, lth) that only need price_to_amount for percentiles + #[traversable(skip)] + pub price_to_amount: Option, + + #[traversable(flatten)] + pub inner: common::Vecs, +} + +impl Vecs { + pub fn forced_import( + db: &Database, + filter: Filter, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: &Path, + state_level: StateLevel, + ) -> Result { + let compute_dollars = price.is_some(); + + let full_name = filter.to_full_name(CohortContext::Utxo); + + Ok(Self { + state_starting_height: None, + + state: if state_level.is_full() { + Some(UTXOCohortState::new( + states_path, + &full_name, + compute_dollars, + )) + } else { + None + }, + + price_to_amount: if state_level.is_price_only() && compute_dollars { + Some(PriceToAmount::create(states_path, &full_name)) + } else { + None + }, + + inner: common::Vecs::forced_import( + db, + filter, + CohortContext::Utxo, + version, + indexes, + price, + )?, + }) + } +} + +impl DynCohortVecs for Vecs { + fn min_height_vecs_len(&self) -> usize { + self.inner.min_height_vecs_len() + } + + fn reset_state_starting_height(&mut self) { + self.state_starting_height = Some(Height::ZERO); + } + + fn import_state(&mut self, starting_height: Height) -> Result { + let starting_height = self + .inner + .import_state(starting_height, self.state.um())?; + + self.state_starting_height = Some(starting_height); + + Ok(starting_height) + } + + fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.inner.validate_computed_versions(base_version) + } + + fn truncate_push(&mut self, height: Height) -> Result<()> { + if self.state_starting_height.unwrap() > height { + return Ok(()); + } + + self.inner + .truncate_push(height, self.state.u()) + } + + fn compute_then_truncate_push_unrealized_states( + &mut self, + height: Height, + height_price: Option, + dateindex: Option, + date_price: Option>, + ) -> Result<()> { + self.inner.compute_then_truncate_push_unrealized_states( + height, + height_price, + dateindex, + date_price, + self.state.um(), + ) + } + + fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.inner + .safe_flush_stateful_vecs(height, exit, self.state.um()) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.inner + .compute_rest_part1(indexes, price, starting_indexes, exit) + } +} + +impl CohortVecs for Vecs { + fn compute_from_stateful( + &mut self, + starting_indexes: &Indexes, + others: &[&Self], + exit: &Exit, + ) -> Result<()> { + self.inner.compute_from_stateful( + starting_indexes, + &others.iter().map(|v| &v.inner).collect::>(), + exit, + ) + } + + #[allow(clippy::too_many_arguments)] + fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.inner.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + } +} + +impl Vecs { + /// Compute percentile prices for aggregate cohorts that have standalone price_to_amount. + /// Returns NaN array if price_to_amount is None or empty. + pub fn compute_percentile_prices_from_standalone( + &self, + supply: Sats, + ) -> [Dollars; PERCENTILES_LEN] { + let mut result = [Dollars::NAN; PERCENTILES_LEN]; + + let price_to_amount = match self.price_to_amount.as_ref() { + Some(p) => p, + None => return result, + }; + + if price_to_amount.is_empty() || supply == Sats::ZERO { + return result; + } + + let total = supply; + let targets = PERCENTILES.map(|p| total * p / 100); + + let mut accumulated = Sats::ZERO; + let mut pct_idx = 0; + + for (&price, &sats) in price_to_amount.iter() { + accumulated += sats; + + while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] { + result[pct_idx] = price; + pct_idx += 1; + } + + if pct_idx >= PERCENTILES_LEN { + break; + } + } + + result + } +} + +impl Deref for Vecs { + type Target = common::Vecs; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl Filtered for Vecs { + fn filter(&self) -> &Filter { + &self.inner.filter + } +} diff --git a/crates/brk_computer/src/stateful_old/utxo_cohorts.rs b/crates/brk_computer/src/stateful_old/utxo_cohorts.rs new file mode 100644 index 000000000..459acc8ce --- /dev/null +++ b/crates/brk_computer/src/stateful_old/utxo_cohorts.rs @@ -0,0 +1,697 @@ +use std::path::Path; + +use brk_error::Result; +use brk_grouper::{ + AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, + ByMaxAge, ByMinAge, BySpendableType, ByTerm, Filter, Filtered, StateLevel, Term, TimeFilter, + UTXOGroups, +}; +use brk_traversable::Traversable; +use brk_types::{ + Bitcoin, CheckedSub, DateIndex, Dollars, HalvingEpoch, Height, ONE_DAY_IN_SEC, OutputType, + Sats, Timestamp, Version, +}; +use derive_deref::{Deref, DerefMut}; +use rayon::prelude::*; +use rustc_hash::FxHashMap; +use vecdb::{Database, Exit, IterableVec, VecIndex}; + +use crate::{ + Indexes, indexes, price, + stateful::{Flushable, HeightFlushable, r#trait::DynCohortVecs}, + states::{BlockState, Transacted}, + utils::OptionExt, +}; + +use super::{r#trait::CohortVecs, utxo_cohort}; + +const VERSION: Version = Version::new(0); + +#[derive(Clone, Deref, DerefMut, Traversable)] +pub struct Vecs(UTXOGroups); + +impl Vecs { + pub fn forced_import( + db: &Database, + version: Version, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + states_path: &Path, + ) -> Result { + let v = version + VERSION + Version::ZERO; + + // Helper to create a cohort - booleans are now derived from filter + let create = |filter: Filter, state_level: StateLevel| -> Result { + utxo_cohort::Vecs::forced_import( + db, + filter, + v, + indexes, + price, + states_path, + state_level, + ) + }; + + let full = |f: Filter| create(f, StateLevel::Full); + let none = |f: Filter| create(f, StateLevel::None); + + Ok(Self(UTXOGroups { + // Special case: all uses Version::ONE + all: utxo_cohort::Vecs::forced_import( + db, + Filter::All, + version + VERSION + Version::ONE, + indexes, + price, + states_path, + StateLevel::PriceOnly, + )?, + + term: ByTerm { + short: create(Filter::Term(Term::Sth), StateLevel::PriceOnly)?, + long: create(Filter::Term(Term::Lth), StateLevel::PriceOnly)?, + }, + + epoch: ByEpoch { + _0: full(Filter::Epoch(HalvingEpoch::new(0)))?, + _1: full(Filter::Epoch(HalvingEpoch::new(1)))?, + _2: full(Filter::Epoch(HalvingEpoch::new(2)))?, + _3: full(Filter::Epoch(HalvingEpoch::new(3)))?, + _4: full(Filter::Epoch(HalvingEpoch::new(4)))?, + }, + + type_: BySpendableType { + p2pk65: full(Filter::Type(OutputType::P2PK65))?, + p2pk33: full(Filter::Type(OutputType::P2PK33))?, + p2pkh: full(Filter::Type(OutputType::P2PKH))?, + p2sh: full(Filter::Type(OutputType::P2SH))?, + p2wpkh: full(Filter::Type(OutputType::P2WPKH))?, + p2wsh: full(Filter::Type(OutputType::P2WSH))?, + p2tr: full(Filter::Type(OutputType::P2TR))?, + p2a: full(Filter::Type(OutputType::P2A))?, + p2ms: full(Filter::Type(OutputType::P2MS))?, + empty: full(Filter::Type(OutputType::Empty))?, + unknown: full(Filter::Type(OutputType::Unknown))?, + }, + + max_age: ByMaxAge { + _1w: none(Filter::Time(TimeFilter::LowerThan(7)))?, + _1m: none(Filter::Time(TimeFilter::LowerThan(30)))?, + _2m: none(Filter::Time(TimeFilter::LowerThan(2 * 30)))?, + _3m: none(Filter::Time(TimeFilter::LowerThan(3 * 30)))?, + _4m: none(Filter::Time(TimeFilter::LowerThan(4 * 30)))?, + _5m: none(Filter::Time(TimeFilter::LowerThan(5 * 30)))?, + _6m: none(Filter::Time(TimeFilter::LowerThan(6 * 30)))?, + _1y: none(Filter::Time(TimeFilter::LowerThan(365)))?, + _2y: none(Filter::Time(TimeFilter::LowerThan(2 * 365)))?, + _3y: none(Filter::Time(TimeFilter::LowerThan(3 * 365)))?, + _4y: none(Filter::Time(TimeFilter::LowerThan(4 * 365)))?, + _5y: none(Filter::Time(TimeFilter::LowerThan(5 * 365)))?, + _6y: none(Filter::Time(TimeFilter::LowerThan(6 * 365)))?, + _7y: none(Filter::Time(TimeFilter::LowerThan(7 * 365)))?, + _8y: none(Filter::Time(TimeFilter::LowerThan(8 * 365)))?, + _10y: none(Filter::Time(TimeFilter::LowerThan(10 * 365)))?, + _12y: none(Filter::Time(TimeFilter::LowerThan(12 * 365)))?, + _15y: none(Filter::Time(TimeFilter::LowerThan(15 * 365)))?, + }, + + min_age: ByMinAge { + _1d: none(Filter::Time(TimeFilter::GreaterOrEqual(1)))?, + _1w: none(Filter::Time(TimeFilter::GreaterOrEqual(7)))?, + _1m: none(Filter::Time(TimeFilter::GreaterOrEqual(30)))?, + _2m: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 30)))?, + _3m: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 30)))?, + _4m: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 30)))?, + _5m: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 30)))?, + _6m: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 30)))?, + _1y: none(Filter::Time(TimeFilter::GreaterOrEqual(365)))?, + _2y: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 365)))?, + _3y: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 365)))?, + _4y: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 365)))?, + _5y: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 365)))?, + _6y: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 365)))?, + _7y: none(Filter::Time(TimeFilter::GreaterOrEqual(7 * 365)))?, + _8y: none(Filter::Time(TimeFilter::GreaterOrEqual(8 * 365)))?, + _10y: none(Filter::Time(TimeFilter::GreaterOrEqual(10 * 365)))?, + _12y: none(Filter::Time(TimeFilter::GreaterOrEqual(12 * 365)))?, + }, + + age_range: ByAgeRange { + up_to_1d: full(Filter::Time(TimeFilter::Range(0..1)))?, + _1d_to_1w: full(Filter::Time(TimeFilter::Range(1..7)))?, + _1w_to_1m: full(Filter::Time(TimeFilter::Range(7..30)))?, + _1m_to_2m: full(Filter::Time(TimeFilter::Range(30..60)))?, + _2m_to_3m: full(Filter::Time(TimeFilter::Range(60..90)))?, + _3m_to_4m: full(Filter::Time(TimeFilter::Range(90..120)))?, + _4m_to_5m: full(Filter::Time(TimeFilter::Range(120..150)))?, + _5m_to_6m: full(Filter::Time(TimeFilter::Range(150..180)))?, + _6m_to_1y: full(Filter::Time(TimeFilter::Range(180..365)))?, + _1y_to_2y: full(Filter::Time(TimeFilter::Range(365..730)))?, + _2y_to_3y: full(Filter::Time(TimeFilter::Range(730..1095)))?, + _3y_to_4y: full(Filter::Time(TimeFilter::Range(1095..1460)))?, + _4y_to_5y: full(Filter::Time(TimeFilter::Range(1460..1825)))?, + _5y_to_6y: full(Filter::Time(TimeFilter::Range(1825..2190)))?, + _6y_to_7y: full(Filter::Time(TimeFilter::Range(2190..2555)))?, + _7y_to_8y: full(Filter::Time(TimeFilter::Range(2555..2920)))?, + _8y_to_10y: full(Filter::Time(TimeFilter::Range(2920..3650)))?, + _10y_to_12y: full(Filter::Time(TimeFilter::Range(3650..4380)))?, + _12y_to_15y: full(Filter::Time(TimeFilter::Range(4380..5475)))?, + from_15y: full(Filter::Time(TimeFilter::GreaterOrEqual(15 * 365)))?, + }, + + amount_range: ByAmountRange { + _0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?, + _1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?, + _10sats_to_100sats: full(Filter::Amount(AmountFilter::Range( + Sats::_10..Sats::_100, + )))?, + _100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_100..Sats::_1K, + )))?, + _1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_1K..Sats::_10K, + )))?, + _10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_10K..Sats::_100K, + )))?, + _100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_100K..Sats::_1M, + )))?, + _1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range( + Sats::_1M..Sats::_10M, + )))?, + _10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10M..Sats::_1BTC, + )))?, + _1btc_to_10btc: full(Filter::Amount(AmountFilter::Range( + Sats::_1BTC..Sats::_10BTC, + )))?, + _10btc_to_100btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10BTC..Sats::_100BTC, + )))?, + _100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_100BTC..Sats::_1K_BTC, + )))?, + _1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_1K_BTC..Sats::_10K_BTC, + )))?, + _10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range( + Sats::_10K_BTC..Sats::_100K_BTC, + )))?, + _100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual( + Sats::_100K_BTC, + )))?, + }, + + lt_amount: ByLowerThanAmount { + _10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?, + _100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?, + _1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?, + _10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?, + _100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?, + _1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?, + _10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?, + _1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?, + _10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?, + _100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?, + _1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?, + _10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?, + _100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?, + }, + + ge_amount: ByGreatEqualAmount { + _1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?, + _10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?, + _100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?, + _1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?, + _10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?, + _100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?, + _1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?, + _10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?, + _1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?, + _10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?, + _100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?, + _1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?, + _10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?, + }, + })) + } + + pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) { + if chain_state.is_empty() { + return; + } + + let prev_timestamp = chain_state.last().unwrap().timestamp; + + // Only blocks whose age % ONE_DAY >= threshold can cross a day boundary. + // Saves 1 subtraction + 2 divisions per block vs computing days_old directly. + let elapsed = (*timestamp).saturating_sub(*prev_timestamp); + let threshold = ONE_DAY_IN_SEC.saturating_sub(elapsed); + + // Extract all mutable references upfront to avoid borrow checker issues + // Use a single destructuring to get non-overlapping mutable borrows + let UTXOGroups { + all, + term, + age_range, + .. + } = &mut self.0; + + let mut vecs = age_range + .iter_mut() + .map(|v| (v.filter().clone(), &mut v.state)) + .collect::>(); + + // Collect aggregate cohorts' filter and p2a for age transitions + let mut aggregate_p2a: Vec<(Filter, Option<&mut crate::PriceToAmount>)> = vec![ + (all.filter().clone(), all.price_to_amount.as_mut()), + ( + term.short.filter().clone(), + term.short.price_to_amount.as_mut(), + ), + ( + term.long.filter().clone(), + term.long.price_to_amount.as_mut(), + ), + ]; + + chain_state + .iter() + .filter(|block_state| { + let age = (*prev_timestamp).saturating_sub(*block_state.timestamp); + age % ONE_DAY_IN_SEC >= threshold + }) + .for_each(|block_state| { + let prev_days_old = + prev_timestamp.difference_in_days_between(block_state.timestamp); + let days_old = timestamp.difference_in_days_between(block_state.timestamp); + + if prev_days_old == days_old { + return; + } + + vecs.iter_mut().for_each(|(filter, state)| { + let is = filter.contains_time(days_old); + let was = filter.contains_time(prev_days_old); + + if is && !was { + state + .as_mut() + .unwrap() + .increment(&block_state.supply, block_state.price); + } else if was && !is { + state + .as_mut() + .unwrap() + .decrement(&block_state.supply, block_state.price); + } + }); + + // Handle age transitions for aggregate cohorts' price_to_amount + // Check which cohorts the UTXO was in vs is now in, and increment/decrement accordingly + // Only process if there's remaining supply (like CohortState::increment/decrement do) + if let Some(price) = block_state.price + && block_state.supply.value > Sats::ZERO + { + aggregate_p2a.iter_mut().for_each(|(filter, p2a)| { + let is = filter.contains_time(days_old); + let was = filter.contains_time(prev_days_old); + + if is && !was { + p2a.um().increment(price, &block_state.supply); + } else if was && !is { + p2a.um().decrement(price, &block_state.supply); + } + }); + } + }); + } + + pub fn send( + &mut self, + height_to_sent: FxHashMap, + chain_state: &mut [BlockState], + ) { + // Extract all mutable references upfront to avoid borrow checker issues + let UTXOGroups { + all, + term, + age_range, + epoch, + type_, + amount_range, + .. + } = &mut self.0; + + let mut time_based_vecs = age_range + .iter_mut() + .chain(epoch.iter_mut()) + .collect::>(); + + // Collect aggregate cohorts' filter and p2a for iteration + let mut aggregate_p2a: Vec<(Filter, Option<&mut crate::PriceToAmount>)> = vec![ + (all.filter().clone(), all.price_to_amount.as_mut()), + ( + term.short.filter().clone(), + term.short.price_to_amount.as_mut(), + ), + ( + term.long.filter().clone(), + term.long.price_to_amount.as_mut(), + ), + ]; + + let last_block = chain_state.last().unwrap(); + let last_timestamp = last_block.timestamp; + let current_price = last_block.price; + + let chain_state_len = chain_state.len(); + + height_to_sent.into_iter().for_each(|(height, sent)| { + chain_state[height.to_usize()].supply -= &sent.spendable_supply; + + let block_state = chain_state.get(height.to_usize()).unwrap(); + + let prev_price = block_state.price; + + let blocks_old = chain_state_len - 1 - height.to_usize(); + + let days_old = last_timestamp.difference_in_days_between(block_state.timestamp); + let days_old_float = + last_timestamp.difference_in_days_between_float(block_state.timestamp); + + let older_than_hour = last_timestamp + .checked_sub(block_state.timestamp) + .unwrap() + .is_more_than_hour(); + + time_based_vecs + .iter_mut() + .filter(|v| match v.filter() { + Filter::Time(TimeFilter::GreaterOrEqual(from)) => *from <= days_old, + Filter::Time(TimeFilter::LowerThan(to)) => *to > days_old, + Filter::Time(TimeFilter::Range(range)) => range.contains(&days_old), + Filter::Epoch(epoch) => *epoch == HalvingEpoch::from(height), + _ => unreachable!(), + }) + .for_each(|vecs| { + vecs.state.um().send( + &sent.spendable_supply, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); + }); + + sent.by_type + .spendable + .iter_typed() + .for_each(|(output_type, supply_state)| { + type_.get_mut(output_type).state.um().send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ) + }); + + sent.by_size_group + .iter_typed() + .for_each(|(group, supply_state)| { + amount_range.get_mut(group).state.um().send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); + }); + + // Update aggregate cohorts' price_to_amount using filter.contains_time() + if let Some(prev_price) = prev_price { + let supply_state = &sent.spendable_supply; + if supply_state.value.is_not_zero() { + aggregate_p2a + .iter_mut() + .filter(|(f, _)| f.contains_time(days_old)) + .map(|(_, p2a)| p2a) + .for_each(|p2a| { + p2a.um().decrement(prev_price, supply_state); + }); + } + } + }); + } + + pub fn receive(&mut self, received: Transacted, height: Height, price: Option) { + let supply_state = received.spendable_supply; + + [ + &mut self.0.age_range.up_to_1d, + self.0.epoch.mut_vec_from_height(height), + ] + .into_iter() + .for_each(|v| { + v.state.um().receive(&supply_state, price); + }); + + // Update aggregate cohorts' price_to_amount + // New UTXOs have days_old = 0, so use filter.contains_time(0) to check applicability + if let Some(price) = price + && supply_state.value.is_not_zero() + { + self.0 + .iter_aggregate_mut() + .filter(|v| v.filter().contains_time(0)) + .for_each(|v| { + v.price_to_amount + .as_mut() + .unwrap() + .increment(price, &supply_state); + }); + } + + self.type_.iter_mut().for_each(|vecs| { + let output_type = match vecs.filter() { + Filter::Type(output_type) => *output_type, + _ => unreachable!(), + }; + vecs.state + .as_mut() + .unwrap() + .receive(received.by_type.get(output_type), price) + }); + + received + .by_size_group + .iter_typed() + .for_each(|(group, supply_state)| { + self.amount_range + .get_mut(group) + .state + .as_mut() + .unwrap() + .receive(supply_state, price); + }); + } + + pub fn compute_overlapping_vecs( + &mut self, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + let by_date_range = &self.0.age_range; + let by_size_range = &self.0.amount_range; + + [(&mut self.0.all, by_date_range.iter().collect::>())] + .into_par_iter() + .chain(self.0.min_age.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_date_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.max_age.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_date_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.term.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_date_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.ge_amount.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_size_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .chain(self.0.lt_amount.par_iter_mut().map(|vecs| { + let filter = vecs.filter().clone(); + ( + vecs, + by_size_range + .iter() + .filter(|other| filter.includes(other.filter())) + .collect::>(), + ) + })) + .try_for_each(|(vecs, stateful)| { + vecs.compute_from_stateful(starting_indexes, &stateful, exit) + }) + } + + pub fn compute_rest_part1( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { + self.par_iter_mut() + .try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit)) + } + + #[allow(clippy::too_many_arguments)] + pub fn compute_rest_part2( + &mut self, + indexes: &indexes::Vecs, + price: Option<&price::Vecs>, + starting_indexes: &Indexes, + height_to_supply: &impl IterableVec, + dateindex_to_supply: &impl IterableVec, + height_to_market_cap: Option<&impl IterableVec>, + dateindex_to_market_cap: Option<&impl IterableVec>, + height_to_realized_cap: Option<&impl IterableVec>, + dateindex_to_realized_cap: Option<&impl IterableVec>, + exit: &Exit, + ) -> Result<()> { + self.par_iter_mut().try_for_each(|v| { + v.compute_rest_part2( + indexes, + price, + starting_indexes, + height_to_supply, + dateindex_to_supply, + height_to_market_cap, + dateindex_to_market_cap, + height_to_realized_cap, + dateindex_to_realized_cap, + exit, + ) + }) + } + + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + // Flush stateful cohorts + self.par_iter_separate_mut() + .try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))?; + + // Flush aggregate cohorts' price_to_amount and price_percentiles + // Using traits ensures we can't forget to flush any field + self.0.par_iter_aggregate_mut().try_for_each(|v| { + v.price_to_amount.flush_at_height(height, exit)?; + v.inner.price_percentiles.safe_write(exit)?; + Ok(()) + }) + } + + /// Reset aggregate cohorts' price_to_amount when starting from scratch + pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> { + self.0 + .iter_aggregate_mut() + .try_for_each(|v| v.price_to_amount.reset()) + } + + /// Import aggregate cohorts' price_to_amount from disk when resuming from a checkpoint. + /// Returns the height to start processing from (checkpoint_height + 1), matching the + /// behavior of `common::import_state` for separate cohorts. + /// + /// Note: We don't check inner.min_height_vecs_len() for aggregate cohorts because their + /// inner vecs (height_to_supply, etc.) are computed post-hoc by compute_overlapping_vecs, + /// not maintained during the main processing loop. + pub fn import_aggregate_price_to_amount(&mut self, height: Height) -> Result { + // Match separate vecs behavior: decrement height to get prev_height + let Some(mut prev_height) = height.decremented() else { + // height is 0, return ZERO (caller will handle this) + return Ok(Height::ZERO); + }; + + for v in self.0.iter_aggregate_mut() { + // Using HeightFlushable trait - if price_to_amount is None, returns height unchanged + prev_height = prev_height.min(v.price_to_amount.import_at_or_before(prev_height)?); + } + // Return prev_height + 1, matching separate vecs behavior + Ok(prev_height.incremented()) + } + + /// Compute and push percentiles for aggregate cohorts (all, sth, lth). + /// Must be called after receive()/send() when price_to_amount is up to date. + pub fn truncate_push_aggregate_percentiles(&mut self, height: Height) -> Result<()> { + let age_range_data: Vec<_> = self + .0 + .age_range + .iter() + .map(|sub| (sub.filter().clone(), sub.state.u().supply.value)) + .collect(); + + let results: Vec<_> = self + .0 + .par_iter_aggregate() + .map(|v| { + if v.price_to_amount.is_none() { + panic!(); + } + let filter = v.filter().clone(); + let supply = age_range_data + .iter() + .filter(|(sub_filter, _)| filter.includes(sub_filter)) + .map(|(_, value)| *value) + .fold(Sats::ZERO, |acc, v| acc + v); + let percentiles = v.compute_percentile_prices_from_standalone(supply); + (filter, percentiles) + }) + .collect(); + + // Push results sequentially (requires &mut) + for (filter, percentiles) in results { + let v = self + .0 + .iter_aggregate_mut() + .find(|v| v.filter() == &filter) + .unwrap(); + + if let Some(pp) = v.inner.price_percentiles.as_mut() { + pp.truncate_push(height, &percentiles)?; + } + } + + Ok(()) + } +} diff --git a/crates/brk_computer/src/stateful_old/withaddressdatasource.rs b/crates/brk_computer/src/stateful_old/withaddressdatasource.rs new file mode 100644 index 000000000..79ac4b7da --- /dev/null +++ b/crates/brk_computer/src/stateful_old/withaddressdatasource.rs @@ -0,0 +1,56 @@ +use brk_types::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex}; + +#[derive(Debug)] +pub enum WithAddressDataSource { + New(T), + FromLoadedAddressDataVec((LoadedAddressIndex, T)), + FromEmptyAddressDataVec((EmptyAddressIndex, T)), +} + +impl WithAddressDataSource { + pub fn is_new(&self) -> bool { + matches!(self, Self::New(_)) + } + + pub fn is_from_emptyaddressdata(&self) -> bool { + matches!(self, Self::FromEmptyAddressDataVec(_)) + } + + pub fn deref_mut(&mut self) -> &mut T { + match self { + Self::New(v) => v, + Self::FromLoadedAddressDataVec((_, v)) => v, + Self::FromEmptyAddressDataVec((_, v)) => v, + } + } +} + +impl From> for WithAddressDataSource { + #[inline] + fn from(value: WithAddressDataSource) -> Self { + match value { + WithAddressDataSource::New(v) => Self::New(v.into()), + WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => { + Self::FromLoadedAddressDataVec((i, v.into())) + } + WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => { + Self::FromEmptyAddressDataVec((i, v.into())) + } + } + } +} + +impl From> for WithAddressDataSource { + #[inline] + fn from(value: WithAddressDataSource) -> Self { + match value { + WithAddressDataSource::New(v) => Self::New(v.into()), + WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => { + Self::FromLoadedAddressDataVec((i, v.into())) + } + WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => { + Self::FromEmptyAddressDataVec((i, v.into())) + } + } + } +} diff --git a/crates/brk_computer/src/states/flushable.rs b/crates/brk_computer/src/states/flushable.rs new file mode 100644 index 000000000..449e93f39 --- /dev/null +++ b/crates/brk_computer/src/states/flushable.rs @@ -0,0 +1,80 @@ +//! Traits for consistent state flushing and importing. +//! +//! These traits ensure all stateful components follow the same patterns +//! for checkpoint/resume operations, preventing bugs where new fields +//! are forgotten during flush operations. + +use brk_error::Result; +use brk_types::Height; +use vecdb::Exit; + +/// Trait for components that can be flushed to disk. +/// +/// This is for simple flush operations that don't require height tracking. +pub trait Flushable { + /// Safely flush data to disk. + fn safe_flush(&mut self, exit: &Exit) -> Result<()>; + + /// Write to mmap without fsync. Data visible to readers immediately but not durable. + fn safe_write(&mut self, exit: &Exit) -> Result<()>; +} + +/// Trait for stateful components that track data indexed by height. +/// +/// This ensures consistent patterns for: +/// - Flushing state at checkpoints +/// - Importing state when resuming from a checkpoint +/// - Resetting state when starting from scratch +pub trait HeightFlushable { + /// Flush state to disk at the given height checkpoint. + fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()>; + + /// Import state from the most recent checkpoint at or before the given height. + /// Returns the actual height that was imported. + fn import_at_or_before(&mut self, height: Height) -> Result; + + /// Reset state for starting from scratch. + fn reset(&mut self) -> Result<()>; +} + +/// Blanket implementation for Option where T: Flushable +impl Flushable for Option { + fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.safe_flush(exit)?; + } + Ok(()) + } + + fn safe_write(&mut self, exit: &Exit) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.safe_write(exit)?; + } + Ok(()) + } +} + +/// Blanket implementation for Option where T: HeightFlushable +impl HeightFlushable for Option { + fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.flush_at_height(height, exit)?; + } + Ok(()) + } + + fn import_at_or_before(&mut self, height: Height) -> Result { + if let Some(inner) = self.as_mut() { + inner.import_at_or_before(height) + } else { + Ok(height) + } + } + + fn reset(&mut self) -> Result<()> { + if let Some(inner) = self.as_mut() { + inner.reset()?; + } + Ok(()) + } +} diff --git a/crates/brk_computer/src/states/mod.rs b/crates/brk_computer/src/states/mod.rs index 5976a47b9..2eed18059 100644 --- a/crates/brk_computer/src/states/mod.rs +++ b/crates/brk_computer/src/states/mod.rs @@ -1,5 +1,6 @@ mod block; mod cohorts; +mod flushable; mod price_to_amount; mod realized; mod supply; @@ -8,6 +9,7 @@ mod unrealized; pub use block::*; pub use cohorts::*; +pub use flushable::*; pub use price_to_amount::*; pub use realized::*; pub use supply::*; diff --git a/crates/brk_computer/src/states/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs index d75a34658..e3a9bfd44 100644 --- a/crates/brk_computer/src/states/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -11,7 +11,9 @@ use pco::standalone::{simple_decompress, simpler_compress}; use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Exit}; -use crate::{stateful::HeightFlushable, states::SupplyState, utils::OptionExt}; +use crate::{states::SupplyState, utils::OptionExt}; + +use super::HeightFlushable; #[derive(Clone, Debug)] pub struct PriceToAmount { diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 78b8635df..4930d30d5 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -23,6 +23,7 @@ brk_traversable = { workspace = true } fjall = { workspace = true } log = { workspace = true } rayon = { workspace = true } +rlimit = "0.10.2" rustc-hash = { workspace = true } vecdb = { workspace = true } diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index d449edeb0..319c8d04e 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -30,6 +30,14 @@ pub struct Indexer { impl Indexer { pub fn forced_import(outputs_dir: &Path) -> Result { + info!("Increasing number of open files limit..."); + let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?; + rlimit::setrlimit( + rlimit::Resource::NOFILE, + no_file_limit.0.max(10_000), + no_file_limit.1, + )?; + info!("Importing indexer..."); let path = outputs_dir.join("indexed"); diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index e6650de5f..f0c50b9bf 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -25,6 +25,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3); pub fn open_database(path: &Path) -> fjall::Result { Database::builder(path.join("fjall")) .cache_size(3 * 1024 * 1024 * 1024) + .max_cached_files(Some(1024)) .open() } diff --git a/crates/brk_types/src/addressdata_source.rs b/crates/brk_types/src/addressdata_source.rs new file mode 100644 index 000000000..1d2af20a7 --- /dev/null +++ b/crates/brk_types/src/addressdata_source.rs @@ -0,0 +1,12 @@ +use crate::{EmptyAddressIndex, LoadedAddressIndex}; + +/// Source of address data update (where the data came from). +#[derive(Clone)] +pub enum AddressDataSource { + /// Brand new address, not in any storage yet. + New(T), + /// From empty address storage. + FromEmpty((EmptyAddressIndex, T)), + /// From loaded address storage. + FromLoaded((LoadedAddressIndex, T)), +} diff --git a/crates/brk_types/src/lib.rs b/crates/brk_types/src/lib.rs index 6a5dabd70..385fa74a8 100644 --- a/crates/brk_types/src/lib.rs +++ b/crates/brk_types/src/lib.rs @@ -5,6 +5,7 @@ pub use vecdb::{CheckedSub, Exit, PrintableIndex, Version}; mod address; mod addressbytes; mod addresschainstats; +mod addressdata_source; mod addresshash; mod addressindexoutpoint; mod addressindextxindex; @@ -103,6 +104,7 @@ mod yearindex; pub use address::*; pub use addressbytes::*; pub use addresschainstats::*; +pub use addressdata_source::*; pub use addresshash::*; pub use addressindexoutpoint::*; pub use addressindextxindex::*;