computer: stateful: refactor part 1

This commit is contained in:
nym21
2025-12-11 11:26:11 +01:00
parent abde9ed162
commit 1cf75b48b5
77 changed files with 12846 additions and 22 deletions
Generated
+11 -11
View File
@@ -631,7 +631,6 @@ dependencies = [
"log",
"pco",
"rayon",
"rlimit",
"rustc-hash",
"serde",
"smallvec",
@@ -695,6 +694,7 @@ dependencies = [
"log",
"mimalloc",
"rayon",
"rlimit",
"rustc-hash",
"vecdb",
]
@@ -2594,9 +2594,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
[[package]]
name = "icu_properties"
version = "2.1.1"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99"
checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
dependencies = [
"icu_collections",
"icu_locale_core",
@@ -2608,9 +2608,9 @@ dependencies = [
[[package]]
name = "icu_properties_data"
version = "2.1.1"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899"
checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
[[package]]
name = "icu_provider"
@@ -2924,9 +2924,9 @@ dependencies = [
[[package]]
name = "libz-rs-sys"
version = "0.5.3"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222"
checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c"
dependencies = [
"zlib-rs",
]
@@ -3287,9 +3287,9 @@ dependencies = [
[[package]]
name = "oxc-browserslist"
version = "2.1.3"
version = "2.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f978be538ca5e2a64326d24b7991dc658cc8495132833ae387212ab3b8abd70a"
checksum = "9bd39c45e1d6bd2abfbd4b89cbcaba34bd315cd3cee23aad623fd075acc1ea01"
dependencies = [
"bincode",
"flate2",
@@ -5973,9 +5973,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.5.3"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7"
checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235"
[[package]]
name = "zopfli"
+1
View File
@@ -1,6 +1,7 @@
[workspace]
resolver = "3"
members = ["crates/*"]
default-members = ["crates/brk_cli"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
-1
View File
@@ -26,7 +26,6 @@ derive_deref = { workspace = true }
log = { workspace = true }
pco = "0.4.7"
rayon = { workspace = true }
rlimit = "0.10.2"
rustc-hash = { workspace = true }
serde = { workspace = true }
smallvec = "1.15.1"
+1 -8
View File
@@ -22,6 +22,7 @@ mod market;
mod pools;
mod price;
mod stateful;
mod stateful_new;
mod states;
mod traits;
mod utils;
@@ -56,14 +57,6 @@ impl Computer {
indexer: &Indexer,
fetcher: Option<Fetcher>,
) -> Result<Self> {
info!("Increasing number of open files...");
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit(
rlimit::Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
info!("Importing computer...");
let import_start = Instant::now();
+1 -1
View File
@@ -73,7 +73,7 @@ mod utxo_cohort;
mod utxo_cohorts;
mod withaddressdatasource;
pub use flushable::{Flushable, HeightFlushable};
pub use crate::states::{Flushable, HeightFlushable};
use address_indexes::{AddressesDataVecs, AnyAddressIndexesVecs};
use addresstype::*;
@@ -0,0 +1,176 @@
//! Address count types per address type.
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version};
use derive_deref::{Deref, DerefMut};
use vecdb::{Database, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec, TypedVecIterator};
use crate::{
Indexes,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes,
};
/// Address count per address type (runtime state).
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
#[inline]
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(ByAddressType {
p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(),
p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(),
p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(),
p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(),
p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(),
p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(),
p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(),
p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(),
})
} else {
Default::default()
}
}
}
/// Address count per address type, indexed by height.
#[derive(Debug, Clone, Deref, DerefMut, Traversable)]
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>);
impl From<ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>>
for AddressTypeToHeightToAddressCount
{
#[inline]
fn from(value: ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>) -> Self {
Self(value)
}
}
impl AddressTypeToHeightToAddressCount {
pub fn forced_import(db: &Database, name: &str, version: Version) -> Result<Self> {
Ok(Self::from(ByAddressType::new_with_name(|type_name| {
Ok(EagerVec::forced_import(
db,
&format!("{type_name}_{name}"),
version,
)?)
})?))
}
pub fn truncate_push(
&mut self,
height: Height,
addresstype_to_usize: &AddressTypeToAddressCount,
) -> Result<()> {
self.p2pk65
.truncate_push(height, addresstype_to_usize.p2pk65.into())?;
self.p2pk33
.truncate_push(height, addresstype_to_usize.p2pk33.into())?;
self.p2pkh
.truncate_push(height, addresstype_to_usize.p2pkh.into())?;
self.p2sh
.truncate_push(height, addresstype_to_usize.p2sh.into())?;
self.p2wpkh
.truncate_push(height, addresstype_to_usize.p2wpkh.into())?;
self.p2wsh
.truncate_push(height, addresstype_to_usize.p2wsh.into())?;
self.p2tr
.truncate_push(height, addresstype_to_usize.p2tr.into())?;
self.p2a
.truncate_push(height, addresstype_to_usize.p2a.into())?;
Ok(())
}
}
/// Address count per address type, indexed by various indexes (dateindex, etc.).
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
#[inline]
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
Self(value)
}
}
impl AddressTypeToIndexesToAddressCount {
pub fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self::from(ByAddressType::new_with_name(|type_name| {
ComputedVecsFromHeight::forced_import(
db,
&format!("{type_name}_{name}"),
Source::None,
version,
indexes,
VecBuilderOptions::default().add_last(),
)
})?))
}
pub fn compute(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
) -> Result<()> {
self.p2pk65.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk65),
)?;
self.p2pk33.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk33),
)?;
self.p2pkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pkh),
)?;
self.p2sh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2sh),
)?;
self.p2wpkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wpkh),
)?;
self.p2wsh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wsh),
)?;
self.p2tr.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2tr),
)?;
self.p2a.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2a),
)?;
Ok(())
}
}
@@ -0,0 +1,98 @@
//! Storage for address indexes by type.
use brk_error::{Error, Result};
use brk_traversable::Traversable;
use brk_types::{
AnyAddressIndex, Height, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex,
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
TypeIndex, Version,
};
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Reader, Stamp,
};
const SAVED_STAMPED_CHANGES: u16 = 10;
/// Macro to define AnyAddressIndexesVecs and its methods.
macro_rules! define_any_address_indexes_vecs {
($(($field:ident, $variant:ident, $index:ty)),* $(,)?) => {
#[derive(Clone, Traversable)]
pub struct AnyAddressIndexesVecs {
$(pub $field: BytesVec<$index, AnyAddressIndex>,)*
}
impl AnyAddressIndexesVecs {
/// Import from database.
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
$($field: BytesVec::forced_import_with(
ImportOptions::new(db, "anyaddressindex", version)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,)*
})
}
/// Get minimum stamped height across all address types.
pub fn min_stamped_height(&self) -> Height {
[$(Height::from(self.$field.stamp()).incremented()),*]
.into_iter()
.min()
.unwrap_or_default()
}
/// Rollback all address types to before the given stamp.
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<Vec<Stamp>> {
Ok(vec![$(self.$field.rollback_before(stamp)?),*])
}
/// Reset all address types.
pub fn reset(&mut self) -> Result<()> {
$(self.$field.reset()?;)*
Ok(())
}
/// Get address index for a given type and typeindex.
pub fn get(&self, address_type: OutputType, typeindex: TypeIndex, reader: &Reader) -> AnyAddressIndex {
match address_type {
$(OutputType::$variant => self.$field.get_pushed_or_read_at_unwrap(typeindex.into(), reader),)*
_ => unreachable!("Invalid address type: {:?}", address_type),
}
}
/// Get address index with single read (no caching).
pub fn get_once(&self, address_type: OutputType, typeindex: TypeIndex) -> Result<AnyAddressIndex> {
match address_type {
$(OutputType::$variant => self.$field.read_at_once(typeindex.into()).map_err(Into::into),)*
_ => Err(Error::UnsupportedType(address_type.to_string())),
}
}
/// Update or push address index for a given type.
pub fn update_or_push(&mut self, address_type: OutputType, typeindex: TypeIndex, index: AnyAddressIndex) -> Result<()> {
match address_type {
$(OutputType::$variant => self.$field.update_or_push(typeindex.into(), index)?,)*
_ => unreachable!("Invalid address type: {:?}", address_type),
}
Ok(())
}
/// Flush all address types with stamp.
pub fn flush(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> {
$(self.$field.stamped_flush_maybe_with_changes(stamp, with_changes)?;)*
Ok(())
}
}
};
}
// Generate the struct and methods
define_any_address_indexes_vecs!(
(p2a, P2A, P2AAddressIndex),
(p2pk33, P2PK33, P2PK33AddressIndex),
(p2pk65, P2PK65, P2PK65AddressIndex),
(p2pkh, P2PKH, P2PKHAddressIndex),
(p2sh, P2SH, P2SHAddressIndex),
(p2tr, P2TR, P2TRAddressIndex),
(p2wpkh, P2WPKH, P2WPKHAddressIndex),
(p2wsh, P2WSH, P2WSHAddressIndex),
);
@@ -0,0 +1,66 @@
//! Storage for address data (loaded and empty addresses).
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{
EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, Version,
};
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Stamp,
};
const SAVED_STAMPED_CHANGES: u16 = 10;
/// Storage for both loaded and empty address data.
#[derive(Clone, Traversable)]
pub struct AddressesDataVecs {
pub loaded: BytesVec<LoadedAddressIndex, LoadedAddressData>,
pub empty: BytesVec<EmptyAddressIndex, EmptyAddressData>,
}
impl AddressesDataVecs {
/// Import from database.
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
loaded: BytesVec::forced_import_with(
ImportOptions::new(db, "loadedaddressdata", version)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
empty: BytesVec::forced_import_with(
ImportOptions::new(db, "emptyaddressdata", version)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
})
}
/// Get minimum stamped height across loaded and empty data.
pub fn min_stamped_height(&self) -> Height {
Height::from(self.loaded.stamp())
.incremented()
.min(Height::from(self.empty.stamp()).incremented())
}
/// Rollback both loaded and empty data to before the given stamp.
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> {
Ok([
self.loaded.rollback_before(stamp)?,
self.empty.rollback_before(stamp)?,
])
}
/// Reset both loaded and empty data.
pub fn reset(&mut self) -> Result<()> {
self.loaded.reset()?;
self.empty.reset()?;
Ok(())
}
/// Flush both loaded and empty data with stamp.
pub fn flush(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> {
self.loaded
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.empty
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
Ok(())
}
}
@@ -0,0 +1,20 @@
//! Height to AddressTypeToVec hashmap.
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;
use super::type_vec::AddressTypeToVec;
/// Hashmap from Height to AddressTypeToVec.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct HeightToAddressTypeToVec<T>(FxHashMap<Height, AddressTypeToVec<T>>);
impl<T> HeightToAddressTypeToVec<T> {
/// Merge another map into this one.
pub fn merge_mut(&mut self, other: Self) {
for (height, vec) in other.0 {
self.entry(height).or_default().merge_mut(vec);
}
}
}
@@ -0,0 +1,28 @@
//! Address handling with macro-generated code for 8 address types.
//!
//! This module provides:
//! - `AnyAddressIndexesVecs` for storing address indexes by type
//! - `AddressesDataVecs` for storing address data (loaded/empty)
//! - `AddressTypeToTypeIndexMap` for per-type hashmaps
//! - `AddressTypeToVec` for per-type vectors
//! - `HeightToAddressTypeToVec` for height-keyed per-type vectors
//! - `AddressTypeToAddressCount` for runtime address counts
//! - `AddressTypeToHeightToAddressCount` for height-indexed address counts
//! - `AddressTypeToIndexesToAddressCount` for computed address counts
mod address_count;
mod any_address_indexes;
mod data;
mod height_type_vec;
mod type_index_map;
mod type_vec;
pub use address_count::{
AddressTypeToAddressCount, AddressTypeToHeightToAddressCount,
AddressTypeToIndexesToAddressCount,
};
pub use any_address_indexes::AnyAddressIndexesVecs;
pub use data::AddressesDataVecs;
pub use height_type_vec::HeightToAddressTypeToVec;
pub use type_index_map::AddressTypeToTypeIndexMap;
pub use type_vec::AddressTypeToVec;
@@ -0,0 +1,106 @@
//! Per-address-type hashmap keyed by TypeIndex.
use std::mem;
use brk_grouper::ByAddressType;
use brk_types::{OutputType, TypeIndex};
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;
use smallvec::{Array, SmallVec};
use std::collections::hash_map::Entry;
/// A hashmap for each address type, keyed by TypeIndex.
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToTypeIndexMap<T>(ByAddressType<FxHashMap<TypeIndex, T>>);
impl<T> Default for AddressTypeToTypeIndexMap<T> {
fn default() -> Self {
Self(ByAddressType {
p2a: FxHashMap::default(),
p2pk33: FxHashMap::default(),
p2pk65: FxHashMap::default(),
p2pkh: FxHashMap::default(),
p2sh: FxHashMap::default(),
p2tr: FxHashMap::default(),
p2wpkh: FxHashMap::default(),
p2wsh: FxHashMap::default(),
})
}
}
impl<T> AddressTypeToTypeIndexMap<T> {
/// Merge two maps, consuming other and extending self.
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_single(&mut self.p2a, &mut other.p2a);
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
self
}
fn merge_single(own: &mut FxHashMap<TypeIndex, T>, other: &mut FxHashMap<TypeIndex, T>) {
if own.len() < other.len() {
mem::swap(own, other);
}
own.extend(other.drain());
}
/// Insert a value for a specific address type and typeindex.
pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) {
self.get_mut(address_type).unwrap().insert(typeindex, value);
}
/// Remove and return a value for a specific address type and typeindex.
pub fn remove_for_type(&mut self, address_type: OutputType, typeindex: &TypeIndex) -> T {
self.get_mut(address_type)
.unwrap()
.remove(typeindex)
.unwrap()
}
/// Iterate over sorted entries by address type.
pub fn into_sorted_iter(self) -> impl Iterator<Item = (OutputType, Vec<(TypeIndex, T)>)> {
self.0.into_iter().map(|(output_type, map)| {
let mut sorted: Vec<_> = map.into_iter().collect();
sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex);
(output_type, sorted)
})
}
/// Consume and iterate over entries by address type.
#[allow(clippy::should_implement_trait)]
pub fn into_iter(self) -> impl Iterator<Item = (OutputType, FxHashMap<TypeIndex, T>)> {
self.0.into_iter()
}
}
impl<T> AddressTypeToTypeIndexMap<SmallVec<T>>
where
T: Array,
{
/// Merge two maps of SmallVec values, concatenating vectors.
pub fn merge_vec(mut self, other: Self) -> Self {
for (address_type, other_map) in other.0.into_iter() {
let self_map = self.0.get_mut_unwrap(address_type);
for (typeindex, mut other_vec) in other_map {
match self_map.entry(typeindex) {
Entry::Occupied(mut entry) => {
let self_vec = entry.get_mut();
if other_vec.len() > self_vec.len() {
mem::swap(self_vec, &mut other_vec);
}
self_vec.extend(other_vec);
}
Entry::Vacant(entry) => {
entry.insert(other_vec);
}
}
}
}
self
}
}
@@ -0,0 +1,66 @@
//! Per-address-type vector.
use std::mem;
use brk_grouper::ByAddressType;
use derive_deref::{Deref, DerefMut};
/// A vector for each address type.
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2a: vec![],
p2pk33: vec![],
p2pk65: vec![],
p2pkh: vec![],
p2sh: vec![],
p2tr: vec![],
p2wpkh: vec![],
p2wsh: vec![],
})
}
}
impl<T> AddressTypeToVec<T> {
/// Merge two AddressTypeToVec, consuming other.
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_single(&mut self.p2a, &mut other.p2a);
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
self
}
/// Merge in place.
pub fn merge_mut(&mut self, mut other: Self) {
Self::merge_single(&mut self.p2a, &mut other.p2a);
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
}
fn merge_single(own: &mut Vec<T>, other: &mut Vec<T>) {
if own.len() >= other.len() {
own.append(other);
} else {
other.append(own);
mem::swap(own, other);
}
}
/// Unwrap the inner ByAddressType.
pub fn unwrap(self) -> ByAddressType<Vec<T>> {
self.0
}
}
@@ -0,0 +1,280 @@
//! Address cohort vectors with metrics and state.
use std::path::Path;
use brk_error::Result;
use brk_grouper::{CohortContext, Filter, Filtered};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec,
PcoVec,
};
use crate::{
Indexes,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, price,
states::AddressCohortState,
};
use super::super::metrics::{CohortMetrics, ImportConfig};
use super::traits::{CohortVecs, DynCohortVecs};
const VERSION: Version = Version::ZERO;
/// Address cohort with metrics and optional runtime state.
#[derive(Clone, Traversable)]
pub struct AddressCohortVecs {
/// Starting height when state was imported
starting_height: Option<Height>,
/// Runtime state for block-by-block processing
#[traversable(skip)]
pub state: Option<AddressCohortState>,
/// Metric vectors
#[traversable(flatten)]
pub metrics: CohortMetrics,
/// Address count at each height
pub height_to_addr_count: EagerVec<PcoVec<Height, StoredU64>>,
/// Address count indexed by various dimensions
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
}
impl AddressCohortVecs {
/// Import address cohort from database.
pub fn forced_import(
db: &Database,
filter: Filter,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: Option<&Path>,
) -> Result<Self> {
let compute_dollars = price.is_some();
let full_name = filter.to_full_name(CohortContext::Address);
let cfg = ImportConfig {
db,
filter,
context: CohortContext::Address,
version,
indexes,
price,
};
Ok(Self {
starting_height: None,
state: states_path
.map(|path| AddressCohortState::new(path, &full_name, compute_dollars)),
metrics: CohortMetrics::forced_import(&cfg)?,
height_to_addr_count: EagerVec::forced_import(
db,
&cfg.name("addr_count"),
version + VERSION + Version::ZERO,
)?,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
db,
&cfg.name("addr_count"),
Source::None,
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
})
}
/// Get the starting height when state was imported.
pub fn starting_height(&self) -> Option<Height> {
self.starting_height
}
/// Set the starting height.
pub fn set_starting_height(&mut self, height: Height) {
self.starting_height = Some(height);
}
/// Reset starting height to zero.
pub fn reset_starting_height(&mut self) {
self.starting_height = Some(Height::ZERO);
}
/// Get minimum length across height-indexed vectors.
pub fn min_len(&self) -> usize {
self.height_to_addr_count
.len()
.min(self.metrics.supply.min_len())
.min(self.metrics.activity.min_len())
}
}
impl Filtered for AddressCohortVecs {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for AddressCohortVecs {
fn min_height_vecs_len(&self) -> usize {
self.min_len()
}
fn reset_state_starting_height(&mut self) {
self.reset_starting_height();
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
// Import state from runtime state if present
if let Some(state) = self.state.as_mut() {
let imported = state.inner.import_at_or_before(starting_height)?;
self.starting_height = Some(imported);
// Restore addr_count from last known value
if let Some(prev_height) = imported.decremented() {
use vecdb::TypedVecIterator;
state.addr_count = *self
.height_to_addr_count
.into_iter()
.get_unwrap(prev_height);
}
Ok(imported)
} else {
self.starting_height = Some(starting_height);
Ok(starting_height)
}
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
use vecdb::GenericStoredVec;
self.height_to_addr_count
.validate_computed_version_or_reset(
base_version + self.height_to_addr_count.inner_version(),
)?;
self.metrics.validate_computed_versions(base_version)?;
Ok(())
}
fn truncate_push(&mut self, height: Height) -> Result<()> {
if self.starting_height.map_or(false, |h| h > height) {
return Ok(());
}
// Push addr_count from state
if let Some(state) = self.state.as_ref() {
self.height_to_addr_count
.truncate_push(height, state.addr_count.into())?;
self.metrics.truncate_push(height, &state.inner)?;
}
Ok(())
}
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()> {
if let Some(state) = self.state.as_ref() {
self.metrics.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
&state.inner,
)?;
}
Ok(())
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.height_to_addr_count.safe_write(exit)?;
self.metrics.safe_flush(exit)?;
if let Some(state) = self.state.as_mut() {
state.inner.commit(height)?;
}
Ok(())
}
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_addr_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_addr_count),
)?;
self.metrics
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
Ok(())
}
}
impl CohortVecs for AddressCohortVecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_addr_count.compute_sum_of_others(
starting_indexes.height,
others
.iter()
.map(|v| &v.height_to_addr_count)
.collect::<Vec<_>>()
.as_slice(),
exit,
)?;
self.metrics.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.metrics.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)?;
Ok(())
}
}
@@ -0,0 +1,230 @@
//! Container for all Address cohorts organized by filter type.
use std::path::Path;
use brk_error::Result;
use brk_grouper::{
AddressGroups, AmountFilter, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, Filter,
Filtered,
};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{Database, Exit, IterableVec};
use crate::{Indexes, indexes, price, stateful_new::DynCohortVecs};
use super::{AddressCohortVecs, CohortVecs};
const VERSION: Version = Version::new(0);
/// All Address cohorts organized by filter type.
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct AddressCohorts(AddressGroups<AddressCohortVecs>);
impl AddressCohorts {
/// Import all Address cohorts from database.
pub fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
let v = version + VERSION + Version::ZERO;
// Helper to create a cohort - only amount_range cohorts have state
let create = |filter: Filter, has_state: bool| -> Result<AddressCohortVecs> {
let states_path = if has_state { Some(states_path) } else { None };
AddressCohortVecs::forced_import(db, filter, v, indexes, price, states_path)
};
let full = |f: Filter| create(f, true);
let none = |f: Filter| create(f, false);
Ok(Self(AddressGroups {
amount_range: ByAmountRange {
_0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?,
_1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?,
_10sats_to_100sats: full(Filter::Amount(AmountFilter::Range(
Sats::_10..Sats::_100,
)))?,
_100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_100..Sats::_1K,
)))?,
_1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_1K..Sats::_10K,
)))?,
_10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_10K..Sats::_100K,
)))?,
_100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_100K..Sats::_1M,
)))?,
_1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_1M..Sats::_10M,
)))?,
_10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10M..Sats::_1BTC,
)))?,
_1btc_to_10btc: full(Filter::Amount(AmountFilter::Range(
Sats::_1BTC..Sats::_10BTC,
)))?,
_10btc_to_100btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10BTC..Sats::_100BTC,
)))?,
_100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_100BTC..Sats::_1K_BTC,
)))?,
_1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_1K_BTC..Sats::_10K_BTC,
)))?,
_10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10K_BTC..Sats::_100K_BTC,
)))?,
_100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual(
Sats::_100K_BTC,
)))?,
},
lt_amount: ByLowerThanAmount {
_10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?,
_100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?,
_1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?,
_10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?,
_100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?,
_1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?,
_10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?,
_1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?,
_10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?,
_100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?,
_1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?,
_10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?,
_100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?,
},
ge_amount: ByGreatEqualAmount {
_1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?,
_10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?,
_100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?,
_1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?,
_10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?,
_100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?,
_1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?,
_10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?,
_1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?,
_10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?,
_100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?,
_1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?,
_10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?,
},
}))
}
/// Compute overlapping cohorts from component amount_range cohorts.
///
/// For example, ">=1 BTC" cohort is computed from sum of amount_range cohorts that match.
pub fn compute_overlapping_vecs(
&mut self,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let by_amount_range = &self.0.amount_range;
// ge_amount cohorts computed from matching amount_range cohorts
[
self.0
.ge_amount
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
// lt_amount cohorts computed from matching amount_range cohorts
self.0
.lt_amount
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
]
.into_iter()
.flatten()
.try_for_each(|(vecs, components)| {
vecs.compute_from_stateful(starting_indexes, &components, exit)
})
}
/// First phase of post-processing: compute index transforms.
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
/// Second phase of post-processing: compute relative metrics.
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2<S, D, HM, DM, HR, DR>(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &S,
dateindex_to_supply: &D,
height_to_market_cap: Option<&HM>,
dateindex_to_market_cap: Option<&DM>,
height_to_realized_cap: Option<&HR>,
dateindex_to_realized_cap: Option<&DR>,
exit: &Exit,
) -> Result<()>
where
S: IterableVec<Height, Bitcoin> + Sync,
D: IterableVec<DateIndex, Bitcoin> + Sync,
HM: IterableVec<Height, Dollars> + Sync,
DM: IterableVec<DateIndex, Dollars> + Sync,
HR: IterableVec<Height, Dollars> + Sync,
DR: IterableVec<DateIndex, Dollars> + Sync,
{
self.0.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
})
}
/// Flush stateful vectors for separate cohorts.
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))
}
}
@@ -0,0 +1,22 @@
//! Cohort management for UTXO and address groupings.
//!
//! Cohorts are groups of UTXOs or addresses filtered by criteria like:
//! - Age (0-1d, 1-7d, etc.)
//! - Amount (< 1 BTC, 1-10 BTC, etc.)
//! - Type (P2PKH, P2SH, etc.)
//! - Term (short-term holder, long-term holder)
mod address;
mod address_cohorts;
mod state;
mod traits;
mod utxo;
mod utxo_cohorts;
pub use address::AddressCohortVecs;
pub use address_cohorts::AddressCohorts;
pub use crate::states::{Flushable, HeightFlushable};
pub use state::CohortState;
pub use traits::{CohortVecs, DynCohortVecs};
pub use utxo::UTXOCohortVecs;
pub use utxo_cohorts::UTXOCohorts;
@@ -0,0 +1,243 @@
//! Cohort state tracking during computation.
//!
//! This state is maintained in memory during block processing and periodically flushed.
use std::cmp::Ordering;
use std::path::Path;
use brk_error::Result;
use brk_types::{CheckedSub, Dollars, Height, Sats};
use crate::{
PriceToAmount, RealizedState, SupplyState, UnrealizedState,
grouped::{PERCENTILES, PERCENTILES_LEN},
utils::OptionExt,
};
/// State tracked for each cohort during computation.
#[derive(Clone)]
pub struct CohortState {
/// Current supply in this cohort
pub supply: SupplyState,
/// Realized cap and profit/loss (requires price data)
pub realized: Option<RealizedState>,
/// Amount sent in current block
pub sent: Sats,
/// Satoshi-blocks destroyed (supply * blocks_old when spent)
pub satblocks_destroyed: Sats,
/// Satoshi-days destroyed (supply * days_old when spent)
pub satdays_destroyed: Sats,
/// Price distribution for percentile calculations (requires price data)
price_to_amount: Option<PriceToAmount>,
}
impl CohortState {
/// Create new cohort state.
pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self {
Self {
supply: SupplyState::default(),
realized: compute_dollars.then_some(RealizedState::NAN),
sent: Sats::ZERO,
satblocks_destroyed: Sats::ZERO,
satdays_destroyed: Sats::ZERO,
price_to_amount: compute_dollars.then_some(PriceToAmount::create(path, name)),
}
}
/// Import state from checkpoint.
pub fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
match self.price_to_amount.as_mut() {
Some(p) => p.import_at_or_before(height),
None => Ok(height),
}
}
/// Reset price_to_amount if needed (for starting fresh).
pub fn reset_price_to_amount(&mut self) -> Result<()> {
if let Some(p) = self.price_to_amount.as_mut() {
p.clean()?;
p.init();
}
Ok(())
}
/// Reset per-block values before processing next block.
pub fn reset_block_values(&mut self) {
self.sent = Sats::ZERO;
self.satdays_destroyed = Sats::ZERO;
self.satblocks_destroyed = Sats::ZERO;
if let Some(realized) = self.realized.as_mut() {
realized.reset_single_iteration_values();
}
}
/// Add supply to this cohort (e.g., when UTXO ages into cohort).
pub fn increment(&mut self, supply: &SupplyState, price: Option<Dollars>) {
self.supply += supply;
if supply.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.increment(supply, price);
self.price_to_amount.as_mut().unwrap().increment(price, supply);
}
}
}
/// Remove supply from this cohort (e.g., when UTXO ages out of cohort).
pub fn decrement(&mut self, supply: &SupplyState, price: Option<Dollars>) {
self.supply -= supply;
if supply.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.decrement(supply, price);
self.price_to_amount.as_mut().unwrap().decrement(price, supply);
}
}
}
/// Process received output (new UTXO in cohort).
pub fn receive(&mut self, supply: &SupplyState, price: Option<Dollars>) {
self.supply += supply;
if supply.value > Sats::ZERO {
if let Some(realized) = self.realized.as_mut() {
let price = price.unwrap();
realized.receive(supply, price);
self.price_to_amount.as_mut().unwrap().increment(price, supply);
}
}
}
/// Process spent input (UTXO leaving cohort).
pub fn send(
&mut self,
supply: &SupplyState,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
blocks_old: usize,
days_old: f64,
older_than_hour: bool,
) {
if supply.utxo_count == 0 {
return;
}
self.supply -= supply;
if supply.value > Sats::ZERO {
self.sent += supply.value;
self.satblocks_destroyed += supply.value * blocks_old;
self.satdays_destroyed +=
Sats::from((u64::from(supply.value) as f64 * days_old).floor() as u64);
if let Some(realized) = self.realized.as_mut() {
let current_price = current_price.unwrap();
let prev_price = prev_price.unwrap();
realized.send(supply, current_price, prev_price, older_than_hour);
self.price_to_amount.as_mut().unwrap().decrement(prev_price, supply);
}
}
}
/// Compute prices at percentile thresholds.
pub fn compute_percentile_prices(&self) -> [Dollars; PERCENTILES_LEN] {
let mut result = [Dollars::NAN; PERCENTILES_LEN];
let price_to_amount = match self.price_to_amount.as_ref() {
Some(p) => p,
None => return result,
};
if price_to_amount.is_empty() || self.supply.value == Sats::ZERO {
return result;
}
let total = u64::from(self.supply.value);
let targets = PERCENTILES.map(|p| total * u64::from(p) / 100);
let mut accumulated = 0u64;
let mut pct_idx = 0;
for (&price, &sats) in price_to_amount.iter() {
accumulated += u64::from(sats);
while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] {
result[pct_idx] = price;
pct_idx += 1;
}
if pct_idx >= PERCENTILES_LEN {
break;
}
}
result
}
/// Compute unrealized profit/loss at current price.
pub fn compute_unrealized(
&self,
height_price: Dollars,
date_price: Option<Dollars>,
) -> (UnrealizedState, Option<UnrealizedState>) {
let price_to_amount = match self.price_to_amount.as_ref() {
Some(p) if !p.is_empty() => p,
_ => return (UnrealizedState::NAN, date_price.map(|_| UnrealizedState::NAN)),
};
let mut height_state = UnrealizedState::ZERO;
let mut date_state = date_price.map(|_| UnrealizedState::ZERO);
for (&price, &sats) in price_to_amount.iter() {
Self::update_unrealized(price, height_price, sats, &mut height_state);
if let Some(date_price) = date_price {
Self::update_unrealized(price, date_price, sats, date_state.um());
}
}
(height_state, date_state)
}
fn update_unrealized(price: Dollars, current: Dollars, sats: Sats, state: &mut UnrealizedState) {
match price.cmp(&current) {
Ordering::Less | Ordering::Equal => {
state.supply_in_profit += sats;
if price < current && price > Dollars::ZERO && current > Dollars::ZERO {
state.unrealized_profit += current.checked_sub(price).unwrap() * sats;
}
}
Ordering::Greater => {
state.supply_in_loss += sats;
if price > Dollars::ZERO && current > Dollars::ZERO {
state.unrealized_loss += price.checked_sub(current).unwrap() * sats;
}
}
}
}
/// Flush state to disk at checkpoint.
pub fn commit(&mut self, height: Height) -> Result<()> {
if let Some(p) = self.price_to_amount.as_mut() {
p.flush(height)?;
}
Ok(())
}
/// Get first (lowest) price in distribution.
pub fn min_price(&self) -> Option<&Dollars> {
self.price_to_amount.as_ref()?.first_key_value().map(|(k, _)| k)
}
/// Get last (highest) price in distribution.
pub fn max_price(&self) -> Option<&Dollars> {
self.price_to_amount.as_ref()?.last_key_value().map(|(k, _)| k)
}
}
@@ -0,0 +1,76 @@
//! Traits for cohort vector operations.
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Exit, IterableVec};
use crate::{indexes, price, Indexes};
/// Dynamic dispatch trait for cohort vectors.
///
/// This trait enables heterogeneous cohort processing via trait objects.
pub trait DynCohortVecs: Send + Sync {
/// Get minimum length across height-indexed vectors.
fn min_height_vecs_len(&self) -> usize;
/// Reset the starting height for state tracking.
fn reset_state_starting_height(&mut self);
/// Import state from checkpoint at or before the given height.
fn import_state(&mut self, starting_height: Height) -> Result<Height>;
/// Validate that computed vectors have correct versions.
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
/// Push state to height-indexed vectors (truncating if needed).
fn truncate_push(&mut self, height: Height) -> Result<()>;
/// Compute and push unrealized profit/loss states.
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()>;
/// Flush stateful vectors to disk.
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>;
/// First phase of post-processing computations.
#[allow(clippy::too_many_arguments)]
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()>;
}
/// Static dispatch trait for cohort vectors with additional methods.
pub trait CohortVecs: DynCohortVecs {
/// Compute aggregate cohort from component cohorts.
fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()>;
/// Second phase of post-processing computations.
#[allow(clippy::too_many_arguments)]
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()>;
}
@@ -0,0 +1,224 @@
//! UTXO cohort vectors with metrics and state.
use std::path::Path;
use brk_error::Result;
use brk_grouper::{CohortContext, Filter, Filtered, StateLevel};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Database, Exit, IterableVec};
use crate::{
Indexes, PriceToAmount, UTXOCohortState, indexes, price,
stateful_new::{CohortVecs, DynCohortVecs},
};
use super::super::metrics::{CohortMetrics, ImportConfig};
/// UTXO cohort with metrics and optional runtime state.
#[derive(Clone, Traversable)]
pub struct UTXOCohortVecs {
/// Starting height when state was imported
state_starting_height: Option<Height>,
/// Runtime state for block-by-block processing
#[traversable(skip)]
pub state: Option<UTXOCohortState>,
/// For aggregate cohorts that only need price_to_amount for percentiles
#[traversable(skip)]
pub price_to_amount: Option<PriceToAmount>,
/// Metric vectors
#[traversable(flatten)]
pub metrics: CohortMetrics,
}
impl UTXOCohortVecs {
/// Import UTXO cohort from database.
pub fn forced_import(
db: &Database,
filter: Filter,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
state_level: StateLevel,
) -> Result<Self> {
let compute_dollars = price.is_some();
let full_name = filter.to_full_name(CohortContext::Utxo);
let cfg = ImportConfig {
db,
filter,
context: CohortContext::Utxo,
version,
indexes,
price,
};
Ok(Self {
state_starting_height: None,
state: if state_level.is_full() {
Some(UTXOCohortState::new(
states_path,
&full_name,
compute_dollars,
))
} else {
None
},
price_to_amount: if state_level.is_price_only() && compute_dollars {
Some(PriceToAmount::create(states_path, &full_name))
} else {
None
},
metrics: CohortMetrics::forced_import(&cfg)?,
})
}
/// Get the starting height when state was imported.
pub fn state_starting_height(&self) -> Option<Height> {
self.state_starting_height
}
/// Set the state starting height.
pub fn set_state_starting_height(&mut self, height: Height) {
self.state_starting_height = Some(height);
}
/// Reset state starting height to zero.
pub fn reset_state_starting_height(&mut self) {
self.state_starting_height = Some(Height::ZERO);
}
}
impl Filtered for UTXOCohortVecs {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for UTXOCohortVecs {
fn min_height_vecs_len(&self) -> usize {
self.metrics.min_len()
}
fn reset_state_starting_height(&mut self) {
self.state_starting_height = Some(Height::ZERO);
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
// Import state from runtime state if present
if let Some(state) = self.state.as_mut() {
let imported = state.import_at_or_before(starting_height)?;
self.state_starting_height = Some(imported);
Ok(imported)
} else {
self.state_starting_height = Some(starting_height);
Ok(starting_height)
}
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.metrics.validate_computed_versions(base_version)
}
fn truncate_push(&mut self, height: Height) -> Result<()> {
if self.state_starting_height.map_or(false, |h| h > height) {
return Ok(());
}
// Push from state to metrics
if let Some(state) = self.state.as_ref() {
self.metrics.truncate_push(height, &state)?;
}
Ok(())
}
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()> {
if let Some(state) = self.state.as_ref() {
self.metrics.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
state,
)?;
}
Ok(())
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.metrics.safe_flush(exit)?;
if let Some(state) = self.state.as_mut() {
state.commit(height)?;
}
Ok(())
}
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(indexes, price, starting_indexes, exit)
}
}
impl CohortVecs for UTXOCohortVecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.metrics.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
exit,
)
}
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.metrics.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
}
}
@@ -0,0 +1,376 @@
//! Container for all UTXO cohorts organized by filter type.
mod receive;
mod send;
mod tick_tock;
use std::path::Path;
use brk_error::Result;
use brk_grouper::{
AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount,
ByMaxAge, ByMinAge, BySpendableType, ByTerm, Filter, Filtered, StateLevel, Term, TimeFilter,
UTXOGroups,
};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, HalvingEpoch, Height, OutputType, Sats, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{Database, Exit, IterableVec};
use crate::{Indexes, indexes, price, stateful_new::DynCohortVecs};
use super::{CohortVecs, HeightFlushable, UTXOCohortVecs};
const VERSION: Version = Version::new(0);
/// All UTXO cohorts organized by filter type.
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct UTXOCohorts(pub(crate) UTXOGroups<UTXOCohortVecs>);
impl UTXOCohorts {
/// Import all UTXO cohorts from database.
pub fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
let v = version + VERSION + Version::ZERO;
let create = |filter: Filter, state_level: StateLevel| -> Result<UTXOCohortVecs> {
UTXOCohortVecs::forced_import(db, filter, v, indexes, price, states_path, state_level)
};
let full = |f: Filter| create(f, StateLevel::Full);
let none = |f: Filter| create(f, StateLevel::None);
Ok(Self(UTXOGroups {
all: UTXOCohortVecs::forced_import(
db,
Filter::All,
version + VERSION + Version::ONE,
indexes,
price,
states_path,
StateLevel::PriceOnly,
)?,
term: ByTerm {
short: create(Filter::Term(Term::Sth), StateLevel::PriceOnly)?,
long: create(Filter::Term(Term::Lth), StateLevel::PriceOnly)?,
},
epoch: ByEpoch {
_0: full(Filter::Epoch(HalvingEpoch::new(0)))?,
_1: full(Filter::Epoch(HalvingEpoch::new(1)))?,
_2: full(Filter::Epoch(HalvingEpoch::new(2)))?,
_3: full(Filter::Epoch(HalvingEpoch::new(3)))?,
_4: full(Filter::Epoch(HalvingEpoch::new(4)))?,
},
type_: BySpendableType {
p2pk65: full(Filter::Type(OutputType::P2PK65))?,
p2pk33: full(Filter::Type(OutputType::P2PK33))?,
p2pkh: full(Filter::Type(OutputType::P2PKH))?,
p2sh: full(Filter::Type(OutputType::P2SH))?,
p2wpkh: full(Filter::Type(OutputType::P2WPKH))?,
p2wsh: full(Filter::Type(OutputType::P2WSH))?,
p2tr: full(Filter::Type(OutputType::P2TR))?,
p2a: full(Filter::Type(OutputType::P2A))?,
p2ms: full(Filter::Type(OutputType::P2MS))?,
empty: full(Filter::Type(OutputType::Empty))?,
unknown: full(Filter::Type(OutputType::Unknown))?,
},
max_age: ByMaxAge {
_1w: none(Filter::Time(TimeFilter::LowerThan(7)))?,
_1m: none(Filter::Time(TimeFilter::LowerThan(30)))?,
_2m: none(Filter::Time(TimeFilter::LowerThan(2 * 30)))?,
_3m: none(Filter::Time(TimeFilter::LowerThan(3 * 30)))?,
_4m: none(Filter::Time(TimeFilter::LowerThan(4 * 30)))?,
_5m: none(Filter::Time(TimeFilter::LowerThan(5 * 30)))?,
_6m: none(Filter::Time(TimeFilter::LowerThan(6 * 30)))?,
_1y: none(Filter::Time(TimeFilter::LowerThan(365)))?,
_2y: none(Filter::Time(TimeFilter::LowerThan(2 * 365)))?,
_3y: none(Filter::Time(TimeFilter::LowerThan(3 * 365)))?,
_4y: none(Filter::Time(TimeFilter::LowerThan(4 * 365)))?,
_5y: none(Filter::Time(TimeFilter::LowerThan(5 * 365)))?,
_6y: none(Filter::Time(TimeFilter::LowerThan(6 * 365)))?,
_7y: none(Filter::Time(TimeFilter::LowerThan(7 * 365)))?,
_8y: none(Filter::Time(TimeFilter::LowerThan(8 * 365)))?,
_10y: none(Filter::Time(TimeFilter::LowerThan(10 * 365)))?,
_12y: none(Filter::Time(TimeFilter::LowerThan(12 * 365)))?,
_15y: none(Filter::Time(TimeFilter::LowerThan(15 * 365)))?,
},
min_age: ByMinAge {
_1d: none(Filter::Time(TimeFilter::GreaterOrEqual(1)))?,
_1w: none(Filter::Time(TimeFilter::GreaterOrEqual(7)))?,
_1m: none(Filter::Time(TimeFilter::GreaterOrEqual(30)))?,
_2m: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 30)))?,
_3m: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 30)))?,
_4m: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 30)))?,
_5m: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 30)))?,
_6m: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 30)))?,
_1y: none(Filter::Time(TimeFilter::GreaterOrEqual(365)))?,
_2y: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 365)))?,
_3y: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 365)))?,
_4y: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 365)))?,
_5y: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 365)))?,
_6y: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 365)))?,
_7y: none(Filter::Time(TimeFilter::GreaterOrEqual(7 * 365)))?,
_8y: none(Filter::Time(TimeFilter::GreaterOrEqual(8 * 365)))?,
_10y: none(Filter::Time(TimeFilter::GreaterOrEqual(10 * 365)))?,
_12y: none(Filter::Time(TimeFilter::GreaterOrEqual(12 * 365)))?,
},
age_range: ByAgeRange {
up_to_1d: full(Filter::Time(TimeFilter::Range(0..1)))?,
_1d_to_1w: full(Filter::Time(TimeFilter::Range(1..7)))?,
_1w_to_1m: full(Filter::Time(TimeFilter::Range(7..30)))?,
_1m_to_2m: full(Filter::Time(TimeFilter::Range(30..2 * 30)))?,
_2m_to_3m: full(Filter::Time(TimeFilter::Range(2 * 30..3 * 30)))?,
_3m_to_4m: full(Filter::Time(TimeFilter::Range(3 * 30..4 * 30)))?,
_4m_to_5m: full(Filter::Time(TimeFilter::Range(4 * 30..5 * 30)))?,
_5m_to_6m: full(Filter::Time(TimeFilter::Range(5 * 30..6 * 30)))?,
_6m_to_1y: full(Filter::Time(TimeFilter::Range(6 * 30..365)))?,
_1y_to_2y: full(Filter::Time(TimeFilter::Range(365..2 * 365)))?,
_2y_to_3y: full(Filter::Time(TimeFilter::Range(2 * 365..3 * 365)))?,
_3y_to_4y: full(Filter::Time(TimeFilter::Range(3 * 365..4 * 365)))?,
_4y_to_5y: full(Filter::Time(TimeFilter::Range(4 * 365..5 * 365)))?,
_5y_to_6y: full(Filter::Time(TimeFilter::Range(5 * 365..6 * 365)))?,
_6y_to_7y: full(Filter::Time(TimeFilter::Range(6 * 365..7 * 365)))?,
_7y_to_8y: full(Filter::Time(TimeFilter::Range(7 * 365..8 * 365)))?,
_8y_to_10y: full(Filter::Time(TimeFilter::Range(8 * 365..10 * 365)))?,
_10y_to_12y: full(Filter::Time(TimeFilter::Range(10 * 365..12 * 365)))?,
_12y_to_15y: full(Filter::Time(TimeFilter::Range(12 * 365..15 * 365)))?,
from_15y: full(Filter::Time(TimeFilter::GreaterOrEqual(15 * 365)))?,
},
amount_range: ByAmountRange {
_0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?,
_1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?,
_10sats_to_100sats: full(Filter::Amount(AmountFilter::Range(
Sats::_10..Sats::_100,
)))?,
_100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_100..Sats::_1K,
)))?,
_1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_1K..Sats::_10K,
)))?,
_10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_10K..Sats::_100K,
)))?,
_100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_100K..Sats::_1M,
)))?,
_1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_1M..Sats::_10M,
)))?,
_10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10M..Sats::_1BTC,
)))?,
_1btc_to_10btc: full(Filter::Amount(AmountFilter::Range(
Sats::_1BTC..Sats::_10BTC,
)))?,
_10btc_to_100btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10BTC..Sats::_100BTC,
)))?,
_100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_100BTC..Sats::_1K_BTC,
)))?,
_1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_1K_BTC..Sats::_10K_BTC,
)))?,
_10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10K_BTC..Sats::_100K_BTC,
)))?,
_100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual(
Sats::_100K_BTC,
)))?,
},
lt_amount: ByLowerThanAmount {
_10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?,
_100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?,
_1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?,
_10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?,
_100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?,
_1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?,
_10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?,
_1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?,
_10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?,
_100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?,
_1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?,
_10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?,
_100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?,
},
ge_amount: ByGreatEqualAmount {
_1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?,
_10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?,
_100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?,
_1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?,
_10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?,
_100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?,
_1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?,
_10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?,
_1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?,
_10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?,
_100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?,
_1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?,
_10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?,
},
}))
}
/// Compute overlapping cohorts from component age/amount range cohorts.
pub fn compute_overlapping_vecs(
&mut self,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let by_age_range = &self.0.age_range;
let by_amount_range = &self.0.amount_range;
[(&mut self.0.all, by_age_range.iter().collect::<Vec<_>>())]
.into_par_iter()
.chain(self.0.min_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_age_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.max_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_age_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.term.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_age_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.ge_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.lt_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.try_for_each(|(vecs, components)| {
vecs.compute_from_stateful(starting_indexes, &components, exit)
})
}
/// First phase of post-processing: compute index transforms.
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
/// Second phase of post-processing: compute relative metrics.
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2<S, D, HM, DM, HR, DR>(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &S,
dateindex_to_supply: &D,
height_to_market_cap: Option<&HM>,
dateindex_to_market_cap: Option<&DM>,
height_to_realized_cap: Option<&HR>,
dateindex_to_realized_cap: Option<&DR>,
exit: &Exit,
) -> Result<()>
where
S: IterableVec<Height, Bitcoin> + Sync,
D: IterableVec<DateIndex, Bitcoin> + Sync,
HM: IterableVec<Height, Dollars> + Sync,
DM: IterableVec<DateIndex, Dollars> + Sync,
HR: IterableVec<Height, Dollars> + Sync,
DR: IterableVec<DateIndex, Dollars> + Sync,
{
self.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
})
}
/// Flush stateful vectors for separate cohorts.
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))?;
self.0
.par_iter_aggregate_mut()
.try_for_each(|v| v.price_to_amount.flush_at_height(height, exit))
}
/// Reset aggregate cohorts' price_to_amount for fresh start.
pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> {
self.0
.iter_aggregate_mut()
.try_for_each(|v| v.price_to_amount.reset())
}
/// Import aggregate cohorts' price_to_amount when resuming from checkpoint.
pub fn import_aggregate_price_to_amount(&mut self, height: Height) -> Result<Height> {
let Some(mut prev_height) = height.decremented() else {
return Ok(Height::ZERO);
};
for v in self.0.iter_aggregate_mut() {
prev_height = prev_height.min(v.price_to_amount.import_at_or_before(prev_height)?);
}
Ok(prev_height.incremented())
}
}
@@ -0,0 +1,72 @@
//! Processing received outputs (new UTXOs).
use brk_grouper::{Filter, Filtered};
use brk_types::{Dollars, Height};
use crate::states::Transacted;
use super::UTXOCohorts;
impl UTXOCohorts {
/// Process received outputs for this block.
///
/// New UTXOs are added to:
/// - The "up_to_1d" age cohort (all new UTXOs start at 0 days old)
/// - The appropriate epoch cohort based on block height
/// - The appropriate output type cohort (P2PKH, P2SH, etc.)
/// - The appropriate amount range cohort based on value
pub fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
let supply_state = received.spendable_supply;
// New UTXOs go into up_to_1d and current epoch
[
&mut self.0.age_range.up_to_1d,
self.0.epoch.mut_vec_from_height(height),
]
.into_iter()
.for_each(|v| {
v.state.as_mut().unwrap().receive(&supply_state, price);
});
// Update aggregate cohorts' price_to_amount
// New UTXOs have days_old = 0, so check if filter includes day 0
if let Some(price) = price
&& supply_state.value.is_not_zero()
{
self.0
.iter_aggregate_mut()
.filter(|v| v.filter().contains_time(0))
.for_each(|v| {
v.price_to_amount
.as_mut()
.unwrap()
.increment(price, &supply_state);
});
}
// Update output type cohorts
self.type_.iter_mut().for_each(|vecs| {
let output_type = match vecs.filter() {
Filter::Type(output_type) => *output_type,
_ => unreachable!(),
};
vecs.state
.as_mut()
.unwrap()
.receive(received.by_type.get(output_type), price)
});
// Update amount range cohorts
received
.by_size_group
.iter_typed()
.for_each(|(group, supply_state)| {
self.amount_range
.get_mut(group)
.state
.as_mut()
.unwrap()
.receive(supply_state, price);
});
}
}
@@ -0,0 +1,135 @@
//! Processing spent inputs (UTXOs being spent).
use brk_grouper::{Filter, Filtered, TimeFilter, UTXOGroups};
use brk_types::{CheckedSub, HalvingEpoch, Height};
use rustc_hash::FxHashMap;
use vecdb::VecIndex;
use crate::{states::{BlockState, Transacted}, utils::OptionExt, PriceToAmount};
use super::UTXOCohorts;
impl UTXOCohorts {
/// Process spent inputs for this block.
///
/// Each input references a UTXO created at some previous height.
/// We need to update the cohort states based on when that UTXO was created.
pub fn send(
&mut self,
height_to_sent: FxHashMap<Height, Transacted>,
chain_state: &mut [BlockState],
) {
let UTXOGroups {
all,
term,
age_range,
epoch,
type_,
amount_range,
..
} = &mut self.0;
// Time-based cohorts: age_range + epoch
let mut time_cohorts: Vec<_> = age_range
.iter_mut()
.chain(epoch.iter_mut())
.collect();
// Aggregate cohorts' price_to_amount
let mut aggregate_p2a: Vec<(Filter, Option<&mut PriceToAmount>)> = vec![
(all.filter().clone(), all.price_to_amount.as_mut()),
(
term.short.filter().clone(),
term.short.price_to_amount.as_mut(),
),
(
term.long.filter().clone(),
term.long.price_to_amount.as_mut(),
),
];
let last_block = chain_state.last().unwrap();
let last_timestamp = last_block.timestamp;
let current_price = last_block.price;
let chain_len = chain_state.len();
for (height, sent) in height_to_sent {
// Update chain_state to reflect spent supply
chain_state[height.to_usize()].supply -= &sent.spendable_supply;
let block_state = &chain_state[height.to_usize()];
let prev_price = block_state.price;
let blocks_old = chain_len - 1 - height.to_usize();
let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
let days_old_float =
last_timestamp.difference_in_days_between_float(block_state.timestamp);
let older_than_hour = last_timestamp
.checked_sub(block_state.timestamp)
.unwrap()
.is_more_than_hour();
// Update time-based cohorts
time_cohorts
.iter_mut()
.filter(|v| match v.filter() {
Filter::Time(TimeFilter::GreaterOrEqual(from)) => *from <= days_old,
Filter::Time(TimeFilter::LowerThan(to)) => *to > days_old,
Filter::Time(TimeFilter::Range(range)) => range.contains(&days_old),
Filter::Epoch(e) => *e == HalvingEpoch::from(height),
_ => unreachable!(),
})
.for_each(|vecs| {
vecs.state.um().send(
&sent.spendable_supply,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
});
// Update output type cohorts
sent.by_type
.spendable
.iter_typed()
.for_each(|(output_type, supply_state)| {
type_.get_mut(output_type).state.um().send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
)
});
// Update amount range cohorts
sent.by_size_group
.iter_typed()
.for_each(|(group, supply_state)| {
amount_range.get_mut(group).state.um().send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
});
// Update aggregate cohorts' price_to_amount
if let Some(prev_price) = prev_price {
let supply_state = &sent.spendable_supply;
if supply_state.value.is_not_zero() {
aggregate_p2a
.iter_mut()
.filter(|(f, _)| f.contains_time(days_old))
.for_each(|(_, p2a)| {
p2a.um().decrement(prev_price, supply_state);
});
}
}
}
}
}
@@ -0,0 +1,107 @@
//! Age-based state transitions for UTXO cohorts.
//!
//! When a new block arrives, UTXOs age. Some cross day boundaries
//! and need to move between age-based cohorts.
use brk_grouper::{Filter, Filtered, UTXOGroups};
use brk_types::{ONE_DAY_IN_SEC, Sats, Timestamp};
use crate::{states::BlockState, utils::OptionExt, PriceToAmount};
use super::UTXOCohorts;
impl UTXOCohorts {
/// Handle age transitions when processing a new block.
///
/// UTXOs age with each block. When they cross day boundaries,
/// they move between age-based cohorts (e.g., from "0-1d" to "1-7d").
pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
if chain_state.is_empty() {
return;
}
let prev_timestamp = chain_state.last().unwrap().timestamp;
// Optimization: Only blocks whose age % ONE_DAY >= threshold can cross a day boundary.
// Saves computation vs checking days_old for every block.
let elapsed = (*timestamp).saturating_sub(*prev_timestamp);
let threshold = ONE_DAY_IN_SEC.saturating_sub(elapsed);
// Extract mutable references to avoid borrow checker issues
let UTXOGroups {
all,
term,
age_range,
..
} = &mut self.0;
// Collect age_range cohorts with their filters and states
let mut age_cohorts: Vec<(Filter, &mut Option<_>)> = age_range
.iter_mut()
.map(|v| (v.filter().clone(), &mut v.state))
.collect();
// Collect aggregate cohorts' price_to_amount for age transitions
let mut aggregate_p2a: Vec<(Filter, Option<&mut PriceToAmount>)> = vec![
(all.filter().clone(), all.price_to_amount.as_mut()),
(
term.short.filter().clone(),
term.short.price_to_amount.as_mut(),
),
(
term.long.filter().clone(),
term.long.price_to_amount.as_mut(),
),
];
// Process blocks that might cross a day boundary
chain_state
.iter()
.filter(|block_state| {
let age = (*prev_timestamp).saturating_sub(*block_state.timestamp);
age % ONE_DAY_IN_SEC >= threshold
})
.for_each(|block_state| {
let prev_days = prev_timestamp.difference_in_days_between(block_state.timestamp);
let curr_days = timestamp.difference_in_days_between(block_state.timestamp);
if prev_days == curr_days {
return;
}
// Update age_range cohort states
age_cohorts.iter_mut().for_each(|(filter, state)| {
let is_now = filter.contains_time(curr_days);
let was_before = filter.contains_time(prev_days);
if is_now && !was_before {
state
.as_mut()
.unwrap()
.increment(&block_state.supply, block_state.price);
} else if was_before && !is_now {
state
.as_mut()
.unwrap()
.decrement(&block_state.supply, block_state.price);
}
});
// Update aggregate cohorts' price_to_amount
if let Some(price) = block_state.price
&& block_state.supply.value > Sats::ZERO
{
aggregate_p2a.iter_mut().for_each(|(filter, p2a)| {
let is_now = filter.contains_time(curr_days);
let was_before = filter.contains_time(prev_days);
if is_now && !was_before {
p2a.um().increment(price, &block_state.supply);
} else if was_before && !is_now {
p2a.um().decrement(price, &block_state.supply);
}
});
}
});
}
}
@@ -0,0 +1,110 @@
//! Aggregate cohort computation.
//!
//! After block processing, compute derived metrics:
//! 1. Overlapping cohorts (e.g., ">=1d" from sum of age_range cohorts)
//! 2. Index-based transforms (height -> dateindex, etc.)
//! 3. Relative metrics (supply ratios, market cap ratios)
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height};
use log::info;
use vecdb::{Exit, IterableVec};
use crate::{Indexes, indexes, price};
use super::super::cohorts::{AddressCohorts, UTXOCohorts};
/// Compute overlapping cohorts from component cohorts.
///
/// For example:
/// - ">=1d" UTXO cohort is computed from sum of age_range cohorts that match
/// - ">=1 BTC" address cohort is computed from sum of amount_range cohorts that match
pub fn compute_overlapping(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
info!("Computing overlapping cohorts...");
utxo_cohorts.compute_overlapping_vecs(starting_indexes, exit)?;
address_cohorts.compute_overlapping_vecs(starting_indexes, exit)?;
Ok(())
}
/// First phase of post-processing: compute index transforms.
///
/// Converts height-indexed data to dateindex-indexed data and other transforms.
pub fn compute_rest_part1(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
info!("Computing rest part 1...");
utxo_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?;
address_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?;
Ok(())
}
/// Second phase of post-processing: compute relative metrics.
///
/// Computes supply ratios, market cap ratios, etc. using total references.
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2<S, D, HM, DM, HR, DR>(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &S,
dateindex_to_supply: &D,
height_to_market_cap: Option<&HM>,
dateindex_to_market_cap: Option<&DM>,
height_to_realized_cap: Option<&HR>,
dateindex_to_realized_cap: Option<&DR>,
exit: &Exit,
) -> Result<()>
where
S: IterableVec<Height, Bitcoin> + Sync,
D: IterableVec<DateIndex, Bitcoin> + Sync,
HM: IterableVec<Height, Dollars> + Sync,
DM: IterableVec<DateIndex, Dollars> + Sync,
HR: IterableVec<Height, Dollars> + Sync,
DR: IterableVec<DateIndex, Dollars> + Sync,
{
info!("Computing rest part 2...");
utxo_cohorts.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)?;
address_cohorts.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)?;
Ok(())
}
@@ -0,0 +1,323 @@
//! Main block processing loop.
//!
//! Iterates through blocks and processes each one:
//! 1. Reset per-block state values
//! 2. Tick-tock age transitions
//! 3. Process outputs (receive) in parallel
//! 4. Process inputs (send) in parallel
//! 5. Push to height-indexed vectors
//! 6. Periodically flush checkpoints
use std::thread;
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_indexer::Indexer;
use brk_types::{DateIndex, Height, OutputType, Sats};
use log::info;
use vecdb::{Exit, GenericStoredVec, IterableVec, VecIndex};
use crate::states::{BlockState, Transacted};
use crate::{chain, indexes, price};
use super::super::cohorts::{AddressCohorts, DynCohortVecs, UTXOCohorts};
use super::super::vecs::Vecs;
use super::{
FLUSH_INTERVAL, IndexerReaders, build_txinindex_to_txindex, build_txoutindex_to_height_map,
build_txoutindex_to_txindex, process_inputs, process_outputs,
};
/// BIP30 duplicate coinbase heights - must handle specially.
const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842;
const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880;
const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812;
const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722;
/// Process all blocks from starting_height to last_height.
#[allow(clippy::too_many_arguments)]
pub fn process_blocks(
vecs: &mut Vecs,
indexer: &Indexer,
indexes: &indexes::Vecs,
chain: &chain::Vecs,
price: Option<&price::Vecs>,
starting_height: Height,
last_height: Height,
chain_state: &mut Vec<BlockState>,
exit: &Exit,
) -> Result<()> {
if starting_height > last_height {
return Ok(());
}
info!(
"Processing blocks {} to {}...",
starting_height, last_height
);
// Pre-compute iterators for fast access
let mut height_to_first_txindex = indexes.height_to_first_txindex.boxed_iter();
let mut height_to_tx_count = chain.height_to_tx_count.boxed_iter();
let mut height_to_first_txoutindex = indexes.height_to_first_txoutindex.boxed_iter();
let mut height_to_output_count = chain.height_to_output_count.boxed_iter();
let mut height_to_first_txinindex = indexes.height_to_first_txinindex.boxed_iter();
let mut height_to_input_count = chain.height_to_input_count.boxed_iter();
let mut height_to_timestamp = chain.height_to_timestamp.boxed_iter();
let mut height_to_unclaimed_rewards = chain.height_to_unclaimed_reward.boxed_iter();
let mut height_to_date = indexes.height_to_date.boxed_iter();
let mut dateindex_to_first_height = indexes.dateindex_to_first_height.boxed_iter();
let mut dateindex_to_height_count = indexes.dateindex_to_height_count.boxed_iter();
let mut txindex_to_output_count = chain.txindex_to_output_count.boxed_iter();
let mut txindex_to_input_count = chain.txindex_to_input_count.boxed_iter();
let mut height_to_price = price.map(|p| p.height_to_close.boxed_iter());
let mut dateindex_to_price = price.map(|p| p.dateindex_to_close.boxed_iter());
// Build txoutindex -> height map for input processing
let txoutindex_to_height = build_txoutindex_to_height_map(&indexes.height_to_first_txoutindex);
// Create readers for parallel data access
let ir = IndexerReaders::new(indexer);
// Track running totals
let mut unspendable_supply = Sats::ZERO;
let mut opreturn_supply = Sats::ZERO;
let mut addresstype_to_addr_count = ByAddressType::<u64>::default();
let mut addresstype_to_empty_addr_count = ByAddressType::<u64>::default();
// Recover initial values if resuming
if starting_height > Height::ZERO {
let prev_height = starting_height.decremented().unwrap();
unspendable_supply = vecs
.height_to_unspendable_supply
.get(prev_height)
.unwrap_or_default();
opreturn_supply = vecs
.height_to_opreturn_supply
.get(prev_height)
.unwrap_or_default();
}
// Main block iteration
for height in starting_height.to_usize()..=last_height.to_usize() {
let height = Height::from(height);
if height.to_usize() % 10000 == 0 {
info!("Processing chain at {}...", height);
}
// Get block metadata
let first_txindex = height_to_first_txindex.get_unwrap(height);
let tx_count = u64::from(height_to_tx_count.get_unwrap(height));
let first_txoutindex = height_to_first_txoutindex.get_unwrap(height).to_usize();
let output_count = u64::from(height_to_output_count.get_unwrap(height)) as usize;
let first_txinindex = height_to_first_txinindex.get_unwrap(height).to_usize();
let input_count = u64::from(height_to_input_count.get_unwrap(height)) as usize;
let timestamp = height_to_timestamp.get_unwrap(height);
let block_price = height_to_price.as_mut().map(|v| v.get_unwrap(height));
// Build txindex mappings for this block
let txoutindex_to_txindex =
build_txoutindex_to_txindex(first_txindex, tx_count, &mut txindex_to_output_count);
let txinindex_to_txindex =
build_txinindex_to_txindex(first_txindex, tx_count, &mut txindex_to_input_count);
// Reset per-block values for all separate cohorts
reset_block_values(&mut vecs.utxo_cohorts, &mut vecs.address_cohorts);
// Process outputs and inputs in parallel with tick-tock
let (outputs_result, inputs_result) = thread::scope(|scope| {
// Tick-tock age transitions in background
scope.spawn(|| {
vecs.utxo_cohorts
.tick_tock_next_block(chain_state, timestamp);
});
// Process outputs (receive)
let outputs_result = process_outputs(
first_txoutindex,
output_count,
&txoutindex_to_txindex,
&indexer.vecs.txoutindex_to_value,
&indexer.vecs.txoutindex_to_outputtype,
&indexer.vecs.txoutindex_to_typeindex,
&ir,
);
// Process inputs (send) - skip coinbase input
let inputs_result = if input_count > 1 {
process_inputs(
first_txinindex + 1, // Skip coinbase
input_count - 1,
&txinindex_to_txindex[1..], // Skip coinbase
&indexer.vecs.txinindex_to_outpoint,
&indexer.vecs.txindex_to_first_txoutindex,
&indexer.vecs.txoutindex_to_value,
&indexer.vecs.txoutindex_to_outputtype,
&indexer.vecs.txoutindex_to_typeindex,
&txoutindex_to_height,
&ir,
)
} else {
super::InputsResult {
height_to_sent: Default::default(),
sent_data: Default::default(),
}
};
(outputs_result, inputs_result)
});
let mut transacted = outputs_result.transacted;
let mut height_to_sent = inputs_result.height_to_sent;
// Update supply tracking
unspendable_supply += transacted.by_type.unspendable.opreturn.value
+ height_to_unclaimed_rewards.get_unwrap(height);
opreturn_supply += transacted.by_type.unspendable.opreturn.value;
// Handle special cases
if height == Height::ZERO {
// Genesis block - reset transacted, add 50 BTC to unspendable
transacted = Transacted::default();
unspendable_supply += Sats::FIFTY_BTC;
} else if height == Height::new(BIP30_DUPLICATE_HEIGHT_1)
|| height == Height::new(BIP30_DUPLICATE_HEIGHT_2)
{
// BIP30: Add 50 BTC to spent from original height
let original_height = if height == Height::new(BIP30_DUPLICATE_HEIGHT_1) {
Height::new(BIP30_ORIGINAL_HEIGHT_1)
} else {
Height::new(BIP30_ORIGINAL_HEIGHT_2)
};
height_to_sent
.entry(original_height)
.or_default()
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
}
// Push current block state before processing cohort updates
chain_state.push(BlockState {
supply: transacted.spendable_supply.clone(),
price: block_price,
timestamp,
});
// Update UTXO cohorts
vecs.utxo_cohorts.receive(transacted, height, block_price);
vecs.utxo_cohorts.send(height_to_sent, chain_state);
// Push to height-indexed vectors
vecs.height_to_unspendable_supply
.truncate_push(height, unspendable_supply)?;
vecs.height_to_opreturn_supply
.truncate_push(height, opreturn_supply)?;
vecs.addresstype_to_height_to_addr_count
.truncate_push(height, &addresstype_to_addr_count)?;
vecs.addresstype_to_height_to_empty_addr_count
.truncate_push(height, &addresstype_to_empty_addr_count)?;
// Get date info for unrealized state computation
let date = height_to_date.get_unwrap(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height.get_unwrap(dateindex);
let date_height_count = dateindex_to_height_count.get_unwrap(dateindex);
let is_date_last_height =
date_first_height + Height::from(date_height_count).decremented().unwrap() == height;
let date_price = dateindex_to_price
.as_mut()
.map(|v| is_date_last_height.then(|| v.get_unwrap(dateindex)));
let dateindex_opt = is_date_last_height.then_some(dateindex);
// Push cohort states and compute unrealized
push_cohort_states(
&mut vecs.utxo_cohorts,
&mut vecs.address_cohorts,
height,
block_price,
dateindex_opt,
date_price,
)?;
// Periodic checkpoint flush
if height != last_height
&& height != Height::ZERO
&& height.to_usize() % FLUSH_INTERVAL == 0
{
let _lock = exit.lock();
flush_checkpoint(vecs, height, exit)?;
}
}
// Final flush
let _lock = exit.lock();
flush_checkpoint(vecs, last_height, exit)?;
Ok(())
}
/// Reset per-block values for all separate cohorts.
fn reset_block_values(utxo_cohorts: &mut UTXOCohorts, address_cohorts: &mut AddressCohorts) {
utxo_cohorts.par_iter_separate_mut().for_each(|v| {
if let Some(state) = v.state.as_mut() {
state.reset_single_iteration_values();
}
});
address_cohorts.par_iter_separate_mut().for_each(|v| {
if let Some(state) = v.state.as_mut() {
state.inner.reset_single_iteration_values();
}
});
}
/// Push cohort states to height-indexed vectors.
fn push_cohort_states(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
height: Height,
height_price: Option<brk_types::Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<brk_types::Dollars>>,
) -> Result<()> {
utxo_cohorts
.par_iter_separate_mut()
.map(|v| v as &mut dyn DynCohortVecs)
.chain(
address_cohorts
.par_iter_separate_mut()
.map(|v| v as &mut dyn DynCohortVecs),
)
.try_for_each(|v| {
v.truncate_push(height)?;
v.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
)
})?;
Ok(())
}
/// Flush checkpoint to disk.
fn flush_checkpoint(vecs: &mut Vecs, height: Height, exit: &Exit) -> Result<()> {
info!("Flushing checkpoint at height {}...", height);
// Flush cohort states
vecs.utxo_cohorts.safe_flush_stateful_vecs(height, exit)?;
vecs.address_cohorts.safe_flush_stateful_vecs(height, exit)?;
// Flush height-indexed vectors
vecs.height_to_unspendable_supply.safe_write(exit)?;
vecs.height_to_opreturn_supply.safe_write(exit)?;
vecs.addresstype_to_height_to_addr_count.safe_flush(exit)?;
vecs.addresstype_to_height_to_empty_addr_count
.safe_flush(exit)?;
// Flush chain state with stamp
vecs.chain_state.safe_write_with_stamp(height.into(), exit)?;
Ok(())
}
@@ -0,0 +1,39 @@
//! Computation context holding shared state during block processing.
use brk_types::{Dollars, Height, Timestamp};
use vecdb::VecIndex;
use crate::price;
/// Context shared across block processing.
pub struct ComputeContext<'a> {
/// Starting height for this computation run
pub starting_height: Height,
/// Last height to process
pub last_height: Height,
/// Whether price data is available
pub compute_dollars: bool,
/// Price data (optional)
pub price: Option<&'a price::Vecs>,
/// Pre-computed height -> timestamp mapping
pub height_to_timestamp: Vec<Timestamp>,
/// Pre-computed height -> price mapping (if available)
pub height_to_price: Option<Vec<Dollars>>,
}
impl<'a> ComputeContext<'a> {
/// Get price at height (None if no price data or height out of range).
pub fn price_at(&self, height: Height) -> Option<Dollars> {
self.height_to_price.as_ref()?.get(height.to_usize()).copied()
}
/// Get timestamp at height.
pub fn timestamp_at(&self, height: Height) -> Timestamp {
self.height_to_timestamp[height.to_usize()]
}
}
@@ -0,0 +1,87 @@
//! State flushing logic for checkpoints.
//!
//! Handles periodic flushing of all stateful data to disk,
//! including cohort states, address data, and chain state.
use brk_error::Result;
use brk_types::{
AddressDataSource, AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, Height,
LoadedAddressData, LoadedAddressIndex,
};
use log::info;
use vecdb::{Exit, Stamp};
use crate::stateful_new::process::{process_empty_addresses, process_loaded_addresses};
use super::super::address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs};
use super::super::cohorts::DynCohortVecs;
/// Flush all cohort stateful vectors.
pub fn flush_cohort_states(
height: Height,
utxo_vecs: &mut [&mut dyn DynCohortVecs],
address_vecs: &mut [&mut dyn DynCohortVecs],
exit: &Exit,
) -> Result<()> {
for v in utxo_vecs.iter_mut() {
v.safe_flush_stateful_vecs(height, exit)?;
}
for v in address_vecs.iter_mut() {
v.safe_flush_stateful_vecs(height, exit)?;
}
Ok(())
}
/// Apply address index updates to the index storage.
fn apply_address_index_updates(
address_indexes: &mut AnyAddressIndexesVecs,
updates: AddressTypeToTypeIndexMap<AnyAddressIndex>,
) -> Result<()> {
for (address_type, sorted) in updates.into_sorted_iter() {
for (typeindex, any_index) in sorted {
address_indexes.update_or_push(address_type, typeindex, any_index)?;
}
}
Ok(())
}
/// Full state flush at a checkpoint.
///
/// This is the main entry point for checkpoint flushing:
/// 1. Flush cohort stateful vectors
/// 2. Process address data updates (empty and loaded)
/// 3. Update address indexes
/// 4. Stamped flush address indexes and data
/// 5. Flush chain state
#[allow(clippy::too_many_arguments)]
pub fn flush_checkpoint(
height: Height,
utxo_vecs: &mut [&mut dyn DynCohortVecs],
address_vecs: &mut [&mut dyn DynCohortVecs],
address_indexes: &mut AnyAddressIndexesVecs,
addresses_data: &mut AddressesDataVecs,
empty_updates: AddressTypeToTypeIndexMap<AddressDataSource<EmptyAddressData>>,
loaded_updates: AddressTypeToTypeIndexMap<AddressDataSource<LoadedAddressData>>,
with_changes: bool,
exit: &Exit,
) -> Result<()> {
info!("Flushing at height {}...", height);
// 1. Flush cohort states
flush_cohort_states(height, utxo_vecs, address_vecs, exit)?;
// 2. Process address updates - empty first, then loaded
let empty_result = process_empty_addresses(addresses_data, empty_updates)?;
let loaded_result = process_loaded_addresses(addresses_data, loaded_updates)?;
let all_updates = empty_result.merge(loaded_result);
// 3. Apply index updates
apply_address_index_updates(address_indexes, all_updates)?;
// 4. Stamped flush
let stamp = Stamp::from(height);
address_indexes.flush(stamp, with_changes)?;
addresses_data.flush(stamp, with_changes)?;
Ok(())
}
@@ -0,0 +1,36 @@
//! Block processing pipeline.
//!
//! This module handles the main computation loop that processes blocks:
//! 1. Recover state from checkpoint or start fresh
//! 2. Process each block's outputs and inputs
//! 3. Update cohort states
//! 4. Periodically flush to disk
//! 5. Compute aggregate cohorts from separate cohorts
mod aggregates;
// mod block_loop;
mod context;
mod flush;
mod readers;
mod recover;
pub use aggregates::{compute_overlapping, compute_rest_part1, compute_rest_part2};
// pub use block_loop::process_blocks;
pub use context::ComputeContext;
pub use flush::{flush_checkpoint, flush_cohort_states};
pub use readers::{
IndexerReaders, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex,
};
pub use recover::{
RecoveredState, StartMode, determine_start_mode, find_min_height,
import_aggregate_price_to_amount, import_cohort_states, reset_all_state, rollback_states,
};
/// Flush checkpoint interval (every N blocks).
pub const FLUSH_INTERVAL: usize = 10_000;
// BIP30 duplicate coinbase heights (special case handling)
pub const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842;
pub const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880;
pub const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812;
pub const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722;
@@ -0,0 +1,119 @@
//! Cached readers for efficient data access during computation.
//!
//! Readers provide mmap-based access to indexed data without repeated syscalls.
use brk_grouper::{ByAddressType, ByAnyAddress};
use brk_indexer::Indexer;
use brk_types::{OutputType, StoredU64, TxIndex};
use vecdb::{BoxedVecIterator, GenericStoredVec, Reader, VecIndex};
use crate::stateful_new::address::{AddressesDataVecs, AnyAddressIndexesVecs};
/// Cached readers for indexer vectors.
pub struct IndexerReaders {
pub txinindex_to_outpoint: Reader,
pub txindex_to_first_txoutindex: Reader,
pub txoutindex_to_value: Reader,
pub txoutindex_to_outputtype: Reader,
pub txoutindex_to_typeindex: Reader,
}
impl IndexerReaders {
pub fn new(indexer: &Indexer) -> Self {
Self {
txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(),
txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(),
txoutindex_to_value: indexer.vecs.txoutindex_to_value.create_reader(),
txoutindex_to_outputtype: indexer.vecs.txoutindex_to_outputtype.create_reader(),
txoutindex_to_typeindex: indexer.vecs.txoutindex_to_typeindex.create_reader(),
}
}
}
/// Cached readers for stateful vectors.
pub struct VecsReaders {
pub addresstypeindex_to_anyaddressindex: ByAddressType<Reader>,
pub anyaddressindex_to_anyaddressdata: ByAnyAddress<Reader>,
}
impl VecsReaders {
pub fn new(
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> Self {
Self {
addresstypeindex_to_anyaddressindex: ByAddressType {
p2a: any_address_indexes.p2a.create_reader(),
p2pk33: any_address_indexes.p2pk33.create_reader(),
p2pk65: any_address_indexes.p2pk65.create_reader(),
p2pkh: any_address_indexes.p2pkh.create_reader(),
p2sh: any_address_indexes.p2sh.create_reader(),
p2tr: any_address_indexes.p2tr.create_reader(),
p2wpkh: any_address_indexes.p2wpkh.create_reader(),
p2wsh: any_address_indexes.p2wsh.create_reader(),
},
anyaddressindex_to_anyaddressdata: ByAnyAddress {
loaded: addresses_data.loaded.create_reader(),
empty: addresses_data.empty.create_reader(),
},
}
}
/// Get reader for specific address type.
pub fn address_reader(&self, address_type: OutputType) -> &Reader {
self.addresstypeindex_to_anyaddressindex
.get_unwrap(address_type)
}
}
/// Build txoutindex -> txindex mapping for a block.
pub fn build_txoutindex_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_output_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
let first = block_first_txindex.to_usize();
let counts: Vec<u64> = (0..block_tx_count as usize)
.map(|offset| {
let txindex = TxIndex::from(first + offset);
u64::from(txindex_to_output_count.get_unwrap(txindex))
})
.collect();
let total: u64 = counts.iter().sum();
let mut result = Vec::with_capacity(total as usize);
for (offset, &count) in counts.iter().enumerate() {
let txindex = TxIndex::from(first + offset);
result.extend(std::iter::repeat(txindex).take(count as usize));
}
result
}
/// Build txinindex -> txindex mapping for a block.
pub fn build_txinindex_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_input_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
let first = block_first_txindex.to_usize();
let counts: Vec<u64> = (0..block_tx_count as usize)
.map(|offset| {
let txindex = TxIndex::from(first + offset);
u64::from(txindex_to_input_count.get_unwrap(txindex))
})
.collect();
let total: u64 = counts.iter().sum();
let mut result = Vec::with_capacity(total as usize);
for (offset, &count) in counts.iter().enumerate() {
let txindex = TxIndex::from(first + offset);
result.extend(std::iter::repeat_n(txindex, count as usize));
}
result
}
@@ -0,0 +1,163 @@
//! State recovery logic for checkpoint/resume.
//!
//! Determines starting height and imports saved state from checkpoints.
use std::cmp::Ordering;
use std::collections::BTreeSet;
use brk_error::Result;
use brk_types::Height;
use vecdb::{AnyVec, Stamp};
use super::super::address::AnyAddressIndexesVecs;
use super::super::cohorts::{DynCohortVecs, UTXOCohorts};
/// Result of state recovery.
pub struct RecoveredState {
/// Height to start processing from.
pub starting_height: Height,
/// Whether state was successfully restored (vs starting fresh).
pub restored: bool,
}
/// Determine starting height from vector lengths.
pub fn find_min_height(
utxo_vecs: &[&mut dyn DynCohortVecs],
address_vecs: &[&mut dyn DynCohortVecs],
chain_state_len: usize,
address_indexes_min_height: Height,
address_data_min_height: Height,
other_vec_lens: &[usize],
) -> Height {
let utxo_min = utxo_vecs
.iter()
.map(|v| Height::from(v.min_height_vecs_len()))
.min()
.unwrap_or_default();
let address_min = address_vecs
.iter()
.map(|v| Height::from(v.min_height_vecs_len()))
.min()
.unwrap_or_default();
let other_min = other_vec_lens
.iter()
.map(|&len| Height::from(len))
.min()
.unwrap_or_default();
utxo_min
.min(address_min)
.min(Height::from(chain_state_len))
.min(address_indexes_min_height)
.min(address_data_min_height)
.min(other_min)
}
/// Check if we can resume from a checkpoint or need to start fresh.
pub fn determine_start_mode(computed_min: Height, chain_state_height: Height) -> StartMode {
match computed_min.cmp(&chain_state_height) {
Ordering::Greater => unreachable!("min height > chain state height"),
Ordering::Equal => StartMode::Resume(chain_state_height),
Ordering::Less => StartMode::Fresh,
}
}
/// Whether to resume from checkpoint or start fresh.
pub enum StartMode {
/// Resume from the given height.
Resume(Height),
/// Start from height 0.
Fresh,
}
/// Rollback state vectors to before a given stamp.
///
/// Returns the consistent starting height if all vectors agree,
/// otherwise returns Height::ZERO (need fresh start).
pub fn rollback_states(
stamp: Stamp,
chain_state_rollback: Result<Stamp>,
address_indexes_rollbacks: Vec<Result<Stamp>>,
address_data_rollbacks: Vec<Result<Stamp>>,
) -> Height {
let mut heights: BTreeSet<Height> = [chain_state_rollback]
.into_iter()
.chain(address_indexes_rollbacks)
.chain(address_data_rollbacks)
.filter_map(|r| r.ok())
.map(Height::from)
.map(Height::incremented)
.collect();
if heights.len() == 1 {
heights.pop_first().unwrap()
} else {
Height::ZERO
}
}
/// Import state for all separate cohorts.
///
/// Returns the starting height if all imports succeed with the same height,
/// otherwise returns Height::ZERO.
pub fn import_cohort_states(
starting_height: Height,
cohorts: &mut [&mut dyn DynCohortVecs],
) -> Height {
if starting_height.is_zero() {
return Height::ZERO;
}
let all_match = cohorts
.iter_mut()
.map(|v| v.import_state(starting_height).unwrap_or_default())
.all(|h| h == starting_height);
if all_match {
starting_height
} else {
Height::ZERO
}
}
/// Import aggregate price_to_amount for UTXO cohorts.
pub fn import_aggregate_price_to_amount(
starting_height: Height,
utxo_cohorts: &mut UTXOCohorts,
) -> Result<Height> {
if starting_height.is_zero() {
return Ok(Height::ZERO);
}
let imported = utxo_cohorts.import_aggregate_price_to_amount(starting_height)?;
Ok(if imported == starting_height {
starting_height
} else {
Height::ZERO
})
}
/// Reset all state for fresh start.
pub fn reset_all_state(
address_indexes: &mut AnyAddressIndexesVecs,
utxo_vecs: &mut [&mut dyn DynCohortVecs],
address_vecs: &mut [&mut dyn DynCohortVecs],
utxo_cohorts: &mut UTXOCohorts,
) -> Result<()> {
address_indexes.reset()?;
for v in utxo_vecs.iter_mut() {
v.reset_state_starting_height();
}
for v in address_vecs.iter_mut() {
v.reset_state_starting_height();
}
utxo_cohorts.reset_aggregate_price_to_amount()?;
Ok(())
}
@@ -0,0 +1,201 @@
//! Transaction activity metrics.
//!
//! These metrics track amounts sent and destruction of satoshi-days/blocks.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Height, Sats, StoredF64, Version};
use vecdb::{AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use crate::{
Indexes,
grouped::{ComputedValueVecsFromHeight, ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, price,
};
use super::ImportConfig;
/// Activity metrics for a cohort.
#[derive(Clone, Traversable)]
pub struct ActivityMetrics {
/// Total satoshis sent at each height
pub height_to_sent: EagerVec<PcoVec<Height, Sats>>,
/// Sent amounts indexed by various dimensions
pub indexes_to_sent: ComputedValueVecsFromHeight,
/// Satoshi-blocks destroyed (supply * blocks_old when spent)
pub height_to_satblocks_destroyed: EagerVec<PcoVec<Height, Sats>>,
/// Satoshi-days destroyed (supply * days_old when spent)
pub height_to_satdays_destroyed: EagerVec<PcoVec<Height, Sats>>,
/// Coin-blocks destroyed (in BTC rather than sats)
pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
/// Coin-days destroyed (in BTC rather than sats)
pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
}
impl ActivityMetrics {
/// Import activity metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let compute_dollars = cfg.compute_dollars();
let sum = VecBuilderOptions::default().add_sum();
Ok(Self {
height_to_sent: EagerVec::forced_import(cfg.db, &cfg.name("sent"), cfg.version + v0)?,
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
cfg.db,
&cfg.name("sent"),
Source::None,
cfg.version + v0,
sum,
compute_dollars,
cfg.indexes,
)?,
height_to_satblocks_destroyed: EagerVec::forced_import(
cfg.db,
&cfg.name("satblocks_destroyed"),
cfg.version + v0,
)?,
height_to_satdays_destroyed: EagerVec::forced_import(
cfg.db,
&cfg.name("satdays_destroyed"),
cfg.version + v0,
)?,
indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("coinblocks_destroyed"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum.clone(),
)?,
indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("coindays_destroyed"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum,
)?,
})
}
/// Get minimum length across height-indexed vectors.
pub fn min_len(&self) -> usize {
self.height_to_sent
.len()
.min(self.height_to_satblocks_destroyed.len())
.min(self.height_to_satdays_destroyed.len())
}
/// Push activity state values to height-indexed vectors.
pub fn truncate_push(
&mut self,
height: Height,
sent: Sats,
satblocks_destroyed: Sats,
satdays_destroyed: Sats,
) -> Result<()> {
self.height_to_sent.truncate_push(height, sent)?;
self.height_to_satblocks_destroyed
.truncate_push(height, satblocks_destroyed)?;
self.height_to_satdays_destroyed
.truncate_push(height, satdays_destroyed)?;
Ok(())
}
/// Flush height-indexed vectors to disk.
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
self.height_to_sent.safe_write(exit)?;
self.height_to_satblocks_destroyed.safe_write(exit)?;
self.height_to_satdays_destroyed.safe_write(exit)?;
Ok(())
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_sent.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_sent).collect::<Vec<_>>(),
exit,
)?;
self.height_to_satblocks_destroyed.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_satblocks_destroyed)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_satdays_destroyed.compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_satdays_destroyed)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_sent.compute_rest(
indexes,
price,
starting_indexes,
exit,
Some(&self.height_to_sent),
)?;
self.indexes_to_coinblocks_destroyed
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_satblocks_destroyed,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)?;
Ok(())
})?;
self.indexes_to_coindays_destroyed
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_satdays_destroyed,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
)?;
Ok(())
})?;
Ok(())
}
}
@@ -0,0 +1,49 @@
//! Configuration for metric imports.
use brk_grouper::{CohortContext, Filter};
use brk_types::Version;
use vecdb::Database;
use crate::{indexes, price};
/// Configuration for importing metrics.
pub struct ImportConfig<'a> {
pub db: &'a Database,
pub filter: Filter,
pub context: CohortContext,
pub version: Version,
pub indexes: &'a indexes::Vecs,
pub price: Option<&'a price::Vecs>,
}
impl<'a> ImportConfig<'a> {
/// Whether price data is available (enables realized/unrealized metrics).
pub fn compute_dollars(&self) -> bool {
self.price.is_some()
}
/// Whether this is an extended cohort (more relative metrics).
pub fn extended(&self) -> bool {
self.filter.is_extended(self.context)
}
/// Whether to compute relative-to-all metrics.
pub fn compute_rel_to_all(&self) -> bool {
self.filter.compute_rel_to_all()
}
/// Whether to compute adjusted metrics (SOPR, etc.).
pub fn compute_adjusted(&self) -> bool {
self.filter.compute_adjusted(self.context)
}
/// Get full metric name with filter prefix.
pub fn name(&self, suffix: &str) -> String {
let prefix = self.filter.to_full_name(self.context);
if prefix.is_empty() {
suffix.to_string()
} else {
format!("{prefix}_{suffix}")
}
}
}
@@ -0,0 +1,293 @@
//! Metric vectors organized by category.
//!
//! Instead of a single 80+ field struct, metrics are grouped into logical categories:
//! - `supply`: Supply and UTXO count metrics (always computed)
//! - `activity`: Transaction activity metrics (always computed)
//! - `realized`: Realized cap, profit/loss, SOPR (requires price)
//! - `unrealized`: Unrealized profit/loss (requires price)
//! - `price`: Price paid metrics and percentiles (requires price)
//! - `relative`: Ratios relative to market cap, etc. (requires price)
mod activity;
mod config;
mod price_paid;
mod realized;
mod relative;
mod supply;
mod unrealized;
pub use activity::ActivityMetrics;
pub use config::ImportConfig;
pub use price_paid::PricePaidMetrics;
pub use realized::RealizedMetrics;
pub use relative::RelativeMetrics;
pub use supply::SupplyMetrics;
pub use unrealized::UnrealizedMetrics;
use brk_error::Result;
use brk_grouper::Filter;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Exit, IterableVec};
use crate::{Indexes, indexes, price, states::CohortState};
/// All metrics for a cohort, organized by category.
#[derive(Clone, Traversable)]
pub struct CohortMetrics {
#[traversable(skip)]
pub filter: Filter,
/// Supply and UTXO count (always computed)
pub supply: SupplyMetrics,
/// Transaction activity (always computed)
pub activity: ActivityMetrics,
/// Realized cap and profit/loss (requires price data)
pub realized: Option<RealizedMetrics>,
/// Unrealized profit/loss (requires price data)
pub unrealized: Option<UnrealizedMetrics>,
/// Price paid metrics (requires price data)
pub price_paid: Option<PricePaidMetrics>,
/// Relative metrics (requires price data)
pub relative: Option<RelativeMetrics>,
}
impl CohortMetrics {
/// Import all metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let compute_dollars = cfg.compute_dollars();
Ok(Self {
filter: cfg.filter.clone(),
supply: SupplyMetrics::forced_import(cfg)?,
activity: ActivityMetrics::forced_import(cfg)?,
realized: compute_dollars
.then(|| RealizedMetrics::forced_import(cfg))
.transpose()?,
unrealized: compute_dollars
.then(|| UnrealizedMetrics::forced_import(cfg))
.transpose()?,
price_paid: compute_dollars
.then(|| PricePaidMetrics::forced_import(cfg))
.transpose()?,
relative: compute_dollars
.then(|| RelativeMetrics::forced_import(cfg))
.transpose()?,
})
}
/// Get minimum length across height-indexed vectors.
pub fn min_len(&self) -> usize {
self.supply.min_len().min(self.activity.min_len())
}
/// Push state values to height-indexed vectors.
pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> {
self.supply.truncate_push(height, &state.supply)?;
self.activity.truncate_push(
height,
state.sent,
state.satblocks_destroyed,
state.satdays_destroyed,
)?;
if let (Some(realized), Some(realized_state)) =
(self.realized.as_mut(), state.realized.as_ref())
{
realized.truncate_push(height, realized_state)?;
}
Ok(())
}
/// Flush height-indexed vectors to disk.
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
self.supply.safe_flush(exit)?;
self.activity.safe_flush(exit)?;
if let Some(realized) = self.realized.as_mut() {
realized.safe_flush(exit)?;
}
if let Some(unrealized) = self.unrealized.as_mut() {
unrealized.safe_flush(exit)?;
}
if let Some(price_paid) = self.price_paid.as_mut() {
price_paid.safe_flush(exit)?;
}
Ok(())
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.supply.validate_computed_versions(base_version)?;
self.activity.validate_computed_versions(base_version)?;
if let Some(realized) = self.realized.as_mut() {
realized.validate_computed_versions(base_version)?;
}
Ok(())
}
/// Compute and push unrealized states.
pub fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
state: &CohortState,
) -> Result<()> {
if let (Some(unrealized), Some(price_paid), Some(height_price)) = (
self.unrealized.as_mut(),
self.price_paid.as_mut(),
height_price,
) {
// Push price paid min/max
price_paid.truncate_push_minmax(height, state)?;
// Compute unrealized states from price_to_amount
let (height_unrealized_state, date_unrealized_state) =
state.compute_unrealized_states(height_price, date_price.unwrap());
unrealized.truncate_push(
height,
dateindex,
&height_unrealized_state,
date_unrealized_state.as_ref(),
)?;
// Compute and push price percentiles
price_paid.truncate_push_percentiles(height, state)?;
}
Ok(())
}
/// Compute aggregate cohort values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.supply.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.supply).collect::<Vec<_>>(),
exit,
)?;
self.activity.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.activity).collect::<Vec<_>>(),
exit,
)?;
if let Some(realized) = self.realized.as_mut() {
realized.compute_from_stateful(
starting_indexes,
&others
.iter()
.filter_map(|v| v.realized.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
}
if let Some(unrealized) = self.unrealized.as_mut() {
unrealized.compute_from_stateful(
starting_indexes,
&others
.iter()
.filter_map(|v| v.unrealized.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
}
if let Some(price_paid) = self.price_paid.as_mut() {
price_paid.compute_from_stateful(
starting_indexes,
&others
.iter()
.filter_map(|v| v.price_paid.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
}
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.supply
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
self.activity
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
if let Some(realized) = self.realized.as_mut() {
realized.compute_rest_part1(indexes, price, starting_indexes, exit)?;
}
Ok(())
}
/// Second phase of computed metrics (ratios, relative values).
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.supply.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
exit,
)?;
if let Some(relative) = self.relative.as_mut() {
relative.compute_rest_part2(
indexes,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
&self.supply,
self.realized.as_ref(),
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,144 @@
//! Price paid metrics and percentiles.
//!
//! Tracks min/max price paid for UTXOs and price distribution percentiles.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Dollars, Height, Version};
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use crate::{
Indexes,
grouped::{ComputedVecsFromHeight, PricePercentiles, Source, VecBuilderOptions},
states::{CohortState, Flushable},
};
use super::ImportConfig;
/// Price paid metrics.
#[derive(Clone, Traversable)]
pub struct PricePaidMetrics {
/// Minimum price paid for any UTXO at this height
pub height_to_min_price_paid: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_min_price_paid: ComputedVecsFromHeight<Dollars>,
/// Maximum price paid for any UTXO at this height
pub height_to_max_price_paid: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_max_price_paid: ComputedVecsFromHeight<Dollars>,
/// Price distribution percentiles (median, quartiles, etc.)
pub price_percentiles: Option<PricePercentiles>,
}
impl PricePaidMetrics {
/// Import price paid metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let extended = cfg.extended();
let last = VecBuilderOptions::default().add_last();
Ok(Self {
height_to_min_price_paid: EagerVec::forced_import(
cfg.db,
&cfg.name("min_price_paid"),
cfg.version + v0,
)?,
indexes_to_min_price_paid: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("min_price_paid"),
Source::None,
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_max_price_paid: EagerVec::forced_import(
cfg.db,
&cfg.name("max_price_paid"),
cfg.version + v0,
)?,
indexes_to_max_price_paid: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("max_price_paid"),
Source::None,
cfg.version + v0,
cfg.indexes,
last,
)?,
price_percentiles: extended
.then(|| {
PricePercentiles::forced_import(
cfg.db,
&cfg.name(""),
cfg.version + v0,
cfg.indexes,
true,
)
})
.transpose()?,
})
}
/// Push min/max price paid from state.
pub fn truncate_push_minmax(&mut self, height: Height, state: &CohortState) -> Result<()> {
self.height_to_min_price_paid.truncate_push(
height,
state
.price_to_amount_first_key_value()
.map(|(&dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
)?;
self.height_to_max_price_paid.truncate_push(
height,
state
.price_to_amount_last_key_value()
.map(|(&dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
)?;
Ok(())
}
/// Push price percentiles from state.
pub fn truncate_push_percentiles(&mut self, height: Height, state: &CohortState) -> Result<()> {
if let Some(price_percentiles) = self.price_percentiles.as_mut() {
let percentile_prices = state.compute_percentile_prices();
price_percentiles.truncate_push(height, &percentile_prices)?;
}
Ok(())
}
/// Flush height-indexed vectors to disk.
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
self.height_to_min_price_paid.safe_write(exit)?;
self.height_to_max_price_paid.safe_write(exit)?;
if let Some(price_percentiles) = self.price_percentiles.as_mut() {
price_percentiles.safe_flush(exit)?;
}
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_min_price_paid.compute_min_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_min_price_paid)
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_max_price_paid.compute_max_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| &v.height_to_max_price_paid)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
}
@@ -0,0 +1,556 @@
//! Realized cap and profit/loss metrics.
//!
//! These metrics require price data and track realized value based on acquisition price.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, StoredF32, StoredF64, Version};
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use crate::{
Indexes,
grouped::{
ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source,
VecBuilderOptions,
},
indexes, price,
states::RealizedState,
utils::OptionExt,
};
use super::ImportConfig;
/// Realized cap and related metrics.
#[derive(Clone, Traversable)]
pub struct RealizedMetrics {
// === Realized Cap ===
pub height_to_realized_cap: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_realized_cap: ComputedVecsFromHeight<Dollars>,
pub indexes_to_realized_price: ComputedVecsFromHeight<Dollars>,
pub indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex,
pub indexes_to_realized_cap_rel_to_own_market_cap: Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex<Dollars>,
// === Realized Profit/Loss ===
pub height_to_realized_profit: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_realized_profit: ComputedVecsFromHeight<Dollars>,
pub height_to_realized_loss: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_realized_loss: ComputedVecsFromHeight<Dollars>,
pub indexes_to_neg_realized_loss: ComputedVecsFromHeight<Dollars>,
pub indexes_to_net_realized_pnl: ComputedVecsFromHeight<Dollars>,
pub indexes_to_realized_value: ComputedVecsFromHeight<Dollars>,
// === Realized vs Realized Cap Ratios ===
pub indexes_to_realized_profit_rel_to_realized_cap: ComputedVecsFromHeight<StoredF32>,
pub indexes_to_realized_loss_rel_to_realized_cap: ComputedVecsFromHeight<StoredF32>,
pub indexes_to_net_realized_pnl_rel_to_realized_cap: ComputedVecsFromHeight<StoredF32>,
// === Total Realized PnL ===
pub height_to_total_realized_pnl: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_total_realized_pnl: ComputedVecsFromDateIndex<Dollars>,
pub dateindex_to_realized_profit_to_loss_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
// === Value Created/Destroyed ===
pub height_to_value_created: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_value_created: ComputedVecsFromHeight<Dollars>,
pub height_to_value_destroyed: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_value_destroyed: ComputedVecsFromHeight<Dollars>,
// === Adjusted Value (optional) ===
pub height_to_adjusted_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_adjusted_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_adjusted_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
// === SOPR (Spent Output Profit Ratio) ===
pub dateindex_to_sopr: EagerVec<PcoVec<DateIndex, StoredF64>>,
pub dateindex_to_sopr_7d_ema: EagerVec<PcoVec<DateIndex, StoredF64>>,
pub dateindex_to_sopr_30d_ema: EagerVec<PcoVec<DateIndex, StoredF64>>,
pub dateindex_to_adjusted_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
// === Sell Side Risk ===
pub dateindex_to_sell_side_risk_ratio: EagerVec<PcoVec<DateIndex, StoredF32>>,
pub dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec<PcoVec<DateIndex, StoredF32>>,
pub dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec<PcoVec<DateIndex, StoredF32>>,
// === Net Realized PnL Deltas ===
pub indexes_to_net_realized_pnl_cumulative_30d_delta: ComputedVecsFromDateIndex<Dollars>,
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
ComputedVecsFromDateIndex<StoredF32>,
}
impl RealizedMetrics {
/// Import realized metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let v1 = Version::ONE;
let v3 = Version::new(3);
let extended = cfg.extended();
let compute_adjusted = cfg.compute_adjusted();
let last = VecBuilderOptions::default().add_last();
let sum = VecBuilderOptions::default().add_sum();
let sum_cum = VecBuilderOptions::default().add_sum().add_cumulative();
Ok(Self {
// === Realized Cap ===
height_to_realized_cap: EagerVec::forced_import(
cfg.db,
&cfg.name("realized_cap"),
cfg.version + v0,
)?,
indexes_to_realized_cap: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_cap"),
Source::None,
cfg.version + v0,
cfg.indexes,
last,
)?,
indexes_to_realized_price: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_price"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("realized_price"),
Source::None,
cfg.version + v0,
cfg.indexes,
extended,
)?,
indexes_to_realized_cap_rel_to_own_market_cap: extended
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_cap_rel_to_own_market_cap"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("realized_cap_30d_delta"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
// === Realized Profit/Loss ===
height_to_realized_profit: EagerVec::forced_import(
cfg.db,
&cfg.name("realized_profit"),
cfg.version + v0,
)?,
indexes_to_realized_profit: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_profit"),
Source::None,
cfg.version + v0,
cfg.indexes,
sum_cum,
)?,
height_to_realized_loss: EagerVec::forced_import(
cfg.db,
&cfg.name("realized_loss"),
cfg.version + v0,
)?,
indexes_to_realized_loss: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_loss"),
Source::None,
cfg.version + v0,
cfg.indexes,
sum_cum,
)?,
indexes_to_neg_realized_loss: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("neg_realized_loss"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
sum_cum,
)?,
indexes_to_net_realized_pnl: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("net_realized_pnl"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum_cum,
)?,
indexes_to_realized_value: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_value"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum,
)?,
// === Realized vs Realized Cap Ratios ===
indexes_to_realized_profit_rel_to_realized_cap: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_profit_rel_to_realized_cap"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum,
)?,
indexes_to_realized_loss_rel_to_realized_cap: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_loss_rel_to_realized_cap"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
sum,
)?,
indexes_to_net_realized_pnl_rel_to_realized_cap: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_rel_to_realized_cap"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
sum,
)?,
// === Total Realized PnL ===
height_to_total_realized_pnl: EagerVec::forced_import(
cfg.db,
&cfg.name("total_realized_pnl"),
cfg.version + v0,
)?,
indexes_to_total_realized_pnl: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("total_realized_pnl"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
sum,
)?,
dateindex_to_realized_profit_to_loss_ratio: extended
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("realized_profit_to_loss_ratio"),
cfg.version + v1,
)
})
.transpose()?,
// === Value Created/Destroyed ===
height_to_value_created: EagerVec::forced_import(
cfg.db,
&cfg.name("value_created"),
cfg.version + v0,
)?,
indexes_to_value_created: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("value_created"),
Source::None,
cfg.version + v0,
cfg.indexes,
sum,
)?,
height_to_value_destroyed: EagerVec::forced_import(
cfg.db,
&cfg.name("value_destroyed"),
cfg.version + v0,
)?,
indexes_to_value_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("value_destroyed"),
Source::None,
cfg.version + v0,
cfg.indexes,
sum,
)?,
// === Adjusted Value (optional) ===
height_to_adjusted_value_created: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
cfg.version + v0,
)
})
.transpose()?,
indexes_to_adjusted_value_created: compute_adjusted
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
Source::None,
cfg.version + v0,
cfg.indexes,
sum,
)
})
.transpose()?,
height_to_adjusted_value_destroyed: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
cfg.version + v0,
)
})
.transpose()?,
indexes_to_adjusted_value_destroyed: compute_adjusted
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
Source::None,
cfg.version + v0,
cfg.indexes,
sum,
)
})
.transpose()?,
// === SOPR ===
dateindex_to_sopr: EagerVec::forced_import(
cfg.db,
&cfg.name("sopr"),
cfg.version + v1,
)?,
dateindex_to_sopr_7d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sopr_7d_ema"),
cfg.version + v1,
)?,
dateindex_to_sopr_30d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sopr_30d_ema"),
cfg.version + v1,
)?,
dateindex_to_adjusted_sopr: compute_adjusted
.then(|| {
EagerVec::forced_import(cfg.db, &cfg.name("adjusted_sopr"), cfg.version + v1)
})
.transpose()?,
dateindex_to_adjusted_sopr_7d_ema: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_sopr_7d_ema"),
cfg.version + v1,
)
})
.transpose()?,
dateindex_to_adjusted_sopr_30d_ema: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_sopr_30d_ema"),
cfg.version + v1,
)
})
.transpose()?,
// === Sell Side Risk ===
dateindex_to_sell_side_risk_ratio: EagerVec::forced_import(
cfg.db,
&cfg.name("sell_side_risk_ratio"),
cfg.version + v1,
)?,
dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sell_side_risk_ratio_7d_ema"),
cfg.version + v1,
)?,
dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec::forced_import(
cfg.db,
&cfg.name("sell_side_risk_ratio_30d_ema"),
cfg.version + v1,
)?,
// === Net Realized PnL Deltas ===
indexes_to_net_realized_pnl_cumulative_30d_delta:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_cumulative_30d_delta"),
Source::Compute,
cfg.version + v3,
cfg.indexes,
last,
)?,
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap"),
Source::Compute,
cfg.version + v3,
cfg.indexes,
last,
)?,
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_market_cap"),
Source::Compute,
cfg.version + v3,
cfg.indexes,
last,
)?,
})
}
/// Push realized state values to height-indexed vectors.
pub fn truncate_push(&mut self, height: Height, state: &RealizedState) -> Result<()> {
self.height_to_realized_cap.truncate_push(height, state.cap)?;
self.height_to_realized_profit.truncate_push(height, state.profit)?;
self.height_to_realized_loss.truncate_push(height, state.loss)?;
self.height_to_value_created.truncate_push(height, state.value_created)?;
self.height_to_value_destroyed.truncate_push(height, state.value_destroyed)?;
if let Some(v) = self.height_to_adjusted_value_created.as_mut() {
v.truncate_push(height, state.adj_value_created)?;
}
if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() {
v.truncate_push(height, state.adj_value_destroyed)?;
}
Ok(())
}
/// Flush height-indexed vectors to disk.
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
self.height_to_realized_cap.safe_write(exit)?;
self.height_to_realized_profit.safe_write(exit)?;
self.height_to_realized_loss.safe_write(exit)?;
self.height_to_value_created.safe_write(exit)?;
self.height_to_value_destroyed.safe_write(exit)?;
self.height_to_adjusted_value_created.um().safe_write(exit)?;
self.height_to_adjusted_value_destroyed.um().safe_write(exit)?;
Ok(())
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_realized_cap.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_realized_cap).collect::<Vec<_>>(),
exit,
)?;
self.height_to_realized_profit.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_realized_profit).collect::<Vec<_>>(),
exit,
)?;
self.height_to_realized_loss.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_realized_loss).collect::<Vec<_>>(),
exit,
)?;
self.height_to_value_created.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_value_created).collect::<Vec<_>>(),
exit,
)?;
self.height_to_value_destroyed.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_value_destroyed).collect::<Vec<_>>(),
exit,
)?;
if self.height_to_adjusted_value_created.is_some() {
self.height_to_adjusted_value_created.um().compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| {
v.height_to_adjusted_value_created
.as_ref()
.unwrap_or(&v.height_to_value_created)
})
.collect::<Vec<_>>(),
exit,
)?;
self.height_to_adjusted_value_destroyed.um().compute_sum_of_others(
starting_indexes.height,
&others
.iter()
.map(|v| {
v.height_to_adjusted_value_destroyed
.as_ref()
.unwrap_or(&v.height_to_value_destroyed)
})
.collect::<Vec<_>>(),
exit,
)?;
}
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
_price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_realized_cap.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_realized_cap),
)?;
self.indexes_to_realized_profit.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_realized_profit),
)?;
self.indexes_to_realized_loss.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_realized_loss),
)?;
self.indexes_to_value_created.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_value_created),
)?;
self.indexes_to_value_destroyed.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_value_destroyed),
)?;
Ok(())
}
}
@@ -0,0 +1,450 @@
//! Relative metrics (ratios to market cap, realized cap, supply, etc.)
//!
//! These are computed ratios comparing cohort metrics to global metrics.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version};
use vecdb::{EagerVec, Exit, ImportableVec, IterableVec, PcoVec};
use crate::{
Indexes,
grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes,
};
use super::{ImportConfig, RealizedMetrics, SupplyMetrics};
/// Relative metrics comparing cohort values to global values.
#[derive(Clone, Traversable)]
pub struct RelativeMetrics {
// === Supply Relative to Circulating Supply ===
pub indexes_to_supply_rel_to_circulating_supply: Option<ComputedVecsFromHeight<StoredF64>>,
// === Supply in Profit/Loss Relative to Own Supply ===
pub height_to_supply_in_profit_rel_to_own_supply: EagerVec<PcoVec<Height, StoredF64>>,
pub height_to_supply_in_loss_rel_to_own_supply: EagerVec<PcoVec<Height, StoredF64>>,
pub indexes_to_supply_in_profit_rel_to_own_supply: ComputedVecsFromDateIndex<StoredF64>,
pub indexes_to_supply_in_loss_rel_to_own_supply: ComputedVecsFromDateIndex<StoredF64>,
// === Supply in Profit/Loss Relative to Circulating Supply ===
pub height_to_supply_in_profit_rel_to_circulating_supply:
Option<EagerVec<PcoVec<Height, StoredF64>>>,
pub height_to_supply_in_loss_rel_to_circulating_supply:
Option<EagerVec<PcoVec<Height, StoredF64>>>,
pub indexes_to_supply_in_profit_rel_to_circulating_supply:
Option<ComputedVecsFromDateIndex<StoredF64>>,
pub indexes_to_supply_in_loss_rel_to_circulating_supply:
Option<ComputedVecsFromDateIndex<StoredF64>>,
// === Unrealized vs Market Cap ===
pub height_to_unrealized_profit_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
pub height_to_unrealized_loss_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
pub height_to_neg_unrealized_loss_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
pub height_to_net_unrealized_pnl_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
pub indexes_to_unrealized_profit_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_neg_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_net_unrealized_pnl_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
// === Unrealized vs Own Market Cap (optional) ===
pub height_to_unrealized_profit_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_unrealized_loss_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_neg_unrealized_loss_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_net_unrealized_pnl_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub indexes_to_unrealized_profit_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_unrealized_loss_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
// === Unrealized vs Own Total Unrealized PnL (optional) ===
pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
}
impl RelativeMetrics {
/// Import relative metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let v1 = Version::ONE;
let v2 = Version::new(2);
let extended = cfg.extended();
let compute_rel_to_all = cfg.compute_rel_to_all();
let last = VecBuilderOptions::default().add_last();
Ok(Self {
// === Supply Relative to Circulating Supply ===
indexes_to_supply_rel_to_circulating_supply: compute_rel_to_all
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("supply_rel_to_circulating_supply"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
// === Supply in Profit/Loss Relative to Own Supply ===
height_to_supply_in_profit_rel_to_own_supply: EagerVec::forced_import(
cfg.db,
&cfg.name("supply_in_profit_rel_to_own_supply"),
cfg.version + v1,
)?,
height_to_supply_in_loss_rel_to_own_supply: EagerVec::forced_import(
cfg.db,
&cfg.name("supply_in_loss_rel_to_own_supply"),
cfg.version + v1,
)?,
indexes_to_supply_in_profit_rel_to_own_supply:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_profit_rel_to_own_supply"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)?,
indexes_to_supply_in_loss_rel_to_own_supply: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_loss_rel_to_own_supply"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)?,
// === Supply in Profit/Loss Relative to Circulating Supply ===
height_to_supply_in_profit_rel_to_circulating_supply: compute_rel_to_all
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("supply_in_profit_rel_to_circulating_supply"),
cfg.version + v1,
)
})
.transpose()?,
height_to_supply_in_loss_rel_to_circulating_supply: compute_rel_to_all
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("supply_in_loss_rel_to_circulating_supply"),
cfg.version + v1,
)
})
.transpose()?,
indexes_to_supply_in_profit_rel_to_circulating_supply: compute_rel_to_all
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_profit_rel_to_circulating_supply"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_supply_in_loss_rel_to_circulating_supply: compute_rel_to_all
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_loss_rel_to_circulating_supply"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
// === Unrealized vs Market Cap ===
height_to_unrealized_profit_rel_to_market_cap: EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_profit_rel_to_market_cap"),
cfg.version + v0,
)?,
height_to_unrealized_loss_rel_to_market_cap: EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_loss_rel_to_market_cap"),
cfg.version + v0,
)?,
height_to_neg_unrealized_loss_rel_to_market_cap: EagerVec::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss_rel_to_market_cap"),
cfg.version + v0,
)?,
height_to_net_unrealized_pnl_rel_to_market_cap: EagerVec::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl_rel_to_market_cap"),
cfg.version + v1,
)?,
indexes_to_unrealized_profit_rel_to_market_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_profit_rel_to_market_cap"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)?,
indexes_to_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_loss_rel_to_market_cap"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)?,
indexes_to_neg_unrealized_loss_rel_to_market_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss_rel_to_market_cap"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)?,
indexes_to_net_unrealized_pnl_rel_to_market_cap:
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl_rel_to_market_cap"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)?,
// === Unrealized vs Own Market Cap (optional) ===
height_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_profit_rel_to_own_market_cap"),
cfg.version + v1,
)
})
.transpose()?,
height_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_loss_rel_to_own_market_cap"),
cfg.version + v1,
)
})
.transpose()?,
height_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss_rel_to_own_market_cap"),
cfg.version + v1,
)
})
.transpose()?,
height_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl_rel_to_own_market_cap"),
cfg.version + v2,
)
})
.transpose()?,
indexes_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_profit_rel_to_own_market_cap"),
Source::Compute,
cfg.version + v2,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_loss_rel_to_own_market_cap"),
Source::Compute,
cfg.version + v2,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss_rel_to_own_market_cap"),
Source::Compute,
cfg.version + v2,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl_rel_to_own_market_cap"),
Source::Compute,
cfg.version + v2,
cfg.indexes,
last,
)
})
.transpose()?,
// === Unrealized vs Own Total Unrealized PnL (optional) ===
height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"),
cfg.version + v0,
)
})
.transpose()?,
height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"),
cfg.version + v0,
)
})
.transpose()?,
height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"),
cfg.version + v0,
)
})
.transpose()?,
height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"),
cfg.version + v1,
)
})
.transpose()?,
indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended
.then(|| {
ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"),
Source::Compute,
cfg.version + v1,
cfg.indexes,
last,
)
})
.transpose()?,
})
}
/// Second phase of computed metrics (ratios, relative values).
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
_height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
_dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
supply: &SupplyMetrics,
_realized: Option<&RealizedMetrics>,
exit: &Exit,
) -> Result<()> {
// Supply relative to circulating supply
if let Some(v) = self.indexes_to_supply_rel_to_circulating_supply.as_mut() {
v.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_percentage(
starting_indexes.height,
&supply.height_to_supply_value.bitcoin,
height_to_supply,
exit,
)?;
Ok(())
})?;
}
let _ = (dateindex_to_supply, height_to_market_cap, dateindex_to_market_cap);
// Additional relative metrics computed here
Ok(())
}
}
@@ -0,0 +1,252 @@
//! Supply and UTXO count metrics.
//!
//! These metrics are always computed regardless of price data availability.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, StoredF64, StoredU64, Version};
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, PcoVec,
TypedVecIterator,
};
use crate::{
Indexes,
grouped::{
ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromHeight, Source,
VecBuilderOptions,
},
indexes, price,
states::SupplyState,
};
use super::ImportConfig;
/// Supply and UTXO count metrics for a cohort.
#[derive(Clone, Traversable)]
pub struct SupplyMetrics {
/// Total supply at each height
pub height_to_supply: EagerVec<PcoVec<Height, Sats>>,
/// Supply value in BTC and USD (computed from height_to_supply)
pub height_to_supply_value: ComputedHeightValueVecs,
/// Supply indexed by date
pub indexes_to_supply: ComputedValueVecsFromDateIndex,
/// UTXO count at each height
pub height_to_utxo_count: EagerVec<PcoVec<Height, StoredU64>>,
/// UTXO count indexed by various dimensions
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
/// Half of supply value (used for computing median)
pub height_to_supply_half_value: ComputedHeightValueVecs,
/// Half of supply indexed by date
pub indexes_to_supply_half: ComputedValueVecsFromDateIndex,
}
impl SupplyMetrics {
/// Import supply metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let v1 = Version::ONE;
let compute_dollars = cfg.compute_dollars();
let last = VecBuilderOptions::default().add_last();
Ok(Self {
height_to_supply: EagerVec::forced_import(
cfg.db,
&cfg.name("supply"),
cfg.version + v0,
)?,
height_to_supply_value: ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply"),
Source::None,
cfg.version + v0,
compute_dollars,
)?,
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply"),
Source::Compute,
cfg.version + v1,
last,
compute_dollars,
cfg.indexes,
)?,
height_to_utxo_count: EagerVec::forced_import(
cfg.db,
&cfg.name("utxo_count"),
cfg.version + v0,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("utxo_count"),
Source::None,
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_supply_half_value: ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply_half"),
Source::Compute,
cfg.version + v0,
compute_dollars,
)?,
indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_half"),
Source::Compute,
cfg.version + v0,
last,
compute_dollars,
cfg.indexes,
)?,
})
}
/// Get minimum length across height-indexed vectors.
pub fn min_len(&self) -> usize {
self.height_to_supply
.len()
.min(self.height_to_utxo_count.len())
}
/// Push supply state values to height-indexed vectors.
pub fn truncate_push(&mut self, height: Height, state: &SupplyState) -> Result<()> {
self.height_to_supply.truncate_push(height, state.value)?;
self.height_to_utxo_count
.truncate_push(height, StoredU64::from(state.utxo_count))?;
Ok(())
}
/// Flush height-indexed vectors to disk.
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
self.height_to_supply.safe_write(exit)?;
self.height_to_utxo_count.safe_write(exit)?;
Ok(())
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_supply.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_supply).collect::<Vec<_>>(),
exit,
)?;
self.height_to_utxo_count.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_utxo_count).collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
/// First phase of computed metrics (indexes from height).
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.height_to_supply_value.compute_rest(
price,
starting_indexes,
exit,
Some(&self.height_to_supply),
)?;
self.indexes_to_supply
.compute_all(price, starting_indexes, exit, |v| {
let mut dateindex_to_height_count_iter =
indexes.dateindex_to_height_count.into_iter();
let mut height_to_supply_iter = self.height_to_supply.into_iter();
v.compute_transform(
starting_indexes.dateindex,
&indexes.dateindex_to_first_height,
|(i, height, ..)| {
let count = dateindex_to_height_count_iter.get_unwrap(i);
if count == StoredU64::default() {
unreachable!()
}
let supply = height_to_supply_iter.get_unwrap(height + (*count - 1));
(i, supply)
},
exit,
)?;
Ok(())
})?;
self.indexes_to_utxo_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_utxo_count),
)?;
self.height_to_supply_half_value
.compute_all(price, starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_supply,
|(h, v, ..)| (h, v / 2),
exit,
)?;
Ok(())
})?;
self.indexes_to_supply_half
.compute_all(price, starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.dateindex,
self.indexes_to_supply.sats.dateindex.as_ref().unwrap(),
|(i, sats, ..)| (i, sats / 2),
exit,
)?;
Ok(())
})?;
Ok(())
}
/// Second phase of computed metrics (ratios, relative values).
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
_dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
let _ = (indexes, price, height_to_supply, height_to_market_cap, dateindex_to_market_cap);
// Supply relative metrics computed here if needed
Ok(())
}
}
@@ -0,0 +1,281 @@
//! Unrealized profit/loss metrics.
//!
//! These metrics track paper gains/losses based on current vs acquisition price.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Sats, Version};
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec};
use crate::{
Indexes,
grouped::{
ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, Source,
VecBuilderOptions,
},
states::UnrealizedState,
};
use super::ImportConfig;
/// Unrealized profit/loss metrics.
#[derive(Clone, Traversable)]
pub struct UnrealizedMetrics {
// === Supply in Profit/Loss ===
pub height_to_supply_in_profit: EagerVec<PcoVec<Height, Sats>>,
pub indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex,
pub height_to_supply_in_loss: EagerVec<PcoVec<Height, Sats>>,
pub indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex,
pub dateindex_to_supply_in_profit: EagerVec<PcoVec<DateIndex, Sats>>,
pub dateindex_to_supply_in_loss: EagerVec<PcoVec<DateIndex, Sats>>,
pub height_to_supply_in_profit_value: ComputedHeightValueVecs,
pub height_to_supply_in_loss_value: ComputedHeightValueVecs,
// === Unrealized Profit/Loss ===
pub height_to_unrealized_profit: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_unrealized_profit: ComputedVecsFromDateIndex<Dollars>,
pub height_to_unrealized_loss: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_unrealized_loss: ComputedVecsFromDateIndex<Dollars>,
pub dateindex_to_unrealized_profit: EagerVec<PcoVec<DateIndex, Dollars>>,
pub dateindex_to_unrealized_loss: EagerVec<PcoVec<DateIndex, Dollars>>,
// === Negated and Net ===
pub height_to_neg_unrealized_loss: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_neg_unrealized_loss: ComputedVecsFromDateIndex<Dollars>,
pub height_to_net_unrealized_pnl: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex<Dollars>,
pub height_to_total_unrealized_pnl: EagerVec<PcoVec<Height, Dollars>>,
pub indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex<Dollars>,
}
impl UnrealizedMetrics {
/// Import unrealized metrics from database.
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v0 = Version::ZERO;
let compute_dollars = cfg.compute_dollars();
let last = VecBuilderOptions::default().add_last();
// Pre-import the dateindex vecs that are used as sources
let dateindex_to_supply_in_profit =
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_profit"), cfg.version + v0)?;
let dateindex_to_supply_in_loss =
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_loss"), cfg.version + v0)?;
let dateindex_to_unrealized_profit =
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_profit"), cfg.version + v0)?;
let dateindex_to_unrealized_loss =
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_loss"), cfg.version + v0)?;
Ok(Self {
// === Supply in Profit/Loss ===
height_to_supply_in_profit: EagerVec::forced_import(
cfg.db,
&cfg.name("supply_in_profit"),
cfg.version + v0,
)?,
indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_profit"),
Source::Vec(dateindex_to_supply_in_profit.boxed_clone()),
cfg.version + v0,
last,
compute_dollars,
cfg.indexes,
)?,
height_to_supply_in_loss: EagerVec::forced_import(
cfg.db,
&cfg.name("supply_in_loss"),
cfg.version + v0,
)?,
indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("supply_in_loss"),
Source::Vec(dateindex_to_supply_in_loss.boxed_clone()),
cfg.version + v0,
last,
compute_dollars,
cfg.indexes,
)?,
dateindex_to_supply_in_profit,
dateindex_to_supply_in_loss,
height_to_supply_in_profit_value: ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply_in_profit"),
Source::None,
cfg.version + v0,
compute_dollars,
)?,
height_to_supply_in_loss_value: ComputedHeightValueVecs::forced_import(
cfg.db,
&cfg.name("supply_in_loss"),
Source::None,
cfg.version + v0,
compute_dollars,
)?,
// === Unrealized Profit/Loss ===
height_to_unrealized_profit: EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_profit"),
cfg.version + v0,
)?,
indexes_to_unrealized_profit: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_profit"),
Source::Vec(dateindex_to_unrealized_profit.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_unrealized_loss: EagerVec::forced_import(
cfg.db,
&cfg.name("unrealized_loss"),
cfg.version + v0,
)?,
indexes_to_unrealized_loss: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("unrealized_loss"),
Source::Vec(dateindex_to_unrealized_loss.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
dateindex_to_unrealized_profit,
dateindex_to_unrealized_loss,
// === Negated and Net ===
height_to_neg_unrealized_loss: EagerVec::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss"),
cfg.version + v0,
)?,
indexes_to_neg_unrealized_loss: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("neg_unrealized_loss"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_net_unrealized_pnl: EagerVec::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl"),
cfg.version + v0,
)?,
indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("net_unrealized_pnl"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_total_unrealized_pnl: EagerVec::forced_import(
cfg.db,
&cfg.name("total_unrealized_pnl"),
cfg.version + v0,
)?,
indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("total_unrealized_pnl"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
})
}
/// Push unrealized state values to height-indexed vectors.
pub fn truncate_push(
&mut self,
height: Height,
dateindex: Option<DateIndex>,
height_state: &UnrealizedState,
date_state: Option<&UnrealizedState>,
) -> Result<()> {
self.height_to_supply_in_profit
.truncate_push(height, height_state.supply_in_profit)?;
self.height_to_supply_in_loss
.truncate_push(height, height_state.supply_in_loss)?;
self.height_to_unrealized_profit
.truncate_push(height, height_state.unrealized_profit)?;
self.height_to_unrealized_loss
.truncate_push(height, height_state.unrealized_loss)?;
if let (Some(dateindex), Some(date_state)) = (dateindex, date_state) {
self.dateindex_to_supply_in_profit
.truncate_push(dateindex, date_state.supply_in_profit)?;
self.dateindex_to_supply_in_loss
.truncate_push(dateindex, date_state.supply_in_loss)?;
self.dateindex_to_unrealized_profit
.truncate_push(dateindex, date_state.unrealized_profit)?;
self.dateindex_to_unrealized_loss
.truncate_push(dateindex, date_state.unrealized_loss)?;
}
Ok(())
}
/// Flush height-indexed vectors to disk.
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
self.height_to_supply_in_profit.safe_write(exit)?;
self.height_to_supply_in_loss.safe_write(exit)?;
self.height_to_unrealized_profit.safe_write(exit)?;
self.height_to_unrealized_loss.safe_write(exit)?;
self.dateindex_to_supply_in_profit.safe_write(exit)?;
self.dateindex_to_supply_in_loss.safe_write(exit)?;
self.dateindex_to_unrealized_profit.safe_write(exit)?;
self.dateindex_to_unrealized_loss.safe_write(exit)?;
Ok(())
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_supply_in_profit.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_supply_in_profit).collect::<Vec<_>>(),
exit,
)?;
self.height_to_supply_in_loss.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_supply_in_loss).collect::<Vec<_>>(),
exit,
)?;
self.height_to_unrealized_profit.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_unrealized_profit).collect::<Vec<_>>(),
exit,
)?;
self.height_to_unrealized_loss.compute_sum_of_others(
starting_indexes.height,
&others.iter().map(|v| &v.height_to_unrealized_loss).collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_supply_in_profit.compute_sum_of_others(
starting_indexes.dateindex,
&others.iter().map(|v| &v.dateindex_to_supply_in_profit).collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_supply_in_loss.compute_sum_of_others(
starting_indexes.dateindex,
&others.iter().map(|v| &v.dateindex_to_supply_in_loss).collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_unrealized_profit.compute_sum_of_others(
starting_indexes.dateindex,
&others.iter().map(|v| &v.dateindex_to_unrealized_profit).collect::<Vec<_>>(),
exit,
)?;
self.dateindex_to_unrealized_loss.compute_sum_of_others(
starting_indexes.dateindex,
&others.iter().map(|v| &v.dateindex_to_unrealized_loss).collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
}
@@ -0,0 +1,57 @@
//! Stateful computation for Bitcoin UTXO and address cohort metrics.
//!
//! This module processes blockchain data to compute metrics for various cohorts
//! (groups of UTXOs or addresses filtered by age, amount, type, etc.).
//!
//! ## Module Structure
//!
//! ```text
//! stateful/
//! ├── address/ # Address type handling (indexes, data storage)
//! ├── cohorts/ # Cohort traits and state management
//! ├── compute/ # Block processing pipeline
//! └── metrics/ # Metric vectors organized by category
//! ```
//!
//! ## Data Flow
//!
//! 1. **Import**: Load from checkpoint or start fresh
//! 2. **Process blocks**: For each block, process outputs/inputs in parallel
//! 3. **Update cohorts**: Track supply, realized/unrealized P&L per cohort
//! 4. **Flush**: Periodically checkpoint state to disk
//! 5. **Compute aggregates**: Derive aggregate cohorts from separate cohorts
pub mod address;
pub mod cohorts;
pub mod compute;
pub mod metrics;
mod process;
mod vecs;
use process::*;
pub use vecs::Vecs;
// Address re-exports
pub use address::{
AddressTypeToTypeIndexMap, AddressTypeToVec, AddressesDataVecs, AnyAddressIndexesVecs,
HeightToAddressTypeToVec,
};
// Cohort re-exports
pub use cohorts::{
AddressCohortVecs, AddressCohorts, CohortState, CohortVecs, DynCohortVecs, Flushable,
HeightFlushable, UTXOCohortVecs, UTXOCohorts,
};
// Compute re-exports
pub use compute::{
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexerReaders, VecsReaders,
};
// Metrics re-exports
pub use metrics::{
ActivityMetrics, CohortMetrics, ImportConfig, PricePaidMetrics, RealizedMetrics,
RelativeMetrics, SupplyMetrics, UnrealizedMetrics,
};
@@ -0,0 +1,44 @@
use brk_error::Result;
use brk_types::{AddressDataSource, AnyAddressIndex, EmptyAddressData};
use crate::stateful_new::{AddressTypeToTypeIndexMap, AddressesDataVecs};
/// Process empty address data updates.
///
/// Handles three cases:
/// - New empty address: push to empty storage
/// - Updated empty address (was empty): update in place
/// - Transition loaded -> empty: delete from loaded, push to empty
pub fn process_empty_addresses(
addresses_data: &mut AddressesDataVecs,
empty_updates: AddressTypeToTypeIndexMap<AddressDataSource<EmptyAddressData>>,
) -> Result<AddressTypeToTypeIndexMap<AnyAddressIndex>> {
let mut result = AddressTypeToTypeIndexMap::default();
for (address_type, sorted) in empty_updates.into_sorted_iter() {
for (typeindex, source) in sorted {
match source {
AddressDataSource::New(data) => {
let index = addresses_data.empty.fill_first_hole_or_push(data)?;
result
.get_mut(address_type)
.unwrap()
.insert(typeindex, AnyAddressIndex::from(index));
}
AddressDataSource::FromEmpty((index, data)) => {
addresses_data.empty.update(index, data)?;
}
AddressDataSource::FromLoaded((loaded_index, data)) => {
addresses_data.loaded.delete(loaded_index);
let empty_index = addresses_data.empty.fill_first_hole_or_push(data)?;
result
.get_mut(address_type)
.unwrap()
.insert(typeindex, AnyAddressIndex::from(empty_index));
}
}
}
}
Ok(result)
}
@@ -0,0 +1,128 @@
//! Parallel input processing.
//!
//! Processes a block's inputs (spent UTXOs) in parallel, building:
//! - height_to_sent: map from creation height -> Transacted for sends
//! - Address data for address cohort tracking (optional)
use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TxOutIndex, TypeIndex};
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use vecdb::{BytesVec, GenericStoredVec, PcoVec};
use crate::{
stateful_new::{IndexerReaders, process::RangeMap},
states::Transacted,
};
use super::super::address::HeightToAddressTypeToVec;
/// Result of processing inputs for a block.
pub struct InputsResult {
/// Map from UTXO creation height -> aggregated sent supply.
pub height_to_sent: FxHashMap<Height, Transacted>,
/// Per-height, per-address-type sent data: (typeindex, value) for each address.
pub sent_data: HeightToAddressTypeToVec<(TypeIndex, Sats)>,
}
/// Process inputs (spent UTXOs) for a block in parallel.
///
/// For each input:
/// 1. Read outpoint, resolve to txoutindex
/// 2. Get the creation height from txoutindex_to_height map
/// 3. Read value and type from the referenced output
/// 4. Accumulate into height_to_sent map
/// 5. Track address-specific data if input references an address type
#[allow(clippy::too_many_arguments)]
pub fn process_inputs(
first_txinindex: usize,
input_count: usize,
txinindex_to_txindex: &[TxIndex],
txinindex_to_outpoint: &PcoVec<TxInIndex, OutPoint>,
txindex_to_first_txoutindex: &BytesVec<TxIndex, TxOutIndex>,
txoutindex_to_value: &BytesVec<TxOutIndex, Sats>,
txoutindex_to_outputtype: &BytesVec<TxOutIndex, OutputType>,
txoutindex_to_typeindex: &BytesVec<TxOutIndex, TypeIndex>,
txoutindex_to_height: &RangeMap<TxOutIndex, Height>,
ir: &IndexerReaders,
) -> InputsResult {
let (height_to_sent, sent_data) = (first_txinindex..first_txinindex + input_count)
.into_par_iter()
.map(|i| {
let txinindex = TxInIndex::from(i);
let local_idx = i - first_txinindex;
let _txindex = txinindex_to_txindex[local_idx];
// Get outpoint and resolve to txoutindex
let outpoint = txinindex_to_outpoint.read_unwrap(txinindex, &ir.txinindex_to_outpoint);
let first_txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &ir.txindex_to_first_txoutindex);
let txoutindex = first_txoutindex + outpoint.vout();
// Get creation height
let prev_height = *txoutindex_to_height.get(txoutindex).unwrap();
// Get value and type from the output being spent
let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value);
let input_type =
txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype);
// Non-address inputs don't need typeindex
if input_type.is_not_address() {
return (prev_height, value, input_type, None);
}
let typeindex =
txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
(prev_height, value, input_type, Some((typeindex, value)))
})
.fold(
|| {
(
FxHashMap::<Height, Transacted>::default(),
HeightToAddressTypeToVec::default(),
)
},
|(mut height_to_sent, mut sent_data), (prev_height, value, output_type, addr_data)| {
height_to_sent
.entry(prev_height)
.or_default()
.iterate(value, output_type);
if let Some((typeindex, value)) = addr_data {
sent_data
.entry(prev_height)
.or_default()
.get_mut(output_type)
.unwrap()
.push((typeindex, value));
}
(height_to_sent, sent_data)
},
)
.reduce(
|| {
(
FxHashMap::<Height, Transacted>::default(),
HeightToAddressTypeToVec::default(),
)
},
|(mut h1, mut s1), (h2, s2)| {
// Merge height_to_sent maps
for (k, v) in h2 {
*h1.entry(k).or_default() += v;
}
// Merge sent_data maps
s1.merge_mut(s2);
(h1, s1)
},
);
InputsResult {
height_to_sent,
sent_data,
}
}
@@ -0,0 +1,44 @@
use brk_error::Result;
use brk_types::{AddressDataSource, AnyAddressIndex, LoadedAddressData};
use crate::stateful_new::{AddressTypeToTypeIndexMap, AddressesDataVecs};
/// Process loaded address data updates.
///
/// Handles three cases:
/// - New loaded address: push to loaded storage
/// - Updated loaded address (was loaded): update in place
/// - Transition empty -> loaded: delete from empty, push to loaded
pub fn process_loaded_addresses(
addresses_data: &mut AddressesDataVecs,
loaded_updates: AddressTypeToTypeIndexMap<AddressDataSource<LoadedAddressData>>,
) -> Result<AddressTypeToTypeIndexMap<AnyAddressIndex>> {
let mut result = AddressTypeToTypeIndexMap::default();
for (address_type, sorted) in loaded_updates.into_sorted_iter() {
for (typeindex, source) in sorted {
match source {
AddressDataSource::New(data) => {
let index = addresses_data.loaded.fill_first_hole_or_push(data)?;
result
.get_mut(address_type)
.unwrap()
.insert(typeindex, AnyAddressIndex::from(index));
}
AddressDataSource::FromLoaded((index, data)) => {
addresses_data.loaded.update(index, data)?;
}
AddressDataSource::FromEmpty((empty_index, data)) => {
addresses_data.empty.delete(empty_index);
let loaded_index = addresses_data.loaded.fill_first_hole_or_push(data)?;
result
.get_mut(address_type)
.unwrap()
.insert(typeindex, AnyAddressIndex::from(loaded_index));
}
}
}
}
Ok(result)
}
@@ -0,0 +1,11 @@
mod empty_addresses;
mod inputs;
mod loaded_addresses;
mod outputs;
mod range_map;
pub use empty_addresses::*;
pub use inputs::*;
pub use loaded_addresses::*;
pub use outputs::*;
pub use range_map::*;
@@ -0,0 +1,83 @@
//! Parallel output processing.
//!
//! Processes a block's outputs (new UTXOs) in parallel, building:
//! - Transacted: aggregated supply by output type and amount range
//! - Address data for address cohort tracking (optional)
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex, TypeIndex};
use rayon::prelude::*;
use vecdb::{BytesVec, GenericStoredVec};
use crate::{stateful_new::IndexerReaders, states::Transacted};
use super::super::address::AddressTypeToVec;
/// Result of processing outputs for a block.
pub struct OutputsResult {
/// Aggregated supply transacted in this block.
pub transacted: Transacted,
/// Per-address-type received data: (typeindex, value) for each address.
pub received_data: AddressTypeToVec<(TypeIndex, Sats)>,
}
/// Process outputs (new UTXOs) for a block in parallel.
///
/// For each output:
/// 1. Read value and output type from indexer
/// 2. Accumulate into Transacted by type and amount
/// 3. Track address-specific data if output is an address type
pub fn process_outputs(
first_txoutindex: usize,
output_count: usize,
txoutindex_to_txindex: &[TxIndex],
txoutindex_to_value: &BytesVec<TxOutIndex, Sats>,
txoutindex_to_outputtype: &BytesVec<TxOutIndex, OutputType>,
txoutindex_to_typeindex: &BytesVec<TxOutIndex, TypeIndex>,
ir: &IndexerReaders,
) -> OutputsResult {
let (transacted, received_data) = (first_txoutindex..first_txoutindex + output_count)
.into_par_iter()
.map(|i| {
let txoutindex = TxOutIndex::from(i);
let local_idx = i - first_txoutindex;
let _txindex = txoutindex_to_txindex[local_idx];
let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value);
let output_type =
txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype);
// Non-address outputs don't need typeindex
if output_type.is_not_address() {
return (value, output_type, None);
}
let typeindex =
txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
(value, output_type, Some((typeindex, value)))
})
.fold(
|| (Transacted::default(), AddressTypeToVec::default()),
|(mut transacted, mut received_data), (value, output_type, addr_data)| {
transacted.iterate(value, output_type);
if let Some((typeindex, value)) = addr_data {
received_data
.get_mut(output_type)
.unwrap()
.push((typeindex, value));
}
(transacted, received_data)
},
)
.reduce(
|| (Transacted::default(), AddressTypeToVec::default()),
|(t1, r1), (t2, r2)| (t1 + t2, r1.merge(r2)),
);
OutputsResult {
transacted,
received_data,
}
}
@@ -0,0 +1,65 @@
//! Main block processing loop.
//!
//! Iterates through blocks, processing outputs (receive) and inputs (send) in parallel.
use std::collections::BTreeMap;
use brk_types::{Height, TxOutIndex};
use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex};
/// Maps ranges of indices to their corresponding height.
/// Used to efficiently look up which block a txoutindex belongs to.
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
impl<I, T> RangeMap<I, T>
where
I: VecIndex,
T: VecIndex,
{
/// Look up value for a key using range search.
/// Returns the value associated with the largest key <= given key.
#[inline]
pub fn get(&self, key: I) -> Option<&T> {
self.0.range(..=key).next_back().map(|(_, value)| value)
}
}
impl<I, T> From<&BytesVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + BytesVecValue,
{
#[inline]
fn from(vec: &BytesVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect(),
)
}
}
impl<I, T> From<&PcoVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + PcoVecValue,
{
#[inline]
fn from(vec: &PcoVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect(),
)
}
}
/// Creates a RangeMap from height_to_first_txoutindex for fast txoutindex -> height lookups.
pub fn build_txoutindex_to_height_map(
height_to_first_txoutindex: &PcoVec<Height, TxOutIndex>,
) -> RangeMap<TxOutIndex, Height> {
RangeMap::from(height_to_first_txoutindex)
}
@@ -0,0 +1,231 @@
//! Main Vecs struct for stateful computation.
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{Dollars, Height, Sats, StoredU64, Version};
use vecdb::{
BytesVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, LazyVecFrom1,
PAGE_SIZE, PcoVec,
};
use crate::{
Indexes, SupplyState, chain,
grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, price,
};
use super::{
AddressCohorts, AddressesDataVecs, AnyAddressIndexesVecs, UTXOCohorts,
address::{AddressTypeToHeightToAddressCount, AddressTypeToIndexesToAddressCount},
};
const VERSION: Version = Version::new(21);
/// Main struct holding all computed vectors and state for stateful computation.
#[derive(Clone, Traversable)]
pub struct Vecs {
#[traversable(skip)]
db: Database,
// ---
// States
// ---
pub chain_state: BytesVec<Height, SupplyState>,
pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs,
pub utxo_cohorts: UTXOCohorts,
pub address_cohorts: AddressCohorts,
pub height_to_unspendable_supply: EagerVec<PcoVec<Height, Sats>>,
pub height_to_opreturn_supply: EagerVec<PcoVec<Height, Sats>>,
pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount,
pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount,
// ---
// Computed
// ---
pub addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount,
pub addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount,
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_empty_addr_count: ComputedVecsFromHeight<StoredU64>,
pub height_to_market_cap: Option<LazyVecFrom1<Height, Dollars, Height, Dollars>>,
pub indexes_to_market_cap: Option<ComputedVecsFromDateIndex<Dollars>>,
}
const SAVED_STAMPED_CHANGES: u16 = 10;
impl Vecs {
pub fn forced_import(
parent: &Path,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let db_path = parent.join("stateful");
let states_path = db_path.join("states");
let db = Database::open(&db_path)?;
db.set_min_len(PAGE_SIZE * 20_000_000)?;
db.set_min_regions(50_000)?;
let compute_dollars = price.is_some();
let v0 = version + VERSION + Version::ZERO;
let v2 = version + VERSION + Version::TWO;
let utxo_cohorts = UTXOCohorts::forced_import(&db, version, indexes, price, &states_path)?;
Ok(Self {
chain_state: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
&db,
"addr_count",
Source::Compute,
v0,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_empty_addr_count: ComputedVecsFromHeight::forced_import(
&db,
"empty_addr_count",
Source::Compute,
v0,
indexes,
VecBuilderOptions::default().add_last(),
)?,
height_to_market_cap: compute_dollars.then(|| {
LazyVecFrom1::init(
"market_cap",
v0,
utxo_cohorts
.all
.metrics
.supply
.height_to_supply_value
.dollars
.as_ref()
.unwrap()
.boxed_clone(),
|height: Height, iter| iter.get(height),
)
}),
indexes_to_market_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
&db,
"market_cap",
Source::Compute,
v2,
indexes,
VecBuilderOptions::default().add_last(),
)
.unwrap()
}),
addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::forced_import(
&db,
"addr_count",
v0,
)?,
addresstype_to_height_to_empty_addr_count:
AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?,
addresstype_to_indexes_to_addr_count:
AddressTypeToIndexesToAddressCount::forced_import(&db, "addr_count", v0, indexes)?,
addresstype_to_indexes_to_empty_addr_count:
AddressTypeToIndexesToAddressCount::forced_import(
&db,
"empty_addr_count",
v0,
indexes,
)?,
utxo_cohorts,
address_cohorts: AddressCohorts::forced_import(
&db,
version,
indexes,
price,
&states_path,
)?,
any_address_indexes: AnyAddressIndexesVecs::forced_import(&db, v0)?,
addresses_data: AddressesDataVecs::forced_import(&db, v0)?,
db,
})
}
/// Main computation loop.
///
/// Processes blocks to compute UTXO and address cohort metrics:
/// 1. Recovers state from checkpoints or starts fresh
/// 2. Iterates through blocks, processing outputs/inputs in parallel
/// 3. Flushes checkpoints periodically
/// 4. Computes aggregate cohorts from separate cohorts
/// 5. Computes derived metrics
///
/// NOTE: This is a placeholder. The full implementation needs to be ported
/// from stateful/mod.rs once all the supporting methods on UTXOCohorts,
/// AddressCohorts, and state types are implemented.
#[allow(clippy::too_many_arguments)]
pub fn compute(
&mut self,
_indexer: &Indexer,
_indexes: &indexes::Vecs,
_chain: &chain::Vecs,
_price: Option<&price::Vecs>,
_starting_indexes: &mut Indexes,
_exit: &Exit,
) -> Result<()> {
// The full compute implementation requires these methods to be implemented:
//
// On UTXOCohorts:
// - tick_tock_next_block(&chain_state, timestamp)
// - receive(transacted, height, price)
// - send(height_to_sent, &mut chain_state)
// - truncate_push_aggregate_percentiles(height)
// - import_aggregate_price_to_amount(height)
// - reset_aggregate_price_to_amount()
//
// On UTXOCohortState:
// - reset_block_values()
// - reset_price_to_amount()
//
// On AddressCohortState:
// - inner.reset_block_values()
// - inner.reset_price_to_amount()
//
// On AddressTypeToHeightToAddressCount:
// - safe_flush(exit)
// - truncate_push(height, &count)
//
// See stateful/mod.rs:368-1397 for the full implementation.
//
// The basic structure is:
// 1. Validate computed versions against base version
// 2. Find min stateful height and recover state
// 3. For each block:
// a. Reset per-block values
// b. Process outputs in parallel (receive)
// c. Process inputs in parallel (send)
// d. Push to height-indexed vectors
// e. Flush checkpoint every 10,000 blocks
// 4. Compute aggregate cohorts from separate cohorts
// 5. Compute rest_part1 (dateindex mappings)
// 6. Compute rest_part2 (ratios and relative metrics)
self.db.compact()?;
Ok(())
}
}
@@ -0,0 +1,239 @@
use std::path::Path;
use brk_error::Result;
use brk_grouper::{CohortContext, Filter, Filtered};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec,
PcoVec, TypedVecIterator,
};
use crate::{
Indexes,
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
indexes, price,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
},
states::AddressCohortState,
utils::OptionExt,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone, Traversable)]
pub struct Vecs {
starting_height: Option<Height>,
#[traversable(skip)]
pub state: Option<AddressCohortState>,
#[traversable(flatten)]
pub inner: common::Vecs,
pub height_to_addr_count: EagerVec<PcoVec<Height, StoredU64>>,
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
}
impl Vecs {
pub fn forced_import(
db: &Database,
filter: Filter,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: Option<&Path>,
) -> Result<Self> {
let compute_dollars = price.is_some();
let full_name = filter.to_full_name(CohortContext::Address);
let suffix = |s: &str| {
if full_name.is_empty() {
s.to_string()
} else {
format!("{full_name}_{s}")
}
};
Ok(Self {
starting_height: None,
state: states_path.map(|states_path| {
AddressCohortState::new(states_path, &full_name, compute_dollars)
}),
height_to_addr_count: EagerVec::forced_import(
db,
&suffix("addr_count"),
version + VERSION + Version::ZERO,
)?,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
db,
&suffix("addr_count"),
Source::None,
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
inner: common::Vecs::forced_import(
db,
filter,
CohortContext::Address,
version,
indexes,
price,
)?,
})
}
}
impl DynCohortVecs for Vecs {
fn min_height_vecs_len(&self) -> usize {
std::cmp::min(
self.height_to_addr_count.len(),
self.inner.min_height_vecs_len(),
)
}
fn reset_state_starting_height(&mut self) {
self.starting_height = Some(Height::ZERO);
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
let starting_height = self
.inner
.import_state(starting_height, &mut self.state.um().inner)?;
self.starting_height = Some(starting_height);
if let Some(prev_height) = starting_height.decremented() {
self.state.um().addr_count = *self
.height_to_addr_count
.into_iter()
.get_unwrap(prev_height);
}
Ok(starting_height)
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.height_to_addr_count
.validate_computed_version_or_reset(
base_version + self.height_to_addr_count.inner_version(),
)?;
self.inner.validate_computed_versions(base_version)
}
fn truncate_push(&mut self, height: Height) -> Result<()> {
if self.starting_height.unwrap() > height {
return Ok(());
}
self.height_to_addr_count
.truncate_push(height, self.state.u().addr_count.into())?;
self.inner
.truncate_push(height, &self.state.u().inner)
}
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()> {
self.inner.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
&self.state.u().inner,
)
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.height_to_addr_count.safe_write(exit)?;
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state.um().inner)
}
#[allow(clippy::too_many_arguments)]
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.indexes_to_addr_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_addr_count),
)?;
self.inner
.compute_rest_part1(indexes, price, starting_indexes, exit)
}
}
impl CohortVecs for Vecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.height_to_addr_count.compute_sum_of_others(
starting_indexes.height,
others
.iter()
.map(|v| &v.height_to_addr_count)
.collect::<Vec<_>>()
.as_slice(),
exit,
)?;
self.inner.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.inner).collect::<Vec<_>>(),
exit,
)
}
#[allow(clippy::too_many_arguments)]
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.inner.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
}
}
impl Filtered for Vecs {
fn filter(&self) -> &Filter {
&self.inner.filter
}
}
@@ -0,0 +1,139 @@
use std::path::Path;
use brk_error::Result;
use brk_grouper::{AddressGroups, AmountFilter, Filter, Filtered};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{Database, Exit, IterableVec};
use crate::{
Indexes, indexes, price,
stateful::{
address_cohort,
r#trait::{CohortVecs, DynCohortVecs},
},
};
const VERSION: Version = Version::new(0);
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct Vecs(AddressGroups<address_cohort::Vecs>);
impl Vecs {
pub fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
Ok(Self(AddressGroups::new(|filter| {
let states_path = match &filter {
Filter::Amount(AmountFilter::Range(_)) => Some(states_path),
_ => None,
};
address_cohort::Vecs::forced_import(
db,
filter,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
)
.unwrap()
})))
}
pub fn compute_overlapping_vecs(
&mut self,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let by_size_range = &self.0.amount_range;
[
self.0
.ge_amount
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_size_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
self.0
.lt_amount
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_size_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
})
.collect::<Vec<_>>(),
]
.into_iter()
.flatten()
.try_for_each(|(vecs, stateful)| {
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
})
}
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.0.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
})
}
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))
}
}
@@ -0,0 +1,226 @@
use brk_error::{Error, Result};
use brk_traversable::Traversable;
use brk_types::{
AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData,
LoadedAddressIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex,
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
TypeIndex,
};
use vecdb::{AnyStoredVec, BytesVec, GenericStoredVec, Reader, Stamp};
#[derive(Clone, Traversable)]
pub struct AnyAddressIndexesVecs {
pub p2pk33: BytesVec<P2PK33AddressIndex, AnyAddressIndex>,
pub p2pk65: BytesVec<P2PK65AddressIndex, AnyAddressIndex>,
pub p2pkh: BytesVec<P2PKHAddressIndex, AnyAddressIndex>,
pub p2sh: BytesVec<P2SHAddressIndex, AnyAddressIndex>,
pub p2tr: BytesVec<P2TRAddressIndex, AnyAddressIndex>,
pub p2wpkh: BytesVec<P2WPKHAddressIndex, AnyAddressIndex>,
pub p2wsh: BytesVec<P2WSHAddressIndex, AnyAddressIndex>,
pub p2a: BytesVec<P2AAddressIndex, AnyAddressIndex>,
}
impl AnyAddressIndexesVecs {
pub fn min_stamped_height(&self) -> Height {
Height::from(self.p2pk33.stamp())
.incremented()
.min(Height::from(self.p2pk65.stamp()).incremented())
.min(Height::from(self.p2pkh.stamp()).incremented())
.min(Height::from(self.p2sh.stamp()).incremented())
.min(Height::from(self.p2tr.stamp()).incremented())
.min(Height::from(self.p2wpkh.stamp()).incremented())
.min(Height::from(self.p2wsh.stamp()).incremented())
.min(Height::from(self.p2a.stamp()).incremented())
}
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 8]> {
Ok([
self.p2pk33.rollback_before(stamp)?,
self.p2pk65.rollback_before(stamp)?,
self.p2pkh.rollback_before(stamp)?,
self.p2sh.rollback_before(stamp)?,
self.p2tr.rollback_before(stamp)?,
self.p2wpkh.rollback_before(stamp)?,
self.p2wsh.rollback_before(stamp)?,
self.p2a.rollback_before(stamp)?,
])
}
pub fn reset(&mut self) -> Result<()> {
self.p2pk33.reset()?;
self.p2pk65.reset()?;
self.p2pkh.reset()?;
self.p2sh.reset()?;
self.p2tr.reset()?;
self.p2wpkh.reset()?;
self.p2wsh.reset()?;
self.p2a.reset()?;
Ok(())
}
pub fn get_anyaddressindex(
&self,
address_type: OutputType,
typeindex: TypeIndex,
reader: &Reader,
) -> AnyAddressIndex {
match address_type {
OutputType::P2PK33 => self
.p2pk33
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2PK65 => self
.p2pk65
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2PKH => self
.p2pkh
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2SH => self
.p2sh
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2TR => self
.p2tr
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2WPKH => self
.p2wpkh
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2WSH => self
.p2wsh
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
OutputType::P2A => self
.p2a
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
_ => unreachable!(),
}
}
pub fn get_anyaddressindex_once(
&self,
address_type: OutputType,
typeindex: TypeIndex,
) -> Result<AnyAddressIndex> {
match address_type {
OutputType::P2PK33 => self
.p2pk33
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2PK65 => self
.p2pk65
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2PKH => self
.p2pkh
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2SH => self
.p2sh
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2TR => self
.p2tr
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2WPKH => self
.p2wpkh
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2WSH => self
.p2wsh
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
OutputType::P2A => self
.p2a
.read_at_once(typeindex.into())
.map_err(|e| e.into()),
_ => Err(Error::UnsupportedType(address_type.to_string())),
}
}
pub fn update_or_push(
&mut self,
address_type: OutputType,
typeindex: TypeIndex,
anyaddressindex: AnyAddressIndex,
) -> Result<()> {
(match address_type {
OutputType::P2PK33 => self
.p2pk33
.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2PK65 => self
.p2pk65
.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2PKH => self.p2pkh.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2SH => self.p2sh.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2TR => self.p2tr.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2WPKH => self
.p2wpkh
.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2WSH => self.p2wsh.update_or_push(typeindex.into(), anyaddressindex),
OutputType::P2A => self.p2a.update_or_push(typeindex.into(), anyaddressindex),
_ => unreachable!(),
})?;
Ok(())
}
pub fn stamped_flush_maybe_with_changes(
&mut self,
stamp: Stamp,
with_changes: bool,
) -> Result<()> {
self.p2pk33
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2pk65
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2pkh
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2sh
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2tr
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2wpkh
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2wsh
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.p2a
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
Ok(())
}
}
#[derive(Clone, Traversable)]
pub struct AddressesDataVecs {
pub loaded: BytesVec<LoadedAddressIndex, LoadedAddressData>,
pub empty: BytesVec<EmptyAddressIndex, EmptyAddressData>,
}
impl AddressesDataVecs {
pub fn min_stamped_height(&self) -> Height {
Height::from(self.loaded.stamp())
.incremented()
.min(Height::from(self.empty.stamp()).incremented())
}
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> {
Ok([
self.loaded.rollback_before(stamp)?,
self.empty.rollback_before(stamp)?,
])
}
pub fn reset(&mut self) -> Result<()> {
self.loaded.reset()?;
self.empty.reset()?;
Ok(())
}
pub fn stamped_flush_maybe_with_changes(
&mut self,
stamp: Stamp,
with_changes: bool,
) -> Result<()> {
self.loaded
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.empty
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
Ok(())
}
}
@@ -0,0 +1,29 @@
use brk_grouper::ByAddressType;
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use vecdb::TypedVecIterator;
use super::AddressTypeToHeightToAddressCount;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
#[inline]
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(ByAddressType {
p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(),
p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(),
p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(),
p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(),
p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(),
p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(),
p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(),
p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(),
})
} else {
Default::default()
}
}
}
@@ -0,0 +1,45 @@
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64};
use derive_deref::{Deref, DerefMut};
use vecdb::{PcoVec, EagerVec, GenericStoredVec};
use super::AddressTypeToAddressCount;
#[derive(Debug, Clone, Deref, DerefMut, Traversable)]
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>);
impl From<ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>> for AddressTypeToHeightToAddressCount {
#[inline]
fn from(value: ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>) -> Self {
Self(value)
}
}
impl AddressTypeToHeightToAddressCount {
pub fn truncate_push(
&mut self,
height: Height,
addresstype_to_usize: &AddressTypeToAddressCount,
) -> Result<()> {
self.p2pk65
.truncate_push(height, addresstype_to_usize.p2pk65.into())?;
self.p2pk33
.truncate_push(height, addresstype_to_usize.p2pk33.into())?;
self.p2pkh
.truncate_push(height, addresstype_to_usize.p2pkh.into())?;
self.p2sh
.truncate_push(height, addresstype_to_usize.p2sh.into())?;
self.p2wpkh
.truncate_push(height, addresstype_to_usize.p2wpkh.into())?;
self.p2wsh
.truncate_push(height, addresstype_to_usize.p2wsh.into())?;
self.p2tr
.truncate_push(height, addresstype_to_usize.p2tr.into())?;
self.p2a
.truncate_push(height, addresstype_to_usize.p2a.into())?;
Ok(())
}
}
@@ -0,0 +1,9 @@
use std::collections::BTreeMap;
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use crate::stateful::AddressTypeToVec;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);
@@ -0,0 +1,80 @@
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_traversable::Traversable;
use brk_types::StoredU64;
use derive_deref::{Deref, DerefMut};
use vecdb::Exit;
use crate::{Indexes, grouped::ComputedVecsFromHeight, indexes};
use super::AddressTypeToHeightToAddressCount;
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
#[inline]
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
Self(value)
}
}
impl AddressTypeToIndexesToAddressCount {
pub fn compute(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
) -> Result<()> {
self.p2pk65.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk65),
)?;
self.p2pk33.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk33),
)?;
self.p2pkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pkh),
)?;
self.p2sh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2sh),
)?;
self.p2wpkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wpkh),
)?;
self.p2wsh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wsh),
)?;
self.p2tr.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2tr),
)?;
self.p2a.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2a),
)?;
Ok(())
}
}
@@ -0,0 +1,13 @@
mod addresscount;
mod height_to_addresscount;
mod height_to_vec;
mod indexes_to_addresscount;
mod typeindex_map;
mod vec;
pub use addresscount::*;
pub use height_to_addresscount::*;
pub use height_to_vec::*;
pub use indexes_to_addresscount::*;
pub use typeindex_map::*;
pub use vec::*;
@@ -0,0 +1,100 @@
use std::{collections::hash_map::Entry, mem};
use brk_grouper::ByAddressType;
use brk_types::{OutputType, TypeIndex};
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;
use smallvec::{Array, SmallVec};
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToTypeIndexMap<T>(ByAddressType<FxHashMap<TypeIndex, T>>);
impl<T> AddressTypeToTypeIndexMap<T> {
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
self
}
fn merge_(own: &mut FxHashMap<TypeIndex, T>, other: &mut FxHashMap<TypeIndex, T>) {
if own.len() < other.len() {
mem::swap(own, other);
}
own.extend(other.drain());
}
// pub fn get_for_type(&self, address_type: OutputType, typeindex: &TypeIndex) -> Option<&T> {
// self.get(address_type).unwrap().get(typeindex)
// }
pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) {
self.get_mut(address_type).unwrap().insert(typeindex, value);
}
pub fn remove_for_type(&mut self, address_type: OutputType, typeindex: &TypeIndex) -> T {
self.get_mut(address_type)
.unwrap()
.remove(typeindex)
.unwrap()
}
pub fn into_sorted_iter(self) -> impl Iterator<Item = (OutputType, Vec<(TypeIndex, T)>)> {
self.0.into_iter().map(|(output_type, map)| {
let mut sorted: Vec<_> = map.into_iter().collect();
sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex);
(output_type, sorted)
})
}
#[allow(clippy::should_implement_trait)]
pub fn into_iter(self) -> impl Iterator<Item = (OutputType, FxHashMap<TypeIndex, T>)> {
self.0.into_iter()
}
}
impl<T> Default for AddressTypeToTypeIndexMap<T> {
fn default() -> Self {
Self(ByAddressType {
p2pk65: FxHashMap::default(),
p2pk33: FxHashMap::default(),
p2pkh: FxHashMap::default(),
p2sh: FxHashMap::default(),
p2wpkh: FxHashMap::default(),
p2wsh: FxHashMap::default(),
p2tr: FxHashMap::default(),
p2a: FxHashMap::default(),
})
}
}
impl<T> AddressTypeToTypeIndexMap<SmallVec<T>>
where
T: Array,
{
pub fn merge_vec(mut self, other: Self) -> Self {
for (address_type, other_map) in other.0.into_iter() {
let self_map = self.0.get_mut_unwrap(address_type);
for (typeindex, mut other_vec) in other_map {
match self_map.entry(typeindex) {
Entry::Occupied(mut entry) => {
let self_vec = entry.get_mut();
if other_vec.len() > self_vec.len() {
mem::swap(self_vec, &mut other_vec);
}
self_vec.extend(other_vec);
}
Entry::Vacant(entry) => {
entry.insert(other_vec);
}
}
}
}
self
}
}
@@ -0,0 +1,60 @@
use std::mem;
use brk_grouper::ByAddressType;
use derive_deref::{Deref, DerefMut};
#[derive(Debug, Deref, DerefMut)]
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
impl<T> AddressTypeToVec<T> {
pub fn merge(mut self, mut other: Self) -> Self {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
self
}
pub fn merge_mut(&mut self, mut other: Self) {
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
Self::merge_(&mut self.p2sh, &mut other.p2sh);
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
Self::merge_(&mut self.p2tr, &mut other.p2tr);
Self::merge_(&mut self.p2a, &mut other.p2a);
}
fn merge_(own: &mut Vec<T>, other: &mut Vec<T>) {
if own.len() >= other.len() {
own.append(other);
} else {
other.append(own);
mem::swap(own, other);
}
}
pub fn unwrap(self) -> ByAddressType<Vec<T>> {
self.0
}
}
impl<T> Default for AddressTypeToVec<T> {
fn default() -> Self {
Self(ByAddressType {
p2pk65: vec![],
p2pk33: vec![],
p2pkh: vec![],
p2sh: vec![],
p2wpkh: vec![],
p2wsh: vec![],
p2tr: vec![],
p2a: vec![],
})
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,914 @@
//! Import and validation methods for Vecs.
//!
//! This module contains methods for:
//! - `forced_import`: Creating a new Vecs instance from database
//! - `import_state`: Importing state when resuming from checkpoint
//! - `validate_computed_versions`: Version validation
//! - `min_height_vecs_len`: Finding minimum vector length
use brk_error::{Error, Result};
use brk_grouper::{CohortContext, Filter};
use brk_types::{DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, Version};
use vecdb::{
AnyVec, Database, EagerVec, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec,
StoredVec, TypedVecIterator,
};
use crate::{
grouped::{
ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
PricePercentiles, Source, VecBuilderOptions,
},
indexes, price,
states::CohortState,
utils::OptionExt,
};
use super::Vecs;
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
db: &Database,
filter: Filter,
context: CohortContext,
parent_version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let compute_dollars = price.is_some();
let extended = filter.is_extended(context);
let compute_rel_to_all = filter.compute_rel_to_all();
let compute_adjusted = filter.compute_adjusted(context);
let version = parent_version + Version::ZERO;
let name_prefix = filter.to_full_name(context);
let suffix = |s: &str| {
if name_prefix.is_empty() {
s.to_string()
} else {
format!("{name_prefix}_{s}")
}
};
// Helper macros for imports
macro_rules! eager {
($idx:ty, $val:ty, $name:expr, $v:expr) => {
EagerVec::<PcoVec<$idx, $val>>::forced_import(db, &suffix($name), version + $v)
.unwrap()
};
}
macro_rules! computed_h {
($name:expr, $source:expr, $v:expr, $opts:expr $(,)?) => {
ComputedVecsFromHeight::forced_import(
db,
&suffix($name),
$source,
version + $v,
indexes,
$opts,
)
.unwrap()
};
}
macro_rules! computed_di {
($name:expr, $source:expr, $v:expr, $opts:expr $(,)?) => {
ComputedVecsFromDateIndex::forced_import(
db,
&suffix($name),
$source,
version + $v,
indexes,
$opts,
)
.unwrap()
};
}
// Common version patterns
let v0 = Version::ZERO;
let v1 = Version::ONE;
let v2 = Version::TWO;
let v3 = Version::new(3);
let last = || VecBuilderOptions::default().add_last();
let sum = || VecBuilderOptions::default().add_sum();
let sum_cum = || VecBuilderOptions::default().add_sum().add_cumulative();
// Pre-create dateindex vecs that are used in computed vecs
let dateindex_to_supply_in_profit =
compute_dollars.then(|| eager!(DateIndex, Sats, "supply_in_profit", v0));
let dateindex_to_supply_in_loss =
compute_dollars.then(|| eager!(DateIndex, Sats, "supply_in_loss", v0));
let dateindex_to_unrealized_profit =
compute_dollars.then(|| eager!(DateIndex, Dollars, "unrealized_profit", v0));
let dateindex_to_unrealized_loss =
compute_dollars.then(|| eager!(DateIndex, Dollars, "unrealized_loss", v0));
Ok(Self {
filter,
// ==================== SUPPLY & UTXO COUNT ====================
height_to_supply: EagerVec::forced_import(db, &suffix("supply"), version + v0)?,
height_to_supply_value: ComputedHeightValueVecs::forced_import(
db,
&suffix("supply"),
Source::None,
version + v0,
compute_dollars,
)?,
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
db,
&suffix("supply"),
Source::Compute,
version + v1,
last(),
compute_dollars,
indexes,
)?,
height_to_utxo_count: EagerVec::forced_import(db, &suffix("utxo_count"), version + v0)?,
indexes_to_utxo_count: computed_h!("utxo_count", Source::None, v0, last()),
height_to_supply_half_value: ComputedHeightValueVecs::forced_import(
db,
&suffix("supply_half"),
Source::Compute,
version + v0,
compute_dollars,
)?,
indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import(
db,
&suffix("supply_half"),
Source::Compute,
version + v0,
last(),
compute_dollars,
indexes,
)?,
// ==================== ACTIVITY ====================
height_to_sent: EagerVec::forced_import(db, &suffix("sent"), version + v0)?,
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
db,
&suffix("sent"),
Source::None,
version + v0,
sum(),
compute_dollars,
indexes,
)?,
height_to_satblocks_destroyed: EagerVec::forced_import(
db,
&suffix("satblocks_destroyed"),
version + v0,
)?,
height_to_satdays_destroyed: EagerVec::forced_import(
db,
&suffix("satdays_destroyed"),
version + v0,
)?,
indexes_to_coinblocks_destroyed: computed_h!(
"coinblocks_destroyed",
Source::Compute,
v2,
sum_cum(),
),
indexes_to_coindays_destroyed: computed_h!(
"coindays_destroyed",
Source::Compute,
v2,
sum_cum(),
),
// ==================== REALIZED CAP & PRICE ====================
height_to_realized_cap: compute_dollars
.then(|| eager!(Height, Dollars, "realized_cap", v0)),
indexes_to_realized_cap: compute_dollars
.then(|| computed_h!("realized_cap", Source::None, v0, last())),
indexes_to_realized_price: compute_dollars
.then(|| computed_h!("realized_price", Source::Compute, v0, last())),
indexes_to_realized_price_extra: compute_dollars.then(|| {
ComputedRatioVecsFromDateIndex::forced_import(
db,
&suffix("realized_price"),
Source::None,
version + v0,
indexes,
extended,
)
.unwrap()
}),
indexes_to_realized_cap_rel_to_own_market_cap: (compute_dollars && extended).then(
|| {
computed_h!(
"realized_cap_rel_to_own_market_cap",
Source::Compute,
v0,
last()
)
},
),
indexes_to_realized_cap_30d_delta: compute_dollars
.then(|| computed_di!("realized_cap_30d_delta", Source::Compute, v0, last())),
// ==================== REALIZED PROFIT & LOSS ====================
height_to_realized_profit: compute_dollars
.then(|| eager!(Height, Dollars, "realized_profit", v0)),
indexes_to_realized_profit: compute_dollars
.then(|| computed_h!("realized_profit", Source::None, v0, sum_cum())),
height_to_realized_loss: compute_dollars
.then(|| eager!(Height, Dollars, "realized_loss", v0)),
indexes_to_realized_loss: compute_dollars
.then(|| computed_h!("realized_loss", Source::None, v0, sum_cum())),
indexes_to_neg_realized_loss: compute_dollars
.then(|| computed_h!("neg_realized_loss", Source::Compute, v1, sum_cum())),
indexes_to_net_realized_pnl: compute_dollars
.then(|| computed_h!("net_realized_pnl", Source::Compute, v0, sum_cum())),
indexes_to_realized_value: compute_dollars
.then(|| computed_h!("realized_value", Source::Compute, v0, sum())),
indexes_to_realized_profit_rel_to_realized_cap: compute_dollars.then(|| {
computed_h!(
"realized_profit_rel_to_realized_cap",
Source::Compute,
v0,
sum()
)
}),
indexes_to_realized_loss_rel_to_realized_cap: compute_dollars.then(|| {
computed_h!(
"realized_loss_rel_to_realized_cap",
Source::Compute,
v0,
sum()
)
}),
indexes_to_net_realized_pnl_rel_to_realized_cap: compute_dollars.then(|| {
computed_h!(
"net_realized_pnl_rel_to_realized_cap",
Source::Compute,
v1,
sum()
)
}),
height_to_total_realized_pnl: compute_dollars
.then(|| eager!(Height, Dollars, "total_realized_pnl", v0)),
indexes_to_total_realized_pnl: compute_dollars
.then(|| computed_di!("total_realized_pnl", Source::Compute, v1, sum())),
dateindex_to_realized_profit_to_loss_ratio: (compute_dollars && extended)
.then(|| eager!(DateIndex, StoredF64, "realized_profit_to_loss_ratio", v1)),
// ==================== VALUE CREATED & DESTROYED ====================
height_to_value_created: compute_dollars
.then(|| eager!(Height, Dollars, "value_created", v0)),
indexes_to_value_created: compute_dollars
.then(|| computed_h!("value_created", Source::None, v0, sum())),
height_to_value_destroyed: compute_dollars
.then(|| eager!(Height, Dollars, "value_destroyed", v0)),
indexes_to_value_destroyed: compute_dollars
.then(|| computed_h!("value_destroyed", Source::None, v0, sum())),
height_to_adjusted_value_created: (compute_dollars && compute_adjusted)
.then(|| eager!(Height, Dollars, "adjusted_value_created", v0)),
indexes_to_adjusted_value_created: (compute_dollars && compute_adjusted)
.then(|| computed_h!("adjusted_value_created", Source::None, v0, sum())),
height_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted)
.then(|| eager!(Height, Dollars, "adjusted_value_destroyed", v0)),
indexes_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted)
.then(|| computed_h!("adjusted_value_destroyed", Source::None, v0, sum())),
// ==================== SOPR ====================
dateindex_to_sopr: compute_dollars.then(|| eager!(DateIndex, StoredF64, "sopr", v1)),
dateindex_to_sopr_7d_ema: compute_dollars
.then(|| eager!(DateIndex, StoredF64, "sopr_7d_ema", v1)),
dateindex_to_sopr_30d_ema: compute_dollars
.then(|| eager!(DateIndex, StoredF64, "sopr_30d_ema", v1)),
dateindex_to_adjusted_sopr: (compute_dollars && compute_adjusted)
.then(|| eager!(DateIndex, StoredF64, "adjusted_sopr", v1)),
dateindex_to_adjusted_sopr_7d_ema: (compute_dollars && compute_adjusted)
.then(|| eager!(DateIndex, StoredF64, "adjusted_sopr_7d_ema", v1)),
dateindex_to_adjusted_sopr_30d_ema: (compute_dollars && compute_adjusted)
.then(|| eager!(DateIndex, StoredF64, "adjusted_sopr_30d_ema", v1)),
// ==================== SELL SIDE RISK ====================
dateindex_to_sell_side_risk_ratio: compute_dollars
.then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio", v1)),
dateindex_to_sell_side_risk_ratio_7d_ema: compute_dollars
.then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio_7d_ema", v1)),
dateindex_to_sell_side_risk_ratio_30d_ema: compute_dollars
.then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio_30d_ema", v1)),
// ==================== SUPPLY IN PROFIT/LOSS ====================
height_to_supply_in_profit: compute_dollars
.then(|| eager!(Height, Sats, "supply_in_profit", v0)),
indexes_to_supply_in_profit: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import(
db,
&suffix("supply_in_profit"),
dateindex_to_supply_in_profit
.as_ref()
.map(|v| v.boxed_clone())
.into(),
version + v0,
last(),
compute_dollars,
indexes,
)
.unwrap()
}),
height_to_supply_in_loss: compute_dollars
.then(|| eager!(Height, Sats, "supply_in_loss", v0)),
indexes_to_supply_in_loss: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import(
db,
&suffix("supply_in_loss"),
dateindex_to_supply_in_loss
.as_ref()
.map(|v| v.boxed_clone())
.into(),
version + v0,
last(),
compute_dollars,
indexes,
)
.unwrap()
}),
dateindex_to_supply_in_profit,
dateindex_to_supply_in_loss,
height_to_supply_in_profit_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import(
db,
&suffix("supply_in_profit"),
Source::None,
version + v0,
compute_dollars,
)
.unwrap()
}),
height_to_supply_in_loss_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import(
db,
&suffix("supply_in_loss"),
Source::None,
version + v0,
compute_dollars,
)
.unwrap()
}),
// ==================== UNREALIZED PROFIT & LOSS ====================
height_to_unrealized_profit: compute_dollars
.then(|| eager!(Height, Dollars, "unrealized_profit", v0)),
indexes_to_unrealized_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
db,
&suffix("unrealized_profit"),
dateindex_to_unrealized_profit
.as_ref()
.map(|v| v.boxed_clone())
.into(),
version + v0,
indexes,
last(),
)
.unwrap()
}),
height_to_unrealized_loss: compute_dollars
.then(|| eager!(Height, Dollars, "unrealized_loss", v0)),
indexes_to_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
db,
&suffix("unrealized_loss"),
dateindex_to_unrealized_loss
.as_ref()
.map(|v| v.boxed_clone())
.into(),
version + v0,
indexes,
last(),
)
.unwrap()
}),
dateindex_to_unrealized_profit,
dateindex_to_unrealized_loss,
height_to_neg_unrealized_loss: compute_dollars
.then(|| eager!(Height, Dollars, "neg_unrealized_loss", v0)),
indexes_to_neg_unrealized_loss: compute_dollars
.then(|| computed_di!("neg_unrealized_loss", Source::Compute, v0, last())),
height_to_net_unrealized_pnl: compute_dollars
.then(|| eager!(Height, Dollars, "net_unrealized_pnl", v0)),
indexes_to_net_unrealized_pnl: compute_dollars
.then(|| computed_di!("net_unrealized_pnl", Source::Compute, v0, last())),
height_to_total_unrealized_pnl: compute_dollars
.then(|| eager!(Height, Dollars, "total_unrealized_pnl", v0)),
indexes_to_total_unrealized_pnl: compute_dollars
.then(|| computed_di!("total_unrealized_pnl", Source::Compute, v0, last())),
// ==================== PRICE PAID ====================
height_to_min_price_paid: compute_dollars
.then(|| eager!(Height, Dollars, "min_price_paid", v0)),
indexes_to_min_price_paid: compute_dollars
.then(|| computed_h!("min_price_paid", Source::None, v0, last())),
height_to_max_price_paid: compute_dollars
.then(|| eager!(Height, Dollars, "max_price_paid", v0)),
indexes_to_max_price_paid: compute_dollars
.then(|| computed_h!("max_price_paid", Source::None, v0, last())),
price_percentiles: (compute_dollars && extended).then(|| {
PricePercentiles::forced_import(db, &suffix(""), version + v0, indexes, true)
.unwrap()
}),
// ==================== RELATIVE METRICS: UNREALIZED vs MARKET CAP ====================
height_to_unrealized_profit_rel_to_market_cap: compute_dollars
.then(|| eager!(Height, StoredF32, "unrealized_profit_rel_to_market_cap", v0)),
height_to_unrealized_loss_rel_to_market_cap: compute_dollars
.then(|| eager!(Height, StoredF32, "unrealized_loss_rel_to_market_cap", v0)),
height_to_neg_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| {
eager!(
Height,
StoredF32,
"neg_unrealized_loss_rel_to_market_cap",
v0
)
}),
height_to_net_unrealized_pnl_rel_to_market_cap: compute_dollars.then(|| {
eager!(
Height,
StoredF32,
"net_unrealized_pnl_rel_to_market_cap",
v1
)
}),
indexes_to_unrealized_profit_rel_to_market_cap: compute_dollars.then(|| {
computed_di!(
"unrealized_profit_rel_to_market_cap",
Source::Compute,
v1,
last()
)
}),
indexes_to_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| {
computed_di!(
"unrealized_loss_rel_to_market_cap",
Source::Compute,
v1,
last()
)
}),
indexes_to_neg_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| {
computed_di!(
"neg_unrealized_loss_rel_to_market_cap",
Source::Compute,
v1,
last()
)
}),
indexes_to_net_unrealized_pnl_rel_to_market_cap: compute_dollars.then(|| {
computed_di!(
"net_unrealized_pnl_rel_to_market_cap",
Source::Compute,
v1,
last()
)
}),
// ==================== RELATIVE METRICS: UNREALIZED vs OWN MARKET CAP ====================
height_to_unrealized_profit_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
eager!(
Height,
StoredF32,
"unrealized_profit_rel_to_own_market_cap",
v1
)
}),
height_to_unrealized_loss_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
eager!(
Height,
StoredF32,
"unrealized_loss_rel_to_own_market_cap",
v1
)
}),
height_to_neg_unrealized_loss_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
eager!(
Height,
StoredF32,
"neg_unrealized_loss_rel_to_own_market_cap",
v1
)
}),
height_to_net_unrealized_pnl_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
eager!(
Height,
StoredF32,
"net_unrealized_pnl_rel_to_own_market_cap",
v2
)
}),
indexes_to_unrealized_profit_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
computed_di!(
"unrealized_profit_rel_to_own_market_cap",
Source::Compute,
v2,
last()
)
}),
indexes_to_unrealized_loss_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
computed_di!(
"unrealized_loss_rel_to_own_market_cap",
Source::Compute,
v2,
last()
)
}),
indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
computed_di!(
"neg_unrealized_loss_rel_to_own_market_cap",
Source::Compute,
v2,
last()
)
}),
indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (compute_dollars
&& extended
&& compute_rel_to_all)
.then(|| {
computed_di!(
"net_unrealized_pnl_rel_to_own_market_cap",
Source::Compute,
v2,
last()
)
}),
// ==================== RELATIVE METRICS: UNREALIZED vs OWN TOTAL UNREALIZED ====================
height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
eager!(
Height,
StoredF32,
"unrealized_profit_rel_to_own_total_unrealized_pnl",
v0
)
}),
height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
eager!(
Height,
StoredF32,
"unrealized_loss_rel_to_own_total_unrealized_pnl",
v0
)
}),
height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
eager!(
Height,
StoredF32,
"neg_unrealized_loss_rel_to_own_total_unrealized_pnl",
v0
)
}),
height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
eager!(
Height,
StoredF32,
"net_unrealized_pnl_rel_to_own_total_unrealized_pnl",
v1
)
}),
indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
computed_di!(
"unrealized_profit_rel_to_own_total_unrealized_pnl",
Source::Compute,
v1,
last()
)
}),
indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
computed_di!(
"unrealized_loss_rel_to_own_total_unrealized_pnl",
Source::Compute,
v1,
last()
)
}),
indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
computed_di!(
"neg_unrealized_loss_rel_to_own_total_unrealized_pnl",
Source::Compute,
v1,
last()
)
}),
indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: (compute_dollars
&& extended)
.then(|| {
computed_di!(
"net_unrealized_pnl_rel_to_own_total_unrealized_pnl",
Source::Compute,
v1,
last()
)
}),
// ==================== RELATIVE METRICS: SUPPLY vs CIRCULATING/OWN ====================
indexes_to_supply_rel_to_circulating_supply: compute_rel_to_all.then(|| {
computed_h!(
"supply_rel_to_circulating_supply",
Source::Compute,
v1,
last()
)
}),
height_to_supply_in_profit_rel_to_own_supply: compute_dollars
.then(|| eager!(Height, StoredF64, "supply_in_profit_rel_to_own_supply", v1)),
height_to_supply_in_loss_rel_to_own_supply: compute_dollars
.then(|| eager!(Height, StoredF64, "supply_in_loss_rel_to_own_supply", v1)),
indexes_to_supply_in_profit_rel_to_own_supply: compute_dollars.then(|| {
computed_di!(
"supply_in_profit_rel_to_own_supply",
Source::Compute,
v1,
last()
)
}),
indexes_to_supply_in_loss_rel_to_own_supply: compute_dollars.then(|| {
computed_di!(
"supply_in_loss_rel_to_own_supply",
Source::Compute,
v1,
last()
)
}),
height_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all
&& compute_dollars)
.then(|| {
eager!(
Height,
StoredF64,
"supply_in_profit_rel_to_circulating_supply",
v1
)
}),
height_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
&& compute_dollars)
.then(|| {
eager!(
Height,
StoredF64,
"supply_in_loss_rel_to_circulating_supply",
v1
)
}),
indexes_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all
&& compute_dollars)
.then(|| {
computed_di!(
"supply_in_profit_rel_to_circulating_supply",
Source::Compute,
v1,
last()
)
}),
indexes_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
&& compute_dollars)
.then(|| {
computed_di!(
"supply_in_loss_rel_to_circulating_supply",
Source::Compute,
v1,
last()
)
}),
// ==================== NET REALIZED PNL DELTAS ====================
indexes_to_net_realized_pnl_cumulative_30d_delta: compute_dollars.then(|| {
computed_di!(
"net_realized_pnl_cumulative_30d_delta",
Source::Compute,
v3,
last()
)
}),
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap: compute_dollars
.then(|| {
computed_di!(
"net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap",
Source::Compute,
v3,
last()
)
}),
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: compute_dollars
.then(|| {
computed_di!(
"net_realized_pnl_cumulative_30d_delta_rel_to_market_cap",
Source::Compute,
v3,
last()
)
}),
})
}
/// Returns the minimum length of all height-indexed vectors.
/// Used to determine the starting point for processing.
pub fn min_height_vecs_len(&self) -> usize {
[
self.height_to_supply.len(),
self.height_to_utxo_count.len(),
self.height_to_realized_cap
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_realized_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_realized_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_value_created
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_adjusted_value_created
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_value_destroyed
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_adjusted_value_destroyed
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_in_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_in_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_unrealized_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_unrealized_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_min_price_paid
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_max_price_paid
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_sent.len(),
self.height_to_satdays_destroyed.len(),
self.height_to_satblocks_destroyed.len(),
]
.into_iter()
.min()
.unwrap()
}
/// Import state from a checkpoint when resuming processing.
/// Returns the next height to process from.
pub fn import_state(
&mut self,
starting_height: Height,
state: &mut CohortState,
) -> Result<Height> {
if let Some(mut prev_height) = starting_height.decremented() {
if self.height_to_realized_cap.as_mut().is_some() {
prev_height = state.import_at_or_before(prev_height)?;
}
state.supply.value = self.height_to_supply.into_iter().get_unwrap(prev_height);
state.supply.utxo_count = *self
.height_to_utxo_count
.into_iter()
.get_unwrap(prev_height);
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
state.realized.um().cap =
height_to_realized_cap.into_iter().get_unwrap(prev_height);
}
Ok(prev_height.incremented())
} else {
Err(Error::Str("Unset"))
}
}
/// Validate that all computed versions match expected values, resetting if needed.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
// Always-present vecs
self.height_to_supply.validate_computed_version_or_reset(
base_version + self.height_to_supply.inner_version(),
)?;
self.height_to_utxo_count
.validate_computed_version_or_reset(
base_version + self.height_to_utxo_count.inner_version(),
)?;
self.height_to_sent.validate_computed_version_or_reset(
base_version + self.height_to_sent.inner_version(),
)?;
self.height_to_satblocks_destroyed
.validate_computed_version_or_reset(
base_version + self.height_to_satblocks_destroyed.inner_version(),
)?;
self.height_to_satdays_destroyed
.validate_computed_version_or_reset(
base_version + self.height_to_satdays_destroyed.inner_version(),
)?;
// Dollar-dependent vecs
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() {
height_to_realized_cap.validate_computed_version_or_reset(
base_version + height_to_realized_cap.inner_version(),
)?;
Self::validate_optional_vec_version(&mut self.height_to_realized_profit, base_version)?;
Self::validate_optional_vec_version(&mut self.height_to_realized_loss, base_version)?;
Self::validate_optional_vec_version(&mut self.height_to_value_created, base_version)?;
Self::validate_optional_vec_version(&mut self.height_to_value_destroyed, base_version)?;
Self::validate_optional_vec_version(
&mut self.height_to_supply_in_profit,
base_version,
)?;
Self::validate_optional_vec_version(&mut self.height_to_supply_in_loss, base_version)?;
Self::validate_optional_vec_version(
&mut self.height_to_unrealized_profit,
base_version,
)?;
Self::validate_optional_vec_version(&mut self.height_to_unrealized_loss, base_version)?;
Self::validate_optional_vec_version(
&mut self.dateindex_to_supply_in_profit,
base_version,
)?;
Self::validate_optional_vec_version(
&mut self.dateindex_to_supply_in_loss,
base_version,
)?;
Self::validate_optional_vec_version(
&mut self.dateindex_to_unrealized_profit,
base_version,
)?;
Self::validate_optional_vec_version(
&mut self.dateindex_to_unrealized_loss,
base_version,
)?;
Self::validate_optional_vec_version(&mut self.height_to_min_price_paid, base_version)?;
Self::validate_optional_vec_version(&mut self.height_to_max_price_paid, base_version)?;
if self.height_to_adjusted_value_created.is_some() {
Self::validate_optional_vec_version(
&mut self.height_to_adjusted_value_created,
base_version,
)?;
Self::validate_optional_vec_version(
&mut self.height_to_adjusted_value_destroyed,
base_version,
)?;
}
}
Ok(())
}
/// Helper to validate an optional vec's version.
fn validate_optional_vec_version<V: StoredVec>(
vec: &mut Option<EagerVec<V>>,
base_version: Version,
) -> Result<()> {
if let Some(v) = vec.as_mut() {
v.validate_computed_version_or_reset(base_version + v.inner_version())?;
}
Ok(())
}
}
@@ -0,0 +1,19 @@
//! Common vector structs and logic shared between UTXO and Address cohorts.
//!
//! This module contains the `Vecs` struct which holds all the computed vectors
//! for a single cohort, along with methods for importing, flushing, and computing.
//!
//! ## Module Organization
//!
//! The implementation is split across multiple files for maintainability:
//! - `vecs.rs`: Struct definition with field documentation
//! - `import.rs`: Import, validation, and initialization methods
//! - `push.rs`: Per-block push and flush methods
//! - `compute.rs`: Post-processing computation methods
mod compute;
mod import;
mod push;
mod vecs;
pub use vecs::Vecs;
@@ -0,0 +1,178 @@
//! Push and flush methods for Vecs.
//!
//! This module contains methods for:
//! - `truncate_push`: Push state values to height-indexed vectors
//! - `compute_then_truncate_push_unrealized_states`: Compute and push unrealized states
//! - `safe_flush_stateful_vecs`: Safely flush all stateful vectors
use brk_error::Result;
use brk_types::{DateIndex, Dollars, Height, StoredU64};
use vecdb::{AnyStoredVec, Exit, GenericStoredVec};
use crate::{stateful::Flushable, states::CohortState, utils::OptionExt};
use super::Vecs;
impl Vecs {
pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> {
self.height_to_supply
.truncate_push(height, state.supply.value)?;
self.height_to_utxo_count
.truncate_push(height, StoredU64::from(state.supply.utxo_count))?;
self.height_to_sent.truncate_push(height, state.sent)?;
self.height_to_satblocks_destroyed
.truncate_push(height, state.satblocks_destroyed)?;
self.height_to_satdays_destroyed
.truncate_push(height, state.satdays_destroyed)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
let realized = state.realized.as_ref().unwrap_or_else(|| {
dbg!((&state.realized, &state.supply));
panic!();
});
height_to_realized_cap.truncate_push(height, realized.cap)?;
self.height_to_realized_profit
.um()
.truncate_push(height, realized.profit)?;
self.height_to_realized_loss
.um()
.truncate_push(height, realized.loss)?;
self.height_to_value_created
.um()
.truncate_push(height, realized.value_created)?;
self.height_to_value_destroyed
.um()
.truncate_push(height, realized.value_destroyed)?;
if self.height_to_adjusted_value_created.is_some() {
self.height_to_adjusted_value_created
.um()
.truncate_push(height, realized.adj_value_created)?;
self.height_to_adjusted_value_destroyed
.um()
.truncate_push(height, realized.adj_value_destroyed)?;
}
}
Ok(())
}
pub fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
state: &CohortState,
) -> Result<()> {
if let Some(height_price) = height_price {
self.height_to_min_price_paid.um().truncate_push(
height,
state
.price_to_amount_first_key_value()
.map(|(&dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
)?;
self.height_to_max_price_paid.um().truncate_push(
height,
state
.price_to_amount_last_key_value()
.map(|(&dollars, _)| dollars)
.unwrap_or(Dollars::NAN),
)?;
let (height_unrealized_state, date_unrealized_state) =
state.compute_unrealized_states(height_price, date_price.unwrap());
self.height_to_supply_in_profit
.um()
.truncate_push(height, height_unrealized_state.supply_in_profit)?;
self.height_to_supply_in_loss
.um()
.truncate_push(height, height_unrealized_state.supply_in_loss)?;
self.height_to_unrealized_profit
.um()
.truncate_push(height, height_unrealized_state.unrealized_profit)?;
self.height_to_unrealized_loss
.um()
.truncate_push(height, height_unrealized_state.unrealized_loss)?;
if let Some(date_unrealized_state) = date_unrealized_state {
let dateindex = dateindex.unwrap();
self.dateindex_to_supply_in_profit
.um()
.truncate_push(dateindex, date_unrealized_state.supply_in_profit)?;
self.dateindex_to_supply_in_loss
.um()
.truncate_push(dateindex, date_unrealized_state.supply_in_loss)?;
self.dateindex_to_unrealized_profit
.um()
.truncate_push(dateindex, date_unrealized_state.unrealized_profit)?;
self.dateindex_to_unrealized_loss
.um()
.truncate_push(dateindex, date_unrealized_state.unrealized_loss)?;
}
// Compute and push price percentiles
if let Some(price_percentiles) = self.price_percentiles.as_mut() {
let percentile_prices = state.compute_percentile_prices();
price_percentiles.truncate_push(height, &percentile_prices)?;
}
}
Ok(())
}
pub fn safe_flush_stateful_vecs(
&mut self,
height: Height,
exit: &Exit,
state: &mut CohortState,
) -> Result<()> {
self.height_to_supply.safe_write(exit)?;
self.height_to_utxo_count.safe_write(exit)?;
self.height_to_sent.safe_write(exit)?;
self.height_to_satdays_destroyed.safe_write(exit)?;
self.height_to_satblocks_destroyed.safe_write(exit)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.safe_write(exit)?;
self.height_to_realized_profit.um().safe_write(exit)?;
self.height_to_realized_loss.um().safe_write(exit)?;
self.height_to_value_created.um().safe_write(exit)?;
self.height_to_value_destroyed.um().safe_write(exit)?;
self.height_to_supply_in_profit.um().safe_write(exit)?;
self.height_to_supply_in_loss.um().safe_write(exit)?;
self.height_to_unrealized_profit.um().safe_write(exit)?;
self.height_to_unrealized_loss.um().safe_write(exit)?;
self.dateindex_to_supply_in_profit.um().safe_write(exit)?;
self.dateindex_to_supply_in_loss.um().safe_write(exit)?;
self.dateindex_to_unrealized_profit.um().safe_write(exit)?;
self.dateindex_to_unrealized_loss.um().safe_write(exit)?;
self.height_to_min_price_paid.um().safe_write(exit)?;
self.height_to_max_price_paid.um().safe_write(exit)?;
if self.height_to_adjusted_value_created.is_some() {
self.height_to_adjusted_value_created
.um()
.safe_write(exit)?;
self.height_to_adjusted_value_destroyed
.um()
.safe_write(exit)?;
}
// Uses Flushable trait - Option<T> impl handles None case
self.price_percentiles.safe_write(exit)?;
}
state.commit(height)?;
Ok(())
}
}
@@ -0,0 +1,210 @@
use brk_grouper::Filter;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64};
use vecdb::{EagerVec, PcoVec};
use crate::grouped::{
ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
PricePercentiles,
};
/// Common vectors shared between UTXO and Address cohorts.
///
/// This struct contains all the computed vectors for a single cohort. The fields are
/// organized into logical groups matching the initialization order in `forced_import`.
///
/// ## Field Groups
/// - **Supply & UTXO count**: Basic supply metrics (always computed)
/// - **Activity**: Sent amounts, satblocks/satdays destroyed
/// - **Realized**: Realized cap, profit/loss, value created/destroyed, SOPR
/// - **Unrealized**: Unrealized profit/loss, supply in profit/loss
/// - **Price**: Min/max price paid, price percentiles
/// - **Relative metrics**: Ratios relative to market cap, realized cap, etc.
#[derive(Clone, Traversable)]
pub struct Vecs {
#[traversable(skip)]
pub filter: Filter,
// ==================== SUPPLY & UTXO COUNT ====================
// Always computed - core supply metrics
pub height_to_supply: EagerVec<PcoVec<Height, Sats>>,
pub height_to_supply_value: ComputedHeightValueVecs,
pub indexes_to_supply: ComputedValueVecsFromDateIndex,
pub height_to_utxo_count: EagerVec<PcoVec<Height, StoredU64>>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
pub height_to_supply_half_value: ComputedHeightValueVecs,
pub indexes_to_supply_half: ComputedValueVecsFromDateIndex,
// ==================== ACTIVITY ====================
// Always computed - transaction activity metrics
pub height_to_sent: EagerVec<PcoVec<Height, Sats>>,
pub indexes_to_sent: ComputedValueVecsFromHeight,
pub height_to_satblocks_destroyed: EagerVec<PcoVec<Height, Sats>>,
pub height_to_satdays_destroyed: EagerVec<PcoVec<Height, Sats>>,
pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
// ==================== REALIZED CAP & PRICE ====================
// Conditional on compute_dollars
pub height_to_realized_cap: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_price: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_price_extra: Option<ComputedRatioVecsFromDateIndex>,
pub indexes_to_realized_cap_rel_to_own_market_cap: Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_realized_cap_30d_delta: Option<ComputedVecsFromDateIndex<Dollars>>,
// ==================== REALIZED PROFIT & LOSS ====================
// Conditional on compute_dollars
pub height_to_realized_profit: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_realized_profit: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_realized_loss: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_realized_loss: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_neg_realized_loss: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_net_realized_pnl: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_value: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_profit_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_realized_loss_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_net_realized_pnl_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
pub height_to_total_realized_pnl: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_total_realized_pnl: Option<ComputedVecsFromDateIndex<Dollars>>,
pub dateindex_to_realized_profit_to_loss_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
// ==================== VALUE CREATED & DESTROYED ====================
// Conditional on compute_dollars
pub height_to_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_adjusted_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_adjusted_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_adjusted_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
// ==================== SOPR ====================
// Spent Output Profit Ratio - conditional on compute_dollars
pub dateindex_to_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
pub dateindex_to_adjusted_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
// ==================== SELL SIDE RISK ====================
// Conditional on compute_dollars
pub dateindex_to_sell_side_risk_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF32>>>,
pub dateindex_to_sell_side_risk_ratio_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF32>>>,
pub dateindex_to_sell_side_risk_ratio_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF32>>>,
// ==================== SUPPLY IN PROFIT/LOSS ====================
// Conditional on compute_dollars
pub height_to_supply_in_profit: Option<EagerVec<PcoVec<Height, Sats>>>,
pub indexes_to_supply_in_profit: Option<ComputedValueVecsFromDateIndex>,
pub height_to_supply_in_loss: Option<EagerVec<PcoVec<Height, Sats>>>,
pub indexes_to_supply_in_loss: Option<ComputedValueVecsFromDateIndex>,
pub dateindex_to_supply_in_profit: Option<EagerVec<PcoVec<DateIndex, Sats>>>,
pub dateindex_to_supply_in_loss: Option<EagerVec<PcoVec<DateIndex, Sats>>>,
pub height_to_supply_in_profit_value: Option<ComputedHeightValueVecs>,
pub height_to_supply_in_loss_value: Option<ComputedHeightValueVecs>,
// ==================== UNREALIZED PROFIT & LOSS ====================
// Conditional on compute_dollars
pub height_to_unrealized_profit: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_unrealized_profit: Option<ComputedVecsFromDateIndex<Dollars>>,
pub height_to_unrealized_loss: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_unrealized_loss: Option<ComputedVecsFromDateIndex<Dollars>>,
pub dateindex_to_unrealized_profit: Option<EagerVec<PcoVec<DateIndex, Dollars>>>,
pub dateindex_to_unrealized_loss: Option<EagerVec<PcoVec<DateIndex, Dollars>>>,
pub height_to_neg_unrealized_loss: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_neg_unrealized_loss: Option<ComputedVecsFromDateIndex<Dollars>>,
pub height_to_net_unrealized_pnl: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_net_unrealized_pnl: Option<ComputedVecsFromDateIndex<Dollars>>,
pub height_to_total_unrealized_pnl: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_total_unrealized_pnl: Option<ComputedVecsFromDateIndex<Dollars>>,
// ==================== PRICE PAID ====================
// Conditional on compute_dollars
pub height_to_min_price_paid: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_min_price_paid: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_max_price_paid: Option<EagerVec<PcoVec<Height, Dollars>>>,
pub indexes_to_max_price_paid: Option<ComputedVecsFromHeight<Dollars>>,
pub price_percentiles: Option<PricePercentiles>,
// ==================== RELATIVE METRICS: UNREALIZED vs MARKET CAP ====================
// Conditional on compute_dollars
pub height_to_unrealized_profit_rel_to_market_cap: Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_unrealized_loss_rel_to_market_cap: Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_neg_unrealized_loss_rel_to_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_net_unrealized_pnl_rel_to_market_cap: Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub indexes_to_unrealized_profit_rel_to_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_unrealized_loss_rel_to_market_cap: Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_neg_unrealized_loss_rel_to_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_net_unrealized_pnl_rel_to_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
// ==================== RELATIVE METRICS: UNREALIZED vs OWN MARKET CAP ====================
// Conditional on compute_dollars && extended && compute_rel_to_all
pub height_to_unrealized_profit_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_unrealized_loss_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_neg_unrealized_loss_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_net_unrealized_pnl_rel_to_own_market_cap:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub indexes_to_unrealized_profit_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_unrealized_loss_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
// ==================== RELATIVE METRICS: UNREALIZED vs OWN TOTAL UNREALIZED ====================
// Conditional on compute_dollars && extended
pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
Option<EagerVec<PcoVec<Height, StoredF32>>>,
pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
Option<ComputedVecsFromDateIndex<StoredF32>>,
// ==================== RELATIVE METRICS: SUPPLY vs CIRCULATING/OWN ====================
// Conditional on compute_dollars
pub indexes_to_supply_rel_to_circulating_supply: Option<ComputedVecsFromHeight<StoredF64>>,
pub height_to_supply_in_profit_rel_to_own_supply: Option<EagerVec<PcoVec<Height, StoredF64>>>,
pub height_to_supply_in_loss_rel_to_own_supply: Option<EagerVec<PcoVec<Height, StoredF64>>>,
pub indexes_to_supply_in_profit_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
pub indexes_to_supply_in_loss_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
pub height_to_supply_in_profit_rel_to_circulating_supply:
Option<EagerVec<PcoVec<Height, StoredF64>>>,
pub height_to_supply_in_loss_rel_to_circulating_supply:
Option<EagerVec<PcoVec<Height, StoredF64>>>,
pub indexes_to_supply_in_profit_rel_to_circulating_supply:
Option<ComputedVecsFromDateIndex<StoredF64>>,
pub indexes_to_supply_in_loss_rel_to_circulating_supply:
Option<ComputedVecsFromDateIndex<StoredF64>>,
// ==================== NET REALIZED PNL DELTAS ====================
// Conditional on compute_dollars
pub indexes_to_net_realized_pnl_cumulative_30d_delta:
Option<ComputedVecsFromDateIndex<Dollars>>,
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
}
@@ -0,0 +1,80 @@
//! Traits for consistent state flushing and importing.
//!
//! These traits ensure all stateful components follow the same patterns
//! for checkpoint/resume operations, preventing bugs where new fields
//! are forgotten during flush operations.
use brk_error::Result;
use brk_types::Height;
use vecdb::Exit;
/// Trait for components that can be flushed to disk.
///
/// This is for simple flush operations that don't require height tracking.
pub trait Flushable {
/// Safely flush data to disk.
fn safe_flush(&mut self, exit: &Exit) -> Result<()>;
/// Write to mmap without fsync. Data visible to readers immediately but not durable.
fn safe_write(&mut self, exit: &Exit) -> Result<()>;
}
/// Trait for stateful components that track data indexed by height.
///
/// This ensures consistent patterns for:
/// - Flushing state at checkpoints
/// - Importing state when resuming from a checkpoint
/// - Resetting state when starting from scratch
pub trait HeightFlushable {
/// Flush state to disk at the given height checkpoint.
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()>;
/// Import state from the most recent checkpoint at or before the given height.
/// Returns the actual height that was imported.
fn import_at_or_before(&mut self, height: Height) -> Result<Height>;
/// Reset state for starting from scratch.
fn reset(&mut self) -> Result<()>;
}
/// Blanket implementation for Option<T> where T: Flushable
impl<T: Flushable> Flushable for Option<T> {
fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.safe_flush(exit)?;
}
Ok(())
}
fn safe_write(&mut self, exit: &Exit) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.safe_write(exit)?;
}
Ok(())
}
}
/// Blanket implementation for Option<T> where T: HeightFlushable
impl<T: HeightFlushable> HeightFlushable for Option<T> {
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.flush_at_height(height, exit)?;
}
Ok(())
}
fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
if let Some(inner) = self.as_mut() {
inner.import_at_or_before(height)
} else {
Ok(height)
}
}
fn reset(&mut self) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.reset()?;
}
Ok(())
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,53 @@
use std::collections::BTreeMap;
use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
impl<I, T> RangeMap<I, T>
where
I: VecIndex,
T: VecIndex,
{
pub fn get(&self, key: I) -> Option<&T> {
self.0.range(..=key).next_back().map(|(&min, value)| {
if min > key {
unreachable!()
}
value
})
}
}
impl<I, T> From<&BytesVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + BytesVecValue,
{
#[inline]
fn from(vec: &BytesVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect::<BTreeMap<_, _>>(),
)
}
}
impl<I, T> From<&PcoVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + PcoVecValue,
{
#[inline]
fn from(vec: &PcoVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect::<BTreeMap<_, _>>(),
)
}
}
@@ -0,0 +1,111 @@
use brk_grouper::{ByAddressType, ByAnyAddress};
use brk_indexer::Indexer;
use brk_types::{OutputType, StoredU64, TxIndex};
use vecdb::{BoxedVecIterator, GenericStoredVec, Reader, VecIndex};
use super::Vecs;
pub struct IndexerReaders {
pub txinindex_to_outpoint: Reader,
pub txindex_to_first_txoutindex: Reader,
pub txoutindex_to_value: Reader,
pub txoutindex_to_outputtype: Reader,
pub txoutindex_to_typeindex: Reader,
}
impl IndexerReaders {
pub fn new(indexer: &Indexer) -> Self {
Self {
txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(),
txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(),
txoutindex_to_value: indexer.vecs.txoutindex_to_value.create_reader(),
txoutindex_to_outputtype: indexer.vecs.txoutindex_to_outputtype.create_reader(),
txoutindex_to_typeindex: indexer.vecs.txoutindex_to_typeindex.create_reader(),
}
}
}
pub struct VecsReaders {
pub addresstypeindex_to_anyaddressindex: ByAddressType<Reader>,
pub anyaddressindex_to_anyaddressdata: ByAnyAddress<Reader>,
}
impl VecsReaders {
pub fn new(vecs: &Vecs) -> Self {
Self {
addresstypeindex_to_anyaddressindex: ByAddressType {
p2pk33: vecs.any_address_indexes.p2pk33.create_reader(),
p2pk65: vecs.any_address_indexes.p2pk65.create_reader(),
p2pkh: vecs.any_address_indexes.p2pkh.create_reader(),
p2sh: vecs.any_address_indexes.p2sh.create_reader(),
p2tr: vecs.any_address_indexes.p2tr.create_reader(),
p2wpkh: vecs.any_address_indexes.p2wpkh.create_reader(),
p2wsh: vecs.any_address_indexes.p2wsh.create_reader(),
p2a: vecs.any_address_indexes.p2a.create_reader(),
},
anyaddressindex_to_anyaddressdata: ByAnyAddress {
loaded: vecs.addresses_data.loaded.create_reader(),
empty: vecs.addresses_data.empty.create_reader(),
},
}
}
pub fn get_anyaddressindex_reader(&self, address_type: OutputType) -> &Reader {
self.addresstypeindex_to_anyaddressindex
.get_unwrap(address_type)
}
}
pub fn build_txoutindex_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_output_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
let block_first_txindex = block_first_txindex.to_usize();
let counts: Vec<_> = (0..block_tx_count as usize)
.map(|tx_offset| {
let txindex = TxIndex::from(block_first_txindex + tx_offset);
u64::from(txindex_to_output_count.get_unwrap(txindex))
})
.collect();
let total: u64 = counts.iter().sum();
let mut vec = Vec::with_capacity(total as usize);
for (tx_offset, &output_count) in counts.iter().enumerate() {
let txindex = TxIndex::from(block_first_txindex + tx_offset);
for _ in 0..output_count {
vec.push(txindex);
}
}
vec
}
pub fn build_txinindex_to_txindex<'a>(
block_first_txindex: TxIndex,
block_tx_count: u64,
txindex_to_input_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
) -> Vec<TxIndex> {
let block_first_txindex = block_first_txindex.to_usize();
let counts: Vec<_> = (0..block_tx_count as usize)
.map(|tx_offset| {
let txindex = TxIndex::from(block_first_txindex + tx_offset);
u64::from(txindex_to_input_count.get_unwrap(txindex))
})
.collect();
let total: u64 = counts.iter().sum();
let mut vec = Vec::with_capacity(total as usize);
for (tx_offset, &input_count) in counts.iter().enumerate() {
let txindex = TxIndex::from(block_first_txindex + tx_offset);
for _ in 0..input_count {
vec.push(txindex);
}
}
vec
}
@@ -0,0 +1,59 @@
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Exit, IterableVec};
use crate::{Indexes, indexes, price};
pub trait DynCohortVecs: Send + Sync {
fn min_height_vecs_len(&self) -> usize;
fn reset_state_starting_height(&mut self);
fn import_state(&mut self, starting_height: Height) -> Result<Height>;
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
fn truncate_push(&mut self, height: Height) -> Result<()>;
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()>;
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>;
#[allow(clippy::too_many_arguments)]
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()>;
}
pub trait CohortVecs: DynCohortVecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()>;
#[allow(clippy::too_many_arguments)]
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()>;
}
@@ -0,0 +1,217 @@
use brk_error::Result;
use brk_grouper::{ByAddressType, Filtered};
use brk_types::{
CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex,
};
use vecdb::VecIndex;
use crate::utils::OptionExt;
use super::{
address_cohorts,
addresstype::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec},
withaddressdatasource::WithAddressDataSource,
};
impl AddressTypeToVec<(TypeIndex, Sats)> {
#[allow(clippy::too_many_arguments)]
pub fn process_received(
self,
vecs: &mut address_cohorts::Vecs,
addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexMap<
WithAddressDataSource<LoadedAddressData>,
>,
addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexMap<
WithAddressDataSource<EmptyAddressData>,
>,
price: Option<Dollars>,
addresstype_to_addr_count: &mut ByAddressType<u64>,
addresstype_to_empty_addr_count: &mut ByAddressType<u64>,
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexMap<
WithAddressDataSource<LoadedAddressData>,
>,
) {
self.unwrap().into_iter().for_each(|(_type, vec)| {
vec.into_iter().for_each(|(type_index, value)| {
let mut is_new = false;
let mut from_any_empty = false;
let addressdata_withsource = addresstype_to_typeindex_to_loadedaddressdata
.get_mut(_type)
.unwrap()
.entry(type_index)
.or_insert_with(|| {
addresstype_to_typeindex_to_emptyaddressdata
.get_mut(_type)
.unwrap()
.remove(&type_index)
.map(|ad| {
from_any_empty = true;
ad.into()
})
.unwrap_or_else(|| {
let addressdata =
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
.remove_for_type(_type, &type_index);
is_new = addressdata.is_new();
from_any_empty = addressdata.is_from_emptyaddressdata();
addressdata
})
});
if is_new || from_any_empty {
(*addresstype_to_addr_count.get_mut(_type).unwrap()) += 1;
if from_any_empty {
(*addresstype_to_empty_addr_count.get_mut(_type).unwrap()) -= 1;
}
}
let addressdata = addressdata_withsource.deref_mut();
let prev_amount = addressdata.balance();
let amount = prev_amount + value;
let filters_differ = vecs.amount_range.get(amount).filter()
!= vecs.amount_range.get(prev_amount).filter();
if is_new || from_any_empty || filters_differ {
if !is_new && !from_any_empty {
vecs.amount_range
.get_mut(prev_amount)
.state
.um()
.subtract(addressdata);
}
addressdata.receive(value, price);
vecs.amount_range
.get_mut(amount)
.state
.um()
.add(addressdata);
} else {
vecs.amount_range
.get_mut(amount)
.state
.um()
.receive(addressdata, value, price);
}
});
});
}
}
impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
#[allow(clippy::too_many_arguments)]
pub fn process_sent(
self,
vecs: &mut address_cohorts::Vecs,
addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexMap<
WithAddressDataSource<LoadedAddressData>,
>,
addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexMap<
WithAddressDataSource<EmptyAddressData>,
>,
price: Option<Dollars>,
addresstype_to_addr_count: &mut ByAddressType<u64>,
addresstype_to_empty_addr_count: &mut ByAddressType<u64>,
height_to_price_close_vec: Option<&Vec<brk_types::Close<Dollars>>>,
height_to_timestamp_fixed_vec: &[Timestamp],
height: Height,
timestamp: Timestamp,
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexMap<
WithAddressDataSource<LoadedAddressData>,
>,
) -> Result<()> {
self.0.into_iter().try_for_each(|(prev_height, v)| {
let prev_price = height_to_price_close_vec
.as_ref()
.map(|v| **v.get(prev_height.to_usize()).unwrap());
let prev_timestamp = *height_to_timestamp_fixed_vec
.get(prev_height.to_usize())
.unwrap();
let blocks_old = height.to_usize() - prev_height.to_usize();
let days_old = timestamp.difference_in_days_between_float(prev_timestamp);
let older_than_hour = timestamp
.checked_sub(prev_timestamp)
.unwrap()
.is_more_than_hour();
v.unwrap().into_iter().try_for_each(|(_type, vec)| {
vec.into_iter().try_for_each(|(type_index, value)| {
let typeindex_to_loadedaddressdata =
addresstype_to_typeindex_to_loadedaddressdata.get_mut_unwrap(_type);
let addressdata_withsource = typeindex_to_loadedaddressdata
.entry(type_index)
.or_insert_with(|| {
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
.remove_for_type(_type, &type_index)
});
let addressdata = addressdata_withsource.deref_mut();
let prev_amount = addressdata.balance();
let amount = prev_amount.checked_sub(value).unwrap();
let will_be_empty = addressdata.has_1_utxos();
let filters_differ = vecs.amount_range.get(amount).filter()
!= vecs.amount_range.get(prev_amount).filter();
if will_be_empty || filters_differ {
vecs.amount_range
.get_mut(prev_amount)
.state
.um()
.subtract(addressdata);
addressdata.send(value, prev_price)?;
if will_be_empty {
if amount.is_not_zero() {
unreachable!()
}
(*addresstype_to_addr_count.get_mut(_type).unwrap()) -= 1;
(*addresstype_to_empty_addr_count.get_mut(_type).unwrap()) += 1;
let addressdata =
typeindex_to_loadedaddressdata.remove(&type_index).unwrap();
addresstype_to_typeindex_to_emptyaddressdata
.get_mut(_type)
.unwrap()
.insert(type_index, addressdata.into());
} else {
vecs.amount_range
.get_mut(amount)
.state
.um()
.add(addressdata);
}
} else {
vecs.amount_range.get_mut(amount).state.um().send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
}
Ok(())
})
})
})
}
}
@@ -0,0 +1,241 @@
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_grouper::{CohortContext, Filter, Filtered, StateLevel};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version};
use vecdb::{Database, Exit, IterableVec};
use crate::{
Indexes, PriceToAmount, UTXOCohortState,
grouped::{PERCENTILES, PERCENTILES_LEN},
indexes, price,
stateful::{
common,
r#trait::{CohortVecs, DynCohortVecs},
},
utils::OptionExt,
};
#[derive(Clone, Traversable)]
pub struct Vecs {
state_starting_height: Option<Height>,
#[traversable(skip)]
pub state: Option<UTXOCohortState>,
/// For aggregate cohorts (all, sth, lth) that only need price_to_amount for percentiles
#[traversable(skip)]
pub price_to_amount: Option<PriceToAmount>,
#[traversable(flatten)]
pub inner: common::Vecs,
}
impl Vecs {
pub fn forced_import(
db: &Database,
filter: Filter,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
state_level: StateLevel,
) -> Result<Self> {
let compute_dollars = price.is_some();
let full_name = filter.to_full_name(CohortContext::Utxo);
Ok(Self {
state_starting_height: None,
state: if state_level.is_full() {
Some(UTXOCohortState::new(
states_path,
&full_name,
compute_dollars,
))
} else {
None
},
price_to_amount: if state_level.is_price_only() && compute_dollars {
Some(PriceToAmount::create(states_path, &full_name))
} else {
None
},
inner: common::Vecs::forced_import(
db,
filter,
CohortContext::Utxo,
version,
indexes,
price,
)?,
})
}
}
impl DynCohortVecs for Vecs {
fn min_height_vecs_len(&self) -> usize {
self.inner.min_height_vecs_len()
}
fn reset_state_starting_height(&mut self) {
self.state_starting_height = Some(Height::ZERO);
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
let starting_height = self
.inner
.import_state(starting_height, self.state.um())?;
self.state_starting_height = Some(starting_height);
Ok(starting_height)
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.inner.validate_computed_versions(base_version)
}
fn truncate_push(&mut self, height: Height) -> Result<()> {
if self.state_starting_height.unwrap() > height {
return Ok(());
}
self.inner
.truncate_push(height, self.state.u())
}
fn compute_then_truncate_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
) -> Result<()> {
self.inner.compute_then_truncate_push_unrealized_states(
height,
height_price,
dateindex,
date_price,
self.state.um(),
)
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.inner
.safe_flush_stateful_vecs(height, exit, self.state.um())
}
#[allow(clippy::too_many_arguments)]
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.inner
.compute_rest_part1(indexes, price, starting_indexes, exit)
}
}
impl CohortVecs for Vecs {
fn compute_from_stateful(
&mut self,
starting_indexes: &Indexes,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.inner.compute_from_stateful(
starting_indexes,
&others.iter().map(|v| &v.inner).collect::<Vec<_>>(),
exit,
)
}
#[allow(clippy::too_many_arguments)]
fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.inner.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
}
}
impl Vecs {
/// Compute percentile prices for aggregate cohorts that have standalone price_to_amount.
/// Returns NaN array if price_to_amount is None or empty.
pub fn compute_percentile_prices_from_standalone(
&self,
supply: Sats,
) -> [Dollars; PERCENTILES_LEN] {
let mut result = [Dollars::NAN; PERCENTILES_LEN];
let price_to_amount = match self.price_to_amount.as_ref() {
Some(p) => p,
None => return result,
};
if price_to_amount.is_empty() || supply == Sats::ZERO {
return result;
}
let total = supply;
let targets = PERCENTILES.map(|p| total * p / 100);
let mut accumulated = Sats::ZERO;
let mut pct_idx = 0;
for (&price, &sats) in price_to_amount.iter() {
accumulated += sats;
while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] {
result[pct_idx] = price;
pct_idx += 1;
}
if pct_idx >= PERCENTILES_LEN {
break;
}
}
result
}
}
impl Deref for Vecs {
type Target = common::Vecs;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl Filtered for Vecs {
fn filter(&self) -> &Filter {
&self.inner.filter
}
}
@@ -0,0 +1,697 @@
use std::path::Path;
use brk_error::Result;
use brk_grouper::{
AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount,
ByMaxAge, ByMinAge, BySpendableType, ByTerm, Filter, Filtered, StateLevel, Term, TimeFilter,
UTXOGroups,
};
use brk_traversable::Traversable;
use brk_types::{
Bitcoin, CheckedSub, DateIndex, Dollars, HalvingEpoch, Height, ONE_DAY_IN_SEC, OutputType,
Sats, Timestamp, Version,
};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use vecdb::{Database, Exit, IterableVec, VecIndex};
use crate::{
Indexes, indexes, price,
stateful::{Flushable, HeightFlushable, r#trait::DynCohortVecs},
states::{BlockState, Transacted},
utils::OptionExt,
};
use super::{r#trait::CohortVecs, utxo_cohort};
const VERSION: Version = Version::new(0);
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct Vecs(UTXOGroups<utxo_cohort::Vecs>);
impl Vecs {
pub fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
let v = version + VERSION + Version::ZERO;
// Helper to create a cohort - booleans are now derived from filter
let create = |filter: Filter, state_level: StateLevel| -> Result<utxo_cohort::Vecs> {
utxo_cohort::Vecs::forced_import(
db,
filter,
v,
indexes,
price,
states_path,
state_level,
)
};
let full = |f: Filter| create(f, StateLevel::Full);
let none = |f: Filter| create(f, StateLevel::None);
Ok(Self(UTXOGroups {
// Special case: all uses Version::ONE
all: utxo_cohort::Vecs::forced_import(
db,
Filter::All,
version + VERSION + Version::ONE,
indexes,
price,
states_path,
StateLevel::PriceOnly,
)?,
term: ByTerm {
short: create(Filter::Term(Term::Sth), StateLevel::PriceOnly)?,
long: create(Filter::Term(Term::Lth), StateLevel::PriceOnly)?,
},
epoch: ByEpoch {
_0: full(Filter::Epoch(HalvingEpoch::new(0)))?,
_1: full(Filter::Epoch(HalvingEpoch::new(1)))?,
_2: full(Filter::Epoch(HalvingEpoch::new(2)))?,
_3: full(Filter::Epoch(HalvingEpoch::new(3)))?,
_4: full(Filter::Epoch(HalvingEpoch::new(4)))?,
},
type_: BySpendableType {
p2pk65: full(Filter::Type(OutputType::P2PK65))?,
p2pk33: full(Filter::Type(OutputType::P2PK33))?,
p2pkh: full(Filter::Type(OutputType::P2PKH))?,
p2sh: full(Filter::Type(OutputType::P2SH))?,
p2wpkh: full(Filter::Type(OutputType::P2WPKH))?,
p2wsh: full(Filter::Type(OutputType::P2WSH))?,
p2tr: full(Filter::Type(OutputType::P2TR))?,
p2a: full(Filter::Type(OutputType::P2A))?,
p2ms: full(Filter::Type(OutputType::P2MS))?,
empty: full(Filter::Type(OutputType::Empty))?,
unknown: full(Filter::Type(OutputType::Unknown))?,
},
max_age: ByMaxAge {
_1w: none(Filter::Time(TimeFilter::LowerThan(7)))?,
_1m: none(Filter::Time(TimeFilter::LowerThan(30)))?,
_2m: none(Filter::Time(TimeFilter::LowerThan(2 * 30)))?,
_3m: none(Filter::Time(TimeFilter::LowerThan(3 * 30)))?,
_4m: none(Filter::Time(TimeFilter::LowerThan(4 * 30)))?,
_5m: none(Filter::Time(TimeFilter::LowerThan(5 * 30)))?,
_6m: none(Filter::Time(TimeFilter::LowerThan(6 * 30)))?,
_1y: none(Filter::Time(TimeFilter::LowerThan(365)))?,
_2y: none(Filter::Time(TimeFilter::LowerThan(2 * 365)))?,
_3y: none(Filter::Time(TimeFilter::LowerThan(3 * 365)))?,
_4y: none(Filter::Time(TimeFilter::LowerThan(4 * 365)))?,
_5y: none(Filter::Time(TimeFilter::LowerThan(5 * 365)))?,
_6y: none(Filter::Time(TimeFilter::LowerThan(6 * 365)))?,
_7y: none(Filter::Time(TimeFilter::LowerThan(7 * 365)))?,
_8y: none(Filter::Time(TimeFilter::LowerThan(8 * 365)))?,
_10y: none(Filter::Time(TimeFilter::LowerThan(10 * 365)))?,
_12y: none(Filter::Time(TimeFilter::LowerThan(12 * 365)))?,
_15y: none(Filter::Time(TimeFilter::LowerThan(15 * 365)))?,
},
min_age: ByMinAge {
_1d: none(Filter::Time(TimeFilter::GreaterOrEqual(1)))?,
_1w: none(Filter::Time(TimeFilter::GreaterOrEqual(7)))?,
_1m: none(Filter::Time(TimeFilter::GreaterOrEqual(30)))?,
_2m: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 30)))?,
_3m: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 30)))?,
_4m: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 30)))?,
_5m: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 30)))?,
_6m: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 30)))?,
_1y: none(Filter::Time(TimeFilter::GreaterOrEqual(365)))?,
_2y: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 365)))?,
_3y: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 365)))?,
_4y: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 365)))?,
_5y: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 365)))?,
_6y: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 365)))?,
_7y: none(Filter::Time(TimeFilter::GreaterOrEqual(7 * 365)))?,
_8y: none(Filter::Time(TimeFilter::GreaterOrEqual(8 * 365)))?,
_10y: none(Filter::Time(TimeFilter::GreaterOrEqual(10 * 365)))?,
_12y: none(Filter::Time(TimeFilter::GreaterOrEqual(12 * 365)))?,
},
age_range: ByAgeRange {
up_to_1d: full(Filter::Time(TimeFilter::Range(0..1)))?,
_1d_to_1w: full(Filter::Time(TimeFilter::Range(1..7)))?,
_1w_to_1m: full(Filter::Time(TimeFilter::Range(7..30)))?,
_1m_to_2m: full(Filter::Time(TimeFilter::Range(30..60)))?,
_2m_to_3m: full(Filter::Time(TimeFilter::Range(60..90)))?,
_3m_to_4m: full(Filter::Time(TimeFilter::Range(90..120)))?,
_4m_to_5m: full(Filter::Time(TimeFilter::Range(120..150)))?,
_5m_to_6m: full(Filter::Time(TimeFilter::Range(150..180)))?,
_6m_to_1y: full(Filter::Time(TimeFilter::Range(180..365)))?,
_1y_to_2y: full(Filter::Time(TimeFilter::Range(365..730)))?,
_2y_to_3y: full(Filter::Time(TimeFilter::Range(730..1095)))?,
_3y_to_4y: full(Filter::Time(TimeFilter::Range(1095..1460)))?,
_4y_to_5y: full(Filter::Time(TimeFilter::Range(1460..1825)))?,
_5y_to_6y: full(Filter::Time(TimeFilter::Range(1825..2190)))?,
_6y_to_7y: full(Filter::Time(TimeFilter::Range(2190..2555)))?,
_7y_to_8y: full(Filter::Time(TimeFilter::Range(2555..2920)))?,
_8y_to_10y: full(Filter::Time(TimeFilter::Range(2920..3650)))?,
_10y_to_12y: full(Filter::Time(TimeFilter::Range(3650..4380)))?,
_12y_to_15y: full(Filter::Time(TimeFilter::Range(4380..5475)))?,
from_15y: full(Filter::Time(TimeFilter::GreaterOrEqual(15 * 365)))?,
},
amount_range: ByAmountRange {
_0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?,
_1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?,
_10sats_to_100sats: full(Filter::Amount(AmountFilter::Range(
Sats::_10..Sats::_100,
)))?,
_100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_100..Sats::_1K,
)))?,
_1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_1K..Sats::_10K,
)))?,
_10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_10K..Sats::_100K,
)))?,
_100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_100K..Sats::_1M,
)))?,
_1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range(
Sats::_1M..Sats::_10M,
)))?,
_10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10M..Sats::_1BTC,
)))?,
_1btc_to_10btc: full(Filter::Amount(AmountFilter::Range(
Sats::_1BTC..Sats::_10BTC,
)))?,
_10btc_to_100btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10BTC..Sats::_100BTC,
)))?,
_100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_100BTC..Sats::_1K_BTC,
)))?,
_1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_1K_BTC..Sats::_10K_BTC,
)))?,
_10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range(
Sats::_10K_BTC..Sats::_100K_BTC,
)))?,
_100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual(
Sats::_100K_BTC,
)))?,
},
lt_amount: ByLowerThanAmount {
_10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?,
_100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?,
_1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?,
_10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?,
_100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?,
_1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?,
_10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?,
_1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?,
_10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?,
_100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?,
_1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?,
_10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?,
_100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?,
},
ge_amount: ByGreatEqualAmount {
_1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?,
_10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?,
_100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?,
_1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?,
_10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?,
_100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?,
_1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?,
_10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?,
_1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?,
_10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?,
_100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?,
_1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?,
_10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?,
},
}))
}
pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
if chain_state.is_empty() {
return;
}
let prev_timestamp = chain_state.last().unwrap().timestamp;
// Only blocks whose age % ONE_DAY >= threshold can cross a day boundary.
// Saves 1 subtraction + 2 divisions per block vs computing days_old directly.
let elapsed = (*timestamp).saturating_sub(*prev_timestamp);
let threshold = ONE_DAY_IN_SEC.saturating_sub(elapsed);
// Extract all mutable references upfront to avoid borrow checker issues
// Use a single destructuring to get non-overlapping mutable borrows
let UTXOGroups {
all,
term,
age_range,
..
} = &mut self.0;
let mut vecs = age_range
.iter_mut()
.map(|v| (v.filter().clone(), &mut v.state))
.collect::<Vec<_>>();
// Collect aggregate cohorts' filter and p2a for age transitions
let mut aggregate_p2a: Vec<(Filter, Option<&mut crate::PriceToAmount>)> = vec![
(all.filter().clone(), all.price_to_amount.as_mut()),
(
term.short.filter().clone(),
term.short.price_to_amount.as_mut(),
),
(
term.long.filter().clone(),
term.long.price_to_amount.as_mut(),
),
];
chain_state
.iter()
.filter(|block_state| {
let age = (*prev_timestamp).saturating_sub(*block_state.timestamp);
age % ONE_DAY_IN_SEC >= threshold
})
.for_each(|block_state| {
let prev_days_old =
prev_timestamp.difference_in_days_between(block_state.timestamp);
let days_old = timestamp.difference_in_days_between(block_state.timestamp);
if prev_days_old == days_old {
return;
}
vecs.iter_mut().for_each(|(filter, state)| {
let is = filter.contains_time(days_old);
let was = filter.contains_time(prev_days_old);
if is && !was {
state
.as_mut()
.unwrap()
.increment(&block_state.supply, block_state.price);
} else if was && !is {
state
.as_mut()
.unwrap()
.decrement(&block_state.supply, block_state.price);
}
});
// Handle age transitions for aggregate cohorts' price_to_amount
// Check which cohorts the UTXO was in vs is now in, and increment/decrement accordingly
// Only process if there's remaining supply (like CohortState::increment/decrement do)
if let Some(price) = block_state.price
&& block_state.supply.value > Sats::ZERO
{
aggregate_p2a.iter_mut().for_each(|(filter, p2a)| {
let is = filter.contains_time(days_old);
let was = filter.contains_time(prev_days_old);
if is && !was {
p2a.um().increment(price, &block_state.supply);
} else if was && !is {
p2a.um().decrement(price, &block_state.supply);
}
});
}
});
}
pub fn send(
&mut self,
height_to_sent: FxHashMap<Height, Transacted>,
chain_state: &mut [BlockState],
) {
// Extract all mutable references upfront to avoid borrow checker issues
let UTXOGroups {
all,
term,
age_range,
epoch,
type_,
amount_range,
..
} = &mut self.0;
let mut time_based_vecs = age_range
.iter_mut()
.chain(epoch.iter_mut())
.collect::<Vec<_>>();
// Collect aggregate cohorts' filter and p2a for iteration
let mut aggregate_p2a: Vec<(Filter, Option<&mut crate::PriceToAmount>)> = vec![
(all.filter().clone(), all.price_to_amount.as_mut()),
(
term.short.filter().clone(),
term.short.price_to_amount.as_mut(),
),
(
term.long.filter().clone(),
term.long.price_to_amount.as_mut(),
),
];
let last_block = chain_state.last().unwrap();
let last_timestamp = last_block.timestamp;
let current_price = last_block.price;
let chain_state_len = chain_state.len();
height_to_sent.into_iter().for_each(|(height, sent)| {
chain_state[height.to_usize()].supply -= &sent.spendable_supply;
let block_state = chain_state.get(height.to_usize()).unwrap();
let prev_price = block_state.price;
let blocks_old = chain_state_len - 1 - height.to_usize();
let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
let days_old_float =
last_timestamp.difference_in_days_between_float(block_state.timestamp);
let older_than_hour = last_timestamp
.checked_sub(block_state.timestamp)
.unwrap()
.is_more_than_hour();
time_based_vecs
.iter_mut()
.filter(|v| match v.filter() {
Filter::Time(TimeFilter::GreaterOrEqual(from)) => *from <= days_old,
Filter::Time(TimeFilter::LowerThan(to)) => *to > days_old,
Filter::Time(TimeFilter::Range(range)) => range.contains(&days_old),
Filter::Epoch(epoch) => *epoch == HalvingEpoch::from(height),
_ => unreachable!(),
})
.for_each(|vecs| {
vecs.state.um().send(
&sent.spendable_supply,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
});
sent.by_type
.spendable
.iter_typed()
.for_each(|(output_type, supply_state)| {
type_.get_mut(output_type).state.um().send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
)
});
sent.by_size_group
.iter_typed()
.for_each(|(group, supply_state)| {
amount_range.get_mut(group).state.um().send(
supply_state,
current_price,
prev_price,
blocks_old,
days_old_float,
older_than_hour,
);
});
// Update aggregate cohorts' price_to_amount using filter.contains_time()
if let Some(prev_price) = prev_price {
let supply_state = &sent.spendable_supply;
if supply_state.value.is_not_zero() {
aggregate_p2a
.iter_mut()
.filter(|(f, _)| f.contains_time(days_old))
.map(|(_, p2a)| p2a)
.for_each(|p2a| {
p2a.um().decrement(prev_price, supply_state);
});
}
}
});
}
pub fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
let supply_state = received.spendable_supply;
[
&mut self.0.age_range.up_to_1d,
self.0.epoch.mut_vec_from_height(height),
]
.into_iter()
.for_each(|v| {
v.state.um().receive(&supply_state, price);
});
// Update aggregate cohorts' price_to_amount
// New UTXOs have days_old = 0, so use filter.contains_time(0) to check applicability
if let Some(price) = price
&& supply_state.value.is_not_zero()
{
self.0
.iter_aggregate_mut()
.filter(|v| v.filter().contains_time(0))
.for_each(|v| {
v.price_to_amount
.as_mut()
.unwrap()
.increment(price, &supply_state);
});
}
self.type_.iter_mut().for_each(|vecs| {
let output_type = match vecs.filter() {
Filter::Type(output_type) => *output_type,
_ => unreachable!(),
};
vecs.state
.as_mut()
.unwrap()
.receive(received.by_type.get(output_type), price)
});
received
.by_size_group
.iter_typed()
.for_each(|(group, supply_state)| {
self.amount_range
.get_mut(group)
.state
.as_mut()
.unwrap()
.receive(supply_state, price);
});
}
pub fn compute_overlapping_vecs(
&mut self,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let by_date_range = &self.0.age_range;
let by_size_range = &self.0.amount_range;
[(&mut self.0.all, by_date_range.iter().collect::<Vec<_>>())]
.into_par_iter()
.chain(self.0.min_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_date_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.max_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_date_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.term.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_date_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.ge_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_size_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.chain(self.0.lt_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_size_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect::<Vec<_>>(),
)
}))
.try_for_each(|(vecs, stateful)| {
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
})
}
pub fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
#[allow(clippy::too_many_arguments)]
pub fn compute_rest_part2(
&mut self,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &Indexes,
height_to_supply: &impl IterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
starting_indexes,
height_to_supply,
dateindex_to_supply,
height_to_market_cap,
dateindex_to_market_cap,
height_to_realized_cap,
dateindex_to_realized_cap,
exit,
)
})
}
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
// Flush stateful cohorts
self.par_iter_separate_mut()
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))?;
// Flush aggregate cohorts' price_to_amount and price_percentiles
// Using traits ensures we can't forget to flush any field
self.0.par_iter_aggregate_mut().try_for_each(|v| {
v.price_to_amount.flush_at_height(height, exit)?;
v.inner.price_percentiles.safe_write(exit)?;
Ok(())
})
}
/// Reset aggregate cohorts' price_to_amount when starting from scratch
pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> {
self.0
.iter_aggregate_mut()
.try_for_each(|v| v.price_to_amount.reset())
}
/// Import aggregate cohorts' price_to_amount from disk when resuming from a checkpoint.
/// Returns the height to start processing from (checkpoint_height + 1), matching the
/// behavior of `common::import_state` for separate cohorts.
///
/// Note: We don't check inner.min_height_vecs_len() for aggregate cohorts because their
/// inner vecs (height_to_supply, etc.) are computed post-hoc by compute_overlapping_vecs,
/// not maintained during the main processing loop.
pub fn import_aggregate_price_to_amount(&mut self, height: Height) -> Result<Height> {
// Match separate vecs behavior: decrement height to get prev_height
let Some(mut prev_height) = height.decremented() else {
// height is 0, return ZERO (caller will handle this)
return Ok(Height::ZERO);
};
for v in self.0.iter_aggregate_mut() {
// Using HeightFlushable trait - if price_to_amount is None, returns height unchanged
prev_height = prev_height.min(v.price_to_amount.import_at_or_before(prev_height)?);
}
// Return prev_height + 1, matching separate vecs behavior
Ok(prev_height.incremented())
}
/// Compute and push percentiles for aggregate cohorts (all, sth, lth).
/// Must be called after receive()/send() when price_to_amount is up to date.
pub fn truncate_push_aggregate_percentiles(&mut self, height: Height) -> Result<()> {
let age_range_data: Vec<_> = self
.0
.age_range
.iter()
.map(|sub| (sub.filter().clone(), sub.state.u().supply.value))
.collect();
let results: Vec<_> = self
.0
.par_iter_aggregate()
.map(|v| {
if v.price_to_amount.is_none() {
panic!();
}
let filter = v.filter().clone();
let supply = age_range_data
.iter()
.filter(|(sub_filter, _)| filter.includes(sub_filter))
.map(|(_, value)| *value)
.fold(Sats::ZERO, |acc, v| acc + v);
let percentiles = v.compute_percentile_prices_from_standalone(supply);
(filter, percentiles)
})
.collect();
// Push results sequentially (requires &mut)
for (filter, percentiles) in results {
let v = self
.0
.iter_aggregate_mut()
.find(|v| v.filter() == &filter)
.unwrap();
if let Some(pp) = v.inner.price_percentiles.as_mut() {
pp.truncate_push(height, &percentiles)?;
}
}
Ok(())
}
}
@@ -0,0 +1,56 @@
use brk_types::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {
New(T),
FromLoadedAddressDataVec((LoadedAddressIndex, T)),
FromEmptyAddressDataVec((EmptyAddressIndex, T)),
}
impl<T> WithAddressDataSource<T> {
pub fn is_new(&self) -> bool {
matches!(self, Self::New(_))
}
pub fn is_from_emptyaddressdata(&self) -> bool {
matches!(self, Self::FromEmptyAddressDataVec(_))
}
pub fn deref_mut(&mut self) -> &mut T {
match self {
Self::New(v) => v,
Self::FromLoadedAddressDataVec((_, v)) => v,
Self::FromEmptyAddressDataVec((_, v)) => v,
}
}
}
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
#[inline]
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
Self::FromLoadedAddressDataVec((i, v.into()))
}
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
Self::FromEmptyAddressDataVec((i, v.into()))
}
}
}
}
impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
#[inline]
fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
match value {
WithAddressDataSource::New(v) => Self::New(v.into()),
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
Self::FromLoadedAddressDataVec((i, v.into()))
}
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
Self::FromEmptyAddressDataVec((i, v.into()))
}
}
}
}
@@ -0,0 +1,80 @@
//! Traits for consistent state flushing and importing.
//!
//! These traits ensure all stateful components follow the same patterns
//! for checkpoint/resume operations, preventing bugs where new fields
//! are forgotten during flush operations.
use brk_error::Result;
use brk_types::Height;
use vecdb::Exit;
/// Trait for components that can be flushed to disk.
///
/// This is for simple flush operations that don't require height tracking.
pub trait Flushable {
/// Safely flush data to disk.
fn safe_flush(&mut self, exit: &Exit) -> Result<()>;
/// Write to mmap without fsync. Data visible to readers immediately but not durable.
fn safe_write(&mut self, exit: &Exit) -> Result<()>;
}
/// Trait for stateful components that track data indexed by height.
///
/// This ensures consistent patterns for:
/// - Flushing state at checkpoints
/// - Importing state when resuming from a checkpoint
/// - Resetting state when starting from scratch
pub trait HeightFlushable {
/// Flush state to disk at the given height checkpoint.
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()>;
/// Import state from the most recent checkpoint at or before the given height.
/// Returns the actual height that was imported.
fn import_at_or_before(&mut self, height: Height) -> Result<Height>;
/// Reset state for starting from scratch.
fn reset(&mut self) -> Result<()>;
}
/// Blanket implementation for Option<T> where T: Flushable
impl<T: Flushable> Flushable for Option<T> {
fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.safe_flush(exit)?;
}
Ok(())
}
fn safe_write(&mut self, exit: &Exit) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.safe_write(exit)?;
}
Ok(())
}
}
/// Blanket implementation for Option<T> where T: HeightFlushable
impl<T: HeightFlushable> HeightFlushable for Option<T> {
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.flush_at_height(height, exit)?;
}
Ok(())
}
fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
if let Some(inner) = self.as_mut() {
inner.import_at_or_before(height)
} else {
Ok(height)
}
}
fn reset(&mut self) -> Result<()> {
if let Some(inner) = self.as_mut() {
inner.reset()?;
}
Ok(())
}
}
+2
View File
@@ -1,5 +1,6 @@
mod block;
mod cohorts;
mod flushable;
mod price_to_amount;
mod realized;
mod supply;
@@ -8,6 +9,7 @@ mod unrealized;
pub use block::*;
pub use cohorts::*;
pub use flushable::*;
pub use price_to_amount::*;
pub use realized::*;
pub use supply::*;
@@ -11,7 +11,9 @@ use pco::standalone::{simple_decompress, simpler_compress};
use serde::{Deserialize, Serialize};
use vecdb::{Bytes, Exit};
use crate::{stateful::HeightFlushable, states::SupplyState, utils::OptionExt};
use crate::{states::SupplyState, utils::OptionExt};
use super::HeightFlushable;
#[derive(Clone, Debug)]
pub struct PriceToAmount {
+1
View File
@@ -23,6 +23,7 @@ brk_traversable = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
rlimit = "0.10.2"
rustc-hash = { workspace = true }
vecdb = { workspace = true }
+8
View File
@@ -30,6 +30,14 @@ pub struct Indexer {
impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
info!("Increasing number of open files limit...");
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit(
rlimit::Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
info!("Importing indexer...");
let path = outputs_dir.join("indexed");
+1
View File
@@ -25,6 +25,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3);
pub fn open_database(path: &Path) -> fjall::Result<Database> {
Database::builder(path.join("fjall"))
.cache_size(3 * 1024 * 1024 * 1024)
.max_cached_files(Some(1024))
.open()
}
@@ -0,0 +1,12 @@
use crate::{EmptyAddressIndex, LoadedAddressIndex};
/// Source of address data update (where the data came from).
#[derive(Clone)]
pub enum AddressDataSource<T> {
/// Brand new address, not in any storage yet.
New(T),
/// From empty address storage.
FromEmpty((EmptyAddressIndex, T)),
/// From loaded address storage.
FromLoaded((LoadedAddressIndex, T)),
}
+2
View File
@@ -5,6 +5,7 @@ pub use vecdb::{CheckedSub, Exit, PrintableIndex, Version};
mod address;
mod addressbytes;
mod addresschainstats;
mod addressdata_source;
mod addresshash;
mod addressindexoutpoint;
mod addressindextxindex;
@@ -103,6 +104,7 @@ mod yearindex;
pub use address::*;
pub use addressbytes::*;
pub use addresschainstats::*;
pub use addressdata_source::*;
pub use addresshash::*;
pub use addressindexoutpoint::*;
pub use addressindextxindex::*;