mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 14:11:56 -07:00
computer: stateful: refactor part 1
This commit is contained in:
Generated
+11
-11
@@ -631,7 +631,6 @@ dependencies = [
|
||||
"log",
|
||||
"pco",
|
||||
"rayon",
|
||||
"rlimit",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"smallvec",
|
||||
@@ -695,6 +694,7 @@ dependencies = [
|
||||
"log",
|
||||
"mimalloc",
|
||||
"rayon",
|
||||
"rlimit",
|
||||
"rustc-hash",
|
||||
"vecdb",
|
||||
]
|
||||
@@ -2594,9 +2594,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "2.1.1"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99"
|
||||
checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_locale_core",
|
||||
@@ -2608,9 +2608,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "2.1.1"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899"
|
||||
checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
@@ -2924,9 +2924,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libz-rs-sys"
|
||||
version = "0.5.3"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222"
|
||||
checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c"
|
||||
dependencies = [
|
||||
"zlib-rs",
|
||||
]
|
||||
@@ -3287,9 +3287,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc-browserslist"
|
||||
version = "2.1.3"
|
||||
version = "2.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f978be538ca5e2a64326d24b7991dc658cc8495132833ae387212ab3b8abd70a"
|
||||
checksum = "9bd39c45e1d6bd2abfbd4b89cbcaba34bd315cd3cee23aad623fd075acc1ea01"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"flate2",
|
||||
@@ -5973,9 +5973,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.5.3"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7"
|
||||
checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/*"]
|
||||
default-members = ["crates/brk_cli"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
|
||||
@@ -26,7 +26,6 @@ derive_deref = { workspace = true }
|
||||
log = { workspace = true }
|
||||
pco = "0.4.7"
|
||||
rayon = { workspace = true }
|
||||
rlimit = "0.10.2"
|
||||
rustc-hash = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
smallvec = "1.15.1"
|
||||
|
||||
@@ -22,6 +22,7 @@ mod market;
|
||||
mod pools;
|
||||
mod price;
|
||||
mod stateful;
|
||||
mod stateful_new;
|
||||
mod states;
|
||||
mod traits;
|
||||
mod utils;
|
||||
@@ -56,14 +57,6 @@ impl Computer {
|
||||
indexer: &Indexer,
|
||||
fetcher: Option<Fetcher>,
|
||||
) -> Result<Self> {
|
||||
info!("Increasing number of open files...");
|
||||
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
|
||||
rlimit::setrlimit(
|
||||
rlimit::Resource::NOFILE,
|
||||
no_file_limit.0.max(10_000),
|
||||
no_file_limit.1,
|
||||
)?;
|
||||
|
||||
info!("Importing computer...");
|
||||
let import_start = Instant::now();
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ mod utxo_cohort;
|
||||
mod utxo_cohorts;
|
||||
mod withaddressdatasource;
|
||||
|
||||
pub use flushable::{Flushable, HeightFlushable};
|
||||
pub use crate::states::{Flushable, HeightFlushable};
|
||||
|
||||
use address_indexes::{AddressesDataVecs, AnyAddressIndexesVecs};
|
||||
use addresstype::*;
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Address count types per address type.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Height, StoredU64, Version};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use vecdb::{Database, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec, TypedVecIterator};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
|
||||
indexes,
|
||||
};
|
||||
|
||||
/// Address count per address type (runtime state).
|
||||
#[derive(Debug, Default, Deref, DerefMut)]
|
||||
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
|
||||
|
||||
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
|
||||
#[inline]
|
||||
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
|
||||
if let Some(prev_height) = starting_height.decremented() {
|
||||
Self(ByAddressType {
|
||||
p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(),
|
||||
p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(),
|
||||
p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(),
|
||||
p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(),
|
||||
})
|
||||
} else {
|
||||
Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Address count per address type, indexed by height.
|
||||
#[derive(Debug, Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>);
|
||||
|
||||
impl From<ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>>
|
||||
for AddressTypeToHeightToAddressCount
|
||||
{
|
||||
#[inline]
|
||||
fn from(value: ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl AddressTypeToHeightToAddressCount {
|
||||
pub fn forced_import(db: &Database, name: &str, version: Version) -> Result<Self> {
|
||||
Ok(Self::from(ByAddressType::new_with_name(|type_name| {
|
||||
Ok(EagerVec::forced_import(
|
||||
db,
|
||||
&format!("{type_name}_{name}"),
|
||||
version,
|
||||
)?)
|
||||
})?))
|
||||
}
|
||||
|
||||
pub fn truncate_push(
|
||||
&mut self,
|
||||
height: Height,
|
||||
addresstype_to_usize: &AddressTypeToAddressCount,
|
||||
) -> Result<()> {
|
||||
self.p2pk65
|
||||
.truncate_push(height, addresstype_to_usize.p2pk65.into())?;
|
||||
self.p2pk33
|
||||
.truncate_push(height, addresstype_to_usize.p2pk33.into())?;
|
||||
self.p2pkh
|
||||
.truncate_push(height, addresstype_to_usize.p2pkh.into())?;
|
||||
self.p2sh
|
||||
.truncate_push(height, addresstype_to_usize.p2sh.into())?;
|
||||
self.p2wpkh
|
||||
.truncate_push(height, addresstype_to_usize.p2wpkh.into())?;
|
||||
self.p2wsh
|
||||
.truncate_push(height, addresstype_to_usize.p2wsh.into())?;
|
||||
self.p2tr
|
||||
.truncate_push(height, addresstype_to_usize.p2tr.into())?;
|
||||
self.p2a
|
||||
.truncate_push(height, addresstype_to_usize.p2a.into())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Address count per address type, indexed by various indexes (dateindex, etc.).
|
||||
#[derive(Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
|
||||
|
||||
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
|
||||
#[inline]
|
||||
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl AddressTypeToIndexesToAddressCount {
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
name: &str,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
) -> Result<Self> {
|
||||
Ok(Self::from(ByAddressType::new_with_name(|type_name| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
db,
|
||||
&format!("{type_name}_{name}"),
|
||||
Source::None,
|
||||
version,
|
||||
indexes,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)
|
||||
})?))
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
|
||||
) -> Result<()> {
|
||||
self.p2pk65.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2pk65),
|
||||
)?;
|
||||
self.p2pk33.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2pk33),
|
||||
)?;
|
||||
self.p2pkh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2pkh),
|
||||
)?;
|
||||
self.p2sh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2sh),
|
||||
)?;
|
||||
self.p2wpkh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2wpkh),
|
||||
)?;
|
||||
self.p2wsh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2wsh),
|
||||
)?;
|
||||
self.p2tr.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2tr),
|
||||
)?;
|
||||
self.p2a.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2a),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Storage for address indexes by type.
|
||||
|
||||
use brk_error::{Error, Result};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{
|
||||
AnyAddressIndex, Height, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex,
|
||||
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
|
||||
TypeIndex, Version,
|
||||
};
|
||||
use vecdb::{
|
||||
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Reader, Stamp,
|
||||
};
|
||||
|
||||
const SAVED_STAMPED_CHANGES: u16 = 10;
|
||||
|
||||
/// Macro to define AnyAddressIndexesVecs and its methods.
|
||||
macro_rules! define_any_address_indexes_vecs {
|
||||
($(($field:ident, $variant:ident, $index:ty)),* $(,)?) => {
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct AnyAddressIndexesVecs {
|
||||
$(pub $field: BytesVec<$index, AnyAddressIndex>,)*
|
||||
}
|
||||
|
||||
impl AnyAddressIndexesVecs {
|
||||
/// Import from database.
|
||||
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
|
||||
Ok(Self {
|
||||
$($field: BytesVec::forced_import_with(
|
||||
ImportOptions::new(db, "anyaddressindex", version)
|
||||
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
|
||||
)?,)*
|
||||
})
|
||||
}
|
||||
|
||||
/// Get minimum stamped height across all address types.
|
||||
pub fn min_stamped_height(&self) -> Height {
|
||||
[$(Height::from(self.$field.stamp()).incremented()),*]
|
||||
.into_iter()
|
||||
.min()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Rollback all address types to before the given stamp.
|
||||
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<Vec<Stamp>> {
|
||||
Ok(vec![$(self.$field.rollback_before(stamp)?),*])
|
||||
}
|
||||
|
||||
/// Reset all address types.
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
$(self.$field.reset()?;)*
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get address index for a given type and typeindex.
|
||||
pub fn get(&self, address_type: OutputType, typeindex: TypeIndex, reader: &Reader) -> AnyAddressIndex {
|
||||
match address_type {
|
||||
$(OutputType::$variant => self.$field.get_pushed_or_read_at_unwrap(typeindex.into(), reader),)*
|
||||
_ => unreachable!("Invalid address type: {:?}", address_type),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get address index with single read (no caching).
|
||||
pub fn get_once(&self, address_type: OutputType, typeindex: TypeIndex) -> Result<AnyAddressIndex> {
|
||||
match address_type {
|
||||
$(OutputType::$variant => self.$field.read_at_once(typeindex.into()).map_err(Into::into),)*
|
||||
_ => Err(Error::UnsupportedType(address_type.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Update or push address index for a given type.
|
||||
pub fn update_or_push(&mut self, address_type: OutputType, typeindex: TypeIndex, index: AnyAddressIndex) -> Result<()> {
|
||||
match address_type {
|
||||
$(OutputType::$variant => self.$field.update_or_push(typeindex.into(), index)?,)*
|
||||
_ => unreachable!("Invalid address type: {:?}", address_type),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush all address types with stamp.
|
||||
pub fn flush(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> {
|
||||
$(self.$field.stamped_flush_maybe_with_changes(stamp, with_changes)?;)*
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Generate the struct and methods
|
||||
define_any_address_indexes_vecs!(
|
||||
(p2a, P2A, P2AAddressIndex),
|
||||
(p2pk33, P2PK33, P2PK33AddressIndex),
|
||||
(p2pk65, P2PK65, P2PK65AddressIndex),
|
||||
(p2pkh, P2PKH, P2PKHAddressIndex),
|
||||
(p2sh, P2SH, P2SHAddressIndex),
|
||||
(p2tr, P2TR, P2TRAddressIndex),
|
||||
(p2wpkh, P2WPKH, P2WPKHAddressIndex),
|
||||
(p2wsh, P2WSH, P2WSHAddressIndex),
|
||||
);
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Storage for address data (loaded and empty addresses).
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{
|
||||
EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, Version,
|
||||
};
|
||||
use vecdb::{
|
||||
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Stamp,
|
||||
};
|
||||
|
||||
const SAVED_STAMPED_CHANGES: u16 = 10;
|
||||
|
||||
/// Storage for both loaded and empty address data.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct AddressesDataVecs {
|
||||
pub loaded: BytesVec<LoadedAddressIndex, LoadedAddressData>,
|
||||
pub empty: BytesVec<EmptyAddressIndex, EmptyAddressData>,
|
||||
}
|
||||
|
||||
impl AddressesDataVecs {
|
||||
/// Import from database.
|
||||
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
|
||||
Ok(Self {
|
||||
loaded: BytesVec::forced_import_with(
|
||||
ImportOptions::new(db, "loadedaddressdata", version)
|
||||
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
|
||||
)?,
|
||||
empty: BytesVec::forced_import_with(
|
||||
ImportOptions::new(db, "emptyaddressdata", version)
|
||||
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get minimum stamped height across loaded and empty data.
|
||||
pub fn min_stamped_height(&self) -> Height {
|
||||
Height::from(self.loaded.stamp())
|
||||
.incremented()
|
||||
.min(Height::from(self.empty.stamp()).incremented())
|
||||
}
|
||||
|
||||
/// Rollback both loaded and empty data to before the given stamp.
|
||||
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> {
|
||||
Ok([
|
||||
self.loaded.rollback_before(stamp)?,
|
||||
self.empty.rollback_before(stamp)?,
|
||||
])
|
||||
}
|
||||
|
||||
/// Reset both loaded and empty data.
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
self.loaded.reset()?;
|
||||
self.empty.reset()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush both loaded and empty data with stamp.
|
||||
pub fn flush(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> {
|
||||
self.loaded
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.empty
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Height to AddressTypeToVec hashmap.
|
||||
|
||||
use brk_types::Height;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use super::type_vec::AddressTypeToVec;
|
||||
|
||||
/// Hashmap from Height to AddressTypeToVec.
|
||||
#[derive(Debug, Default, Deref, DerefMut)]
|
||||
pub struct HeightToAddressTypeToVec<T>(FxHashMap<Height, AddressTypeToVec<T>>);
|
||||
|
||||
impl<T> HeightToAddressTypeToVec<T> {
|
||||
/// Merge another map into this one.
|
||||
pub fn merge_mut(&mut self, other: Self) {
|
||||
for (height, vec) in other.0 {
|
||||
self.entry(height).or_default().merge_mut(vec);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
//! Address handling with macro-generated code for 8 address types.
|
||||
//!
|
||||
//! This module provides:
|
||||
//! - `AnyAddressIndexesVecs` for storing address indexes by type
|
||||
//! - `AddressesDataVecs` for storing address data (loaded/empty)
|
||||
//! - `AddressTypeToTypeIndexMap` for per-type hashmaps
|
||||
//! - `AddressTypeToVec` for per-type vectors
|
||||
//! - `HeightToAddressTypeToVec` for height-keyed per-type vectors
|
||||
//! - `AddressTypeToAddressCount` for runtime address counts
|
||||
//! - `AddressTypeToHeightToAddressCount` for height-indexed address counts
|
||||
//! - `AddressTypeToIndexesToAddressCount` for computed address counts
|
||||
|
||||
mod address_count;
|
||||
mod any_address_indexes;
|
||||
mod data;
|
||||
mod height_type_vec;
|
||||
mod type_index_map;
|
||||
mod type_vec;
|
||||
|
||||
pub use address_count::{
|
||||
AddressTypeToAddressCount, AddressTypeToHeightToAddressCount,
|
||||
AddressTypeToIndexesToAddressCount,
|
||||
};
|
||||
pub use any_address_indexes::AnyAddressIndexesVecs;
|
||||
pub use data::AddressesDataVecs;
|
||||
pub use height_type_vec::HeightToAddressTypeToVec;
|
||||
pub use type_index_map::AddressTypeToTypeIndexMap;
|
||||
pub use type_vec::AddressTypeToVec;
|
||||
@@ -0,0 +1,106 @@
|
||||
//! Per-address-type hashmap keyed by TypeIndex.
|
||||
|
||||
use std::mem;
|
||||
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_types::{OutputType, TypeIndex};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rustc_hash::FxHashMap;
|
||||
use smallvec::{Array, SmallVec};
|
||||
use std::collections::hash_map::Entry;
|
||||
|
||||
/// A hashmap for each address type, keyed by TypeIndex.
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct AddressTypeToTypeIndexMap<T>(ByAddressType<FxHashMap<TypeIndex, T>>);
|
||||
|
||||
impl<T> Default for AddressTypeToTypeIndexMap<T> {
|
||||
fn default() -> Self {
|
||||
Self(ByAddressType {
|
||||
p2a: FxHashMap::default(),
|
||||
p2pk33: FxHashMap::default(),
|
||||
p2pk65: FxHashMap::default(),
|
||||
p2pkh: FxHashMap::default(),
|
||||
p2sh: FxHashMap::default(),
|
||||
p2tr: FxHashMap::default(),
|
||||
p2wpkh: FxHashMap::default(),
|
||||
p2wsh: FxHashMap::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> AddressTypeToTypeIndexMap<T> {
|
||||
/// Merge two maps, consuming other and extending self.
|
||||
pub fn merge(mut self, mut other: Self) -> Self {
|
||||
Self::merge_single(&mut self.p2a, &mut other.p2a);
|
||||
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
|
||||
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
|
||||
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
|
||||
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
|
||||
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
|
||||
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
|
||||
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
|
||||
self
|
||||
}
|
||||
|
||||
fn merge_single(own: &mut FxHashMap<TypeIndex, T>, other: &mut FxHashMap<TypeIndex, T>) {
|
||||
if own.len() < other.len() {
|
||||
mem::swap(own, other);
|
||||
}
|
||||
own.extend(other.drain());
|
||||
}
|
||||
|
||||
/// Insert a value for a specific address type and typeindex.
|
||||
pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) {
|
||||
self.get_mut(address_type).unwrap().insert(typeindex, value);
|
||||
}
|
||||
|
||||
/// Remove and return a value for a specific address type and typeindex.
|
||||
pub fn remove_for_type(&mut self, address_type: OutputType, typeindex: &TypeIndex) -> T {
|
||||
self.get_mut(address_type)
|
||||
.unwrap()
|
||||
.remove(typeindex)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Iterate over sorted entries by address type.
|
||||
pub fn into_sorted_iter(self) -> impl Iterator<Item = (OutputType, Vec<(TypeIndex, T)>)> {
|
||||
self.0.into_iter().map(|(output_type, map)| {
|
||||
let mut sorted: Vec<_> = map.into_iter().collect();
|
||||
sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex);
|
||||
(output_type, sorted)
|
||||
})
|
||||
}
|
||||
|
||||
/// Consume and iterate over entries by address type.
|
||||
#[allow(clippy::should_implement_trait)]
|
||||
pub fn into_iter(self) -> impl Iterator<Item = (OutputType, FxHashMap<TypeIndex, T>)> {
|
||||
self.0.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> AddressTypeToTypeIndexMap<SmallVec<T>>
|
||||
where
|
||||
T: Array,
|
||||
{
|
||||
/// Merge two maps of SmallVec values, concatenating vectors.
|
||||
pub fn merge_vec(mut self, other: Self) -> Self {
|
||||
for (address_type, other_map) in other.0.into_iter() {
|
||||
let self_map = self.0.get_mut_unwrap(address_type);
|
||||
for (typeindex, mut other_vec) in other_map {
|
||||
match self_map.entry(typeindex) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
let self_vec = entry.get_mut();
|
||||
if other_vec.len() > self_vec.len() {
|
||||
mem::swap(self_vec, &mut other_vec);
|
||||
}
|
||||
self_vec.extend(other_vec);
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(other_vec);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Per-address-type vector.
|
||||
|
||||
use std::mem;
|
||||
|
||||
use brk_grouper::ByAddressType;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
/// A vector for each address type.
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
|
||||
|
||||
impl<T> Default for AddressTypeToVec<T> {
|
||||
fn default() -> Self {
|
||||
Self(ByAddressType {
|
||||
p2a: vec![],
|
||||
p2pk33: vec![],
|
||||
p2pk65: vec![],
|
||||
p2pkh: vec![],
|
||||
p2sh: vec![],
|
||||
p2tr: vec![],
|
||||
p2wpkh: vec![],
|
||||
p2wsh: vec![],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> AddressTypeToVec<T> {
|
||||
/// Merge two AddressTypeToVec, consuming other.
|
||||
pub fn merge(mut self, mut other: Self) -> Self {
|
||||
Self::merge_single(&mut self.p2a, &mut other.p2a);
|
||||
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
|
||||
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
|
||||
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
|
||||
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
|
||||
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
|
||||
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
|
||||
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
|
||||
self
|
||||
}
|
||||
|
||||
/// Merge in place.
|
||||
pub fn merge_mut(&mut self, mut other: Self) {
|
||||
Self::merge_single(&mut self.p2a, &mut other.p2a);
|
||||
Self::merge_single(&mut self.p2pk33, &mut other.p2pk33);
|
||||
Self::merge_single(&mut self.p2pk65, &mut other.p2pk65);
|
||||
Self::merge_single(&mut self.p2pkh, &mut other.p2pkh);
|
||||
Self::merge_single(&mut self.p2sh, &mut other.p2sh);
|
||||
Self::merge_single(&mut self.p2tr, &mut other.p2tr);
|
||||
Self::merge_single(&mut self.p2wpkh, &mut other.p2wpkh);
|
||||
Self::merge_single(&mut self.p2wsh, &mut other.p2wsh);
|
||||
}
|
||||
|
||||
fn merge_single(own: &mut Vec<T>, other: &mut Vec<T>) {
|
||||
if own.len() >= other.len() {
|
||||
own.append(other);
|
||||
} else {
|
||||
other.append(own);
|
||||
mem::swap(own, other);
|
||||
}
|
||||
}
|
||||
|
||||
/// Unwrap the inner ByAddressType.
|
||||
pub fn unwrap(self) -> ByAddressType<Vec<T>> {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
//! Address cohort vectors with metrics and state.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{CohortContext, Filter, Filtered};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
|
||||
use vecdb::{
|
||||
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec,
|
||||
PcoVec,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
|
||||
indexes, price,
|
||||
states::AddressCohortState,
|
||||
};
|
||||
|
||||
use super::super::metrics::{CohortMetrics, ImportConfig};
|
||||
use super::traits::{CohortVecs, DynCohortVecs};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
/// Address cohort with metrics and optional runtime state.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct AddressCohortVecs {
|
||||
/// Starting height when state was imported
|
||||
starting_height: Option<Height>,
|
||||
|
||||
/// Runtime state for block-by-block processing
|
||||
#[traversable(skip)]
|
||||
pub state: Option<AddressCohortState>,
|
||||
|
||||
/// Metric vectors
|
||||
#[traversable(flatten)]
|
||||
pub metrics: CohortMetrics,
|
||||
|
||||
/// Address count at each height
|
||||
pub height_to_addr_count: EagerVec<PcoVec<Height, StoredU64>>,
|
||||
|
||||
/// Address count indexed by various dimensions
|
||||
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
|
||||
}
|
||||
|
||||
impl AddressCohortVecs {
|
||||
/// Import address cohort from database.
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
filter: Filter,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: Option<&Path>,
|
||||
) -> Result<Self> {
|
||||
let compute_dollars = price.is_some();
|
||||
let full_name = filter.to_full_name(CohortContext::Address);
|
||||
|
||||
let cfg = ImportConfig {
|
||||
db,
|
||||
filter,
|
||||
context: CohortContext::Address,
|
||||
version,
|
||||
indexes,
|
||||
price,
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
starting_height: None,
|
||||
|
||||
state: states_path
|
||||
.map(|path| AddressCohortState::new(path, &full_name, compute_dollars)),
|
||||
|
||||
metrics: CohortMetrics::forced_import(&cfg)?,
|
||||
|
||||
height_to_addr_count: EagerVec::forced_import(
|
||||
db,
|
||||
&cfg.name("addr_count"),
|
||||
version + VERSION + Version::ZERO,
|
||||
)?,
|
||||
|
||||
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
|
||||
db,
|
||||
&cfg.name("addr_count"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexes,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the starting height when state was imported.
|
||||
pub fn starting_height(&self) -> Option<Height> {
|
||||
self.starting_height
|
||||
}
|
||||
|
||||
/// Set the starting height.
|
||||
pub fn set_starting_height(&mut self, height: Height) {
|
||||
self.starting_height = Some(height);
|
||||
}
|
||||
|
||||
/// Reset starting height to zero.
|
||||
pub fn reset_starting_height(&mut self) {
|
||||
self.starting_height = Some(Height::ZERO);
|
||||
}
|
||||
|
||||
/// Get minimum length across height-indexed vectors.
|
||||
pub fn min_len(&self) -> usize {
|
||||
self.height_to_addr_count
|
||||
.len()
|
||||
.min(self.metrics.supply.min_len())
|
||||
.min(self.metrics.activity.min_len())
|
||||
}
|
||||
}
|
||||
|
||||
impl Filtered for AddressCohortVecs {
|
||||
fn filter(&self) -> &Filter {
|
||||
&self.metrics.filter
|
||||
}
|
||||
}
|
||||
|
||||
impl DynCohortVecs for AddressCohortVecs {
|
||||
fn min_height_vecs_len(&self) -> usize {
|
||||
self.min_len()
|
||||
}
|
||||
|
||||
fn reset_state_starting_height(&mut self) {
|
||||
self.reset_starting_height();
|
||||
}
|
||||
|
||||
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
|
||||
// Import state from runtime state if present
|
||||
if let Some(state) = self.state.as_mut() {
|
||||
let imported = state.inner.import_at_or_before(starting_height)?;
|
||||
self.starting_height = Some(imported);
|
||||
|
||||
// Restore addr_count from last known value
|
||||
if let Some(prev_height) = imported.decremented() {
|
||||
use vecdb::TypedVecIterator;
|
||||
state.addr_count = *self
|
||||
.height_to_addr_count
|
||||
.into_iter()
|
||||
.get_unwrap(prev_height);
|
||||
}
|
||||
|
||||
Ok(imported)
|
||||
} else {
|
||||
self.starting_height = Some(starting_height);
|
||||
Ok(starting_height)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
use vecdb::GenericStoredVec;
|
||||
self.height_to_addr_count
|
||||
.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_addr_count.inner_version(),
|
||||
)?;
|
||||
self.metrics.validate_computed_versions(base_version)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn truncate_push(&mut self, height: Height) -> Result<()> {
|
||||
if self.starting_height.map_or(false, |h| h > height) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Push addr_count from state
|
||||
if let Some(state) = self.state.as_ref() {
|
||||
self.height_to_addr_count
|
||||
.truncate_push(height, state.addr_count.into())?;
|
||||
self.metrics.truncate_push(height, &state.inner)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
) -> Result<()> {
|
||||
if let Some(state) = self.state.as_ref() {
|
||||
self.metrics.compute_then_truncate_push_unrealized_states(
|
||||
height,
|
||||
height_price,
|
||||
dateindex,
|
||||
date_price,
|
||||
&state.inner,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.height_to_addr_count.safe_write(exit)?;
|
||||
self.metrics.safe_flush(exit)?;
|
||||
|
||||
if let Some(state) = self.state.as_mut() {
|
||||
state.inner.commit(height)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.indexes_to_addr_count.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_addr_count),
|
||||
)?;
|
||||
self.metrics
|
||||
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl CohortVecs for AddressCohortVecs {
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_addr_count.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_addr_count)
|
||||
.collect::<Vec<_>>()
|
||||
.as_slice(),
|
||||
exit,
|
||||
)?;
|
||||
self.metrics.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.metrics.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
//! Container for all Address cohorts organized by filter type.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{
|
||||
AddressGroups, AmountFilter, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, Filter,
|
||||
Filtered,
|
||||
};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{Database, Exit, IterableVec};
|
||||
|
||||
use crate::{Indexes, indexes, price, stateful_new::DynCohortVecs};
|
||||
|
||||
use super::{AddressCohortVecs, CohortVecs};
|
||||
|
||||
const VERSION: Version = Version::new(0);
|
||||
|
||||
/// All Address cohorts organized by filter type.
|
||||
#[derive(Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct AddressCohorts(AddressGroups<AddressCohortVecs>);
|
||||
|
||||
impl AddressCohorts {
|
||||
/// Import all Address cohorts from database.
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: &Path,
|
||||
) -> Result<Self> {
|
||||
let v = version + VERSION + Version::ZERO;
|
||||
|
||||
// Helper to create a cohort - only amount_range cohorts have state
|
||||
let create = |filter: Filter, has_state: bool| -> Result<AddressCohortVecs> {
|
||||
let states_path = if has_state { Some(states_path) } else { None };
|
||||
AddressCohortVecs::forced_import(db, filter, v, indexes, price, states_path)
|
||||
};
|
||||
|
||||
let full = |f: Filter| create(f, true);
|
||||
let none = |f: Filter| create(f, false);
|
||||
|
||||
Ok(Self(AddressGroups {
|
||||
amount_range: ByAmountRange {
|
||||
_0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?,
|
||||
_1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?,
|
||||
_10sats_to_100sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10..Sats::_100,
|
||||
)))?,
|
||||
_100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100..Sats::_1K,
|
||||
)))?,
|
||||
_1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1K..Sats::_10K,
|
||||
)))?,
|
||||
_10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10K..Sats::_100K,
|
||||
)))?,
|
||||
_100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100K..Sats::_1M,
|
||||
)))?,
|
||||
_1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1M..Sats::_10M,
|
||||
)))?,
|
||||
_10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10M..Sats::_1BTC,
|
||||
)))?,
|
||||
_1btc_to_10btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1BTC..Sats::_10BTC,
|
||||
)))?,
|
||||
_10btc_to_100btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10BTC..Sats::_100BTC,
|
||||
)))?,
|
||||
_100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100BTC..Sats::_1K_BTC,
|
||||
)))?,
|
||||
_1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1K_BTC..Sats::_10K_BTC,
|
||||
)))?,
|
||||
_10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10K_BTC..Sats::_100K_BTC,
|
||||
)))?,
|
||||
_100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual(
|
||||
Sats::_100K_BTC,
|
||||
)))?,
|
||||
},
|
||||
|
||||
lt_amount: ByLowerThanAmount {
|
||||
_10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?,
|
||||
_100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?,
|
||||
_1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?,
|
||||
_10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?,
|
||||
_100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?,
|
||||
_1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?,
|
||||
_10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?,
|
||||
_1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?,
|
||||
_10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?,
|
||||
_100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?,
|
||||
_1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?,
|
||||
_10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?,
|
||||
_100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?,
|
||||
},
|
||||
|
||||
ge_amount: ByGreatEqualAmount {
|
||||
_1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?,
|
||||
_10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?,
|
||||
_100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?,
|
||||
_1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?,
|
||||
_10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?,
|
||||
_100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?,
|
||||
_1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?,
|
||||
_10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?,
|
||||
_1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?,
|
||||
_10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?,
|
||||
_100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?,
|
||||
_1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?,
|
||||
_10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
/// Compute overlapping cohorts from component amount_range cohorts.
|
||||
///
|
||||
/// For example, ">=1 BTC" cohort is computed from sum of amount_range cohorts that match.
|
||||
pub fn compute_overlapping_vecs(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let by_amount_range = &self.0.amount_range;
|
||||
|
||||
// ge_amount cohorts computed from matching amount_range cohorts
|
||||
[
|
||||
self.0
|
||||
.ge_amount
|
||||
.par_iter_mut()
|
||||
.map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_amount_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
// lt_amount cohorts computed from matching amount_range cohorts
|
||||
self.0
|
||||
.lt_amount
|
||||
.par_iter_mut()
|
||||
.map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_amount_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.try_for_each(|(vecs, components)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &components, exit)
|
||||
})
|
||||
}
|
||||
|
||||
/// First phase of post-processing: compute index transforms.
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.par_iter_mut()
|
||||
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
|
||||
}
|
||||
|
||||
/// Second phase of post-processing: compute relative metrics.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2<S, D, HM, DM, HR, DR>(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &S,
|
||||
dateindex_to_supply: &D,
|
||||
height_to_market_cap: Option<&HM>,
|
||||
dateindex_to_market_cap: Option<&DM>,
|
||||
height_to_realized_cap: Option<&HR>,
|
||||
dateindex_to_realized_cap: Option<&DR>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: IterableVec<Height, Bitcoin> + Sync,
|
||||
D: IterableVec<DateIndex, Bitcoin> + Sync,
|
||||
HM: IterableVec<Height, Dollars> + Sync,
|
||||
DM: IterableVec<DateIndex, Dollars> + Sync,
|
||||
HR: IterableVec<Height, Dollars> + Sync,
|
||||
DR: IterableVec<DateIndex, Dollars> + Sync,
|
||||
{
|
||||
self.0.par_iter_mut().try_for_each(|v| {
|
||||
v.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Flush stateful vectors for separate cohorts.
|
||||
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.par_iter_separate_mut()
|
||||
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//! Cohort management for UTXO and address groupings.
|
||||
//!
|
||||
//! Cohorts are groups of UTXOs or addresses filtered by criteria like:
|
||||
//! - Age (0-1d, 1-7d, etc.)
|
||||
//! - Amount (< 1 BTC, 1-10 BTC, etc.)
|
||||
//! - Type (P2PKH, P2SH, etc.)
|
||||
//! - Term (short-term holder, long-term holder)
|
||||
|
||||
mod address;
|
||||
mod address_cohorts;
|
||||
mod state;
|
||||
mod traits;
|
||||
mod utxo;
|
||||
mod utxo_cohorts;
|
||||
|
||||
pub use address::AddressCohortVecs;
|
||||
pub use address_cohorts::AddressCohorts;
|
||||
pub use crate::states::{Flushable, HeightFlushable};
|
||||
pub use state::CohortState;
|
||||
pub use traits::{CohortVecs, DynCohortVecs};
|
||||
pub use utxo::UTXOCohortVecs;
|
||||
pub use utxo_cohorts::UTXOCohorts;
|
||||
@@ -0,0 +1,243 @@
|
||||
//! Cohort state tracking during computation.
|
||||
//!
|
||||
//! This state is maintained in memory during block processing and periodically flushed.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{CheckedSub, Dollars, Height, Sats};
|
||||
|
||||
use crate::{
|
||||
PriceToAmount, RealizedState, SupplyState, UnrealizedState,
|
||||
grouped::{PERCENTILES, PERCENTILES_LEN},
|
||||
utils::OptionExt,
|
||||
};
|
||||
|
||||
/// State tracked for each cohort during computation.
|
||||
#[derive(Clone)]
|
||||
pub struct CohortState {
|
||||
/// Current supply in this cohort
|
||||
pub supply: SupplyState,
|
||||
|
||||
/// Realized cap and profit/loss (requires price data)
|
||||
pub realized: Option<RealizedState>,
|
||||
|
||||
/// Amount sent in current block
|
||||
pub sent: Sats,
|
||||
|
||||
/// Satoshi-blocks destroyed (supply * blocks_old when spent)
|
||||
pub satblocks_destroyed: Sats,
|
||||
|
||||
/// Satoshi-days destroyed (supply * days_old when spent)
|
||||
pub satdays_destroyed: Sats,
|
||||
|
||||
/// Price distribution for percentile calculations (requires price data)
|
||||
price_to_amount: Option<PriceToAmount>,
|
||||
}
|
||||
|
||||
impl CohortState {
|
||||
/// Create new cohort state.
|
||||
pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self {
|
||||
Self {
|
||||
supply: SupplyState::default(),
|
||||
realized: compute_dollars.then_some(RealizedState::NAN),
|
||||
sent: Sats::ZERO,
|
||||
satblocks_destroyed: Sats::ZERO,
|
||||
satdays_destroyed: Sats::ZERO,
|
||||
price_to_amount: compute_dollars.then_some(PriceToAmount::create(path, name)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Import state from checkpoint.
|
||||
pub fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
|
||||
match self.price_to_amount.as_mut() {
|
||||
Some(p) => p.import_at_or_before(height),
|
||||
None => Ok(height),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset price_to_amount if needed (for starting fresh).
|
||||
pub fn reset_price_to_amount(&mut self) -> Result<()> {
|
||||
if let Some(p) = self.price_to_amount.as_mut() {
|
||||
p.clean()?;
|
||||
p.init();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset per-block values before processing next block.
|
||||
pub fn reset_block_values(&mut self) {
|
||||
self.sent = Sats::ZERO;
|
||||
self.satdays_destroyed = Sats::ZERO;
|
||||
self.satblocks_destroyed = Sats::ZERO;
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.reset_single_iteration_values();
|
||||
}
|
||||
}
|
||||
|
||||
/// Add supply to this cohort (e.g., when UTXO ages into cohort).
|
||||
pub fn increment(&mut self, supply: &SupplyState, price: Option<Dollars>) {
|
||||
self.supply += supply;
|
||||
|
||||
if supply.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let price = price.unwrap();
|
||||
realized.increment(supply, price);
|
||||
self.price_to_amount.as_mut().unwrap().increment(price, supply);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove supply from this cohort (e.g., when UTXO ages out of cohort).
|
||||
pub fn decrement(&mut self, supply: &SupplyState, price: Option<Dollars>) {
|
||||
self.supply -= supply;
|
||||
|
||||
if supply.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let price = price.unwrap();
|
||||
realized.decrement(supply, price);
|
||||
self.price_to_amount.as_mut().unwrap().decrement(price, supply);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process received output (new UTXO in cohort).
|
||||
pub fn receive(&mut self, supply: &SupplyState, price: Option<Dollars>) {
|
||||
self.supply += supply;
|
||||
|
||||
if supply.value > Sats::ZERO {
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let price = price.unwrap();
|
||||
realized.receive(supply, price);
|
||||
self.price_to_amount.as_mut().unwrap().increment(price, supply);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process spent input (UTXO leaving cohort).
|
||||
pub fn send(
|
||||
&mut self,
|
||||
supply: &SupplyState,
|
||||
current_price: Option<Dollars>,
|
||||
prev_price: Option<Dollars>,
|
||||
blocks_old: usize,
|
||||
days_old: f64,
|
||||
older_than_hour: bool,
|
||||
) {
|
||||
if supply.utxo_count == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
self.supply -= supply;
|
||||
|
||||
if supply.value > Sats::ZERO {
|
||||
self.sent += supply.value;
|
||||
self.satblocks_destroyed += supply.value * blocks_old;
|
||||
self.satdays_destroyed +=
|
||||
Sats::from((u64::from(supply.value) as f64 * days_old).floor() as u64);
|
||||
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
let current_price = current_price.unwrap();
|
||||
let prev_price = prev_price.unwrap();
|
||||
realized.send(supply, current_price, prev_price, older_than_hour);
|
||||
self.price_to_amount.as_mut().unwrap().decrement(prev_price, supply);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute prices at percentile thresholds.
|
||||
pub fn compute_percentile_prices(&self) -> [Dollars; PERCENTILES_LEN] {
|
||||
let mut result = [Dollars::NAN; PERCENTILES_LEN];
|
||||
|
||||
let price_to_amount = match self.price_to_amount.as_ref() {
|
||||
Some(p) => p,
|
||||
None => return result,
|
||||
};
|
||||
|
||||
if price_to_amount.is_empty() || self.supply.value == Sats::ZERO {
|
||||
return result;
|
||||
}
|
||||
|
||||
let total = u64::from(self.supply.value);
|
||||
let targets = PERCENTILES.map(|p| total * u64::from(p) / 100);
|
||||
|
||||
let mut accumulated = 0u64;
|
||||
let mut pct_idx = 0;
|
||||
|
||||
for (&price, &sats) in price_to_amount.iter() {
|
||||
accumulated += u64::from(sats);
|
||||
|
||||
while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] {
|
||||
result[pct_idx] = price;
|
||||
pct_idx += 1;
|
||||
}
|
||||
|
||||
if pct_idx >= PERCENTILES_LEN {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Compute unrealized profit/loss at current price.
|
||||
pub fn compute_unrealized(
|
||||
&self,
|
||||
height_price: Dollars,
|
||||
date_price: Option<Dollars>,
|
||||
) -> (UnrealizedState, Option<UnrealizedState>) {
|
||||
let price_to_amount = match self.price_to_amount.as_ref() {
|
||||
Some(p) if !p.is_empty() => p,
|
||||
_ => return (UnrealizedState::NAN, date_price.map(|_| UnrealizedState::NAN)),
|
||||
};
|
||||
|
||||
let mut height_state = UnrealizedState::ZERO;
|
||||
let mut date_state = date_price.map(|_| UnrealizedState::ZERO);
|
||||
|
||||
for (&price, &sats) in price_to_amount.iter() {
|
||||
Self::update_unrealized(price, height_price, sats, &mut height_state);
|
||||
|
||||
if let Some(date_price) = date_price {
|
||||
Self::update_unrealized(price, date_price, sats, date_state.um());
|
||||
}
|
||||
}
|
||||
|
||||
(height_state, date_state)
|
||||
}
|
||||
|
||||
fn update_unrealized(price: Dollars, current: Dollars, sats: Sats, state: &mut UnrealizedState) {
|
||||
match price.cmp(¤t) {
|
||||
Ordering::Less | Ordering::Equal => {
|
||||
state.supply_in_profit += sats;
|
||||
if price < current && price > Dollars::ZERO && current > Dollars::ZERO {
|
||||
state.unrealized_profit += current.checked_sub(price).unwrap() * sats;
|
||||
}
|
||||
}
|
||||
Ordering::Greater => {
|
||||
state.supply_in_loss += sats;
|
||||
if price > Dollars::ZERO && current > Dollars::ZERO {
|
||||
state.unrealized_loss += price.checked_sub(current).unwrap() * sats;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Flush state to disk at checkpoint.
|
||||
pub fn commit(&mut self, height: Height) -> Result<()> {
|
||||
if let Some(p) = self.price_to_amount.as_mut() {
|
||||
p.flush(height)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get first (lowest) price in distribution.
|
||||
pub fn min_price(&self) -> Option<&Dollars> {
|
||||
self.price_to_amount.as_ref()?.first_key_value().map(|(k, _)| k)
|
||||
}
|
||||
|
||||
/// Get last (highest) price in distribution.
|
||||
pub fn max_price(&self) -> Option<&Dollars> {
|
||||
self.price_to_amount.as_ref()?.last_key_value().map(|(k, _)| k)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
//! Traits for cohort vector operations.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
|
||||
use vecdb::{Exit, IterableVec};
|
||||
|
||||
use crate::{indexes, price, Indexes};
|
||||
|
||||
/// Dynamic dispatch trait for cohort vectors.
|
||||
///
|
||||
/// This trait enables heterogeneous cohort processing via trait objects.
|
||||
pub trait DynCohortVecs: Send + Sync {
|
||||
/// Get minimum length across height-indexed vectors.
|
||||
fn min_height_vecs_len(&self) -> usize;
|
||||
|
||||
/// Reset the starting height for state tracking.
|
||||
fn reset_state_starting_height(&mut self);
|
||||
|
||||
/// Import state from checkpoint at or before the given height.
|
||||
fn import_state(&mut self, starting_height: Height) -> Result<Height>;
|
||||
|
||||
/// Validate that computed vectors have correct versions.
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
|
||||
|
||||
/// Push state to height-indexed vectors (truncating if needed).
|
||||
fn truncate_push(&mut self, height: Height) -> Result<()>;
|
||||
|
||||
/// Compute and push unrealized profit/loss states.
|
||||
fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Flush stateful vectors to disk.
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>;
|
||||
|
||||
/// First phase of post-processing computations.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Static dispatch trait for cohort vectors with additional methods.
|
||||
pub trait CohortVecs: DynCohortVecs {
|
||||
/// Compute aggregate cohort from component cohorts.
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Second phase of post-processing computations.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
//! UTXO cohort vectors with metrics and state.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{CohortContext, Filter, Filtered, StateLevel};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
|
||||
use vecdb::{Database, Exit, IterableVec};
|
||||
|
||||
use crate::{
|
||||
Indexes, PriceToAmount, UTXOCohortState, indexes, price,
|
||||
stateful_new::{CohortVecs, DynCohortVecs},
|
||||
};
|
||||
|
||||
use super::super::metrics::{CohortMetrics, ImportConfig};
|
||||
|
||||
/// UTXO cohort with metrics and optional runtime state.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct UTXOCohortVecs {
|
||||
/// Starting height when state was imported
|
||||
state_starting_height: Option<Height>,
|
||||
|
||||
/// Runtime state for block-by-block processing
|
||||
#[traversable(skip)]
|
||||
pub state: Option<UTXOCohortState>,
|
||||
|
||||
/// For aggregate cohorts that only need price_to_amount for percentiles
|
||||
#[traversable(skip)]
|
||||
pub price_to_amount: Option<PriceToAmount>,
|
||||
|
||||
/// Metric vectors
|
||||
#[traversable(flatten)]
|
||||
pub metrics: CohortMetrics,
|
||||
}
|
||||
|
||||
impl UTXOCohortVecs {
|
||||
/// Import UTXO cohort from database.
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
filter: Filter,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: &Path,
|
||||
state_level: StateLevel,
|
||||
) -> Result<Self> {
|
||||
let compute_dollars = price.is_some();
|
||||
let full_name = filter.to_full_name(CohortContext::Utxo);
|
||||
|
||||
let cfg = ImportConfig {
|
||||
db,
|
||||
filter,
|
||||
context: CohortContext::Utxo,
|
||||
version,
|
||||
indexes,
|
||||
price,
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
state_starting_height: None,
|
||||
|
||||
state: if state_level.is_full() {
|
||||
Some(UTXOCohortState::new(
|
||||
states_path,
|
||||
&full_name,
|
||||
compute_dollars,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
|
||||
price_to_amount: if state_level.is_price_only() && compute_dollars {
|
||||
Some(PriceToAmount::create(states_path, &full_name))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
|
||||
metrics: CohortMetrics::forced_import(&cfg)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the starting height when state was imported.
|
||||
pub fn state_starting_height(&self) -> Option<Height> {
|
||||
self.state_starting_height
|
||||
}
|
||||
|
||||
/// Set the state starting height.
|
||||
pub fn set_state_starting_height(&mut self, height: Height) {
|
||||
self.state_starting_height = Some(height);
|
||||
}
|
||||
|
||||
/// Reset state starting height to zero.
|
||||
pub fn reset_state_starting_height(&mut self) {
|
||||
self.state_starting_height = Some(Height::ZERO);
|
||||
}
|
||||
}
|
||||
|
||||
impl Filtered for UTXOCohortVecs {
|
||||
fn filter(&self) -> &Filter {
|
||||
&self.metrics.filter
|
||||
}
|
||||
}
|
||||
|
||||
impl DynCohortVecs for UTXOCohortVecs {
|
||||
fn min_height_vecs_len(&self) -> usize {
|
||||
self.metrics.min_len()
|
||||
}
|
||||
|
||||
fn reset_state_starting_height(&mut self) {
|
||||
self.state_starting_height = Some(Height::ZERO);
|
||||
}
|
||||
|
||||
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
|
||||
// Import state from runtime state if present
|
||||
if let Some(state) = self.state.as_mut() {
|
||||
let imported = state.import_at_or_before(starting_height)?;
|
||||
self.state_starting_height = Some(imported);
|
||||
Ok(imported)
|
||||
} else {
|
||||
self.state_starting_height = Some(starting_height);
|
||||
Ok(starting_height)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
self.metrics.validate_computed_versions(base_version)
|
||||
}
|
||||
|
||||
fn truncate_push(&mut self, height: Height) -> Result<()> {
|
||||
if self.state_starting_height.map_or(false, |h| h > height) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Push from state to metrics
|
||||
if let Some(state) = self.state.as_ref() {
|
||||
self.metrics.truncate_push(height, &state)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
) -> Result<()> {
|
||||
if let Some(state) = self.state.as_ref() {
|
||||
self.metrics.compute_then_truncate_push_unrealized_states(
|
||||
height,
|
||||
height_price,
|
||||
dateindex,
|
||||
date_price,
|
||||
state,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.metrics.safe_flush(exit)?;
|
||||
|
||||
if let Some(state) = self.state.as_mut() {
|
||||
state.commit(height)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.metrics
|
||||
.compute_rest_part1(indexes, price, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl CohortVecs for UTXOCohortVecs {
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.metrics.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)
|
||||
}
|
||||
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.metrics.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
//! Container for all UTXO cohorts organized by filter type.
|
||||
|
||||
mod receive;
|
||||
mod send;
|
||||
mod tick_tock;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{
|
||||
AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount,
|
||||
ByMaxAge, ByMinAge, BySpendableType, ByTerm, Filter, Filtered, StateLevel, Term, TimeFilter,
|
||||
UTXOGroups,
|
||||
};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, HalvingEpoch, Height, OutputType, Sats, Version};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{Database, Exit, IterableVec};
|
||||
|
||||
use crate::{Indexes, indexes, price, stateful_new::DynCohortVecs};
|
||||
|
||||
use super::{CohortVecs, HeightFlushable, UTXOCohortVecs};
|
||||
|
||||
const VERSION: Version = Version::new(0);
|
||||
|
||||
/// All UTXO cohorts organized by filter type.
|
||||
#[derive(Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct UTXOCohorts(pub(crate) UTXOGroups<UTXOCohortVecs>);
|
||||
|
||||
impl UTXOCohorts {
|
||||
/// Import all UTXO cohorts from database.
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: &Path,
|
||||
) -> Result<Self> {
|
||||
let v = version + VERSION + Version::ZERO;
|
||||
|
||||
let create = |filter: Filter, state_level: StateLevel| -> Result<UTXOCohortVecs> {
|
||||
UTXOCohortVecs::forced_import(db, filter, v, indexes, price, states_path, state_level)
|
||||
};
|
||||
|
||||
let full = |f: Filter| create(f, StateLevel::Full);
|
||||
let none = |f: Filter| create(f, StateLevel::None);
|
||||
|
||||
Ok(Self(UTXOGroups {
|
||||
all: UTXOCohortVecs::forced_import(
|
||||
db,
|
||||
Filter::All,
|
||||
version + VERSION + Version::ONE,
|
||||
indexes,
|
||||
price,
|
||||
states_path,
|
||||
StateLevel::PriceOnly,
|
||||
)?,
|
||||
|
||||
term: ByTerm {
|
||||
short: create(Filter::Term(Term::Sth), StateLevel::PriceOnly)?,
|
||||
long: create(Filter::Term(Term::Lth), StateLevel::PriceOnly)?,
|
||||
},
|
||||
|
||||
epoch: ByEpoch {
|
||||
_0: full(Filter::Epoch(HalvingEpoch::new(0)))?,
|
||||
_1: full(Filter::Epoch(HalvingEpoch::new(1)))?,
|
||||
_2: full(Filter::Epoch(HalvingEpoch::new(2)))?,
|
||||
_3: full(Filter::Epoch(HalvingEpoch::new(3)))?,
|
||||
_4: full(Filter::Epoch(HalvingEpoch::new(4)))?,
|
||||
},
|
||||
|
||||
type_: BySpendableType {
|
||||
p2pk65: full(Filter::Type(OutputType::P2PK65))?,
|
||||
p2pk33: full(Filter::Type(OutputType::P2PK33))?,
|
||||
p2pkh: full(Filter::Type(OutputType::P2PKH))?,
|
||||
p2sh: full(Filter::Type(OutputType::P2SH))?,
|
||||
p2wpkh: full(Filter::Type(OutputType::P2WPKH))?,
|
||||
p2wsh: full(Filter::Type(OutputType::P2WSH))?,
|
||||
p2tr: full(Filter::Type(OutputType::P2TR))?,
|
||||
p2a: full(Filter::Type(OutputType::P2A))?,
|
||||
p2ms: full(Filter::Type(OutputType::P2MS))?,
|
||||
empty: full(Filter::Type(OutputType::Empty))?,
|
||||
unknown: full(Filter::Type(OutputType::Unknown))?,
|
||||
},
|
||||
|
||||
max_age: ByMaxAge {
|
||||
_1w: none(Filter::Time(TimeFilter::LowerThan(7)))?,
|
||||
_1m: none(Filter::Time(TimeFilter::LowerThan(30)))?,
|
||||
_2m: none(Filter::Time(TimeFilter::LowerThan(2 * 30)))?,
|
||||
_3m: none(Filter::Time(TimeFilter::LowerThan(3 * 30)))?,
|
||||
_4m: none(Filter::Time(TimeFilter::LowerThan(4 * 30)))?,
|
||||
_5m: none(Filter::Time(TimeFilter::LowerThan(5 * 30)))?,
|
||||
_6m: none(Filter::Time(TimeFilter::LowerThan(6 * 30)))?,
|
||||
_1y: none(Filter::Time(TimeFilter::LowerThan(365)))?,
|
||||
_2y: none(Filter::Time(TimeFilter::LowerThan(2 * 365)))?,
|
||||
_3y: none(Filter::Time(TimeFilter::LowerThan(3 * 365)))?,
|
||||
_4y: none(Filter::Time(TimeFilter::LowerThan(4 * 365)))?,
|
||||
_5y: none(Filter::Time(TimeFilter::LowerThan(5 * 365)))?,
|
||||
_6y: none(Filter::Time(TimeFilter::LowerThan(6 * 365)))?,
|
||||
_7y: none(Filter::Time(TimeFilter::LowerThan(7 * 365)))?,
|
||||
_8y: none(Filter::Time(TimeFilter::LowerThan(8 * 365)))?,
|
||||
_10y: none(Filter::Time(TimeFilter::LowerThan(10 * 365)))?,
|
||||
_12y: none(Filter::Time(TimeFilter::LowerThan(12 * 365)))?,
|
||||
_15y: none(Filter::Time(TimeFilter::LowerThan(15 * 365)))?,
|
||||
},
|
||||
|
||||
min_age: ByMinAge {
|
||||
_1d: none(Filter::Time(TimeFilter::GreaterOrEqual(1)))?,
|
||||
_1w: none(Filter::Time(TimeFilter::GreaterOrEqual(7)))?,
|
||||
_1m: none(Filter::Time(TimeFilter::GreaterOrEqual(30)))?,
|
||||
_2m: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 30)))?,
|
||||
_3m: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 30)))?,
|
||||
_4m: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 30)))?,
|
||||
_5m: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 30)))?,
|
||||
_6m: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 30)))?,
|
||||
_1y: none(Filter::Time(TimeFilter::GreaterOrEqual(365)))?,
|
||||
_2y: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 365)))?,
|
||||
_3y: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 365)))?,
|
||||
_4y: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 365)))?,
|
||||
_5y: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 365)))?,
|
||||
_6y: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 365)))?,
|
||||
_7y: none(Filter::Time(TimeFilter::GreaterOrEqual(7 * 365)))?,
|
||||
_8y: none(Filter::Time(TimeFilter::GreaterOrEqual(8 * 365)))?,
|
||||
_10y: none(Filter::Time(TimeFilter::GreaterOrEqual(10 * 365)))?,
|
||||
_12y: none(Filter::Time(TimeFilter::GreaterOrEqual(12 * 365)))?,
|
||||
},
|
||||
|
||||
age_range: ByAgeRange {
|
||||
up_to_1d: full(Filter::Time(TimeFilter::Range(0..1)))?,
|
||||
_1d_to_1w: full(Filter::Time(TimeFilter::Range(1..7)))?,
|
||||
_1w_to_1m: full(Filter::Time(TimeFilter::Range(7..30)))?,
|
||||
_1m_to_2m: full(Filter::Time(TimeFilter::Range(30..2 * 30)))?,
|
||||
_2m_to_3m: full(Filter::Time(TimeFilter::Range(2 * 30..3 * 30)))?,
|
||||
_3m_to_4m: full(Filter::Time(TimeFilter::Range(3 * 30..4 * 30)))?,
|
||||
_4m_to_5m: full(Filter::Time(TimeFilter::Range(4 * 30..5 * 30)))?,
|
||||
_5m_to_6m: full(Filter::Time(TimeFilter::Range(5 * 30..6 * 30)))?,
|
||||
_6m_to_1y: full(Filter::Time(TimeFilter::Range(6 * 30..365)))?,
|
||||
_1y_to_2y: full(Filter::Time(TimeFilter::Range(365..2 * 365)))?,
|
||||
_2y_to_3y: full(Filter::Time(TimeFilter::Range(2 * 365..3 * 365)))?,
|
||||
_3y_to_4y: full(Filter::Time(TimeFilter::Range(3 * 365..4 * 365)))?,
|
||||
_4y_to_5y: full(Filter::Time(TimeFilter::Range(4 * 365..5 * 365)))?,
|
||||
_5y_to_6y: full(Filter::Time(TimeFilter::Range(5 * 365..6 * 365)))?,
|
||||
_6y_to_7y: full(Filter::Time(TimeFilter::Range(6 * 365..7 * 365)))?,
|
||||
_7y_to_8y: full(Filter::Time(TimeFilter::Range(7 * 365..8 * 365)))?,
|
||||
_8y_to_10y: full(Filter::Time(TimeFilter::Range(8 * 365..10 * 365)))?,
|
||||
_10y_to_12y: full(Filter::Time(TimeFilter::Range(10 * 365..12 * 365)))?,
|
||||
_12y_to_15y: full(Filter::Time(TimeFilter::Range(12 * 365..15 * 365)))?,
|
||||
from_15y: full(Filter::Time(TimeFilter::GreaterOrEqual(15 * 365)))?,
|
||||
},
|
||||
|
||||
amount_range: ByAmountRange {
|
||||
_0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?,
|
||||
_1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?,
|
||||
_10sats_to_100sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10..Sats::_100,
|
||||
)))?,
|
||||
_100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100..Sats::_1K,
|
||||
)))?,
|
||||
_1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1K..Sats::_10K,
|
||||
)))?,
|
||||
_10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10K..Sats::_100K,
|
||||
)))?,
|
||||
_100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100K..Sats::_1M,
|
||||
)))?,
|
||||
_1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1M..Sats::_10M,
|
||||
)))?,
|
||||
_10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10M..Sats::_1BTC,
|
||||
)))?,
|
||||
_1btc_to_10btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1BTC..Sats::_10BTC,
|
||||
)))?,
|
||||
_10btc_to_100btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10BTC..Sats::_100BTC,
|
||||
)))?,
|
||||
_100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100BTC..Sats::_1K_BTC,
|
||||
)))?,
|
||||
_1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1K_BTC..Sats::_10K_BTC,
|
||||
)))?,
|
||||
_10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10K_BTC..Sats::_100K_BTC,
|
||||
)))?,
|
||||
_100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual(
|
||||
Sats::_100K_BTC,
|
||||
)))?,
|
||||
},
|
||||
|
||||
lt_amount: ByLowerThanAmount {
|
||||
_10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?,
|
||||
_100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?,
|
||||
_1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?,
|
||||
_10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?,
|
||||
_100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?,
|
||||
_1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?,
|
||||
_10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?,
|
||||
_1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?,
|
||||
_10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?,
|
||||
_100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?,
|
||||
_1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?,
|
||||
_10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?,
|
||||
_100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?,
|
||||
},
|
||||
|
||||
ge_amount: ByGreatEqualAmount {
|
||||
_1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?,
|
||||
_10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?,
|
||||
_100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?,
|
||||
_1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?,
|
||||
_10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?,
|
||||
_100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?,
|
||||
_1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?,
|
||||
_10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?,
|
||||
_1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?,
|
||||
_10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?,
|
||||
_100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?,
|
||||
_1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?,
|
||||
_10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
/// Compute overlapping cohorts from component age/amount range cohorts.
|
||||
pub fn compute_overlapping_vecs(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let by_age_range = &self.0.age_range;
|
||||
let by_amount_range = &self.0.amount_range;
|
||||
|
||||
[(&mut self.0.all, by_age_range.iter().collect::<Vec<_>>())]
|
||||
.into_par_iter()
|
||||
.chain(self.0.min_age.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_age_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.max_age.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_age_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.term.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_age_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.ge_amount.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_amount_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.lt_amount.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_amount_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.try_for_each(|(vecs, components)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &components, exit)
|
||||
})
|
||||
}
|
||||
|
||||
/// First phase of post-processing: compute index transforms.
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.par_iter_mut()
|
||||
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
|
||||
}
|
||||
|
||||
/// Second phase of post-processing: compute relative metrics.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2<S, D, HM, DM, HR, DR>(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &S,
|
||||
dateindex_to_supply: &D,
|
||||
height_to_market_cap: Option<&HM>,
|
||||
dateindex_to_market_cap: Option<&DM>,
|
||||
height_to_realized_cap: Option<&HR>,
|
||||
dateindex_to_realized_cap: Option<&DR>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: IterableVec<Height, Bitcoin> + Sync,
|
||||
D: IterableVec<DateIndex, Bitcoin> + Sync,
|
||||
HM: IterableVec<Height, Dollars> + Sync,
|
||||
DM: IterableVec<DateIndex, Dollars> + Sync,
|
||||
HR: IterableVec<Height, Dollars> + Sync,
|
||||
DR: IterableVec<DateIndex, Dollars> + Sync,
|
||||
{
|
||||
self.par_iter_mut().try_for_each(|v| {
|
||||
v.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Flush stateful vectors for separate cohorts.
|
||||
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.par_iter_separate_mut()
|
||||
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))?;
|
||||
|
||||
self.0
|
||||
.par_iter_aggregate_mut()
|
||||
.try_for_each(|v| v.price_to_amount.flush_at_height(height, exit))
|
||||
}
|
||||
|
||||
/// Reset aggregate cohorts' price_to_amount for fresh start.
|
||||
pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> {
|
||||
self.0
|
||||
.iter_aggregate_mut()
|
||||
.try_for_each(|v| v.price_to_amount.reset())
|
||||
}
|
||||
|
||||
/// Import aggregate cohorts' price_to_amount when resuming from checkpoint.
|
||||
pub fn import_aggregate_price_to_amount(&mut self, height: Height) -> Result<Height> {
|
||||
let Some(mut prev_height) = height.decremented() else {
|
||||
return Ok(Height::ZERO);
|
||||
};
|
||||
|
||||
for v in self.0.iter_aggregate_mut() {
|
||||
prev_height = prev_height.min(v.price_to_amount.import_at_or_before(prev_height)?);
|
||||
}
|
||||
|
||||
Ok(prev_height.incremented())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
//! Processing received outputs (new UTXOs).
|
||||
|
||||
use brk_grouper::{Filter, Filtered};
|
||||
use brk_types::{Dollars, Height};
|
||||
|
||||
use crate::states::Transacted;
|
||||
|
||||
use super::UTXOCohorts;
|
||||
|
||||
impl UTXOCohorts {
|
||||
/// Process received outputs for this block.
|
||||
///
|
||||
/// New UTXOs are added to:
|
||||
/// - The "up_to_1d" age cohort (all new UTXOs start at 0 days old)
|
||||
/// - The appropriate epoch cohort based on block height
|
||||
/// - The appropriate output type cohort (P2PKH, P2SH, etc.)
|
||||
/// - The appropriate amount range cohort based on value
|
||||
pub fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
|
||||
let supply_state = received.spendable_supply;
|
||||
|
||||
// New UTXOs go into up_to_1d and current epoch
|
||||
[
|
||||
&mut self.0.age_range.up_to_1d,
|
||||
self.0.epoch.mut_vec_from_height(height),
|
||||
]
|
||||
.into_iter()
|
||||
.for_each(|v| {
|
||||
v.state.as_mut().unwrap().receive(&supply_state, price);
|
||||
});
|
||||
|
||||
// Update aggregate cohorts' price_to_amount
|
||||
// New UTXOs have days_old = 0, so check if filter includes day 0
|
||||
if let Some(price) = price
|
||||
&& supply_state.value.is_not_zero()
|
||||
{
|
||||
self.0
|
||||
.iter_aggregate_mut()
|
||||
.filter(|v| v.filter().contains_time(0))
|
||||
.for_each(|v| {
|
||||
v.price_to_amount
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.increment(price, &supply_state);
|
||||
});
|
||||
}
|
||||
|
||||
// Update output type cohorts
|
||||
self.type_.iter_mut().for_each(|vecs| {
|
||||
let output_type = match vecs.filter() {
|
||||
Filter::Type(output_type) => *output_type,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
vecs.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.receive(received.by_type.get(output_type), price)
|
||||
});
|
||||
|
||||
// Update amount range cohorts
|
||||
received
|
||||
.by_size_group
|
||||
.iter_typed()
|
||||
.for_each(|(group, supply_state)| {
|
||||
self.amount_range
|
||||
.get_mut(group)
|
||||
.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.receive(supply_state, price);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//! Processing spent inputs (UTXOs being spent).
|
||||
|
||||
use brk_grouper::{Filter, Filtered, TimeFilter, UTXOGroups};
|
||||
use brk_types::{CheckedSub, HalvingEpoch, Height};
|
||||
use rustc_hash::FxHashMap;
|
||||
use vecdb::VecIndex;
|
||||
|
||||
use crate::{states::{BlockState, Transacted}, utils::OptionExt, PriceToAmount};
|
||||
|
||||
use super::UTXOCohorts;
|
||||
|
||||
impl UTXOCohorts {
|
||||
/// Process spent inputs for this block.
|
||||
///
|
||||
/// Each input references a UTXO created at some previous height.
|
||||
/// We need to update the cohort states based on when that UTXO was created.
|
||||
pub fn send(
|
||||
&mut self,
|
||||
height_to_sent: FxHashMap<Height, Transacted>,
|
||||
chain_state: &mut [BlockState],
|
||||
) {
|
||||
let UTXOGroups {
|
||||
all,
|
||||
term,
|
||||
age_range,
|
||||
epoch,
|
||||
type_,
|
||||
amount_range,
|
||||
..
|
||||
} = &mut self.0;
|
||||
|
||||
// Time-based cohorts: age_range + epoch
|
||||
let mut time_cohorts: Vec<_> = age_range
|
||||
.iter_mut()
|
||||
.chain(epoch.iter_mut())
|
||||
.collect();
|
||||
|
||||
// Aggregate cohorts' price_to_amount
|
||||
let mut aggregate_p2a: Vec<(Filter, Option<&mut PriceToAmount>)> = vec![
|
||||
(all.filter().clone(), all.price_to_amount.as_mut()),
|
||||
(
|
||||
term.short.filter().clone(),
|
||||
term.short.price_to_amount.as_mut(),
|
||||
),
|
||||
(
|
||||
term.long.filter().clone(),
|
||||
term.long.price_to_amount.as_mut(),
|
||||
),
|
||||
];
|
||||
|
||||
let last_block = chain_state.last().unwrap();
|
||||
let last_timestamp = last_block.timestamp;
|
||||
let current_price = last_block.price;
|
||||
let chain_len = chain_state.len();
|
||||
|
||||
for (height, sent) in height_to_sent {
|
||||
// Update chain_state to reflect spent supply
|
||||
chain_state[height.to_usize()].supply -= &sent.spendable_supply;
|
||||
|
||||
let block_state = &chain_state[height.to_usize()];
|
||||
let prev_price = block_state.price;
|
||||
let blocks_old = chain_len - 1 - height.to_usize();
|
||||
let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
|
||||
let days_old_float =
|
||||
last_timestamp.difference_in_days_between_float(block_state.timestamp);
|
||||
let older_than_hour = last_timestamp
|
||||
.checked_sub(block_state.timestamp)
|
||||
.unwrap()
|
||||
.is_more_than_hour();
|
||||
|
||||
// Update time-based cohorts
|
||||
time_cohorts
|
||||
.iter_mut()
|
||||
.filter(|v| match v.filter() {
|
||||
Filter::Time(TimeFilter::GreaterOrEqual(from)) => *from <= days_old,
|
||||
Filter::Time(TimeFilter::LowerThan(to)) => *to > days_old,
|
||||
Filter::Time(TimeFilter::Range(range)) => range.contains(&days_old),
|
||||
Filter::Epoch(e) => *e == HalvingEpoch::from(height),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
.for_each(|vecs| {
|
||||
vecs.state.um().send(
|
||||
&sent.spendable_supply,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_float,
|
||||
older_than_hour,
|
||||
);
|
||||
});
|
||||
|
||||
// Update output type cohorts
|
||||
sent.by_type
|
||||
.spendable
|
||||
.iter_typed()
|
||||
.for_each(|(output_type, supply_state)| {
|
||||
type_.get_mut(output_type).state.um().send(
|
||||
supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_float,
|
||||
older_than_hour,
|
||||
)
|
||||
});
|
||||
|
||||
// Update amount range cohorts
|
||||
sent.by_size_group
|
||||
.iter_typed()
|
||||
.for_each(|(group, supply_state)| {
|
||||
amount_range.get_mut(group).state.um().send(
|
||||
supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_float,
|
||||
older_than_hour,
|
||||
);
|
||||
});
|
||||
|
||||
// Update aggregate cohorts' price_to_amount
|
||||
if let Some(prev_price) = prev_price {
|
||||
let supply_state = &sent.spendable_supply;
|
||||
if supply_state.value.is_not_zero() {
|
||||
aggregate_p2a
|
||||
.iter_mut()
|
||||
.filter(|(f, _)| f.contains_time(days_old))
|
||||
.for_each(|(_, p2a)| {
|
||||
p2a.um().decrement(prev_price, supply_state);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
//! Age-based state transitions for UTXO cohorts.
|
||||
//!
|
||||
//! When a new block arrives, UTXOs age. Some cross day boundaries
|
||||
//! and need to move between age-based cohorts.
|
||||
|
||||
use brk_grouper::{Filter, Filtered, UTXOGroups};
|
||||
use brk_types::{ONE_DAY_IN_SEC, Sats, Timestamp};
|
||||
|
||||
use crate::{states::BlockState, utils::OptionExt, PriceToAmount};
|
||||
|
||||
use super::UTXOCohorts;
|
||||
|
||||
impl UTXOCohorts {
|
||||
/// Handle age transitions when processing a new block.
|
||||
///
|
||||
/// UTXOs age with each block. When they cross day boundaries,
|
||||
/// they move between age-based cohorts (e.g., from "0-1d" to "1-7d").
|
||||
pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
|
||||
if chain_state.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let prev_timestamp = chain_state.last().unwrap().timestamp;
|
||||
|
||||
// Optimization: Only blocks whose age % ONE_DAY >= threshold can cross a day boundary.
|
||||
// Saves computation vs checking days_old for every block.
|
||||
let elapsed = (*timestamp).saturating_sub(*prev_timestamp);
|
||||
let threshold = ONE_DAY_IN_SEC.saturating_sub(elapsed);
|
||||
|
||||
// Extract mutable references to avoid borrow checker issues
|
||||
let UTXOGroups {
|
||||
all,
|
||||
term,
|
||||
age_range,
|
||||
..
|
||||
} = &mut self.0;
|
||||
|
||||
// Collect age_range cohorts with their filters and states
|
||||
let mut age_cohorts: Vec<(Filter, &mut Option<_>)> = age_range
|
||||
.iter_mut()
|
||||
.map(|v| (v.filter().clone(), &mut v.state))
|
||||
.collect();
|
||||
|
||||
// Collect aggregate cohorts' price_to_amount for age transitions
|
||||
let mut aggregate_p2a: Vec<(Filter, Option<&mut PriceToAmount>)> = vec![
|
||||
(all.filter().clone(), all.price_to_amount.as_mut()),
|
||||
(
|
||||
term.short.filter().clone(),
|
||||
term.short.price_to_amount.as_mut(),
|
||||
),
|
||||
(
|
||||
term.long.filter().clone(),
|
||||
term.long.price_to_amount.as_mut(),
|
||||
),
|
||||
];
|
||||
|
||||
// Process blocks that might cross a day boundary
|
||||
chain_state
|
||||
.iter()
|
||||
.filter(|block_state| {
|
||||
let age = (*prev_timestamp).saturating_sub(*block_state.timestamp);
|
||||
age % ONE_DAY_IN_SEC >= threshold
|
||||
})
|
||||
.for_each(|block_state| {
|
||||
let prev_days = prev_timestamp.difference_in_days_between(block_state.timestamp);
|
||||
let curr_days = timestamp.difference_in_days_between(block_state.timestamp);
|
||||
|
||||
if prev_days == curr_days {
|
||||
return;
|
||||
}
|
||||
|
||||
// Update age_range cohort states
|
||||
age_cohorts.iter_mut().for_each(|(filter, state)| {
|
||||
let is_now = filter.contains_time(curr_days);
|
||||
let was_before = filter.contains_time(prev_days);
|
||||
|
||||
if is_now && !was_before {
|
||||
state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.increment(&block_state.supply, block_state.price);
|
||||
} else if was_before && !is_now {
|
||||
state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.decrement(&block_state.supply, block_state.price);
|
||||
}
|
||||
});
|
||||
|
||||
// Update aggregate cohorts' price_to_amount
|
||||
if let Some(price) = block_state.price
|
||||
&& block_state.supply.value > Sats::ZERO
|
||||
{
|
||||
aggregate_p2a.iter_mut().for_each(|(filter, p2a)| {
|
||||
let is_now = filter.contains_time(curr_days);
|
||||
let was_before = filter.contains_time(prev_days);
|
||||
|
||||
if is_now && !was_before {
|
||||
p2a.um().increment(price, &block_state.supply);
|
||||
} else if was_before && !is_now {
|
||||
p2a.um().decrement(price, &block_state.supply);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
//! Aggregate cohort computation.
|
||||
//!
|
||||
//! After block processing, compute derived metrics:
|
||||
//! 1. Overlapping cohorts (e.g., ">=1d" from sum of age_range cohorts)
|
||||
//! 2. Index-based transforms (height -> dateindex, etc.)
|
||||
//! 3. Relative metrics (supply ratios, market cap ratios)
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height};
|
||||
use log::info;
|
||||
use vecdb::{Exit, IterableVec};
|
||||
|
||||
use crate::{Indexes, indexes, price};
|
||||
|
||||
use super::super::cohorts::{AddressCohorts, UTXOCohorts};
|
||||
|
||||
/// Compute overlapping cohorts from component cohorts.
|
||||
///
|
||||
/// For example:
|
||||
/// - ">=1d" UTXO cohort is computed from sum of age_range cohorts that match
|
||||
/// - ">=1 BTC" address cohort is computed from sum of amount_range cohorts that match
|
||||
pub fn compute_overlapping(
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
address_cohorts: &mut AddressCohorts,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
info!("Computing overlapping cohorts...");
|
||||
|
||||
utxo_cohorts.compute_overlapping_vecs(starting_indexes, exit)?;
|
||||
address_cohorts.compute_overlapping_vecs(starting_indexes, exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// First phase of post-processing: compute index transforms.
|
||||
///
|
||||
/// Converts height-indexed data to dateindex-indexed data and other transforms.
|
||||
pub fn compute_rest_part1(
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
address_cohorts: &mut AddressCohorts,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
info!("Computing rest part 1...");
|
||||
|
||||
utxo_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?;
|
||||
address_cohorts.compute_rest_part1(indexes, price, starting_indexes, exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Second phase of post-processing: compute relative metrics.
|
||||
///
|
||||
/// Computes supply ratios, market cap ratios, etc. using total references.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2<S, D, HM, DM, HR, DR>(
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
address_cohorts: &mut AddressCohorts,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &S,
|
||||
dateindex_to_supply: &D,
|
||||
height_to_market_cap: Option<&HM>,
|
||||
dateindex_to_market_cap: Option<&DM>,
|
||||
height_to_realized_cap: Option<&HR>,
|
||||
dateindex_to_realized_cap: Option<&DR>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: IterableVec<Height, Bitcoin> + Sync,
|
||||
D: IterableVec<DateIndex, Bitcoin> + Sync,
|
||||
HM: IterableVec<Height, Dollars> + Sync,
|
||||
DM: IterableVec<DateIndex, Dollars> + Sync,
|
||||
HR: IterableVec<Height, Dollars> + Sync,
|
||||
DR: IterableVec<DateIndex, Dollars> + Sync,
|
||||
{
|
||||
info!("Computing rest part 2...");
|
||||
|
||||
utxo_cohorts.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
address_cohorts.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
//! Main block processing loop.
|
||||
//!
|
||||
//! Iterates through blocks and processes each one:
|
||||
//! 1. Reset per-block state values
|
||||
//! 2. Tick-tock age transitions
|
||||
//! 3. Process outputs (receive) in parallel
|
||||
//! 4. Process inputs (send) in parallel
|
||||
//! 5. Push to height-indexed vectors
|
||||
//! 6. Periodically flush checkpoints
|
||||
|
||||
use std::thread;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{DateIndex, Height, OutputType, Sats};
|
||||
use log::info;
|
||||
use vecdb::{Exit, GenericStoredVec, IterableVec, VecIndex};
|
||||
|
||||
use crate::states::{BlockState, Transacted};
|
||||
use crate::{chain, indexes, price};
|
||||
|
||||
use super::super::cohorts::{AddressCohorts, DynCohortVecs, UTXOCohorts};
|
||||
use super::super::vecs::Vecs;
|
||||
use super::{
|
||||
FLUSH_INTERVAL, IndexerReaders, build_txinindex_to_txindex, build_txoutindex_to_height_map,
|
||||
build_txoutindex_to_txindex, process_inputs, process_outputs,
|
||||
};
|
||||
|
||||
/// BIP30 duplicate coinbase heights - must handle specially.
|
||||
const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842;
|
||||
const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880;
|
||||
const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812;
|
||||
const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722;
|
||||
|
||||
/// Process all blocks from starting_height to last_height.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn process_blocks(
|
||||
vecs: &mut Vecs,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
chain: &chain::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_height: Height,
|
||||
last_height: Height,
|
||||
chain_state: &mut Vec<BlockState>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
if starting_height > last_height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(
|
||||
"Processing blocks {} to {}...",
|
||||
starting_height, last_height
|
||||
);
|
||||
|
||||
// Pre-compute iterators for fast access
|
||||
let mut height_to_first_txindex = indexes.height_to_first_txindex.boxed_iter();
|
||||
let mut height_to_tx_count = chain.height_to_tx_count.boxed_iter();
|
||||
let mut height_to_first_txoutindex = indexes.height_to_first_txoutindex.boxed_iter();
|
||||
let mut height_to_output_count = chain.height_to_output_count.boxed_iter();
|
||||
let mut height_to_first_txinindex = indexes.height_to_first_txinindex.boxed_iter();
|
||||
let mut height_to_input_count = chain.height_to_input_count.boxed_iter();
|
||||
let mut height_to_timestamp = chain.height_to_timestamp.boxed_iter();
|
||||
let mut height_to_unclaimed_rewards = chain.height_to_unclaimed_reward.boxed_iter();
|
||||
let mut height_to_date = indexes.height_to_date.boxed_iter();
|
||||
let mut dateindex_to_first_height = indexes.dateindex_to_first_height.boxed_iter();
|
||||
let mut dateindex_to_height_count = indexes.dateindex_to_height_count.boxed_iter();
|
||||
let mut txindex_to_output_count = chain.txindex_to_output_count.boxed_iter();
|
||||
let mut txindex_to_input_count = chain.txindex_to_input_count.boxed_iter();
|
||||
|
||||
let mut height_to_price = price.map(|p| p.height_to_close.boxed_iter());
|
||||
let mut dateindex_to_price = price.map(|p| p.dateindex_to_close.boxed_iter());
|
||||
|
||||
// Build txoutindex -> height map for input processing
|
||||
let txoutindex_to_height = build_txoutindex_to_height_map(&indexes.height_to_first_txoutindex);
|
||||
|
||||
// Create readers for parallel data access
|
||||
let ir = IndexerReaders::new(indexer);
|
||||
|
||||
// Track running totals
|
||||
let mut unspendable_supply = Sats::ZERO;
|
||||
let mut opreturn_supply = Sats::ZERO;
|
||||
let mut addresstype_to_addr_count = ByAddressType::<u64>::default();
|
||||
let mut addresstype_to_empty_addr_count = ByAddressType::<u64>::default();
|
||||
|
||||
// Recover initial values if resuming
|
||||
if starting_height > Height::ZERO {
|
||||
let prev_height = starting_height.decremented().unwrap();
|
||||
unspendable_supply = vecs
|
||||
.height_to_unspendable_supply
|
||||
.get(prev_height)
|
||||
.unwrap_or_default();
|
||||
opreturn_supply = vecs
|
||||
.height_to_opreturn_supply
|
||||
.get(prev_height)
|
||||
.unwrap_or_default();
|
||||
}
|
||||
|
||||
// Main block iteration
|
||||
for height in starting_height.to_usize()..=last_height.to_usize() {
|
||||
let height = Height::from(height);
|
||||
|
||||
if height.to_usize() % 10000 == 0 {
|
||||
info!("Processing chain at {}...", height);
|
||||
}
|
||||
|
||||
// Get block metadata
|
||||
let first_txindex = height_to_first_txindex.get_unwrap(height);
|
||||
let tx_count = u64::from(height_to_tx_count.get_unwrap(height));
|
||||
let first_txoutindex = height_to_first_txoutindex.get_unwrap(height).to_usize();
|
||||
let output_count = u64::from(height_to_output_count.get_unwrap(height)) as usize;
|
||||
let first_txinindex = height_to_first_txinindex.get_unwrap(height).to_usize();
|
||||
let input_count = u64::from(height_to_input_count.get_unwrap(height)) as usize;
|
||||
let timestamp = height_to_timestamp.get_unwrap(height);
|
||||
let block_price = height_to_price.as_mut().map(|v| v.get_unwrap(height));
|
||||
|
||||
// Build txindex mappings for this block
|
||||
let txoutindex_to_txindex =
|
||||
build_txoutindex_to_txindex(first_txindex, tx_count, &mut txindex_to_output_count);
|
||||
let txinindex_to_txindex =
|
||||
build_txinindex_to_txindex(first_txindex, tx_count, &mut txindex_to_input_count);
|
||||
|
||||
// Reset per-block values for all separate cohorts
|
||||
reset_block_values(&mut vecs.utxo_cohorts, &mut vecs.address_cohorts);
|
||||
|
||||
// Process outputs and inputs in parallel with tick-tock
|
||||
let (outputs_result, inputs_result) = thread::scope(|scope| {
|
||||
// Tick-tock age transitions in background
|
||||
scope.spawn(|| {
|
||||
vecs.utxo_cohorts
|
||||
.tick_tock_next_block(chain_state, timestamp);
|
||||
});
|
||||
|
||||
// Process outputs (receive)
|
||||
let outputs_result = process_outputs(
|
||||
first_txoutindex,
|
||||
output_count,
|
||||
&txoutindex_to_txindex,
|
||||
&indexer.vecs.txoutindex_to_value,
|
||||
&indexer.vecs.txoutindex_to_outputtype,
|
||||
&indexer.vecs.txoutindex_to_typeindex,
|
||||
&ir,
|
||||
);
|
||||
|
||||
// Process inputs (send) - skip coinbase input
|
||||
let inputs_result = if input_count > 1 {
|
||||
process_inputs(
|
||||
first_txinindex + 1, // Skip coinbase
|
||||
input_count - 1,
|
||||
&txinindex_to_txindex[1..], // Skip coinbase
|
||||
&indexer.vecs.txinindex_to_outpoint,
|
||||
&indexer.vecs.txindex_to_first_txoutindex,
|
||||
&indexer.vecs.txoutindex_to_value,
|
||||
&indexer.vecs.txoutindex_to_outputtype,
|
||||
&indexer.vecs.txoutindex_to_typeindex,
|
||||
&txoutindex_to_height,
|
||||
&ir,
|
||||
)
|
||||
} else {
|
||||
super::InputsResult {
|
||||
height_to_sent: Default::default(),
|
||||
sent_data: Default::default(),
|
||||
}
|
||||
};
|
||||
|
||||
(outputs_result, inputs_result)
|
||||
});
|
||||
|
||||
let mut transacted = outputs_result.transacted;
|
||||
let mut height_to_sent = inputs_result.height_to_sent;
|
||||
|
||||
// Update supply tracking
|
||||
unspendable_supply += transacted.by_type.unspendable.opreturn.value
|
||||
+ height_to_unclaimed_rewards.get_unwrap(height);
|
||||
opreturn_supply += transacted.by_type.unspendable.opreturn.value;
|
||||
|
||||
// Handle special cases
|
||||
if height == Height::ZERO {
|
||||
// Genesis block - reset transacted, add 50 BTC to unspendable
|
||||
transacted = Transacted::default();
|
||||
unspendable_supply += Sats::FIFTY_BTC;
|
||||
} else if height == Height::new(BIP30_DUPLICATE_HEIGHT_1)
|
||||
|| height == Height::new(BIP30_DUPLICATE_HEIGHT_2)
|
||||
{
|
||||
// BIP30: Add 50 BTC to spent from original height
|
||||
let original_height = if height == Height::new(BIP30_DUPLICATE_HEIGHT_1) {
|
||||
Height::new(BIP30_ORIGINAL_HEIGHT_1)
|
||||
} else {
|
||||
Height::new(BIP30_ORIGINAL_HEIGHT_2)
|
||||
};
|
||||
height_to_sent
|
||||
.entry(original_height)
|
||||
.or_default()
|
||||
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
|
||||
}
|
||||
|
||||
// Push current block state before processing cohort updates
|
||||
chain_state.push(BlockState {
|
||||
supply: transacted.spendable_supply.clone(),
|
||||
price: block_price,
|
||||
timestamp,
|
||||
});
|
||||
|
||||
// Update UTXO cohorts
|
||||
vecs.utxo_cohorts.receive(transacted, height, block_price);
|
||||
vecs.utxo_cohorts.send(height_to_sent, chain_state);
|
||||
|
||||
// Push to height-indexed vectors
|
||||
vecs.height_to_unspendable_supply
|
||||
.truncate_push(height, unspendable_supply)?;
|
||||
vecs.height_to_opreturn_supply
|
||||
.truncate_push(height, opreturn_supply)?;
|
||||
vecs.addresstype_to_height_to_addr_count
|
||||
.truncate_push(height, &addresstype_to_addr_count)?;
|
||||
vecs.addresstype_to_height_to_empty_addr_count
|
||||
.truncate_push(height, &addresstype_to_empty_addr_count)?;
|
||||
|
||||
// Get date info for unrealized state computation
|
||||
let date = height_to_date.get_unwrap(height);
|
||||
let dateindex = DateIndex::try_from(date).unwrap();
|
||||
let date_first_height = dateindex_to_first_height.get_unwrap(dateindex);
|
||||
let date_height_count = dateindex_to_height_count.get_unwrap(dateindex);
|
||||
let is_date_last_height =
|
||||
date_first_height + Height::from(date_height_count).decremented().unwrap() == height;
|
||||
let date_price = dateindex_to_price
|
||||
.as_mut()
|
||||
.map(|v| is_date_last_height.then(|| v.get_unwrap(dateindex)));
|
||||
let dateindex_opt = is_date_last_height.then_some(dateindex);
|
||||
|
||||
// Push cohort states and compute unrealized
|
||||
push_cohort_states(
|
||||
&mut vecs.utxo_cohorts,
|
||||
&mut vecs.address_cohorts,
|
||||
height,
|
||||
block_price,
|
||||
dateindex_opt,
|
||||
date_price,
|
||||
)?;
|
||||
|
||||
// Periodic checkpoint flush
|
||||
if height != last_height
|
||||
&& height != Height::ZERO
|
||||
&& height.to_usize() % FLUSH_INTERVAL == 0
|
||||
{
|
||||
let _lock = exit.lock();
|
||||
flush_checkpoint(vecs, height, exit)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Final flush
|
||||
let _lock = exit.lock();
|
||||
flush_checkpoint(vecs, last_height, exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset per-block values for all separate cohorts.
|
||||
fn reset_block_values(utxo_cohorts: &mut UTXOCohorts, address_cohorts: &mut AddressCohorts) {
|
||||
utxo_cohorts.par_iter_separate_mut().for_each(|v| {
|
||||
if let Some(state) = v.state.as_mut() {
|
||||
state.reset_single_iteration_values();
|
||||
}
|
||||
});
|
||||
|
||||
address_cohorts.par_iter_separate_mut().for_each(|v| {
|
||||
if let Some(state) = v.state.as_mut() {
|
||||
state.inner.reset_single_iteration_values();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Push cohort states to height-indexed vectors.
|
||||
fn push_cohort_states(
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
address_cohorts: &mut AddressCohorts,
|
||||
height: Height,
|
||||
height_price: Option<brk_types::Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<brk_types::Dollars>>,
|
||||
) -> Result<()> {
|
||||
utxo_cohorts
|
||||
.par_iter_separate_mut()
|
||||
.map(|v| v as &mut dyn DynCohortVecs)
|
||||
.chain(
|
||||
address_cohorts
|
||||
.par_iter_separate_mut()
|
||||
.map(|v| v as &mut dyn DynCohortVecs),
|
||||
)
|
||||
.try_for_each(|v| {
|
||||
v.truncate_push(height)?;
|
||||
v.compute_then_truncate_push_unrealized_states(
|
||||
height,
|
||||
height_price,
|
||||
dateindex,
|
||||
date_price,
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush checkpoint to disk.
|
||||
fn flush_checkpoint(vecs: &mut Vecs, height: Height, exit: &Exit) -> Result<()> {
|
||||
info!("Flushing checkpoint at height {}...", height);
|
||||
|
||||
// Flush cohort states
|
||||
vecs.utxo_cohorts.safe_flush_stateful_vecs(height, exit)?;
|
||||
vecs.address_cohorts.safe_flush_stateful_vecs(height, exit)?;
|
||||
|
||||
// Flush height-indexed vectors
|
||||
vecs.height_to_unspendable_supply.safe_write(exit)?;
|
||||
vecs.height_to_opreturn_supply.safe_write(exit)?;
|
||||
vecs.addresstype_to_height_to_addr_count.safe_flush(exit)?;
|
||||
vecs.addresstype_to_height_to_empty_addr_count
|
||||
.safe_flush(exit)?;
|
||||
|
||||
// Flush chain state with stamp
|
||||
vecs.chain_state.safe_write_with_stamp(height.into(), exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
//! Computation context holding shared state during block processing.
|
||||
|
||||
use brk_types::{Dollars, Height, Timestamp};
|
||||
use vecdb::VecIndex;
|
||||
|
||||
use crate::price;
|
||||
|
||||
/// Context shared across block processing.
|
||||
pub struct ComputeContext<'a> {
|
||||
/// Starting height for this computation run
|
||||
pub starting_height: Height,
|
||||
|
||||
/// Last height to process
|
||||
pub last_height: Height,
|
||||
|
||||
/// Whether price data is available
|
||||
pub compute_dollars: bool,
|
||||
|
||||
/// Price data (optional)
|
||||
pub price: Option<&'a price::Vecs>,
|
||||
|
||||
/// Pre-computed height -> timestamp mapping
|
||||
pub height_to_timestamp: Vec<Timestamp>,
|
||||
|
||||
/// Pre-computed height -> price mapping (if available)
|
||||
pub height_to_price: Option<Vec<Dollars>>,
|
||||
}
|
||||
|
||||
impl<'a> ComputeContext<'a> {
|
||||
/// Get price at height (None if no price data or height out of range).
|
||||
pub fn price_at(&self, height: Height) -> Option<Dollars> {
|
||||
self.height_to_price.as_ref()?.get(height.to_usize()).copied()
|
||||
}
|
||||
|
||||
/// Get timestamp at height.
|
||||
pub fn timestamp_at(&self, height: Height) -> Timestamp {
|
||||
self.height_to_timestamp[height.to_usize()]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! State flushing logic for checkpoints.
|
||||
//!
|
||||
//! Handles periodic flushing of all stateful data to disk,
|
||||
//! including cohort states, address data, and chain state.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{
|
||||
AddressDataSource, AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, Height,
|
||||
LoadedAddressData, LoadedAddressIndex,
|
||||
};
|
||||
use log::info;
|
||||
use vecdb::{Exit, Stamp};
|
||||
|
||||
use crate::stateful_new::process::{process_empty_addresses, process_loaded_addresses};
|
||||
|
||||
use super::super::address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs};
|
||||
use super::super::cohorts::DynCohortVecs;
|
||||
|
||||
/// Flush all cohort stateful vectors.
|
||||
pub fn flush_cohort_states(
|
||||
height: Height,
|
||||
utxo_vecs: &mut [&mut dyn DynCohortVecs],
|
||||
address_vecs: &mut [&mut dyn DynCohortVecs],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
for v in utxo_vecs.iter_mut() {
|
||||
v.safe_flush_stateful_vecs(height, exit)?;
|
||||
}
|
||||
for v in address_vecs.iter_mut() {
|
||||
v.safe_flush_stateful_vecs(height, exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply address index updates to the index storage.
|
||||
fn apply_address_index_updates(
|
||||
address_indexes: &mut AnyAddressIndexesVecs,
|
||||
updates: AddressTypeToTypeIndexMap<AnyAddressIndex>,
|
||||
) -> Result<()> {
|
||||
for (address_type, sorted) in updates.into_sorted_iter() {
|
||||
for (typeindex, any_index) in sorted {
|
||||
address_indexes.update_or_push(address_type, typeindex, any_index)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full state flush at a checkpoint.
|
||||
///
|
||||
/// This is the main entry point for checkpoint flushing:
|
||||
/// 1. Flush cohort stateful vectors
|
||||
/// 2. Process address data updates (empty and loaded)
|
||||
/// 3. Update address indexes
|
||||
/// 4. Stamped flush address indexes and data
|
||||
/// 5. Flush chain state
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn flush_checkpoint(
|
||||
height: Height,
|
||||
utxo_vecs: &mut [&mut dyn DynCohortVecs],
|
||||
address_vecs: &mut [&mut dyn DynCohortVecs],
|
||||
address_indexes: &mut AnyAddressIndexesVecs,
|
||||
addresses_data: &mut AddressesDataVecs,
|
||||
empty_updates: AddressTypeToTypeIndexMap<AddressDataSource<EmptyAddressData>>,
|
||||
loaded_updates: AddressTypeToTypeIndexMap<AddressDataSource<LoadedAddressData>>,
|
||||
with_changes: bool,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
info!("Flushing at height {}...", height);
|
||||
|
||||
// 1. Flush cohort states
|
||||
flush_cohort_states(height, utxo_vecs, address_vecs, exit)?;
|
||||
|
||||
// 2. Process address updates - empty first, then loaded
|
||||
let empty_result = process_empty_addresses(addresses_data, empty_updates)?;
|
||||
let loaded_result = process_loaded_addresses(addresses_data, loaded_updates)?;
|
||||
let all_updates = empty_result.merge(loaded_result);
|
||||
|
||||
// 3. Apply index updates
|
||||
apply_address_index_updates(address_indexes, all_updates)?;
|
||||
|
||||
// 4. Stamped flush
|
||||
let stamp = Stamp::from(height);
|
||||
address_indexes.flush(stamp, with_changes)?;
|
||||
addresses_data.flush(stamp, with_changes)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
//! Block processing pipeline.
|
||||
//!
|
||||
//! This module handles the main computation loop that processes blocks:
|
||||
//! 1. Recover state from checkpoint or start fresh
|
||||
//! 2. Process each block's outputs and inputs
|
||||
//! 3. Update cohort states
|
||||
//! 4. Periodically flush to disk
|
||||
//! 5. Compute aggregate cohorts from separate cohorts
|
||||
|
||||
mod aggregates;
|
||||
// mod block_loop;
|
||||
mod context;
|
||||
mod flush;
|
||||
mod readers;
|
||||
mod recover;
|
||||
|
||||
pub use aggregates::{compute_overlapping, compute_rest_part1, compute_rest_part2};
|
||||
// pub use block_loop::process_blocks;
|
||||
pub use context::ComputeContext;
|
||||
pub use flush::{flush_checkpoint, flush_cohort_states};
|
||||
pub use readers::{
|
||||
IndexerReaders, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex,
|
||||
};
|
||||
pub use recover::{
|
||||
RecoveredState, StartMode, determine_start_mode, find_min_height,
|
||||
import_aggregate_price_to_amount, import_cohort_states, reset_all_state, rollback_states,
|
||||
};
|
||||
|
||||
/// Flush checkpoint interval (every N blocks).
|
||||
pub const FLUSH_INTERVAL: usize = 10_000;
|
||||
|
||||
// BIP30 duplicate coinbase heights (special case handling)
|
||||
pub const BIP30_DUPLICATE_HEIGHT_1: u32 = 91_842;
|
||||
pub const BIP30_DUPLICATE_HEIGHT_2: u32 = 91_880;
|
||||
pub const BIP30_ORIGINAL_HEIGHT_1: u32 = 91_812;
|
||||
pub const BIP30_ORIGINAL_HEIGHT_2: u32 = 91_722;
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Cached readers for efficient data access during computation.
|
||||
//!
|
||||
//! Readers provide mmap-based access to indexed data without repeated syscalls.
|
||||
|
||||
use brk_grouper::{ByAddressType, ByAnyAddress};
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{OutputType, StoredU64, TxIndex};
|
||||
use vecdb::{BoxedVecIterator, GenericStoredVec, Reader, VecIndex};
|
||||
|
||||
use crate::stateful_new::address::{AddressesDataVecs, AnyAddressIndexesVecs};
|
||||
|
||||
/// Cached readers for indexer vectors.
|
||||
pub struct IndexerReaders {
|
||||
pub txinindex_to_outpoint: Reader,
|
||||
pub txindex_to_first_txoutindex: Reader,
|
||||
pub txoutindex_to_value: Reader,
|
||||
pub txoutindex_to_outputtype: Reader,
|
||||
pub txoutindex_to_typeindex: Reader,
|
||||
}
|
||||
|
||||
impl IndexerReaders {
|
||||
pub fn new(indexer: &Indexer) -> Self {
|
||||
Self {
|
||||
txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(),
|
||||
txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(),
|
||||
txoutindex_to_value: indexer.vecs.txoutindex_to_value.create_reader(),
|
||||
txoutindex_to_outputtype: indexer.vecs.txoutindex_to_outputtype.create_reader(),
|
||||
txoutindex_to_typeindex: indexer.vecs.txoutindex_to_typeindex.create_reader(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached readers for stateful vectors.
|
||||
pub struct VecsReaders {
|
||||
pub addresstypeindex_to_anyaddressindex: ByAddressType<Reader>,
|
||||
pub anyaddressindex_to_anyaddressdata: ByAnyAddress<Reader>,
|
||||
}
|
||||
|
||||
impl VecsReaders {
|
||||
pub fn new(
|
||||
any_address_indexes: &AnyAddressIndexesVecs,
|
||||
addresses_data: &AddressesDataVecs,
|
||||
) -> Self {
|
||||
Self {
|
||||
addresstypeindex_to_anyaddressindex: ByAddressType {
|
||||
p2a: any_address_indexes.p2a.create_reader(),
|
||||
p2pk33: any_address_indexes.p2pk33.create_reader(),
|
||||
p2pk65: any_address_indexes.p2pk65.create_reader(),
|
||||
p2pkh: any_address_indexes.p2pkh.create_reader(),
|
||||
p2sh: any_address_indexes.p2sh.create_reader(),
|
||||
p2tr: any_address_indexes.p2tr.create_reader(),
|
||||
p2wpkh: any_address_indexes.p2wpkh.create_reader(),
|
||||
p2wsh: any_address_indexes.p2wsh.create_reader(),
|
||||
},
|
||||
anyaddressindex_to_anyaddressdata: ByAnyAddress {
|
||||
loaded: addresses_data.loaded.create_reader(),
|
||||
empty: addresses_data.empty.create_reader(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Get reader for specific address type.
|
||||
pub fn address_reader(&self, address_type: OutputType) -> &Reader {
|
||||
self.addresstypeindex_to_anyaddressindex
|
||||
.get_unwrap(address_type)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build txoutindex -> txindex mapping for a block.
|
||||
pub fn build_txoutindex_to_txindex<'a>(
|
||||
block_first_txindex: TxIndex,
|
||||
block_tx_count: u64,
|
||||
txindex_to_output_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
|
||||
) -> Vec<TxIndex> {
|
||||
let first = block_first_txindex.to_usize();
|
||||
|
||||
let counts: Vec<u64> = (0..block_tx_count as usize)
|
||||
.map(|offset| {
|
||||
let txindex = TxIndex::from(first + offset);
|
||||
u64::from(txindex_to_output_count.get_unwrap(txindex))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total: u64 = counts.iter().sum();
|
||||
let mut result = Vec::with_capacity(total as usize);
|
||||
|
||||
for (offset, &count) in counts.iter().enumerate() {
|
||||
let txindex = TxIndex::from(first + offset);
|
||||
result.extend(std::iter::repeat(txindex).take(count as usize));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Build txinindex -> txindex mapping for a block.
|
||||
pub fn build_txinindex_to_txindex<'a>(
|
||||
block_first_txindex: TxIndex,
|
||||
block_tx_count: u64,
|
||||
txindex_to_input_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
|
||||
) -> Vec<TxIndex> {
|
||||
let first = block_first_txindex.to_usize();
|
||||
|
||||
let counts: Vec<u64> = (0..block_tx_count as usize)
|
||||
.map(|offset| {
|
||||
let txindex = TxIndex::from(first + offset);
|
||||
u64::from(txindex_to_input_count.get_unwrap(txindex))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total: u64 = counts.iter().sum();
|
||||
let mut result = Vec::with_capacity(total as usize);
|
||||
|
||||
for (offset, &count) in counts.iter().enumerate() {
|
||||
let txindex = TxIndex::from(first + offset);
|
||||
result.extend(std::iter::repeat_n(txindex, count as usize));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
//! State recovery logic for checkpoint/resume.
|
||||
//!
|
||||
//! Determines starting height and imports saved state from checkpoints.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::Height;
|
||||
use vecdb::{AnyVec, Stamp};
|
||||
|
||||
use super::super::address::AnyAddressIndexesVecs;
|
||||
use super::super::cohorts::{DynCohortVecs, UTXOCohorts};
|
||||
|
||||
/// Result of state recovery.
|
||||
pub struct RecoveredState {
|
||||
/// Height to start processing from.
|
||||
pub starting_height: Height,
|
||||
/// Whether state was successfully restored (vs starting fresh).
|
||||
pub restored: bool,
|
||||
}
|
||||
|
||||
/// Determine starting height from vector lengths.
|
||||
pub fn find_min_height(
|
||||
utxo_vecs: &[&mut dyn DynCohortVecs],
|
||||
address_vecs: &[&mut dyn DynCohortVecs],
|
||||
chain_state_len: usize,
|
||||
address_indexes_min_height: Height,
|
||||
address_data_min_height: Height,
|
||||
other_vec_lens: &[usize],
|
||||
) -> Height {
|
||||
let utxo_min = utxo_vecs
|
||||
.iter()
|
||||
.map(|v| Height::from(v.min_height_vecs_len()))
|
||||
.min()
|
||||
.unwrap_or_default();
|
||||
|
||||
let address_min = address_vecs
|
||||
.iter()
|
||||
.map(|v| Height::from(v.min_height_vecs_len()))
|
||||
.min()
|
||||
.unwrap_or_default();
|
||||
|
||||
let other_min = other_vec_lens
|
||||
.iter()
|
||||
.map(|&len| Height::from(len))
|
||||
.min()
|
||||
.unwrap_or_default();
|
||||
|
||||
utxo_min
|
||||
.min(address_min)
|
||||
.min(Height::from(chain_state_len))
|
||||
.min(address_indexes_min_height)
|
||||
.min(address_data_min_height)
|
||||
.min(other_min)
|
||||
}
|
||||
|
||||
/// Check if we can resume from a checkpoint or need to start fresh.
|
||||
pub fn determine_start_mode(computed_min: Height, chain_state_height: Height) -> StartMode {
|
||||
match computed_min.cmp(&chain_state_height) {
|
||||
Ordering::Greater => unreachable!("min height > chain state height"),
|
||||
Ordering::Equal => StartMode::Resume(chain_state_height),
|
||||
Ordering::Less => StartMode::Fresh,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether to resume from checkpoint or start fresh.
|
||||
pub enum StartMode {
|
||||
/// Resume from the given height.
|
||||
Resume(Height),
|
||||
/// Start from height 0.
|
||||
Fresh,
|
||||
}
|
||||
|
||||
/// Rollback state vectors to before a given stamp.
|
||||
///
|
||||
/// Returns the consistent starting height if all vectors agree,
|
||||
/// otherwise returns Height::ZERO (need fresh start).
|
||||
pub fn rollback_states(
|
||||
stamp: Stamp,
|
||||
chain_state_rollback: Result<Stamp>,
|
||||
address_indexes_rollbacks: Vec<Result<Stamp>>,
|
||||
address_data_rollbacks: Vec<Result<Stamp>>,
|
||||
) -> Height {
|
||||
let mut heights: BTreeSet<Height> = [chain_state_rollback]
|
||||
.into_iter()
|
||||
.chain(address_indexes_rollbacks)
|
||||
.chain(address_data_rollbacks)
|
||||
.filter_map(|r| r.ok())
|
||||
.map(Height::from)
|
||||
.map(Height::incremented)
|
||||
.collect();
|
||||
|
||||
if heights.len() == 1 {
|
||||
heights.pop_first().unwrap()
|
||||
} else {
|
||||
Height::ZERO
|
||||
}
|
||||
}
|
||||
|
||||
/// Import state for all separate cohorts.
|
||||
///
|
||||
/// Returns the starting height if all imports succeed with the same height,
|
||||
/// otherwise returns Height::ZERO.
|
||||
pub fn import_cohort_states(
|
||||
starting_height: Height,
|
||||
cohorts: &mut [&mut dyn DynCohortVecs],
|
||||
) -> Height {
|
||||
if starting_height.is_zero() {
|
||||
return Height::ZERO;
|
||||
}
|
||||
|
||||
let all_match = cohorts
|
||||
.iter_mut()
|
||||
.map(|v| v.import_state(starting_height).unwrap_or_default())
|
||||
.all(|h| h == starting_height);
|
||||
|
||||
if all_match {
|
||||
starting_height
|
||||
} else {
|
||||
Height::ZERO
|
||||
}
|
||||
}
|
||||
|
||||
/// Import aggregate price_to_amount for UTXO cohorts.
|
||||
pub fn import_aggregate_price_to_amount(
|
||||
starting_height: Height,
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
) -> Result<Height> {
|
||||
if starting_height.is_zero() {
|
||||
return Ok(Height::ZERO);
|
||||
}
|
||||
|
||||
let imported = utxo_cohorts.import_aggregate_price_to_amount(starting_height)?;
|
||||
|
||||
Ok(if imported == starting_height {
|
||||
starting_height
|
||||
} else {
|
||||
Height::ZERO
|
||||
})
|
||||
}
|
||||
|
||||
/// Reset all state for fresh start.
|
||||
pub fn reset_all_state(
|
||||
address_indexes: &mut AnyAddressIndexesVecs,
|
||||
utxo_vecs: &mut [&mut dyn DynCohortVecs],
|
||||
address_vecs: &mut [&mut dyn DynCohortVecs],
|
||||
utxo_cohorts: &mut UTXOCohorts,
|
||||
) -> Result<()> {
|
||||
address_indexes.reset()?;
|
||||
|
||||
for v in utxo_vecs.iter_mut() {
|
||||
v.reset_state_starting_height();
|
||||
}
|
||||
|
||||
for v in address_vecs.iter_mut() {
|
||||
v.reset_state_starting_height();
|
||||
}
|
||||
|
||||
utxo_cohorts.reset_aggregate_price_to_amount()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
//! Transaction activity metrics.
|
||||
//!
|
||||
//! These metrics track amounts sent and destruction of satoshi-days/blocks.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, Height, Sats, StoredF64, Version};
|
||||
use vecdb::{AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{ComputedValueVecsFromHeight, ComputedVecsFromHeight, Source, VecBuilderOptions},
|
||||
indexes, price,
|
||||
};
|
||||
|
||||
use super::ImportConfig;
|
||||
|
||||
/// Activity metrics for a cohort.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct ActivityMetrics {
|
||||
/// Total satoshis sent at each height
|
||||
pub height_to_sent: EagerVec<PcoVec<Height, Sats>>,
|
||||
|
||||
/// Sent amounts indexed by various dimensions
|
||||
pub indexes_to_sent: ComputedValueVecsFromHeight,
|
||||
|
||||
/// Satoshi-blocks destroyed (supply * blocks_old when spent)
|
||||
pub height_to_satblocks_destroyed: EagerVec<PcoVec<Height, Sats>>,
|
||||
|
||||
/// Satoshi-days destroyed (supply * days_old when spent)
|
||||
pub height_to_satdays_destroyed: EagerVec<PcoVec<Height, Sats>>,
|
||||
|
||||
/// Coin-blocks destroyed (in BTC rather than sats)
|
||||
pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
|
||||
|
||||
/// Coin-days destroyed (in BTC rather than sats)
|
||||
pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
|
||||
}
|
||||
|
||||
impl ActivityMetrics {
|
||||
/// Import activity metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let v0 = Version::ZERO;
|
||||
let compute_dollars = cfg.compute_dollars();
|
||||
let sum = VecBuilderOptions::default().add_sum();
|
||||
|
||||
Ok(Self {
|
||||
height_to_sent: EagerVec::forced_import(cfg.db, &cfg.name("sent"), cfg.version + v0)?,
|
||||
|
||||
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sent"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
sum,
|
||||
compute_dollars,
|
||||
cfg.indexes,
|
||||
)?,
|
||||
|
||||
height_to_satblocks_destroyed: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("satblocks_destroyed"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
|
||||
height_to_satdays_destroyed: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("satdays_destroyed"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
|
||||
indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("coinblocks_destroyed"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum.clone(),
|
||||
)?,
|
||||
|
||||
indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("coindays_destroyed"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get minimum length across height-indexed vectors.
|
||||
pub fn min_len(&self) -> usize {
|
||||
self.height_to_sent
|
||||
.len()
|
||||
.min(self.height_to_satblocks_destroyed.len())
|
||||
.min(self.height_to_satdays_destroyed.len())
|
||||
}
|
||||
|
||||
/// Push activity state values to height-indexed vectors.
|
||||
pub fn truncate_push(
|
||||
&mut self,
|
||||
height: Height,
|
||||
sent: Sats,
|
||||
satblocks_destroyed: Sats,
|
||||
satdays_destroyed: Sats,
|
||||
) -> Result<()> {
|
||||
self.height_to_sent.truncate_push(height, sent)?;
|
||||
self.height_to_satblocks_destroyed
|
||||
.truncate_push(height, satblocks_destroyed)?;
|
||||
self.height_to_satdays_destroyed
|
||||
.truncate_push(height, satdays_destroyed)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush height-indexed vectors to disk.
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
self.height_to_sent.safe_write(exit)?;
|
||||
self.height_to_satblocks_destroyed.safe_write(exit)?;
|
||||
self.height_to_satdays_destroyed.safe_write(exit)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate computed versions against base version.
|
||||
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
|
||||
// Validation logic for computed vecs
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute aggregate values from separate cohorts.
|
||||
pub fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_sent.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_sent).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_satblocks_destroyed.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_satblocks_destroyed)
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_satdays_destroyed.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_satdays_destroyed)
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// First phase of computed metrics (indexes from height).
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.indexes_to_sent.compute_rest(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_sent),
|
||||
)?;
|
||||
|
||||
self.indexes_to_coinblocks_destroyed
|
||||
.compute_all(indexes, starting_indexes, exit, |v| {
|
||||
v.compute_transform(
|
||||
starting_indexes.height,
|
||||
&self.height_to_satblocks_destroyed,
|
||||
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.indexes_to_coindays_destroyed
|
||||
.compute_all(indexes, starting_indexes, exit, |v| {
|
||||
v.compute_transform(
|
||||
starting_indexes.height,
|
||||
&self.height_to_satdays_destroyed,
|
||||
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//! Configuration for metric imports.
|
||||
|
||||
use brk_grouper::{CohortContext, Filter};
|
||||
use brk_types::Version;
|
||||
use vecdb::Database;
|
||||
|
||||
use crate::{indexes, price};
|
||||
|
||||
/// Configuration for importing metrics.
|
||||
pub struct ImportConfig<'a> {
|
||||
pub db: &'a Database,
|
||||
pub filter: Filter,
|
||||
pub context: CohortContext,
|
||||
pub version: Version,
|
||||
pub indexes: &'a indexes::Vecs,
|
||||
pub price: Option<&'a price::Vecs>,
|
||||
}
|
||||
|
||||
impl<'a> ImportConfig<'a> {
|
||||
/// Whether price data is available (enables realized/unrealized metrics).
|
||||
pub fn compute_dollars(&self) -> bool {
|
||||
self.price.is_some()
|
||||
}
|
||||
|
||||
/// Whether this is an extended cohort (more relative metrics).
|
||||
pub fn extended(&self) -> bool {
|
||||
self.filter.is_extended(self.context)
|
||||
}
|
||||
|
||||
/// Whether to compute relative-to-all metrics.
|
||||
pub fn compute_rel_to_all(&self) -> bool {
|
||||
self.filter.compute_rel_to_all()
|
||||
}
|
||||
|
||||
/// Whether to compute adjusted metrics (SOPR, etc.).
|
||||
pub fn compute_adjusted(&self) -> bool {
|
||||
self.filter.compute_adjusted(self.context)
|
||||
}
|
||||
|
||||
/// Get full metric name with filter prefix.
|
||||
pub fn name(&self, suffix: &str) -> String {
|
||||
let prefix = self.filter.to_full_name(self.context);
|
||||
if prefix.is_empty() {
|
||||
suffix.to_string()
|
||||
} else {
|
||||
format!("{prefix}_{suffix}")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
//! Metric vectors organized by category.
|
||||
//!
|
||||
//! Instead of a single 80+ field struct, metrics are grouped into logical categories:
|
||||
//! - `supply`: Supply and UTXO count metrics (always computed)
|
||||
//! - `activity`: Transaction activity metrics (always computed)
|
||||
//! - `realized`: Realized cap, profit/loss, SOPR (requires price)
|
||||
//! - `unrealized`: Unrealized profit/loss (requires price)
|
||||
//! - `price`: Price paid metrics and percentiles (requires price)
|
||||
//! - `relative`: Ratios relative to market cap, etc. (requires price)
|
||||
|
||||
mod activity;
|
||||
mod config;
|
||||
mod price_paid;
|
||||
mod realized;
|
||||
mod relative;
|
||||
mod supply;
|
||||
mod unrealized;
|
||||
|
||||
pub use activity::ActivityMetrics;
|
||||
pub use config::ImportConfig;
|
||||
pub use price_paid::PricePaidMetrics;
|
||||
pub use realized::RealizedMetrics;
|
||||
pub use relative::RelativeMetrics;
|
||||
pub use supply::SupplyMetrics;
|
||||
pub use unrealized::UnrealizedMetrics;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::Filter;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
|
||||
use vecdb::{Exit, IterableVec};
|
||||
|
||||
use crate::{Indexes, indexes, price, states::CohortState};
|
||||
|
||||
/// All metrics for a cohort, organized by category.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct CohortMetrics {
|
||||
#[traversable(skip)]
|
||||
pub filter: Filter,
|
||||
|
||||
/// Supply and UTXO count (always computed)
|
||||
pub supply: SupplyMetrics,
|
||||
|
||||
/// Transaction activity (always computed)
|
||||
pub activity: ActivityMetrics,
|
||||
|
||||
/// Realized cap and profit/loss (requires price data)
|
||||
pub realized: Option<RealizedMetrics>,
|
||||
|
||||
/// Unrealized profit/loss (requires price data)
|
||||
pub unrealized: Option<UnrealizedMetrics>,
|
||||
|
||||
/// Price paid metrics (requires price data)
|
||||
pub price_paid: Option<PricePaidMetrics>,
|
||||
|
||||
/// Relative metrics (requires price data)
|
||||
pub relative: Option<RelativeMetrics>,
|
||||
}
|
||||
|
||||
impl CohortMetrics {
|
||||
/// Import all metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let compute_dollars = cfg.compute_dollars();
|
||||
|
||||
Ok(Self {
|
||||
filter: cfg.filter.clone(),
|
||||
supply: SupplyMetrics::forced_import(cfg)?,
|
||||
activity: ActivityMetrics::forced_import(cfg)?,
|
||||
realized: compute_dollars
|
||||
.then(|| RealizedMetrics::forced_import(cfg))
|
||||
.transpose()?,
|
||||
unrealized: compute_dollars
|
||||
.then(|| UnrealizedMetrics::forced_import(cfg))
|
||||
.transpose()?,
|
||||
price_paid: compute_dollars
|
||||
.then(|| PricePaidMetrics::forced_import(cfg))
|
||||
.transpose()?,
|
||||
relative: compute_dollars
|
||||
.then(|| RelativeMetrics::forced_import(cfg))
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get minimum length across height-indexed vectors.
|
||||
pub fn min_len(&self) -> usize {
|
||||
self.supply.min_len().min(self.activity.min_len())
|
||||
}
|
||||
|
||||
/// Push state values to height-indexed vectors.
|
||||
pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> {
|
||||
self.supply.truncate_push(height, &state.supply)?;
|
||||
self.activity.truncate_push(
|
||||
height,
|
||||
state.sent,
|
||||
state.satblocks_destroyed,
|
||||
state.satdays_destroyed,
|
||||
)?;
|
||||
|
||||
if let (Some(realized), Some(realized_state)) =
|
||||
(self.realized.as_mut(), state.realized.as_ref())
|
||||
{
|
||||
realized.truncate_push(height, realized_state)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush height-indexed vectors to disk.
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
self.supply.safe_flush(exit)?;
|
||||
self.activity.safe_flush(exit)?;
|
||||
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.safe_flush(exit)?;
|
||||
}
|
||||
|
||||
if let Some(unrealized) = self.unrealized.as_mut() {
|
||||
unrealized.safe_flush(exit)?;
|
||||
}
|
||||
|
||||
if let Some(price_paid) = self.price_paid.as_mut() {
|
||||
price_paid.safe_flush(exit)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate computed versions against base version.
|
||||
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
self.supply.validate_computed_versions(base_version)?;
|
||||
self.activity.validate_computed_versions(base_version)?;
|
||||
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.validate_computed_versions(base_version)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute and push unrealized states.
|
||||
pub fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
state: &CohortState,
|
||||
) -> Result<()> {
|
||||
if let (Some(unrealized), Some(price_paid), Some(height_price)) = (
|
||||
self.unrealized.as_mut(),
|
||||
self.price_paid.as_mut(),
|
||||
height_price,
|
||||
) {
|
||||
// Push price paid min/max
|
||||
price_paid.truncate_push_minmax(height, state)?;
|
||||
|
||||
// Compute unrealized states from price_to_amount
|
||||
let (height_unrealized_state, date_unrealized_state) =
|
||||
state.compute_unrealized_states(height_price, date_price.unwrap());
|
||||
|
||||
unrealized.truncate_push(
|
||||
height,
|
||||
dateindex,
|
||||
&height_unrealized_state,
|
||||
date_unrealized_state.as_ref(),
|
||||
)?;
|
||||
|
||||
// Compute and push price percentiles
|
||||
price_paid.truncate_push_percentiles(height, state)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute aggregate cohort values from separate cohorts.
|
||||
pub fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.supply.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.supply).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.activity.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.activity).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others
|
||||
.iter()
|
||||
.filter_map(|v| v.realized.as_ref())
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(unrealized) = self.unrealized.as_mut() {
|
||||
unrealized.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others
|
||||
.iter()
|
||||
.filter_map(|v| v.unrealized.as_ref())
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(price_paid) = self.price_paid.as_mut() {
|
||||
price_paid.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others
|
||||
.iter()
|
||||
.filter_map(|v| v.price_paid.as_ref())
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// First phase of computed metrics (indexes from height).
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.supply
|
||||
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
|
||||
self.activity
|
||||
.compute_rest_part1(indexes, price, starting_indexes, exit)?;
|
||||
|
||||
if let Some(realized) = self.realized.as_mut() {
|
||||
realized.compute_rest_part1(indexes, price, starting_indexes, exit)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Second phase of computed metrics (ratios, relative values).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.supply.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
if let Some(relative) = self.relative.as_mut() {
|
||||
relative.compute_rest_part2(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
&self.supply,
|
||||
self.realized.as_ref(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Price paid metrics and percentiles.
|
||||
//!
|
||||
//! Tracks min/max price paid for UTXOs and price distribution percentiles.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Dollars, Height, Version};
|
||||
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, PricePercentiles, Source, VecBuilderOptions},
|
||||
states::{CohortState, Flushable},
|
||||
};
|
||||
|
||||
use super::ImportConfig;
|
||||
|
||||
/// Price paid metrics.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct PricePaidMetrics {
|
||||
/// Minimum price paid for any UTXO at this height
|
||||
pub height_to_min_price_paid: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_min_price_paid: ComputedVecsFromHeight<Dollars>,
|
||||
|
||||
/// Maximum price paid for any UTXO at this height
|
||||
pub height_to_max_price_paid: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_max_price_paid: ComputedVecsFromHeight<Dollars>,
|
||||
|
||||
/// Price distribution percentiles (median, quartiles, etc.)
|
||||
pub price_percentiles: Option<PricePercentiles>,
|
||||
}
|
||||
|
||||
impl PricePaidMetrics {
|
||||
/// Import price paid metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let v0 = Version::ZERO;
|
||||
let extended = cfg.extended();
|
||||
let last = VecBuilderOptions::default().add_last();
|
||||
|
||||
Ok(Self {
|
||||
height_to_min_price_paid: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("min_price_paid"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_min_price_paid: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("min_price_paid"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
height_to_max_price_paid: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("max_price_paid"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_max_price_paid: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("max_price_paid"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
price_percentiles: extended
|
||||
.then(|| {
|
||||
PricePercentiles::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name(""),
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
true,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Push min/max price paid from state.
|
||||
pub fn truncate_push_minmax(&mut self, height: Height, state: &CohortState) -> Result<()> {
|
||||
self.height_to_min_price_paid.truncate_push(
|
||||
height,
|
||||
state
|
||||
.price_to_amount_first_key_value()
|
||||
.map(|(&dollars, _)| dollars)
|
||||
.unwrap_or(Dollars::NAN),
|
||||
)?;
|
||||
self.height_to_max_price_paid.truncate_push(
|
||||
height,
|
||||
state
|
||||
.price_to_amount_last_key_value()
|
||||
.map(|(&dollars, _)| dollars)
|
||||
.unwrap_or(Dollars::NAN),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Push price percentiles from state.
|
||||
pub fn truncate_push_percentiles(&mut self, height: Height, state: &CohortState) -> Result<()> {
|
||||
if let Some(price_percentiles) = self.price_percentiles.as_mut() {
|
||||
let percentile_prices = state.compute_percentile_prices();
|
||||
price_percentiles.truncate_push(height, &percentile_prices)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush height-indexed vectors to disk.
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
self.height_to_min_price_paid.safe_write(exit)?;
|
||||
self.height_to_max_price_paid.safe_write(exit)?;
|
||||
if let Some(price_percentiles) = self.price_percentiles.as_mut() {
|
||||
price_percentiles.safe_flush(exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute aggregate values from separate cohorts.
|
||||
pub fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_min_price_paid.compute_min_of_others(
|
||||
starting_indexes.height,
|
||||
&others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_min_price_paid)
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_max_price_paid.compute_max_of_others(
|
||||
starting_indexes.height,
|
||||
&others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_max_price_paid)
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
//! Realized cap and profit/loss metrics.
|
||||
//!
|
||||
//! These metrics require price data and track realized value based on acquisition price.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{DateIndex, Dollars, Height, StoredF32, StoredF64, Version};
|
||||
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{
|
||||
ComputedRatioVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source,
|
||||
VecBuilderOptions,
|
||||
},
|
||||
indexes, price,
|
||||
states::RealizedState,
|
||||
utils::OptionExt,
|
||||
};
|
||||
|
||||
use super::ImportConfig;
|
||||
|
||||
/// Realized cap and related metrics.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct RealizedMetrics {
|
||||
// === Realized Cap ===
|
||||
pub height_to_realized_cap: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_realized_cap: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_realized_price: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex,
|
||||
pub indexes_to_realized_cap_rel_to_own_market_cap: Option<ComputedVecsFromHeight<StoredF32>>,
|
||||
pub indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex<Dollars>,
|
||||
|
||||
// === Realized Profit/Loss ===
|
||||
pub height_to_realized_profit: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_realized_profit: ComputedVecsFromHeight<Dollars>,
|
||||
pub height_to_realized_loss: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_realized_loss: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_neg_realized_loss: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_net_realized_pnl: ComputedVecsFromHeight<Dollars>,
|
||||
pub indexes_to_realized_value: ComputedVecsFromHeight<Dollars>,
|
||||
|
||||
// === Realized vs Realized Cap Ratios ===
|
||||
pub indexes_to_realized_profit_rel_to_realized_cap: ComputedVecsFromHeight<StoredF32>,
|
||||
pub indexes_to_realized_loss_rel_to_realized_cap: ComputedVecsFromHeight<StoredF32>,
|
||||
pub indexes_to_net_realized_pnl_rel_to_realized_cap: ComputedVecsFromHeight<StoredF32>,
|
||||
|
||||
// === Total Realized PnL ===
|
||||
pub height_to_total_realized_pnl: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_total_realized_pnl: ComputedVecsFromDateIndex<Dollars>,
|
||||
pub dateindex_to_realized_profit_to_loss_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
|
||||
// === Value Created/Destroyed ===
|
||||
pub height_to_value_created: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_value_created: ComputedVecsFromHeight<Dollars>,
|
||||
pub height_to_value_destroyed: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_value_destroyed: ComputedVecsFromHeight<Dollars>,
|
||||
|
||||
// === Adjusted Value (optional) ===
|
||||
pub height_to_adjusted_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub height_to_adjusted_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_adjusted_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
|
||||
// === SOPR (Spent Output Profit Ratio) ===
|
||||
pub dateindex_to_sopr: EagerVec<PcoVec<DateIndex, StoredF64>>,
|
||||
pub dateindex_to_sopr_7d_ema: EagerVec<PcoVec<DateIndex, StoredF64>>,
|
||||
pub dateindex_to_sopr_30d_ema: EagerVec<PcoVec<DateIndex, StoredF64>>,
|
||||
pub dateindex_to_adjusted_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_adjusted_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_adjusted_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
|
||||
// === Sell Side Risk ===
|
||||
pub dateindex_to_sell_side_risk_ratio: EagerVec<PcoVec<DateIndex, StoredF32>>,
|
||||
pub dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec<PcoVec<DateIndex, StoredF32>>,
|
||||
pub dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec<PcoVec<DateIndex, StoredF32>>,
|
||||
|
||||
// === Net Realized PnL Deltas ===
|
||||
pub indexes_to_net_realized_pnl_cumulative_30d_delta: ComputedVecsFromDateIndex<Dollars>,
|
||||
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
|
||||
ComputedVecsFromDateIndex<StoredF32>,
|
||||
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
|
||||
ComputedVecsFromDateIndex<StoredF32>,
|
||||
}
|
||||
|
||||
impl RealizedMetrics {
|
||||
/// Import realized metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let v0 = Version::ZERO;
|
||||
let v1 = Version::ONE;
|
||||
let v3 = Version::new(3);
|
||||
let extended = cfg.extended();
|
||||
let compute_adjusted = cfg.compute_adjusted();
|
||||
let last = VecBuilderOptions::default().add_last();
|
||||
let sum = VecBuilderOptions::default().add_sum();
|
||||
let sum_cum = VecBuilderOptions::default().add_sum().add_cumulative();
|
||||
|
||||
Ok(Self {
|
||||
// === Realized Cap ===
|
||||
height_to_realized_cap: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_cap"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_realized_cap: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_cap"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_realized_price: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_price"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_price"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
extended,
|
||||
)?,
|
||||
indexes_to_realized_cap_rel_to_own_market_cap: extended
|
||||
.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_cap_rel_to_own_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_realized_cap_30d_delta: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_cap_30d_delta"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
|
||||
// === Realized Profit/Loss ===
|
||||
height_to_realized_profit: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_profit"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_realized_profit: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_profit"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum_cum,
|
||||
)?,
|
||||
height_to_realized_loss: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_loss"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_realized_loss: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_loss"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum_cum,
|
||||
)?,
|
||||
indexes_to_neg_realized_loss: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_realized_loss"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
sum_cum,
|
||||
)?,
|
||||
indexes_to_net_realized_pnl: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_realized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum_cum,
|
||||
)?,
|
||||
indexes_to_realized_value: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_value"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
|
||||
// === Realized vs Realized Cap Ratios ===
|
||||
indexes_to_realized_profit_rel_to_realized_cap: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_profit_rel_to_realized_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
indexes_to_realized_loss_rel_to_realized_cap: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_loss_rel_to_realized_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
indexes_to_net_realized_pnl_rel_to_realized_cap: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_realized_pnl_rel_to_realized_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
|
||||
// === Total Realized PnL ===
|
||||
height_to_total_realized_pnl: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("total_realized_pnl"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_total_realized_pnl: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("total_realized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
dateindex_to_realized_profit_to_loss_ratio: extended
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("realized_profit_to_loss_ratio"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
|
||||
// === Value Created/Destroyed ===
|
||||
height_to_value_created: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("value_created"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_value_created: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("value_created"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
height_to_value_destroyed: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("value_destroyed"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_value_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("value_destroyed"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)?,
|
||||
|
||||
// === Adjusted Value (optional) ===
|
||||
height_to_adjusted_value_created: compute_adjusted
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("adjusted_value_created"),
|
||||
cfg.version + v0,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_adjusted_value_created: compute_adjusted
|
||||
.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("adjusted_value_created"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_adjusted_value_destroyed: compute_adjusted
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("adjusted_value_destroyed"),
|
||||
cfg.version + v0,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_adjusted_value_destroyed: compute_adjusted
|
||||
.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("adjusted_value_destroyed"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
sum,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
|
||||
// === SOPR ===
|
||||
dateindex_to_sopr: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sopr"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
dateindex_to_sopr_7d_ema: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sopr_7d_ema"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
dateindex_to_sopr_30d_ema: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sopr_30d_ema"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
dateindex_to_adjusted_sopr: compute_adjusted
|
||||
.then(|| {
|
||||
EagerVec::forced_import(cfg.db, &cfg.name("adjusted_sopr"), cfg.version + v1)
|
||||
})
|
||||
.transpose()?,
|
||||
dateindex_to_adjusted_sopr_7d_ema: compute_adjusted
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("adjusted_sopr_7d_ema"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
dateindex_to_adjusted_sopr_30d_ema: compute_adjusted
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("adjusted_sopr_30d_ema"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
|
||||
// === Sell Side Risk ===
|
||||
dateindex_to_sell_side_risk_ratio: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sell_side_risk_ratio"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
dateindex_to_sell_side_risk_ratio_7d_ema: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sell_side_risk_ratio_7d_ema"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
dateindex_to_sell_side_risk_ratio_30d_ema: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("sell_side_risk_ratio_30d_ema"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
|
||||
// === Net Realized PnL Deltas ===
|
||||
indexes_to_net_realized_pnl_cumulative_30d_delta:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_realized_pnl_cumulative_30d_delta"),
|
||||
Source::Compute,
|
||||
cfg.version + v3,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v3,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_realized_pnl_cumulative_30d_delta_rel_to_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v3,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Push realized state values to height-indexed vectors.
|
||||
pub fn truncate_push(&mut self, height: Height, state: &RealizedState) -> Result<()> {
|
||||
self.height_to_realized_cap.truncate_push(height, state.cap)?;
|
||||
self.height_to_realized_profit.truncate_push(height, state.profit)?;
|
||||
self.height_to_realized_loss.truncate_push(height, state.loss)?;
|
||||
self.height_to_value_created.truncate_push(height, state.value_created)?;
|
||||
self.height_to_value_destroyed.truncate_push(height, state.value_destroyed)?;
|
||||
|
||||
if let Some(v) = self.height_to_adjusted_value_created.as_mut() {
|
||||
v.truncate_push(height, state.adj_value_created)?;
|
||||
}
|
||||
if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() {
|
||||
v.truncate_push(height, state.adj_value_destroyed)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush height-indexed vectors to disk.
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
self.height_to_realized_cap.safe_write(exit)?;
|
||||
self.height_to_realized_profit.safe_write(exit)?;
|
||||
self.height_to_realized_loss.safe_write(exit)?;
|
||||
self.height_to_value_created.safe_write(exit)?;
|
||||
self.height_to_value_destroyed.safe_write(exit)?;
|
||||
self.height_to_adjusted_value_created.um().safe_write(exit)?;
|
||||
self.height_to_adjusted_value_destroyed.um().safe_write(exit)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate computed versions against base version.
|
||||
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
|
||||
// Validation logic for computed vecs
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute aggregate values from separate cohorts.
|
||||
pub fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_realized_cap.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_realized_cap).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_realized_profit.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_realized_profit).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_realized_loss.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_realized_loss).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_value_created.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_value_created).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_value_destroyed.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_value_destroyed).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
|
||||
if self.height_to_adjusted_value_created.is_some() {
|
||||
self.height_to_adjusted_value_created.um().compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others
|
||||
.iter()
|
||||
.map(|v| {
|
||||
v.height_to_adjusted_value_created
|
||||
.as_ref()
|
||||
.unwrap_or(&v.height_to_value_created)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_adjusted_value_destroyed.um().compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others
|
||||
.iter()
|
||||
.map(|v| {
|
||||
v.height_to_adjusted_value_destroyed
|
||||
.as_ref()
|
||||
.unwrap_or(&v.height_to_value_destroyed)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// First phase of computed metrics (indexes from height).
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
_price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.indexes_to_realized_cap.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_realized_cap),
|
||||
)?;
|
||||
|
||||
self.indexes_to_realized_profit.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_realized_profit),
|
||||
)?;
|
||||
|
||||
self.indexes_to_realized_loss.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_realized_loss),
|
||||
)?;
|
||||
|
||||
self.indexes_to_value_created.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_value_created),
|
||||
)?;
|
||||
|
||||
self.indexes_to_value_destroyed.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_value_destroyed),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,450 @@
|
||||
//! Relative metrics (ratios to market cap, realized cap, supply, etc.)
|
||||
//!
|
||||
//! These are computed ratios comparing cohort metrics to global metrics.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version};
|
||||
use vecdb::{EagerVec, Exit, ImportableVec, IterableVec, PcoVec};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions},
|
||||
indexes,
|
||||
};
|
||||
|
||||
use super::{ImportConfig, RealizedMetrics, SupplyMetrics};
|
||||
|
||||
/// Relative metrics comparing cohort values to global values.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct RelativeMetrics {
|
||||
// === Supply Relative to Circulating Supply ===
|
||||
pub indexes_to_supply_rel_to_circulating_supply: Option<ComputedVecsFromHeight<StoredF64>>,
|
||||
|
||||
// === Supply in Profit/Loss Relative to Own Supply ===
|
||||
pub height_to_supply_in_profit_rel_to_own_supply: EagerVec<PcoVec<Height, StoredF64>>,
|
||||
pub height_to_supply_in_loss_rel_to_own_supply: EagerVec<PcoVec<Height, StoredF64>>,
|
||||
pub indexes_to_supply_in_profit_rel_to_own_supply: ComputedVecsFromDateIndex<StoredF64>,
|
||||
pub indexes_to_supply_in_loss_rel_to_own_supply: ComputedVecsFromDateIndex<StoredF64>,
|
||||
|
||||
// === Supply in Profit/Loss Relative to Circulating Supply ===
|
||||
pub height_to_supply_in_profit_rel_to_circulating_supply:
|
||||
Option<EagerVec<PcoVec<Height, StoredF64>>>,
|
||||
pub height_to_supply_in_loss_rel_to_circulating_supply:
|
||||
Option<EagerVec<PcoVec<Height, StoredF64>>>,
|
||||
pub indexes_to_supply_in_profit_rel_to_circulating_supply:
|
||||
Option<ComputedVecsFromDateIndex<StoredF64>>,
|
||||
pub indexes_to_supply_in_loss_rel_to_circulating_supply:
|
||||
Option<ComputedVecsFromDateIndex<StoredF64>>,
|
||||
|
||||
// === Unrealized vs Market Cap ===
|
||||
pub height_to_unrealized_profit_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
|
||||
pub height_to_unrealized_loss_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
|
||||
pub height_to_neg_unrealized_loss_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
|
||||
pub height_to_net_unrealized_pnl_rel_to_market_cap: EagerVec<PcoVec<Height, StoredF32>>,
|
||||
pub indexes_to_unrealized_profit_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
|
||||
pub indexes_to_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
|
||||
pub indexes_to_neg_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
|
||||
pub indexes_to_net_unrealized_pnl_rel_to_market_cap: ComputedVecsFromDateIndex<StoredF32>,
|
||||
|
||||
// === Unrealized vs Own Market Cap (optional) ===
|
||||
pub height_to_unrealized_profit_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_neg_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_net_unrealized_pnl_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub indexes_to_unrealized_profit_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
|
||||
// === Unrealized vs Own Total Unrealized PnL (optional) ===
|
||||
pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
}
|
||||
|
||||
impl RelativeMetrics {
|
||||
/// Import relative metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let v0 = Version::ZERO;
|
||||
let v1 = Version::ONE;
|
||||
let v2 = Version::new(2);
|
||||
let extended = cfg.extended();
|
||||
let compute_rel_to_all = cfg.compute_rel_to_all();
|
||||
let last = VecBuilderOptions::default().add_last();
|
||||
|
||||
Ok(Self {
|
||||
// === Supply Relative to Circulating Supply ===
|
||||
indexes_to_supply_rel_to_circulating_supply: compute_rel_to_all
|
||||
.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_rel_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
|
||||
// === Supply in Profit/Loss Relative to Own Supply ===
|
||||
height_to_supply_in_profit_rel_to_own_supply: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit_rel_to_own_supply"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
height_to_supply_in_loss_rel_to_own_supply: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss_rel_to_own_supply"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
indexes_to_supply_in_profit_rel_to_own_supply:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit_rel_to_own_supply"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_supply_in_loss_rel_to_own_supply: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss_rel_to_own_supply"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
|
||||
// === Supply in Profit/Loss Relative to Circulating Supply ===
|
||||
height_to_supply_in_profit_rel_to_circulating_supply: compute_rel_to_all
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit_rel_to_circulating_supply"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_supply_in_loss_rel_to_circulating_supply: compute_rel_to_all
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss_rel_to_circulating_supply"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_supply_in_profit_rel_to_circulating_supply: compute_rel_to_all
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit_rel_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_supply_in_loss_rel_to_circulating_supply: compute_rel_to_all
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss_rel_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
|
||||
// === Unrealized vs Market Cap ===
|
||||
height_to_unrealized_profit_rel_to_market_cap: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit_rel_to_market_cap"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
height_to_unrealized_loss_rel_to_market_cap: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss_rel_to_market_cap"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
height_to_neg_unrealized_loss_rel_to_market_cap: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss_rel_to_market_cap"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
height_to_net_unrealized_pnl_rel_to_market_cap: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl_rel_to_market_cap"),
|
||||
cfg.version + v1,
|
||||
)?,
|
||||
indexes_to_unrealized_profit_rel_to_market_cap:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit_rel_to_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_unrealized_loss_rel_to_market_cap: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss_rel_to_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_neg_unrealized_loss_rel_to_market_cap:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss_rel_to_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
indexes_to_net_unrealized_pnl_rel_to_market_cap:
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl_rel_to_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
|
||||
// === Unrealized vs Own Market Cap (optional) ===
|
||||
height_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit_rel_to_own_market_cap"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss_rel_to_own_market_cap"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss_rel_to_own_market_cap"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl_rel_to_own_market_cap"),
|
||||
cfg.version + v2,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_unrealized_profit_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit_rel_to_own_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v2,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss_rel_to_own_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v2,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss_rel_to_own_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v2,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (extended && compute_rel_to_all)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl_rel_to_own_market_cap"),
|
||||
Source::Compute,
|
||||
cfg.version + v2,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
|
||||
// === Unrealized vs Own Total Unrealized PnL (optional) ===
|
||||
height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"),
|
||||
cfg.version + v0,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"),
|
||||
cfg.version + v0,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"),
|
||||
cfg.version + v0,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"),
|
||||
cfg.version + v1,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit_rel_to_own_total_unrealized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss_rel_to_own_total_unrealized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss_rel_to_own_total_unrealized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: extended
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl_rel_to_own_total_unrealized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)
|
||||
})
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Second phase of computed metrics (ratios, relative values).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
_height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
_dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
supply: &SupplyMetrics,
|
||||
_realized: Option<&RealizedMetrics>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
// Supply relative to circulating supply
|
||||
if let Some(v) = self.indexes_to_supply_rel_to_circulating_supply.as_mut() {
|
||||
v.compute_all(indexes, starting_indexes, exit, |v| {
|
||||
v.compute_percentage(
|
||||
starting_indexes.height,
|
||||
&supply.height_to_supply_value.bitcoin,
|
||||
height_to_supply,
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
let _ = (dateindex_to_supply, height_to_market_cap, dateindex_to_market_cap);
|
||||
|
||||
// Additional relative metrics computed here
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
//! Supply and UTXO count metrics.
|
||||
//!
|
||||
//! These metrics are always computed regardless of price data availability.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, StoredF64, StoredU64, Version};
|
||||
use vecdb::{
|
||||
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, PcoVec,
|
||||
TypedVecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{
|
||||
ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromHeight, Source,
|
||||
VecBuilderOptions,
|
||||
},
|
||||
indexes, price,
|
||||
states::SupplyState,
|
||||
};
|
||||
|
||||
use super::ImportConfig;
|
||||
|
||||
/// Supply and UTXO count metrics for a cohort.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct SupplyMetrics {
|
||||
/// Total supply at each height
|
||||
pub height_to_supply: EagerVec<PcoVec<Height, Sats>>,
|
||||
|
||||
/// Supply value in BTC and USD (computed from height_to_supply)
|
||||
pub height_to_supply_value: ComputedHeightValueVecs,
|
||||
|
||||
/// Supply indexed by date
|
||||
pub indexes_to_supply: ComputedValueVecsFromDateIndex,
|
||||
|
||||
/// UTXO count at each height
|
||||
pub height_to_utxo_count: EagerVec<PcoVec<Height, StoredU64>>,
|
||||
|
||||
/// UTXO count indexed by various dimensions
|
||||
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
|
||||
|
||||
/// Half of supply value (used for computing median)
|
||||
pub height_to_supply_half_value: ComputedHeightValueVecs,
|
||||
|
||||
/// Half of supply indexed by date
|
||||
pub indexes_to_supply_half: ComputedValueVecsFromDateIndex,
|
||||
}
|
||||
|
||||
impl SupplyMetrics {
|
||||
/// Import supply metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let v0 = Version::ZERO;
|
||||
let v1 = Version::ONE;
|
||||
let compute_dollars = cfg.compute_dollars();
|
||||
let last = VecBuilderOptions::default().add_last();
|
||||
|
||||
Ok(Self {
|
||||
height_to_supply: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
|
||||
height_to_supply_value: ComputedHeightValueVecs::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
compute_dollars,
|
||||
)?,
|
||||
|
||||
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply"),
|
||||
Source::Compute,
|
||||
cfg.version + v1,
|
||||
last,
|
||||
compute_dollars,
|
||||
cfg.indexes,
|
||||
)?,
|
||||
|
||||
height_to_utxo_count: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("utxo_count"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
|
||||
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("utxo_count"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
|
||||
height_to_supply_half_value: ComputedHeightValueVecs::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_half"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
compute_dollars,
|
||||
)?,
|
||||
|
||||
indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_half"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
last,
|
||||
compute_dollars,
|
||||
cfg.indexes,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get minimum length across height-indexed vectors.
|
||||
pub fn min_len(&self) -> usize {
|
||||
self.height_to_supply
|
||||
.len()
|
||||
.min(self.height_to_utxo_count.len())
|
||||
}
|
||||
|
||||
/// Push supply state values to height-indexed vectors.
|
||||
pub fn truncate_push(&mut self, height: Height, state: &SupplyState) -> Result<()> {
|
||||
self.height_to_supply.truncate_push(height, state.value)?;
|
||||
self.height_to_utxo_count
|
||||
.truncate_push(height, StoredU64::from(state.utxo_count))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush height-indexed vectors to disk.
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
self.height_to_supply.safe_write(exit)?;
|
||||
self.height_to_utxo_count.safe_write(exit)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate computed versions against base version.
|
||||
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
|
||||
// Validation logic for computed vecs
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute aggregate values from separate cohorts.
|
||||
pub fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_supply.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_supply).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_utxo_count.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_utxo_count).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// First phase of computed metrics (indexes from height).
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_supply_value.compute_rest(
|
||||
price,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_supply),
|
||||
)?;
|
||||
|
||||
self.indexes_to_supply
|
||||
.compute_all(price, starting_indexes, exit, |v| {
|
||||
let mut dateindex_to_height_count_iter =
|
||||
indexes.dateindex_to_height_count.into_iter();
|
||||
let mut height_to_supply_iter = self.height_to_supply.into_iter();
|
||||
v.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&indexes.dateindex_to_first_height,
|
||||
|(i, height, ..)| {
|
||||
let count = dateindex_to_height_count_iter.get_unwrap(i);
|
||||
if count == StoredU64::default() {
|
||||
unreachable!()
|
||||
}
|
||||
let supply = height_to_supply_iter.get_unwrap(height + (*count - 1));
|
||||
(i, supply)
|
||||
},
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.indexes_to_utxo_count.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_utxo_count),
|
||||
)?;
|
||||
|
||||
self.height_to_supply_half_value
|
||||
.compute_all(price, starting_indexes, exit, |v| {
|
||||
v.compute_transform(
|
||||
starting_indexes.height,
|
||||
&self.height_to_supply,
|
||||
|(h, v, ..)| (h, v / 2),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.indexes_to_supply_half
|
||||
.compute_all(price, starting_indexes, exit, |v| {
|
||||
v.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
self.indexes_to_supply.sats.dateindex.as_ref().unwrap(),
|
||||
|(i, sats, ..)| (i, sats / 2),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Second phase of computed metrics (ratios, relative values).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
_dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let _ = (indexes, price, height_to_supply, height_to_market_cap, dateindex_to_market_cap);
|
||||
|
||||
// Supply relative metrics computed here if needed
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
//! Unrealized profit/loss metrics.
|
||||
//!
|
||||
//! These metrics track paper gains/losses based on current vs acquisition price.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{DateIndex, Dollars, Height, Sats, Version};
|
||||
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{
|
||||
ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, Source,
|
||||
VecBuilderOptions,
|
||||
},
|
||||
states::UnrealizedState,
|
||||
};
|
||||
|
||||
use super::ImportConfig;
|
||||
|
||||
/// Unrealized profit/loss metrics.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct UnrealizedMetrics {
|
||||
// === Supply in Profit/Loss ===
|
||||
pub height_to_supply_in_profit: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex,
|
||||
pub height_to_supply_in_loss: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex,
|
||||
pub dateindex_to_supply_in_profit: EagerVec<PcoVec<DateIndex, Sats>>,
|
||||
pub dateindex_to_supply_in_loss: EagerVec<PcoVec<DateIndex, Sats>>,
|
||||
pub height_to_supply_in_profit_value: ComputedHeightValueVecs,
|
||||
pub height_to_supply_in_loss_value: ComputedHeightValueVecs,
|
||||
|
||||
// === Unrealized Profit/Loss ===
|
||||
pub height_to_unrealized_profit: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_unrealized_profit: ComputedVecsFromDateIndex<Dollars>,
|
||||
pub height_to_unrealized_loss: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_unrealized_loss: ComputedVecsFromDateIndex<Dollars>,
|
||||
pub dateindex_to_unrealized_profit: EagerVec<PcoVec<DateIndex, Dollars>>,
|
||||
pub dateindex_to_unrealized_loss: EagerVec<PcoVec<DateIndex, Dollars>>,
|
||||
|
||||
// === Negated and Net ===
|
||||
pub height_to_neg_unrealized_loss: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_neg_unrealized_loss: ComputedVecsFromDateIndex<Dollars>,
|
||||
pub height_to_net_unrealized_pnl: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex<Dollars>,
|
||||
pub height_to_total_unrealized_pnl: EagerVec<PcoVec<Height, Dollars>>,
|
||||
pub indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex<Dollars>,
|
||||
}
|
||||
|
||||
impl UnrealizedMetrics {
|
||||
/// Import unrealized metrics from database.
|
||||
pub fn forced_import(cfg: &ImportConfig) -> Result<Self> {
|
||||
let v0 = Version::ZERO;
|
||||
let compute_dollars = cfg.compute_dollars();
|
||||
let last = VecBuilderOptions::default().add_last();
|
||||
|
||||
// Pre-import the dateindex vecs that are used as sources
|
||||
let dateindex_to_supply_in_profit =
|
||||
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_profit"), cfg.version + v0)?;
|
||||
let dateindex_to_supply_in_loss =
|
||||
EagerVec::forced_import(cfg.db, &cfg.name("supply_in_loss"), cfg.version + v0)?;
|
||||
let dateindex_to_unrealized_profit =
|
||||
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_profit"), cfg.version + v0)?;
|
||||
let dateindex_to_unrealized_loss =
|
||||
EagerVec::forced_import(cfg.db, &cfg.name("unrealized_loss"), cfg.version + v0)?;
|
||||
|
||||
Ok(Self {
|
||||
// === Supply in Profit/Loss ===
|
||||
height_to_supply_in_profit: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_supply_in_profit: ComputedValueVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit"),
|
||||
Source::Vec(dateindex_to_supply_in_profit.boxed_clone()),
|
||||
cfg.version + v0,
|
||||
last,
|
||||
compute_dollars,
|
||||
cfg.indexes,
|
||||
)?,
|
||||
height_to_supply_in_loss: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_supply_in_loss: ComputedValueVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss"),
|
||||
Source::Vec(dateindex_to_supply_in_loss.boxed_clone()),
|
||||
cfg.version + v0,
|
||||
last,
|
||||
compute_dollars,
|
||||
cfg.indexes,
|
||||
)?,
|
||||
dateindex_to_supply_in_profit,
|
||||
dateindex_to_supply_in_loss,
|
||||
height_to_supply_in_profit_value: ComputedHeightValueVecs::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_profit"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
compute_dollars,
|
||||
)?,
|
||||
height_to_supply_in_loss_value: ComputedHeightValueVecs::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("supply_in_loss"),
|
||||
Source::None,
|
||||
cfg.version + v0,
|
||||
compute_dollars,
|
||||
)?,
|
||||
|
||||
// === Unrealized Profit/Loss ===
|
||||
height_to_unrealized_profit: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_unrealized_profit: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_profit"),
|
||||
Source::Vec(dateindex_to_unrealized_profit.boxed_clone()),
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
height_to_unrealized_loss: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_unrealized_loss: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("unrealized_loss"),
|
||||
Source::Vec(dateindex_to_unrealized_loss.boxed_clone()),
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
dateindex_to_unrealized_profit,
|
||||
dateindex_to_unrealized_loss,
|
||||
|
||||
// === Negated and Net ===
|
||||
height_to_neg_unrealized_loss: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_neg_unrealized_loss: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("neg_unrealized_loss"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
height_to_net_unrealized_pnl: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_net_unrealized_pnl: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("net_unrealized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
height_to_total_unrealized_pnl: EagerVec::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("total_unrealized_pnl"),
|
||||
cfg.version + v0,
|
||||
)?,
|
||||
indexes_to_total_unrealized_pnl: ComputedVecsFromDateIndex::forced_import(
|
||||
cfg.db,
|
||||
&cfg.name("total_unrealized_pnl"),
|
||||
Source::Compute,
|
||||
cfg.version + v0,
|
||||
cfg.indexes,
|
||||
last,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Push unrealized state values to height-indexed vectors.
|
||||
pub fn truncate_push(
|
||||
&mut self,
|
||||
height: Height,
|
||||
dateindex: Option<DateIndex>,
|
||||
height_state: &UnrealizedState,
|
||||
date_state: Option<&UnrealizedState>,
|
||||
) -> Result<()> {
|
||||
self.height_to_supply_in_profit
|
||||
.truncate_push(height, height_state.supply_in_profit)?;
|
||||
self.height_to_supply_in_loss
|
||||
.truncate_push(height, height_state.supply_in_loss)?;
|
||||
self.height_to_unrealized_profit
|
||||
.truncate_push(height, height_state.unrealized_profit)?;
|
||||
self.height_to_unrealized_loss
|
||||
.truncate_push(height, height_state.unrealized_loss)?;
|
||||
|
||||
if let (Some(dateindex), Some(date_state)) = (dateindex, date_state) {
|
||||
self.dateindex_to_supply_in_profit
|
||||
.truncate_push(dateindex, date_state.supply_in_profit)?;
|
||||
self.dateindex_to_supply_in_loss
|
||||
.truncate_push(dateindex, date_state.supply_in_loss)?;
|
||||
self.dateindex_to_unrealized_profit
|
||||
.truncate_push(dateindex, date_state.unrealized_profit)?;
|
||||
self.dateindex_to_unrealized_loss
|
||||
.truncate_push(dateindex, date_state.unrealized_loss)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush height-indexed vectors to disk.
|
||||
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
self.height_to_supply_in_profit.safe_write(exit)?;
|
||||
self.height_to_supply_in_loss.safe_write(exit)?;
|
||||
self.height_to_unrealized_profit.safe_write(exit)?;
|
||||
self.height_to_unrealized_loss.safe_write(exit)?;
|
||||
self.dateindex_to_supply_in_profit.safe_write(exit)?;
|
||||
self.dateindex_to_supply_in_loss.safe_write(exit)?;
|
||||
self.dateindex_to_unrealized_profit.safe_write(exit)?;
|
||||
self.dateindex_to_unrealized_loss.safe_write(exit)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compute aggregate values from separate cohorts.
|
||||
pub fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_supply_in_profit.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_supply_in_profit).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_supply_in_loss.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_supply_in_loss).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_unrealized_profit.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_unrealized_profit).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.height_to_unrealized_loss.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
&others.iter().map(|v| &v.height_to_unrealized_loss).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.dateindex_to_supply_in_profit.compute_sum_of_others(
|
||||
starting_indexes.dateindex,
|
||||
&others.iter().map(|v| &v.dateindex_to_supply_in_profit).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.dateindex_to_supply_in_loss.compute_sum_of_others(
|
||||
starting_indexes.dateindex,
|
||||
&others.iter().map(|v| &v.dateindex_to_supply_in_loss).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.dateindex_to_unrealized_profit.compute_sum_of_others(
|
||||
starting_indexes.dateindex,
|
||||
&others.iter().map(|v| &v.dateindex_to_unrealized_profit).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
self.dateindex_to_unrealized_loss.compute_sum_of_others(
|
||||
starting_indexes.dateindex,
|
||||
&others.iter().map(|v| &v.dateindex_to_unrealized_loss).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
//! Stateful computation for Bitcoin UTXO and address cohort metrics.
|
||||
//!
|
||||
//! This module processes blockchain data to compute metrics for various cohorts
|
||||
//! (groups of UTXOs or addresses filtered by age, amount, type, etc.).
|
||||
//!
|
||||
//! ## Module Structure
|
||||
//!
|
||||
//! ```text
|
||||
//! stateful/
|
||||
//! ├── address/ # Address type handling (indexes, data storage)
|
||||
//! ├── cohorts/ # Cohort traits and state management
|
||||
//! ├── compute/ # Block processing pipeline
|
||||
//! └── metrics/ # Metric vectors organized by category
|
||||
//! ```
|
||||
//!
|
||||
//! ## Data Flow
|
||||
//!
|
||||
//! 1. **Import**: Load from checkpoint or start fresh
|
||||
//! 2. **Process blocks**: For each block, process outputs/inputs in parallel
|
||||
//! 3. **Update cohorts**: Track supply, realized/unrealized P&L per cohort
|
||||
//! 4. **Flush**: Periodically checkpoint state to disk
|
||||
//! 5. **Compute aggregates**: Derive aggregate cohorts from separate cohorts
|
||||
|
||||
pub mod address;
|
||||
pub mod cohorts;
|
||||
pub mod compute;
|
||||
pub mod metrics;
|
||||
mod process;
|
||||
mod vecs;
|
||||
|
||||
use process::*;
|
||||
|
||||
pub use vecs::Vecs;
|
||||
|
||||
// Address re-exports
|
||||
pub use address::{
|
||||
AddressTypeToTypeIndexMap, AddressTypeToVec, AddressesDataVecs, AnyAddressIndexesVecs,
|
||||
HeightToAddressTypeToVec,
|
||||
};
|
||||
|
||||
// Cohort re-exports
|
||||
pub use cohorts::{
|
||||
AddressCohortVecs, AddressCohorts, CohortState, CohortVecs, DynCohortVecs, Flushable,
|
||||
HeightFlushable, UTXOCohortVecs, UTXOCohorts,
|
||||
};
|
||||
|
||||
// Compute re-exports
|
||||
pub use compute::{
|
||||
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
|
||||
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexerReaders, VecsReaders,
|
||||
};
|
||||
|
||||
// Metrics re-exports
|
||||
pub use metrics::{
|
||||
ActivityMetrics, CohortMetrics, ImportConfig, PricePaidMetrics, RealizedMetrics,
|
||||
RelativeMetrics, SupplyMetrics, UnrealizedMetrics,
|
||||
};
|
||||
@@ -0,0 +1,44 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{AddressDataSource, AnyAddressIndex, EmptyAddressData};
|
||||
|
||||
use crate::stateful_new::{AddressTypeToTypeIndexMap, AddressesDataVecs};
|
||||
|
||||
/// Process empty address data updates.
|
||||
///
|
||||
/// Handles three cases:
|
||||
/// - New empty address: push to empty storage
|
||||
/// - Updated empty address (was empty): update in place
|
||||
/// - Transition loaded -> empty: delete from loaded, push to empty
|
||||
pub fn process_empty_addresses(
|
||||
addresses_data: &mut AddressesDataVecs,
|
||||
empty_updates: AddressTypeToTypeIndexMap<AddressDataSource<EmptyAddressData>>,
|
||||
) -> Result<AddressTypeToTypeIndexMap<AnyAddressIndex>> {
|
||||
let mut result = AddressTypeToTypeIndexMap::default();
|
||||
|
||||
for (address_type, sorted) in empty_updates.into_sorted_iter() {
|
||||
for (typeindex, source) in sorted {
|
||||
match source {
|
||||
AddressDataSource::New(data) => {
|
||||
let index = addresses_data.empty.fill_first_hole_or_push(data)?;
|
||||
result
|
||||
.get_mut(address_type)
|
||||
.unwrap()
|
||||
.insert(typeindex, AnyAddressIndex::from(index));
|
||||
}
|
||||
AddressDataSource::FromEmpty((index, data)) => {
|
||||
addresses_data.empty.update(index, data)?;
|
||||
}
|
||||
AddressDataSource::FromLoaded((loaded_index, data)) => {
|
||||
addresses_data.loaded.delete(loaded_index);
|
||||
let empty_index = addresses_data.empty.fill_first_hole_or_push(data)?;
|
||||
result
|
||||
.get_mut(address_type)
|
||||
.unwrap()
|
||||
.insert(typeindex, AnyAddressIndex::from(empty_index));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Parallel input processing.
|
||||
//!
|
||||
//! Processes a block's inputs (spent UTXOs) in parallel, building:
|
||||
//! - height_to_sent: map from creation height -> Transacted for sends
|
||||
//! - Address data for address cohort tracking (optional)
|
||||
|
||||
use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TxOutIndex, TypeIndex};
|
||||
use rayon::prelude::*;
|
||||
use rustc_hash::FxHashMap;
|
||||
use vecdb::{BytesVec, GenericStoredVec, PcoVec};
|
||||
|
||||
use crate::{
|
||||
stateful_new::{IndexerReaders, process::RangeMap},
|
||||
states::Transacted,
|
||||
};
|
||||
|
||||
use super::super::address::HeightToAddressTypeToVec;
|
||||
|
||||
/// Result of processing inputs for a block.
|
||||
pub struct InputsResult {
|
||||
/// Map from UTXO creation height -> aggregated sent supply.
|
||||
pub height_to_sent: FxHashMap<Height, Transacted>,
|
||||
/// Per-height, per-address-type sent data: (typeindex, value) for each address.
|
||||
pub sent_data: HeightToAddressTypeToVec<(TypeIndex, Sats)>,
|
||||
}
|
||||
|
||||
/// Process inputs (spent UTXOs) for a block in parallel.
|
||||
///
|
||||
/// For each input:
|
||||
/// 1. Read outpoint, resolve to txoutindex
|
||||
/// 2. Get the creation height from txoutindex_to_height map
|
||||
/// 3. Read value and type from the referenced output
|
||||
/// 4. Accumulate into height_to_sent map
|
||||
/// 5. Track address-specific data if input references an address type
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn process_inputs(
|
||||
first_txinindex: usize,
|
||||
input_count: usize,
|
||||
txinindex_to_txindex: &[TxIndex],
|
||||
txinindex_to_outpoint: &PcoVec<TxInIndex, OutPoint>,
|
||||
txindex_to_first_txoutindex: &BytesVec<TxIndex, TxOutIndex>,
|
||||
txoutindex_to_value: &BytesVec<TxOutIndex, Sats>,
|
||||
txoutindex_to_outputtype: &BytesVec<TxOutIndex, OutputType>,
|
||||
txoutindex_to_typeindex: &BytesVec<TxOutIndex, TypeIndex>,
|
||||
txoutindex_to_height: &RangeMap<TxOutIndex, Height>,
|
||||
ir: &IndexerReaders,
|
||||
) -> InputsResult {
|
||||
let (height_to_sent, sent_data) = (first_txinindex..first_txinindex + input_count)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let txinindex = TxInIndex::from(i);
|
||||
let local_idx = i - first_txinindex;
|
||||
let _txindex = txinindex_to_txindex[local_idx];
|
||||
|
||||
// Get outpoint and resolve to txoutindex
|
||||
let outpoint = txinindex_to_outpoint.read_unwrap(txinindex, &ir.txinindex_to_outpoint);
|
||||
let first_txoutindex = txindex_to_first_txoutindex
|
||||
.read_unwrap(outpoint.txindex(), &ir.txindex_to_first_txoutindex);
|
||||
let txoutindex = first_txoutindex + outpoint.vout();
|
||||
|
||||
// Get creation height
|
||||
let prev_height = *txoutindex_to_height.get(txoutindex).unwrap();
|
||||
|
||||
// Get value and type from the output being spent
|
||||
let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value);
|
||||
let input_type =
|
||||
txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype);
|
||||
|
||||
// Non-address inputs don't need typeindex
|
||||
if input_type.is_not_address() {
|
||||
return (prev_height, value, input_type, None);
|
||||
}
|
||||
|
||||
let typeindex =
|
||||
txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
|
||||
|
||||
(prev_height, value, input_type, Some((typeindex, value)))
|
||||
})
|
||||
.fold(
|
||||
|| {
|
||||
(
|
||||
FxHashMap::<Height, Transacted>::default(),
|
||||
HeightToAddressTypeToVec::default(),
|
||||
)
|
||||
},
|
||||
|(mut height_to_sent, mut sent_data), (prev_height, value, output_type, addr_data)| {
|
||||
height_to_sent
|
||||
.entry(prev_height)
|
||||
.or_default()
|
||||
.iterate(value, output_type);
|
||||
|
||||
if let Some((typeindex, value)) = addr_data {
|
||||
sent_data
|
||||
.entry(prev_height)
|
||||
.or_default()
|
||||
.get_mut(output_type)
|
||||
.unwrap()
|
||||
.push((typeindex, value));
|
||||
}
|
||||
|
||||
(height_to_sent, sent_data)
|
||||
},
|
||||
)
|
||||
.reduce(
|
||||
|| {
|
||||
(
|
||||
FxHashMap::<Height, Transacted>::default(),
|
||||
HeightToAddressTypeToVec::default(),
|
||||
)
|
||||
},
|
||||
|(mut h1, mut s1), (h2, s2)| {
|
||||
// Merge height_to_sent maps
|
||||
for (k, v) in h2 {
|
||||
*h1.entry(k).or_default() += v;
|
||||
}
|
||||
|
||||
// Merge sent_data maps
|
||||
s1.merge_mut(s2);
|
||||
|
||||
(h1, s1)
|
||||
},
|
||||
);
|
||||
|
||||
InputsResult {
|
||||
height_to_sent,
|
||||
sent_data,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{AddressDataSource, AnyAddressIndex, LoadedAddressData};
|
||||
|
||||
use crate::stateful_new::{AddressTypeToTypeIndexMap, AddressesDataVecs};
|
||||
|
||||
/// Process loaded address data updates.
|
||||
///
|
||||
/// Handles three cases:
|
||||
/// - New loaded address: push to loaded storage
|
||||
/// - Updated loaded address (was loaded): update in place
|
||||
/// - Transition empty -> loaded: delete from empty, push to loaded
|
||||
pub fn process_loaded_addresses(
|
||||
addresses_data: &mut AddressesDataVecs,
|
||||
loaded_updates: AddressTypeToTypeIndexMap<AddressDataSource<LoadedAddressData>>,
|
||||
) -> Result<AddressTypeToTypeIndexMap<AnyAddressIndex>> {
|
||||
let mut result = AddressTypeToTypeIndexMap::default();
|
||||
|
||||
for (address_type, sorted) in loaded_updates.into_sorted_iter() {
|
||||
for (typeindex, source) in sorted {
|
||||
match source {
|
||||
AddressDataSource::New(data) => {
|
||||
let index = addresses_data.loaded.fill_first_hole_or_push(data)?;
|
||||
result
|
||||
.get_mut(address_type)
|
||||
.unwrap()
|
||||
.insert(typeindex, AnyAddressIndex::from(index));
|
||||
}
|
||||
AddressDataSource::FromLoaded((index, data)) => {
|
||||
addresses_data.loaded.update(index, data)?;
|
||||
}
|
||||
AddressDataSource::FromEmpty((empty_index, data)) => {
|
||||
addresses_data.empty.delete(empty_index);
|
||||
let loaded_index = addresses_data.loaded.fill_first_hole_or_push(data)?;
|
||||
result
|
||||
.get_mut(address_type)
|
||||
.unwrap()
|
||||
.insert(typeindex, AnyAddressIndex::from(loaded_index));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
mod empty_addresses;
|
||||
mod inputs;
|
||||
mod loaded_addresses;
|
||||
mod outputs;
|
||||
mod range_map;
|
||||
|
||||
pub use empty_addresses::*;
|
||||
pub use inputs::*;
|
||||
pub use loaded_addresses::*;
|
||||
pub use outputs::*;
|
||||
pub use range_map::*;
|
||||
@@ -0,0 +1,83 @@
|
||||
//! Parallel output processing.
|
||||
//!
|
||||
//! Processes a block's outputs (new UTXOs) in parallel, building:
|
||||
//! - Transacted: aggregated supply by output type and amount range
|
||||
//! - Address data for address cohort tracking (optional)
|
||||
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex, TypeIndex};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{BytesVec, GenericStoredVec};
|
||||
|
||||
use crate::{stateful_new::IndexerReaders, states::Transacted};
|
||||
|
||||
use super::super::address::AddressTypeToVec;
|
||||
|
||||
/// Result of processing outputs for a block.
|
||||
pub struct OutputsResult {
|
||||
/// Aggregated supply transacted in this block.
|
||||
pub transacted: Transacted,
|
||||
/// Per-address-type received data: (typeindex, value) for each address.
|
||||
pub received_data: AddressTypeToVec<(TypeIndex, Sats)>,
|
||||
}
|
||||
|
||||
/// Process outputs (new UTXOs) for a block in parallel.
|
||||
///
|
||||
/// For each output:
|
||||
/// 1. Read value and output type from indexer
|
||||
/// 2. Accumulate into Transacted by type and amount
|
||||
/// 3. Track address-specific data if output is an address type
|
||||
pub fn process_outputs(
|
||||
first_txoutindex: usize,
|
||||
output_count: usize,
|
||||
txoutindex_to_txindex: &[TxIndex],
|
||||
txoutindex_to_value: &BytesVec<TxOutIndex, Sats>,
|
||||
txoutindex_to_outputtype: &BytesVec<TxOutIndex, OutputType>,
|
||||
txoutindex_to_typeindex: &BytesVec<TxOutIndex, TypeIndex>,
|
||||
ir: &IndexerReaders,
|
||||
) -> OutputsResult {
|
||||
let (transacted, received_data) = (first_txoutindex..first_txoutindex + output_count)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let txoutindex = TxOutIndex::from(i);
|
||||
let local_idx = i - first_txoutindex;
|
||||
let _txindex = txoutindex_to_txindex[local_idx];
|
||||
|
||||
let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value);
|
||||
let output_type =
|
||||
txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype);
|
||||
|
||||
// Non-address outputs don't need typeindex
|
||||
if output_type.is_not_address() {
|
||||
return (value, output_type, None);
|
||||
}
|
||||
|
||||
let typeindex =
|
||||
txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
|
||||
|
||||
(value, output_type, Some((typeindex, value)))
|
||||
})
|
||||
.fold(
|
||||
|| (Transacted::default(), AddressTypeToVec::default()),
|
||||
|(mut transacted, mut received_data), (value, output_type, addr_data)| {
|
||||
transacted.iterate(value, output_type);
|
||||
|
||||
if let Some((typeindex, value)) = addr_data {
|
||||
received_data
|
||||
.get_mut(output_type)
|
||||
.unwrap()
|
||||
.push((typeindex, value));
|
||||
}
|
||||
|
||||
(transacted, received_data)
|
||||
},
|
||||
)
|
||||
.reduce(
|
||||
|| (Transacted::default(), AddressTypeToVec::default()),
|
||||
|(t1, r1), (t2, r2)| (t1 + t2, r1.merge(r2)),
|
||||
);
|
||||
|
||||
OutputsResult {
|
||||
transacted,
|
||||
received_data,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Main block processing loop.
|
||||
//!
|
||||
//! Iterates through blocks, processing outputs (receive) and inputs (send) in parallel.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use brk_types::{Height, TxOutIndex};
|
||||
use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex};
|
||||
|
||||
/// Maps ranges of indices to their corresponding height.
|
||||
/// Used to efficiently look up which block a txoutindex belongs to.
|
||||
#[derive(Debug)]
|
||||
pub struct RangeMap<I, T>(BTreeMap<I, T>);
|
||||
|
||||
impl<I, T> RangeMap<I, T>
|
||||
where
|
||||
I: VecIndex,
|
||||
T: VecIndex,
|
||||
{
|
||||
/// Look up value for a key using range search.
|
||||
/// Returns the value associated with the largest key <= given key.
|
||||
#[inline]
|
||||
pub fn get(&self, key: I) -> Option<&T> {
|
||||
self.0.range(..=key).next_back().map(|(_, value)| value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> From<&BytesVec<I, T>> for RangeMap<T, I>
|
||||
where
|
||||
I: VecIndex,
|
||||
T: VecIndex + BytesVecValue,
|
||||
{
|
||||
#[inline]
|
||||
fn from(vec: &BytesVec<I, T>) -> Self {
|
||||
Self(
|
||||
vec.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (v, I::from(i)))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> From<&PcoVec<I, T>> for RangeMap<T, I>
|
||||
where
|
||||
I: VecIndex,
|
||||
T: VecIndex + PcoVecValue,
|
||||
{
|
||||
#[inline]
|
||||
fn from(vec: &PcoVec<I, T>) -> Self {
|
||||
Self(
|
||||
vec.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (v, I::from(i)))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a RangeMap from height_to_first_txoutindex for fast txoutindex -> height lookups.
|
||||
pub fn build_txoutindex_to_height_map(
|
||||
height_to_first_txoutindex: &PcoVec<Height, TxOutIndex>,
|
||||
) -> RangeMap<TxOutIndex, Height> {
|
||||
RangeMap::from(height_to_first_txoutindex)
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
//! Main Vecs struct for stateful computation.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Dollars, Height, Sats, StoredU64, Version};
|
||||
use vecdb::{
|
||||
BytesVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, LazyVecFrom1,
|
||||
PAGE_SIZE, PcoVec,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
Indexes, SupplyState, chain,
|
||||
grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions},
|
||||
indexes, price,
|
||||
};
|
||||
|
||||
use super::{
|
||||
AddressCohorts, AddressesDataVecs, AnyAddressIndexesVecs, UTXOCohorts,
|
||||
address::{AddressTypeToHeightToAddressCount, AddressTypeToIndexesToAddressCount},
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::new(21);
|
||||
|
||||
/// Main struct holding all computed vectors and state for stateful computation.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct Vecs {
|
||||
#[traversable(skip)]
|
||||
db: Database,
|
||||
|
||||
// ---
|
||||
// States
|
||||
// ---
|
||||
pub chain_state: BytesVec<Height, SupplyState>,
|
||||
pub any_address_indexes: AnyAddressIndexesVecs,
|
||||
pub addresses_data: AddressesDataVecs,
|
||||
pub utxo_cohorts: UTXOCohorts,
|
||||
pub address_cohorts: AddressCohorts,
|
||||
|
||||
pub height_to_unspendable_supply: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub height_to_opreturn_supply: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount,
|
||||
pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount,
|
||||
|
||||
// ---
|
||||
// Computed
|
||||
// ---
|
||||
pub addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount,
|
||||
pub addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount,
|
||||
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
|
||||
pub indexes_to_empty_addr_count: ComputedVecsFromHeight<StoredU64>,
|
||||
pub height_to_market_cap: Option<LazyVecFrom1<Height, Dollars, Height, Dollars>>,
|
||||
pub indexes_to_market_cap: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
}
|
||||
|
||||
const SAVED_STAMPED_CHANGES: u16 = 10;
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
parent: &Path,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
) -> Result<Self> {
|
||||
let db_path = parent.join("stateful");
|
||||
let states_path = db_path.join("states");
|
||||
|
||||
let db = Database::open(&db_path)?;
|
||||
db.set_min_len(PAGE_SIZE * 20_000_000)?;
|
||||
db.set_min_regions(50_000)?;
|
||||
|
||||
let compute_dollars = price.is_some();
|
||||
let v0 = version + VERSION + Version::ZERO;
|
||||
let v2 = version + VERSION + Version::TWO;
|
||||
|
||||
let utxo_cohorts = UTXOCohorts::forced_import(&db, version, indexes, price, &states_path)?;
|
||||
|
||||
Ok(Self {
|
||||
chain_state: BytesVec::forced_import_with(
|
||||
vecdb::ImportOptions::new(&db, "chain", v0)
|
||||
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
|
||||
)?,
|
||||
|
||||
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
|
||||
height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?,
|
||||
|
||||
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
|
||||
&db,
|
||||
"addr_count",
|
||||
Source::Compute,
|
||||
v0,
|
||||
indexes,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_empty_addr_count: ComputedVecsFromHeight::forced_import(
|
||||
&db,
|
||||
"empty_addr_count",
|
||||
Source::Compute,
|
||||
v0,
|
||||
indexes,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
|
||||
height_to_market_cap: compute_dollars.then(|| {
|
||||
LazyVecFrom1::init(
|
||||
"market_cap",
|
||||
v0,
|
||||
utxo_cohorts
|
||||
.all
|
||||
.metrics
|
||||
.supply
|
||||
.height_to_supply_value
|
||||
.dollars
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.boxed_clone(),
|
||||
|height: Height, iter| iter.get(height),
|
||||
)
|
||||
}),
|
||||
|
||||
indexes_to_market_cap: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
&db,
|
||||
"market_cap",
|
||||
Source::Compute,
|
||||
v2,
|
||||
indexes,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
|
||||
addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::forced_import(
|
||||
&db,
|
||||
"addr_count",
|
||||
v0,
|
||||
)?,
|
||||
addresstype_to_height_to_empty_addr_count:
|
||||
AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?,
|
||||
addresstype_to_indexes_to_addr_count:
|
||||
AddressTypeToIndexesToAddressCount::forced_import(&db, "addr_count", v0, indexes)?,
|
||||
addresstype_to_indexes_to_empty_addr_count:
|
||||
AddressTypeToIndexesToAddressCount::forced_import(
|
||||
&db,
|
||||
"empty_addr_count",
|
||||
v0,
|
||||
indexes,
|
||||
)?,
|
||||
|
||||
utxo_cohorts,
|
||||
|
||||
address_cohorts: AddressCohorts::forced_import(
|
||||
&db,
|
||||
version,
|
||||
indexes,
|
||||
price,
|
||||
&states_path,
|
||||
)?,
|
||||
|
||||
any_address_indexes: AnyAddressIndexesVecs::forced_import(&db, v0)?,
|
||||
addresses_data: AddressesDataVecs::forced_import(&db, v0)?,
|
||||
|
||||
db,
|
||||
})
|
||||
}
|
||||
|
||||
/// Main computation loop.
|
||||
///
|
||||
/// Processes blocks to compute UTXO and address cohort metrics:
|
||||
/// 1. Recovers state from checkpoints or starts fresh
|
||||
/// 2. Iterates through blocks, processing outputs/inputs in parallel
|
||||
/// 3. Flushes checkpoints periodically
|
||||
/// 4. Computes aggregate cohorts from separate cohorts
|
||||
/// 5. Computes derived metrics
|
||||
///
|
||||
/// NOTE: This is a placeholder. The full implementation needs to be ported
|
||||
/// from stateful/mod.rs once all the supporting methods on UTXOCohorts,
|
||||
/// AddressCohorts, and state types are implemented.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
_indexer: &Indexer,
|
||||
_indexes: &indexes::Vecs,
|
||||
_chain: &chain::Vecs,
|
||||
_price: Option<&price::Vecs>,
|
||||
_starting_indexes: &mut Indexes,
|
||||
_exit: &Exit,
|
||||
) -> Result<()> {
|
||||
// The full compute implementation requires these methods to be implemented:
|
||||
//
|
||||
// On UTXOCohorts:
|
||||
// - tick_tock_next_block(&chain_state, timestamp)
|
||||
// - receive(transacted, height, price)
|
||||
// - send(height_to_sent, &mut chain_state)
|
||||
// - truncate_push_aggregate_percentiles(height)
|
||||
// - import_aggregate_price_to_amount(height)
|
||||
// - reset_aggregate_price_to_amount()
|
||||
//
|
||||
// On UTXOCohortState:
|
||||
// - reset_block_values()
|
||||
// - reset_price_to_amount()
|
||||
//
|
||||
// On AddressCohortState:
|
||||
// - inner.reset_block_values()
|
||||
// - inner.reset_price_to_amount()
|
||||
//
|
||||
// On AddressTypeToHeightToAddressCount:
|
||||
// - safe_flush(exit)
|
||||
// - truncate_push(height, &count)
|
||||
//
|
||||
// See stateful/mod.rs:368-1397 for the full implementation.
|
||||
//
|
||||
// The basic structure is:
|
||||
// 1. Validate computed versions against base version
|
||||
// 2. Find min stateful height and recover state
|
||||
// 3. For each block:
|
||||
// a. Reset per-block values
|
||||
// b. Process outputs in parallel (receive)
|
||||
// c. Process inputs in parallel (send)
|
||||
// d. Push to height-indexed vectors
|
||||
// e. Flush checkpoint every 10,000 blocks
|
||||
// 4. Compute aggregate cohorts from separate cohorts
|
||||
// 5. Compute rest_part1 (dateindex mappings)
|
||||
// 6. Compute rest_part2 (ratios and relative metrics)
|
||||
|
||||
self.db.compact()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{CohortContext, Filter, Filtered};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
|
||||
use vecdb::{
|
||||
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec,
|
||||
PcoVec, TypedVecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
Indexes,
|
||||
grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
|
||||
indexes, price,
|
||||
stateful::{
|
||||
common,
|
||||
r#trait::{CohortVecs, DynCohortVecs},
|
||||
},
|
||||
states::AddressCohortState,
|
||||
utils::OptionExt,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct Vecs {
|
||||
starting_height: Option<Height>,
|
||||
|
||||
#[traversable(skip)]
|
||||
pub state: Option<AddressCohortState>,
|
||||
|
||||
#[traversable(flatten)]
|
||||
pub inner: common::Vecs,
|
||||
|
||||
pub height_to_addr_count: EagerVec<PcoVec<Height, StoredU64>>,
|
||||
pub indexes_to_addr_count: ComputedVecsFromHeight<StoredU64>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
filter: Filter,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: Option<&Path>,
|
||||
) -> Result<Self> {
|
||||
let compute_dollars = price.is_some();
|
||||
|
||||
let full_name = filter.to_full_name(CohortContext::Address);
|
||||
let suffix = |s: &str| {
|
||||
if full_name.is_empty() {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{full_name}_{s}")
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
starting_height: None,
|
||||
state: states_path.map(|states_path| {
|
||||
AddressCohortState::new(states_path, &full_name, compute_dollars)
|
||||
}),
|
||||
height_to_addr_count: EagerVec::forced_import(
|
||||
db,
|
||||
&suffix("addr_count"),
|
||||
version + VERSION + Version::ZERO,
|
||||
)?,
|
||||
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
|
||||
db,
|
||||
&suffix("addr_count"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexes,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
inner: common::Vecs::forced_import(
|
||||
db,
|
||||
filter,
|
||||
CohortContext::Address,
|
||||
version,
|
||||
indexes,
|
||||
price,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl DynCohortVecs for Vecs {
|
||||
fn min_height_vecs_len(&self) -> usize {
|
||||
std::cmp::min(
|
||||
self.height_to_addr_count.len(),
|
||||
self.inner.min_height_vecs_len(),
|
||||
)
|
||||
}
|
||||
|
||||
fn reset_state_starting_height(&mut self) {
|
||||
self.starting_height = Some(Height::ZERO);
|
||||
}
|
||||
|
||||
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
|
||||
let starting_height = self
|
||||
.inner
|
||||
.import_state(starting_height, &mut self.state.um().inner)?;
|
||||
|
||||
self.starting_height = Some(starting_height);
|
||||
|
||||
if let Some(prev_height) = starting_height.decremented() {
|
||||
self.state.um().addr_count = *self
|
||||
.height_to_addr_count
|
||||
.into_iter()
|
||||
.get_unwrap(prev_height);
|
||||
}
|
||||
|
||||
Ok(starting_height)
|
||||
}
|
||||
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
self.height_to_addr_count
|
||||
.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_addr_count.inner_version(),
|
||||
)?;
|
||||
|
||||
self.inner.validate_computed_versions(base_version)
|
||||
}
|
||||
|
||||
fn truncate_push(&mut self, height: Height) -> Result<()> {
|
||||
if self.starting_height.unwrap() > height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.height_to_addr_count
|
||||
.truncate_push(height, self.state.u().addr_count.into())?;
|
||||
|
||||
self.inner
|
||||
.truncate_push(height, &self.state.u().inner)
|
||||
}
|
||||
|
||||
fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
) -> Result<()> {
|
||||
self.inner.compute_then_truncate_push_unrealized_states(
|
||||
height,
|
||||
height_price,
|
||||
dateindex,
|
||||
date_price,
|
||||
&self.state.u().inner,
|
||||
)
|
||||
}
|
||||
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.height_to_addr_count.safe_write(exit)?;
|
||||
|
||||
self.inner
|
||||
.safe_flush_stateful_vecs(height, exit, &mut self.state.um().inner)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.indexes_to_addr_count.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&self.height_to_addr_count),
|
||||
)?;
|
||||
|
||||
self.inner
|
||||
.compute_rest_part1(indexes, price, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl CohortVecs for Vecs {
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.height_to_addr_count.compute_sum_of_others(
|
||||
starting_indexes.height,
|
||||
others
|
||||
.iter()
|
||||
.map(|v| &v.height_to_addr_count)
|
||||
.collect::<Vec<_>>()
|
||||
.as_slice(),
|
||||
exit,
|
||||
)?;
|
||||
self.inner.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.inner).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.inner.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Filtered for Vecs {
|
||||
fn filter(&self) -> &Filter {
|
||||
&self.inner.filter
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{AddressGroups, AmountFilter, Filter, Filtered};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{Database, Exit, IterableVec};
|
||||
|
||||
use crate::{
|
||||
Indexes, indexes, price,
|
||||
stateful::{
|
||||
address_cohort,
|
||||
r#trait::{CohortVecs, DynCohortVecs},
|
||||
},
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::new(0);
|
||||
|
||||
#[derive(Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct Vecs(AddressGroups<address_cohort::Vecs>);
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: &Path,
|
||||
) -> Result<Self> {
|
||||
Ok(Self(AddressGroups::new(|filter| {
|
||||
let states_path = match &filter {
|
||||
Filter::Amount(AmountFilter::Range(_)) => Some(states_path),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
address_cohort::Vecs::forced_import(
|
||||
db,
|
||||
filter,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexes,
|
||||
price,
|
||||
states_path,
|
||||
)
|
||||
.unwrap()
|
||||
})))
|
||||
}
|
||||
|
||||
pub fn compute_overlapping_vecs(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let by_size_range = &self.0.amount_range;
|
||||
|
||||
[
|
||||
self.0
|
||||
.ge_amount
|
||||
.par_iter_mut()
|
||||
.map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
self.0
|
||||
.lt_amount
|
||||
.par_iter_mut()
|
||||
.map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.try_for_each(|(vecs, stateful)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.par_iter_mut()
|
||||
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.0.par_iter_mut().try_for_each(|v| {
|
||||
v.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.par_iter_separate_mut()
|
||||
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{
|
||||
AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData,
|
||||
LoadedAddressIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex,
|
||||
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
|
||||
TypeIndex,
|
||||
};
|
||||
use vecdb::{AnyStoredVec, BytesVec, GenericStoredVec, Reader, Stamp};
|
||||
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct AnyAddressIndexesVecs {
|
||||
pub p2pk33: BytesVec<P2PK33AddressIndex, AnyAddressIndex>,
|
||||
pub p2pk65: BytesVec<P2PK65AddressIndex, AnyAddressIndex>,
|
||||
pub p2pkh: BytesVec<P2PKHAddressIndex, AnyAddressIndex>,
|
||||
pub p2sh: BytesVec<P2SHAddressIndex, AnyAddressIndex>,
|
||||
pub p2tr: BytesVec<P2TRAddressIndex, AnyAddressIndex>,
|
||||
pub p2wpkh: BytesVec<P2WPKHAddressIndex, AnyAddressIndex>,
|
||||
pub p2wsh: BytesVec<P2WSHAddressIndex, AnyAddressIndex>,
|
||||
pub p2a: BytesVec<P2AAddressIndex, AnyAddressIndex>,
|
||||
}
|
||||
|
||||
impl AnyAddressIndexesVecs {
|
||||
pub fn min_stamped_height(&self) -> Height {
|
||||
Height::from(self.p2pk33.stamp())
|
||||
.incremented()
|
||||
.min(Height::from(self.p2pk65.stamp()).incremented())
|
||||
.min(Height::from(self.p2pkh.stamp()).incremented())
|
||||
.min(Height::from(self.p2sh.stamp()).incremented())
|
||||
.min(Height::from(self.p2tr.stamp()).incremented())
|
||||
.min(Height::from(self.p2wpkh.stamp()).incremented())
|
||||
.min(Height::from(self.p2wsh.stamp()).incremented())
|
||||
.min(Height::from(self.p2a.stamp()).incremented())
|
||||
}
|
||||
|
||||
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 8]> {
|
||||
Ok([
|
||||
self.p2pk33.rollback_before(stamp)?,
|
||||
self.p2pk65.rollback_before(stamp)?,
|
||||
self.p2pkh.rollback_before(stamp)?,
|
||||
self.p2sh.rollback_before(stamp)?,
|
||||
self.p2tr.rollback_before(stamp)?,
|
||||
self.p2wpkh.rollback_before(stamp)?,
|
||||
self.p2wsh.rollback_before(stamp)?,
|
||||
self.p2a.rollback_before(stamp)?,
|
||||
])
|
||||
}
|
||||
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
self.p2pk33.reset()?;
|
||||
self.p2pk65.reset()?;
|
||||
self.p2pkh.reset()?;
|
||||
self.p2sh.reset()?;
|
||||
self.p2tr.reset()?;
|
||||
self.p2wpkh.reset()?;
|
||||
self.p2wsh.reset()?;
|
||||
self.p2a.reset()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_anyaddressindex(
|
||||
&self,
|
||||
address_type: OutputType,
|
||||
typeindex: TypeIndex,
|
||||
reader: &Reader,
|
||||
) -> AnyAddressIndex {
|
||||
match address_type {
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2PKH => self
|
||||
.p2pkh
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2SH => self
|
||||
.p2sh
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2TR => self
|
||||
.p2tr
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkh
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2WSH => self
|
||||
.p2wsh
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
OutputType::P2A => self
|
||||
.p2a
|
||||
.get_pushed_or_read_at_unwrap(typeindex.into(), reader),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_anyaddressindex_once(
|
||||
&self,
|
||||
address_type: OutputType,
|
||||
typeindex: TypeIndex,
|
||||
) -> Result<AnyAddressIndex> {
|
||||
match address_type {
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2PKH => self
|
||||
.p2pkh
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2SH => self
|
||||
.p2sh
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2TR => self
|
||||
.p2tr
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkh
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2WSH => self
|
||||
.p2wsh
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
OutputType::P2A => self
|
||||
.p2a
|
||||
.read_at_once(typeindex.into())
|
||||
.map_err(|e| e.into()),
|
||||
_ => Err(Error::UnsupportedType(address_type.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_or_push(
|
||||
&mut self,
|
||||
address_type: OutputType,
|
||||
typeindex: TypeIndex,
|
||||
anyaddressindex: AnyAddressIndex,
|
||||
) -> Result<()> {
|
||||
(match address_type {
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33
|
||||
.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65
|
||||
.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2PKH => self.p2pkh.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2SH => self.p2sh.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2TR => self.p2tr.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkh
|
||||
.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2WSH => self.p2wsh.update_or_push(typeindex.into(), anyaddressindex),
|
||||
OutputType::P2A => self.p2a.update_or_push(typeindex.into(), anyaddressindex),
|
||||
_ => unreachable!(),
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn stamped_flush_maybe_with_changes(
|
||||
&mut self,
|
||||
stamp: Stamp,
|
||||
with_changes: bool,
|
||||
) -> Result<()> {
|
||||
self.p2pk33
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2pk65
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2pkh
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2sh
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2tr
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2wpkh
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2wsh
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.p2a
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct AddressesDataVecs {
|
||||
pub loaded: BytesVec<LoadedAddressIndex, LoadedAddressData>,
|
||||
pub empty: BytesVec<EmptyAddressIndex, EmptyAddressData>,
|
||||
}
|
||||
|
||||
impl AddressesDataVecs {
|
||||
pub fn min_stamped_height(&self) -> Height {
|
||||
Height::from(self.loaded.stamp())
|
||||
.incremented()
|
||||
.min(Height::from(self.empty.stamp()).incremented())
|
||||
}
|
||||
|
||||
pub fn rollback_before(&mut self, stamp: Stamp) -> Result<[Stamp; 2]> {
|
||||
Ok([
|
||||
self.loaded.rollback_before(stamp)?,
|
||||
self.empty.rollback_before(stamp)?,
|
||||
])
|
||||
}
|
||||
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
self.loaded.reset()?;
|
||||
self.empty.reset()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn stamped_flush_maybe_with_changes(
|
||||
&mut self,
|
||||
stamp: Stamp,
|
||||
with_changes: bool,
|
||||
) -> Result<()> {
|
||||
self.loaded
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
self.empty
|
||||
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_types::Height;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use vecdb::TypedVecIterator;
|
||||
|
||||
use super::AddressTypeToHeightToAddressCount;
|
||||
|
||||
#[derive(Debug, Default, Deref, DerefMut)]
|
||||
pub struct AddressTypeToAddressCount(ByAddressType<u64>);
|
||||
|
||||
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
|
||||
#[inline]
|
||||
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
|
||||
if let Some(prev_height) = starting_height.decremented() {
|
||||
Self(ByAddressType {
|
||||
p2pk65: groups.p2pk65.into_iter().get_unwrap(prev_height).into(),
|
||||
p2pk33: groups.p2pk33.into_iter().get_unwrap(prev_height).into(),
|
||||
p2pkh: groups.p2pkh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2sh: groups.p2sh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2wpkh: groups.p2wpkh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2wsh: groups.p2wsh.into_iter().get_unwrap(prev_height).into(),
|
||||
p2tr: groups.p2tr.into_iter().get_unwrap(prev_height).into(),
|
||||
p2a: groups.p2a.into_iter().get_unwrap(prev_height).into(),
|
||||
})
|
||||
} else {
|
||||
Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use brk_error::Result;
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Height, StoredU64};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use vecdb::{PcoVec, EagerVec, GenericStoredVec};
|
||||
|
||||
use super::AddressTypeToAddressCount;
|
||||
|
||||
#[derive(Debug, Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>);
|
||||
|
||||
impl From<ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>> for AddressTypeToHeightToAddressCount {
|
||||
#[inline]
|
||||
fn from(value: ByAddressType<EagerVec<PcoVec<Height, StoredU64>>>) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl AddressTypeToHeightToAddressCount {
|
||||
pub fn truncate_push(
|
||||
&mut self,
|
||||
height: Height,
|
||||
addresstype_to_usize: &AddressTypeToAddressCount,
|
||||
) -> Result<()> {
|
||||
self.p2pk65
|
||||
.truncate_push(height, addresstype_to_usize.p2pk65.into())?;
|
||||
self.p2pk33
|
||||
.truncate_push(height, addresstype_to_usize.p2pk33.into())?;
|
||||
self.p2pkh
|
||||
.truncate_push(height, addresstype_to_usize.p2pkh.into())?;
|
||||
self.p2sh
|
||||
.truncate_push(height, addresstype_to_usize.p2sh.into())?;
|
||||
self.p2wpkh
|
||||
.truncate_push(height, addresstype_to_usize.p2wpkh.into())?;
|
||||
self.p2wsh
|
||||
.truncate_push(height, addresstype_to_usize.p2wsh.into())?;
|
||||
self.p2tr
|
||||
.truncate_push(height, addresstype_to_usize.p2tr.into())?;
|
||||
self.p2a
|
||||
.truncate_push(height, addresstype_to_usize.p2a.into())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use brk_types::Height;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::stateful::AddressTypeToVec;
|
||||
|
||||
#[derive(Debug, Default, Deref, DerefMut)]
|
||||
pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);
|
||||
@@ -0,0 +1,80 @@
|
||||
use brk_error::Result;
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::StoredU64;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use vecdb::Exit;
|
||||
|
||||
use crate::{Indexes, grouped::ComputedVecsFromHeight, indexes};
|
||||
|
||||
use super::AddressTypeToHeightToAddressCount;
|
||||
|
||||
#[derive(Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct AddressTypeToIndexesToAddressCount(ByAddressType<ComputedVecsFromHeight<StoredU64>>);
|
||||
|
||||
impl From<ByAddressType<ComputedVecsFromHeight<StoredU64>>> for AddressTypeToIndexesToAddressCount {
|
||||
#[inline]
|
||||
fn from(value: ByAddressType<ComputedVecsFromHeight<StoredU64>>) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl AddressTypeToIndexesToAddressCount {
|
||||
pub fn compute(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
|
||||
) -> Result<()> {
|
||||
self.p2pk65.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2pk65),
|
||||
)?;
|
||||
self.p2pk33.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2pk33),
|
||||
)?;
|
||||
self.p2pkh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2pkh),
|
||||
)?;
|
||||
self.p2sh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2sh),
|
||||
)?;
|
||||
self.p2wpkh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2wpkh),
|
||||
)?;
|
||||
self.p2wsh.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2wsh),
|
||||
)?;
|
||||
self.p2tr.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2tr),
|
||||
)?;
|
||||
self.p2a.compute_rest(
|
||||
indexes,
|
||||
starting_indexes,
|
||||
exit,
|
||||
Some(&addresstype_to_height_to_addresscount.p2a),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
mod addresscount;
|
||||
mod height_to_addresscount;
|
||||
mod height_to_vec;
|
||||
mod indexes_to_addresscount;
|
||||
mod typeindex_map;
|
||||
mod vec;
|
||||
|
||||
pub use addresscount::*;
|
||||
pub use height_to_addresscount::*;
|
||||
pub use height_to_vec::*;
|
||||
pub use indexes_to_addresscount::*;
|
||||
pub use typeindex_map::*;
|
||||
pub use vec::*;
|
||||
@@ -0,0 +1,100 @@
|
||||
use std::{collections::hash_map::Entry, mem};
|
||||
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_types::{OutputType, TypeIndex};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rustc_hash::FxHashMap;
|
||||
use smallvec::{Array, SmallVec};
|
||||
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct AddressTypeToTypeIndexMap<T>(ByAddressType<FxHashMap<TypeIndex, T>>);
|
||||
|
||||
impl<T> AddressTypeToTypeIndexMap<T> {
|
||||
pub fn merge(mut self, mut other: Self) -> Self {
|
||||
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
|
||||
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
|
||||
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
|
||||
Self::merge_(&mut self.p2sh, &mut other.p2sh);
|
||||
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
|
||||
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
|
||||
Self::merge_(&mut self.p2tr, &mut other.p2tr);
|
||||
Self::merge_(&mut self.p2a, &mut other.p2a);
|
||||
self
|
||||
}
|
||||
|
||||
fn merge_(own: &mut FxHashMap<TypeIndex, T>, other: &mut FxHashMap<TypeIndex, T>) {
|
||||
if own.len() < other.len() {
|
||||
mem::swap(own, other);
|
||||
}
|
||||
own.extend(other.drain());
|
||||
}
|
||||
|
||||
// pub fn get_for_type(&self, address_type: OutputType, typeindex: &TypeIndex) -> Option<&T> {
|
||||
// self.get(address_type).unwrap().get(typeindex)
|
||||
// }
|
||||
|
||||
pub fn insert_for_type(&mut self, address_type: OutputType, typeindex: TypeIndex, value: T) {
|
||||
self.get_mut(address_type).unwrap().insert(typeindex, value);
|
||||
}
|
||||
|
||||
pub fn remove_for_type(&mut self, address_type: OutputType, typeindex: &TypeIndex) -> T {
|
||||
self.get_mut(address_type)
|
||||
.unwrap()
|
||||
.remove(typeindex)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn into_sorted_iter(self) -> impl Iterator<Item = (OutputType, Vec<(TypeIndex, T)>)> {
|
||||
self.0.into_iter().map(|(output_type, map)| {
|
||||
let mut sorted: Vec<_> = map.into_iter().collect();
|
||||
sorted.sort_unstable_by_key(|(typeindex, _)| *typeindex);
|
||||
(output_type, sorted)
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::should_implement_trait)]
|
||||
pub fn into_iter(self) -> impl Iterator<Item = (OutputType, FxHashMap<TypeIndex, T>)> {
|
||||
self.0.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for AddressTypeToTypeIndexMap<T> {
|
||||
fn default() -> Self {
|
||||
Self(ByAddressType {
|
||||
p2pk65: FxHashMap::default(),
|
||||
p2pk33: FxHashMap::default(),
|
||||
p2pkh: FxHashMap::default(),
|
||||
p2sh: FxHashMap::default(),
|
||||
p2wpkh: FxHashMap::default(),
|
||||
p2wsh: FxHashMap::default(),
|
||||
p2tr: FxHashMap::default(),
|
||||
p2a: FxHashMap::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> AddressTypeToTypeIndexMap<SmallVec<T>>
|
||||
where
|
||||
T: Array,
|
||||
{
|
||||
pub fn merge_vec(mut self, other: Self) -> Self {
|
||||
for (address_type, other_map) in other.0.into_iter() {
|
||||
let self_map = self.0.get_mut_unwrap(address_type);
|
||||
for (typeindex, mut other_vec) in other_map {
|
||||
match self_map.entry(typeindex) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
let self_vec = entry.get_mut();
|
||||
if other_vec.len() > self_vec.len() {
|
||||
mem::swap(self_vec, &mut other_vec);
|
||||
}
|
||||
self_vec.extend(other_vec);
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(other_vec);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
use std::mem;
|
||||
|
||||
use brk_grouper::ByAddressType;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
|
||||
|
||||
impl<T> AddressTypeToVec<T> {
|
||||
pub fn merge(mut self, mut other: Self) -> Self {
|
||||
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
|
||||
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
|
||||
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
|
||||
Self::merge_(&mut self.p2sh, &mut other.p2sh);
|
||||
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
|
||||
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
|
||||
Self::merge_(&mut self.p2tr, &mut other.p2tr);
|
||||
Self::merge_(&mut self.p2a, &mut other.p2a);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn merge_mut(&mut self, mut other: Self) {
|
||||
Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
|
||||
Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
|
||||
Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
|
||||
Self::merge_(&mut self.p2sh, &mut other.p2sh);
|
||||
Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
|
||||
Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
|
||||
Self::merge_(&mut self.p2tr, &mut other.p2tr);
|
||||
Self::merge_(&mut self.p2a, &mut other.p2a);
|
||||
}
|
||||
|
||||
fn merge_(own: &mut Vec<T>, other: &mut Vec<T>) {
|
||||
if own.len() >= other.len() {
|
||||
own.append(other);
|
||||
} else {
|
||||
other.append(own);
|
||||
mem::swap(own, other);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unwrap(self) -> ByAddressType<Vec<T>> {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for AddressTypeToVec<T> {
|
||||
fn default() -> Self {
|
||||
Self(ByAddressType {
|
||||
p2pk65: vec![],
|
||||
p2pk33: vec![],
|
||||
p2pkh: vec![],
|
||||
p2sh: vec![],
|
||||
p2wpkh: vec![],
|
||||
p2wsh: vec![],
|
||||
p2tr: vec![],
|
||||
p2a: vec![],
|
||||
})
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,914 @@
|
||||
//! Import and validation methods for Vecs.
|
||||
//!
|
||||
//! This module contains methods for:
|
||||
//! - `forced_import`: Creating a new Vecs instance from database
|
||||
//! - `import_state`: Importing state when resuming from checkpoint
|
||||
//! - `validate_computed_versions`: Version validation
|
||||
//! - `min_height_vecs_len`: Finding minimum vector length
|
||||
|
||||
use brk_error::{Error, Result};
|
||||
use brk_grouper::{CohortContext, Filter};
|
||||
use brk_types::{DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, Version};
|
||||
use vecdb::{
|
||||
AnyVec, Database, EagerVec, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec,
|
||||
StoredVec, TypedVecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
grouped::{
|
||||
ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
|
||||
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
|
||||
PricePercentiles, Source, VecBuilderOptions,
|
||||
},
|
||||
indexes, price,
|
||||
states::CohortState,
|
||||
utils::OptionExt,
|
||||
};
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
filter: Filter,
|
||||
context: CohortContext,
|
||||
parent_version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
) -> Result<Self> {
|
||||
let compute_dollars = price.is_some();
|
||||
let extended = filter.is_extended(context);
|
||||
let compute_rel_to_all = filter.compute_rel_to_all();
|
||||
let compute_adjusted = filter.compute_adjusted(context);
|
||||
|
||||
let version = parent_version + Version::ZERO;
|
||||
|
||||
let name_prefix = filter.to_full_name(context);
|
||||
let suffix = |s: &str| {
|
||||
if name_prefix.is_empty() {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{name_prefix}_{s}")
|
||||
}
|
||||
};
|
||||
|
||||
// Helper macros for imports
|
||||
macro_rules! eager {
|
||||
($idx:ty, $val:ty, $name:expr, $v:expr) => {
|
||||
EagerVec::<PcoVec<$idx, $val>>::forced_import(db, &suffix($name), version + $v)
|
||||
.unwrap()
|
||||
};
|
||||
}
|
||||
macro_rules! computed_h {
|
||||
($name:expr, $source:expr, $v:expr, $opts:expr $(,)?) => {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
db,
|
||||
&suffix($name),
|
||||
$source,
|
||||
version + $v,
|
||||
indexes,
|
||||
$opts,
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
}
|
||||
macro_rules! computed_di {
|
||||
($name:expr, $source:expr, $v:expr, $opts:expr $(,)?) => {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix($name),
|
||||
$source,
|
||||
version + $v,
|
||||
indexes,
|
||||
$opts,
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
}
|
||||
|
||||
// Common version patterns
|
||||
let v0 = Version::ZERO;
|
||||
let v1 = Version::ONE;
|
||||
let v2 = Version::TWO;
|
||||
let v3 = Version::new(3);
|
||||
let last = || VecBuilderOptions::default().add_last();
|
||||
let sum = || VecBuilderOptions::default().add_sum();
|
||||
let sum_cum = || VecBuilderOptions::default().add_sum().add_cumulative();
|
||||
|
||||
// Pre-create dateindex vecs that are used in computed vecs
|
||||
let dateindex_to_supply_in_profit =
|
||||
compute_dollars.then(|| eager!(DateIndex, Sats, "supply_in_profit", v0));
|
||||
let dateindex_to_supply_in_loss =
|
||||
compute_dollars.then(|| eager!(DateIndex, Sats, "supply_in_loss", v0));
|
||||
let dateindex_to_unrealized_profit =
|
||||
compute_dollars.then(|| eager!(DateIndex, Dollars, "unrealized_profit", v0));
|
||||
let dateindex_to_unrealized_loss =
|
||||
compute_dollars.then(|| eager!(DateIndex, Dollars, "unrealized_loss", v0));
|
||||
|
||||
Ok(Self {
|
||||
filter,
|
||||
|
||||
// ==================== SUPPLY & UTXO COUNT ====================
|
||||
height_to_supply: EagerVec::forced_import(db, &suffix("supply"), version + v0)?,
|
||||
height_to_supply_value: ComputedHeightValueVecs::forced_import(
|
||||
db,
|
||||
&suffix("supply"),
|
||||
Source::None,
|
||||
version + v0,
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("supply"),
|
||||
Source::Compute,
|
||||
version + v1,
|
||||
last(),
|
||||
compute_dollars,
|
||||
indexes,
|
||||
)?,
|
||||
height_to_utxo_count: EagerVec::forced_import(db, &suffix("utxo_count"), version + v0)?,
|
||||
indexes_to_utxo_count: computed_h!("utxo_count", Source::None, v0, last()),
|
||||
height_to_supply_half_value: ComputedHeightValueVecs::forced_import(
|
||||
db,
|
||||
&suffix("supply_half"),
|
||||
Source::Compute,
|
||||
version + v0,
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("supply_half"),
|
||||
Source::Compute,
|
||||
version + v0,
|
||||
last(),
|
||||
compute_dollars,
|
||||
indexes,
|
||||
)?,
|
||||
|
||||
// ==================== ACTIVITY ====================
|
||||
height_to_sent: EagerVec::forced_import(db, &suffix("sent"), version + v0)?,
|
||||
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
|
||||
db,
|
||||
&suffix("sent"),
|
||||
Source::None,
|
||||
version + v0,
|
||||
sum(),
|
||||
compute_dollars,
|
||||
indexes,
|
||||
)?,
|
||||
height_to_satblocks_destroyed: EagerVec::forced_import(
|
||||
db,
|
||||
&suffix("satblocks_destroyed"),
|
||||
version + v0,
|
||||
)?,
|
||||
height_to_satdays_destroyed: EagerVec::forced_import(
|
||||
db,
|
||||
&suffix("satdays_destroyed"),
|
||||
version + v0,
|
||||
)?,
|
||||
indexes_to_coinblocks_destroyed: computed_h!(
|
||||
"coinblocks_destroyed",
|
||||
Source::Compute,
|
||||
v2,
|
||||
sum_cum(),
|
||||
),
|
||||
indexes_to_coindays_destroyed: computed_h!(
|
||||
"coindays_destroyed",
|
||||
Source::Compute,
|
||||
v2,
|
||||
sum_cum(),
|
||||
),
|
||||
|
||||
// ==================== REALIZED CAP & PRICE ====================
|
||||
height_to_realized_cap: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "realized_cap", v0)),
|
||||
indexes_to_realized_cap: compute_dollars
|
||||
.then(|| computed_h!("realized_cap", Source::None, v0, last())),
|
||||
indexes_to_realized_price: compute_dollars
|
||||
.then(|| computed_h!("realized_price", Source::Compute, v0, last())),
|
||||
indexes_to_realized_price_extra: compute_dollars.then(|| {
|
||||
ComputedRatioVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("realized_price"),
|
||||
Source::None,
|
||||
version + v0,
|
||||
indexes,
|
||||
extended,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
indexes_to_realized_cap_rel_to_own_market_cap: (compute_dollars && extended).then(
|
||||
|| {
|
||||
computed_h!(
|
||||
"realized_cap_rel_to_own_market_cap",
|
||||
Source::Compute,
|
||||
v0,
|
||||
last()
|
||||
)
|
||||
},
|
||||
),
|
||||
indexes_to_realized_cap_30d_delta: compute_dollars
|
||||
.then(|| computed_di!("realized_cap_30d_delta", Source::Compute, v0, last())),
|
||||
|
||||
// ==================== REALIZED PROFIT & LOSS ====================
|
||||
height_to_realized_profit: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "realized_profit", v0)),
|
||||
indexes_to_realized_profit: compute_dollars
|
||||
.then(|| computed_h!("realized_profit", Source::None, v0, sum_cum())),
|
||||
height_to_realized_loss: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "realized_loss", v0)),
|
||||
indexes_to_realized_loss: compute_dollars
|
||||
.then(|| computed_h!("realized_loss", Source::None, v0, sum_cum())),
|
||||
indexes_to_neg_realized_loss: compute_dollars
|
||||
.then(|| computed_h!("neg_realized_loss", Source::Compute, v1, sum_cum())),
|
||||
indexes_to_net_realized_pnl: compute_dollars
|
||||
.then(|| computed_h!("net_realized_pnl", Source::Compute, v0, sum_cum())),
|
||||
indexes_to_realized_value: compute_dollars
|
||||
.then(|| computed_h!("realized_value", Source::Compute, v0, sum())),
|
||||
indexes_to_realized_profit_rel_to_realized_cap: compute_dollars.then(|| {
|
||||
computed_h!(
|
||||
"realized_profit_rel_to_realized_cap",
|
||||
Source::Compute,
|
||||
v0,
|
||||
sum()
|
||||
)
|
||||
}),
|
||||
indexes_to_realized_loss_rel_to_realized_cap: compute_dollars.then(|| {
|
||||
computed_h!(
|
||||
"realized_loss_rel_to_realized_cap",
|
||||
Source::Compute,
|
||||
v0,
|
||||
sum()
|
||||
)
|
||||
}),
|
||||
indexes_to_net_realized_pnl_rel_to_realized_cap: compute_dollars.then(|| {
|
||||
computed_h!(
|
||||
"net_realized_pnl_rel_to_realized_cap",
|
||||
Source::Compute,
|
||||
v1,
|
||||
sum()
|
||||
)
|
||||
}),
|
||||
height_to_total_realized_pnl: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "total_realized_pnl", v0)),
|
||||
indexes_to_total_realized_pnl: compute_dollars
|
||||
.then(|| computed_di!("total_realized_pnl", Source::Compute, v1, sum())),
|
||||
dateindex_to_realized_profit_to_loss_ratio: (compute_dollars && extended)
|
||||
.then(|| eager!(DateIndex, StoredF64, "realized_profit_to_loss_ratio", v1)),
|
||||
|
||||
// ==================== VALUE CREATED & DESTROYED ====================
|
||||
height_to_value_created: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "value_created", v0)),
|
||||
indexes_to_value_created: compute_dollars
|
||||
.then(|| computed_h!("value_created", Source::None, v0, sum())),
|
||||
height_to_value_destroyed: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "value_destroyed", v0)),
|
||||
indexes_to_value_destroyed: compute_dollars
|
||||
.then(|| computed_h!("value_destroyed", Source::None, v0, sum())),
|
||||
height_to_adjusted_value_created: (compute_dollars && compute_adjusted)
|
||||
.then(|| eager!(Height, Dollars, "adjusted_value_created", v0)),
|
||||
indexes_to_adjusted_value_created: (compute_dollars && compute_adjusted)
|
||||
.then(|| computed_h!("adjusted_value_created", Source::None, v0, sum())),
|
||||
height_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted)
|
||||
.then(|| eager!(Height, Dollars, "adjusted_value_destroyed", v0)),
|
||||
indexes_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted)
|
||||
.then(|| computed_h!("adjusted_value_destroyed", Source::None, v0, sum())),
|
||||
|
||||
// ==================== SOPR ====================
|
||||
dateindex_to_sopr: compute_dollars.then(|| eager!(DateIndex, StoredF64, "sopr", v1)),
|
||||
dateindex_to_sopr_7d_ema: compute_dollars
|
||||
.then(|| eager!(DateIndex, StoredF64, "sopr_7d_ema", v1)),
|
||||
dateindex_to_sopr_30d_ema: compute_dollars
|
||||
.then(|| eager!(DateIndex, StoredF64, "sopr_30d_ema", v1)),
|
||||
dateindex_to_adjusted_sopr: (compute_dollars && compute_adjusted)
|
||||
.then(|| eager!(DateIndex, StoredF64, "adjusted_sopr", v1)),
|
||||
dateindex_to_adjusted_sopr_7d_ema: (compute_dollars && compute_adjusted)
|
||||
.then(|| eager!(DateIndex, StoredF64, "adjusted_sopr_7d_ema", v1)),
|
||||
dateindex_to_adjusted_sopr_30d_ema: (compute_dollars && compute_adjusted)
|
||||
.then(|| eager!(DateIndex, StoredF64, "adjusted_sopr_30d_ema", v1)),
|
||||
|
||||
// ==================== SELL SIDE RISK ====================
|
||||
dateindex_to_sell_side_risk_ratio: compute_dollars
|
||||
.then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio", v1)),
|
||||
dateindex_to_sell_side_risk_ratio_7d_ema: compute_dollars
|
||||
.then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio_7d_ema", v1)),
|
||||
dateindex_to_sell_side_risk_ratio_30d_ema: compute_dollars
|
||||
.then(|| eager!(DateIndex, StoredF32, "sell_side_risk_ratio_30d_ema", v1)),
|
||||
|
||||
// ==================== SUPPLY IN PROFIT/LOSS ====================
|
||||
height_to_supply_in_profit: compute_dollars
|
||||
.then(|| eager!(Height, Sats, "supply_in_profit", v0)),
|
||||
indexes_to_supply_in_profit: compute_dollars.then(|| {
|
||||
ComputedValueVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("supply_in_profit"),
|
||||
dateindex_to_supply_in_profit
|
||||
.as_ref()
|
||||
.map(|v| v.boxed_clone())
|
||||
.into(),
|
||||
version + v0,
|
||||
last(),
|
||||
compute_dollars,
|
||||
indexes,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
height_to_supply_in_loss: compute_dollars
|
||||
.then(|| eager!(Height, Sats, "supply_in_loss", v0)),
|
||||
indexes_to_supply_in_loss: compute_dollars.then(|| {
|
||||
ComputedValueVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("supply_in_loss"),
|
||||
dateindex_to_supply_in_loss
|
||||
.as_ref()
|
||||
.map(|v| v.boxed_clone())
|
||||
.into(),
|
||||
version + v0,
|
||||
last(),
|
||||
compute_dollars,
|
||||
indexes,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
dateindex_to_supply_in_profit,
|
||||
dateindex_to_supply_in_loss,
|
||||
height_to_supply_in_profit_value: compute_dollars.then(|| {
|
||||
ComputedHeightValueVecs::forced_import(
|
||||
db,
|
||||
&suffix("supply_in_profit"),
|
||||
Source::None,
|
||||
version + v0,
|
||||
compute_dollars,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
height_to_supply_in_loss_value: compute_dollars.then(|| {
|
||||
ComputedHeightValueVecs::forced_import(
|
||||
db,
|
||||
&suffix("supply_in_loss"),
|
||||
Source::None,
|
||||
version + v0,
|
||||
compute_dollars,
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
|
||||
// ==================== UNREALIZED PROFIT & LOSS ====================
|
||||
height_to_unrealized_profit: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "unrealized_profit", v0)),
|
||||
indexes_to_unrealized_profit: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("unrealized_profit"),
|
||||
dateindex_to_unrealized_profit
|
||||
.as_ref()
|
||||
.map(|v| v.boxed_clone())
|
||||
.into(),
|
||||
version + v0,
|
||||
indexes,
|
||||
last(),
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
height_to_unrealized_loss: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "unrealized_loss", v0)),
|
||||
indexes_to_unrealized_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
db,
|
||||
&suffix("unrealized_loss"),
|
||||
dateindex_to_unrealized_loss
|
||||
.as_ref()
|
||||
.map(|v| v.boxed_clone())
|
||||
.into(),
|
||||
version + v0,
|
||||
indexes,
|
||||
last(),
|
||||
)
|
||||
.unwrap()
|
||||
}),
|
||||
dateindex_to_unrealized_profit,
|
||||
dateindex_to_unrealized_loss,
|
||||
height_to_neg_unrealized_loss: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "neg_unrealized_loss", v0)),
|
||||
indexes_to_neg_unrealized_loss: compute_dollars
|
||||
.then(|| computed_di!("neg_unrealized_loss", Source::Compute, v0, last())),
|
||||
height_to_net_unrealized_pnl: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "net_unrealized_pnl", v0)),
|
||||
indexes_to_net_unrealized_pnl: compute_dollars
|
||||
.then(|| computed_di!("net_unrealized_pnl", Source::Compute, v0, last())),
|
||||
height_to_total_unrealized_pnl: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "total_unrealized_pnl", v0)),
|
||||
indexes_to_total_unrealized_pnl: compute_dollars
|
||||
.then(|| computed_di!("total_unrealized_pnl", Source::Compute, v0, last())),
|
||||
|
||||
// ==================== PRICE PAID ====================
|
||||
height_to_min_price_paid: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "min_price_paid", v0)),
|
||||
indexes_to_min_price_paid: compute_dollars
|
||||
.then(|| computed_h!("min_price_paid", Source::None, v0, last())),
|
||||
height_to_max_price_paid: compute_dollars
|
||||
.then(|| eager!(Height, Dollars, "max_price_paid", v0)),
|
||||
indexes_to_max_price_paid: compute_dollars
|
||||
.then(|| computed_h!("max_price_paid", Source::None, v0, last())),
|
||||
price_percentiles: (compute_dollars && extended).then(|| {
|
||||
PricePercentiles::forced_import(db, &suffix(""), version + v0, indexes, true)
|
||||
.unwrap()
|
||||
}),
|
||||
|
||||
// ==================== RELATIVE METRICS: UNREALIZED vs MARKET CAP ====================
|
||||
height_to_unrealized_profit_rel_to_market_cap: compute_dollars
|
||||
.then(|| eager!(Height, StoredF32, "unrealized_profit_rel_to_market_cap", v0)),
|
||||
height_to_unrealized_loss_rel_to_market_cap: compute_dollars
|
||||
.then(|| eager!(Height, StoredF32, "unrealized_loss_rel_to_market_cap", v0)),
|
||||
height_to_neg_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"neg_unrealized_loss_rel_to_market_cap",
|
||||
v0
|
||||
)
|
||||
}),
|
||||
height_to_net_unrealized_pnl_rel_to_market_cap: compute_dollars.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"net_unrealized_pnl_rel_to_market_cap",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
indexes_to_unrealized_profit_rel_to_market_cap: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"unrealized_profit_rel_to_market_cap",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"unrealized_loss_rel_to_market_cap",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_neg_unrealized_loss_rel_to_market_cap: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"neg_unrealized_loss_rel_to_market_cap",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_net_unrealized_pnl_rel_to_market_cap: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"net_unrealized_pnl_rel_to_market_cap",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
|
||||
// ==================== RELATIVE METRICS: UNREALIZED vs OWN MARKET CAP ====================
|
||||
height_to_unrealized_profit_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"unrealized_profit_rel_to_own_market_cap",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
height_to_unrealized_loss_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"unrealized_loss_rel_to_own_market_cap",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
height_to_neg_unrealized_loss_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"neg_unrealized_loss_rel_to_own_market_cap",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
height_to_net_unrealized_pnl_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"net_unrealized_pnl_rel_to_own_market_cap",
|
||||
v2
|
||||
)
|
||||
}),
|
||||
indexes_to_unrealized_profit_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"unrealized_profit_rel_to_own_market_cap",
|
||||
Source::Compute,
|
||||
v2,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_unrealized_loss_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"unrealized_loss_rel_to_own_market_cap",
|
||||
Source::Compute,
|
||||
v2,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_neg_unrealized_loss_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"neg_unrealized_loss_rel_to_own_market_cap",
|
||||
Source::Compute,
|
||||
v2,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_net_unrealized_pnl_rel_to_own_market_cap: (compute_dollars
|
||||
&& extended
|
||||
&& compute_rel_to_all)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"net_unrealized_pnl_rel_to_own_market_cap",
|
||||
Source::Compute,
|
||||
v2,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
|
||||
// ==================== RELATIVE METRICS: UNREALIZED vs OWN TOTAL UNREALIZED ====================
|
||||
height_to_unrealized_profit_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"unrealized_profit_rel_to_own_total_unrealized_pnl",
|
||||
v0
|
||||
)
|
||||
}),
|
||||
height_to_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"unrealized_loss_rel_to_own_total_unrealized_pnl",
|
||||
v0
|
||||
)
|
||||
}),
|
||||
height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"neg_unrealized_loss_rel_to_own_total_unrealized_pnl",
|
||||
v0
|
||||
)
|
||||
}),
|
||||
height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF32,
|
||||
"net_unrealized_pnl_rel_to_own_total_unrealized_pnl",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"unrealized_profit_rel_to_own_total_unrealized_pnl",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"unrealized_loss_rel_to_own_total_unrealized_pnl",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"neg_unrealized_loss_rel_to_own_total_unrealized_pnl",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl: (compute_dollars
|
||||
&& extended)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"net_unrealized_pnl_rel_to_own_total_unrealized_pnl",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
|
||||
// ==================== RELATIVE METRICS: SUPPLY vs CIRCULATING/OWN ====================
|
||||
indexes_to_supply_rel_to_circulating_supply: compute_rel_to_all.then(|| {
|
||||
computed_h!(
|
||||
"supply_rel_to_circulating_supply",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
height_to_supply_in_profit_rel_to_own_supply: compute_dollars
|
||||
.then(|| eager!(Height, StoredF64, "supply_in_profit_rel_to_own_supply", v1)),
|
||||
height_to_supply_in_loss_rel_to_own_supply: compute_dollars
|
||||
.then(|| eager!(Height, StoredF64, "supply_in_loss_rel_to_own_supply", v1)),
|
||||
indexes_to_supply_in_profit_rel_to_own_supply: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"supply_in_profit_rel_to_own_supply",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_supply_in_loss_rel_to_own_supply: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"supply_in_loss_rel_to_own_supply",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
height_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF64,
|
||||
"supply_in_profit_rel_to_circulating_supply",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
height_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
eager!(
|
||||
Height,
|
||||
StoredF64,
|
||||
"supply_in_loss_rel_to_circulating_supply",
|
||||
v1
|
||||
)
|
||||
}),
|
||||
indexes_to_supply_in_profit_rel_to_circulating_supply: (compute_rel_to_all
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"supply_in_profit_rel_to_circulating_supply",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"supply_in_loss_rel_to_circulating_supply",
|
||||
Source::Compute,
|
||||
v1,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
|
||||
// ==================== NET REALIZED PNL DELTAS ====================
|
||||
indexes_to_net_realized_pnl_cumulative_30d_delta: compute_dollars.then(|| {
|
||||
computed_di!(
|
||||
"net_realized_pnl_cumulative_30d_delta",
|
||||
Source::Compute,
|
||||
v3,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap: compute_dollars
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap",
|
||||
Source::Compute,
|
||||
v3,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap: compute_dollars
|
||||
.then(|| {
|
||||
computed_di!(
|
||||
"net_realized_pnl_cumulative_30d_delta_rel_to_market_cap",
|
||||
Source::Compute,
|
||||
v3,
|
||||
last()
|
||||
)
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the minimum length of all height-indexed vectors.
|
||||
/// Used to determine the starting point for processing.
|
||||
pub fn min_height_vecs_len(&self) -> usize {
|
||||
[
|
||||
self.height_to_supply.len(),
|
||||
self.height_to_utxo_count.len(),
|
||||
self.height_to_realized_cap
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_realized_profit
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_realized_loss
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_value_created
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_adjusted_value_created
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_value_destroyed
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_adjusted_value_destroyed
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_supply_in_profit
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_supply_in_loss
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_unrealized_profit
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_unrealized_loss
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_min_price_paid
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_max_price_paid
|
||||
.as_ref()
|
||||
.map_or(usize::MAX, |v| v.len()),
|
||||
self.height_to_sent.len(),
|
||||
self.height_to_satdays_destroyed.len(),
|
||||
self.height_to_satblocks_destroyed.len(),
|
||||
]
|
||||
.into_iter()
|
||||
.min()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Import state from a checkpoint when resuming processing.
|
||||
/// Returns the next height to process from.
|
||||
pub fn import_state(
|
||||
&mut self,
|
||||
starting_height: Height,
|
||||
state: &mut CohortState,
|
||||
) -> Result<Height> {
|
||||
if let Some(mut prev_height) = starting_height.decremented() {
|
||||
if self.height_to_realized_cap.as_mut().is_some() {
|
||||
prev_height = state.import_at_or_before(prev_height)?;
|
||||
}
|
||||
|
||||
state.supply.value = self.height_to_supply.into_iter().get_unwrap(prev_height);
|
||||
state.supply.utxo_count = *self
|
||||
.height_to_utxo_count
|
||||
.into_iter()
|
||||
.get_unwrap(prev_height);
|
||||
|
||||
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
|
||||
state.realized.um().cap =
|
||||
height_to_realized_cap.into_iter().get_unwrap(prev_height);
|
||||
}
|
||||
|
||||
Ok(prev_height.incremented())
|
||||
} else {
|
||||
Err(Error::Str("Unset"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that all computed versions match expected values, resetting if needed.
|
||||
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
// Always-present vecs
|
||||
self.height_to_supply.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_supply.inner_version(),
|
||||
)?;
|
||||
self.height_to_utxo_count
|
||||
.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_utxo_count.inner_version(),
|
||||
)?;
|
||||
self.height_to_sent.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_sent.inner_version(),
|
||||
)?;
|
||||
self.height_to_satblocks_destroyed
|
||||
.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_satblocks_destroyed.inner_version(),
|
||||
)?;
|
||||
self.height_to_satdays_destroyed
|
||||
.validate_computed_version_or_reset(
|
||||
base_version + self.height_to_satdays_destroyed.inner_version(),
|
||||
)?;
|
||||
|
||||
// Dollar-dependent vecs
|
||||
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() {
|
||||
height_to_realized_cap.validate_computed_version_or_reset(
|
||||
base_version + height_to_realized_cap.inner_version(),
|
||||
)?;
|
||||
|
||||
Self::validate_optional_vec_version(&mut self.height_to_realized_profit, base_version)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_realized_loss, base_version)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_value_created, base_version)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_value_destroyed, base_version)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.height_to_supply_in_profit,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_supply_in_loss, base_version)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.height_to_unrealized_profit,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_unrealized_loss, base_version)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.dateindex_to_supply_in_profit,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.dateindex_to_supply_in_loss,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.dateindex_to_unrealized_profit,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.dateindex_to_unrealized_loss,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_min_price_paid, base_version)?;
|
||||
Self::validate_optional_vec_version(&mut self.height_to_max_price_paid, base_version)?;
|
||||
|
||||
if self.height_to_adjusted_value_created.is_some() {
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.height_to_adjusted_value_created,
|
||||
base_version,
|
||||
)?;
|
||||
Self::validate_optional_vec_version(
|
||||
&mut self.height_to_adjusted_value_destroyed,
|
||||
base_version,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper to validate an optional vec's version.
|
||||
fn validate_optional_vec_version<V: StoredVec>(
|
||||
vec: &mut Option<EagerVec<V>>,
|
||||
base_version: Version,
|
||||
) -> Result<()> {
|
||||
if let Some(v) = vec.as_mut() {
|
||||
v.validate_computed_version_or_reset(base_version + v.inner_version())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//! Common vector structs and logic shared between UTXO and Address cohorts.
|
||||
//!
|
||||
//! This module contains the `Vecs` struct which holds all the computed vectors
|
||||
//! for a single cohort, along with methods for importing, flushing, and computing.
|
||||
//!
|
||||
//! ## Module Organization
|
||||
//!
|
||||
//! The implementation is split across multiple files for maintainability:
|
||||
//! - `vecs.rs`: Struct definition with field documentation
|
||||
//! - `import.rs`: Import, validation, and initialization methods
|
||||
//! - `push.rs`: Per-block push and flush methods
|
||||
//! - `compute.rs`: Post-processing computation methods
|
||||
|
||||
mod compute;
|
||||
mod import;
|
||||
mod push;
|
||||
mod vecs;
|
||||
|
||||
pub use vecs::Vecs;
|
||||
@@ -0,0 +1,178 @@
|
||||
//! Push and flush methods for Vecs.
|
||||
//!
|
||||
//! This module contains methods for:
|
||||
//! - `truncate_push`: Push state values to height-indexed vectors
|
||||
//! - `compute_then_truncate_push_unrealized_states`: Compute and push unrealized states
|
||||
//! - `safe_flush_stateful_vecs`: Safely flush all stateful vectors
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{DateIndex, Dollars, Height, StoredU64};
|
||||
use vecdb::{AnyStoredVec, Exit, GenericStoredVec};
|
||||
|
||||
use crate::{stateful::Flushable, states::CohortState, utils::OptionExt};
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
impl Vecs {
|
||||
pub fn truncate_push(&mut self, height: Height, state: &CohortState) -> Result<()> {
|
||||
self.height_to_supply
|
||||
.truncate_push(height, state.supply.value)?;
|
||||
|
||||
self.height_to_utxo_count
|
||||
.truncate_push(height, StoredU64::from(state.supply.utxo_count))?;
|
||||
|
||||
self.height_to_sent.truncate_push(height, state.sent)?;
|
||||
|
||||
self.height_to_satblocks_destroyed
|
||||
.truncate_push(height, state.satblocks_destroyed)?;
|
||||
|
||||
self.height_to_satdays_destroyed
|
||||
.truncate_push(height, state.satdays_destroyed)?;
|
||||
|
||||
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
|
||||
let realized = state.realized.as_ref().unwrap_or_else(|| {
|
||||
dbg!((&state.realized, &state.supply));
|
||||
panic!();
|
||||
});
|
||||
|
||||
height_to_realized_cap.truncate_push(height, realized.cap)?;
|
||||
|
||||
self.height_to_realized_profit
|
||||
.um()
|
||||
.truncate_push(height, realized.profit)?;
|
||||
self.height_to_realized_loss
|
||||
.um()
|
||||
.truncate_push(height, realized.loss)?;
|
||||
self.height_to_value_created
|
||||
.um()
|
||||
.truncate_push(height, realized.value_created)?;
|
||||
self.height_to_value_destroyed
|
||||
.um()
|
||||
.truncate_push(height, realized.value_destroyed)?;
|
||||
|
||||
if self.height_to_adjusted_value_created.is_some() {
|
||||
self.height_to_adjusted_value_created
|
||||
.um()
|
||||
.truncate_push(height, realized.adj_value_created)?;
|
||||
self.height_to_adjusted_value_destroyed
|
||||
.um()
|
||||
.truncate_push(height, realized.adj_value_destroyed)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
state: &CohortState,
|
||||
) -> Result<()> {
|
||||
if let Some(height_price) = height_price {
|
||||
self.height_to_min_price_paid.um().truncate_push(
|
||||
height,
|
||||
state
|
||||
.price_to_amount_first_key_value()
|
||||
.map(|(&dollars, _)| dollars)
|
||||
.unwrap_or(Dollars::NAN),
|
||||
)?;
|
||||
self.height_to_max_price_paid.um().truncate_push(
|
||||
height,
|
||||
state
|
||||
.price_to_amount_last_key_value()
|
||||
.map(|(&dollars, _)| dollars)
|
||||
.unwrap_or(Dollars::NAN),
|
||||
)?;
|
||||
|
||||
let (height_unrealized_state, date_unrealized_state) =
|
||||
state.compute_unrealized_states(height_price, date_price.unwrap());
|
||||
|
||||
self.height_to_supply_in_profit
|
||||
.um()
|
||||
.truncate_push(height, height_unrealized_state.supply_in_profit)?;
|
||||
self.height_to_supply_in_loss
|
||||
.um()
|
||||
.truncate_push(height, height_unrealized_state.supply_in_loss)?;
|
||||
self.height_to_unrealized_profit
|
||||
.um()
|
||||
.truncate_push(height, height_unrealized_state.unrealized_profit)?;
|
||||
self.height_to_unrealized_loss
|
||||
.um()
|
||||
.truncate_push(height, height_unrealized_state.unrealized_loss)?;
|
||||
|
||||
if let Some(date_unrealized_state) = date_unrealized_state {
|
||||
let dateindex = dateindex.unwrap();
|
||||
|
||||
self.dateindex_to_supply_in_profit
|
||||
.um()
|
||||
.truncate_push(dateindex, date_unrealized_state.supply_in_profit)?;
|
||||
self.dateindex_to_supply_in_loss
|
||||
.um()
|
||||
.truncate_push(dateindex, date_unrealized_state.supply_in_loss)?;
|
||||
self.dateindex_to_unrealized_profit
|
||||
.um()
|
||||
.truncate_push(dateindex, date_unrealized_state.unrealized_profit)?;
|
||||
self.dateindex_to_unrealized_loss
|
||||
.um()
|
||||
.truncate_push(dateindex, date_unrealized_state.unrealized_loss)?;
|
||||
}
|
||||
|
||||
// Compute and push price percentiles
|
||||
if let Some(price_percentiles) = self.price_percentiles.as_mut() {
|
||||
let percentile_prices = state.compute_percentile_prices();
|
||||
price_percentiles.truncate_push(height, &percentile_prices)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn safe_flush_stateful_vecs(
|
||||
&mut self,
|
||||
height: Height,
|
||||
exit: &Exit,
|
||||
state: &mut CohortState,
|
||||
) -> Result<()> {
|
||||
self.height_to_supply.safe_write(exit)?;
|
||||
self.height_to_utxo_count.safe_write(exit)?;
|
||||
self.height_to_sent.safe_write(exit)?;
|
||||
self.height_to_satdays_destroyed.safe_write(exit)?;
|
||||
self.height_to_satblocks_destroyed.safe_write(exit)?;
|
||||
|
||||
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
|
||||
height_to_realized_cap.safe_write(exit)?;
|
||||
self.height_to_realized_profit.um().safe_write(exit)?;
|
||||
self.height_to_realized_loss.um().safe_write(exit)?;
|
||||
self.height_to_value_created.um().safe_write(exit)?;
|
||||
self.height_to_value_destroyed.um().safe_write(exit)?;
|
||||
self.height_to_supply_in_profit.um().safe_write(exit)?;
|
||||
self.height_to_supply_in_loss.um().safe_write(exit)?;
|
||||
self.height_to_unrealized_profit.um().safe_write(exit)?;
|
||||
self.height_to_unrealized_loss.um().safe_write(exit)?;
|
||||
self.dateindex_to_supply_in_profit.um().safe_write(exit)?;
|
||||
self.dateindex_to_supply_in_loss.um().safe_write(exit)?;
|
||||
self.dateindex_to_unrealized_profit.um().safe_write(exit)?;
|
||||
self.dateindex_to_unrealized_loss.um().safe_write(exit)?;
|
||||
self.height_to_min_price_paid.um().safe_write(exit)?;
|
||||
self.height_to_max_price_paid.um().safe_write(exit)?;
|
||||
|
||||
if self.height_to_adjusted_value_created.is_some() {
|
||||
self.height_to_adjusted_value_created
|
||||
.um()
|
||||
.safe_write(exit)?;
|
||||
self.height_to_adjusted_value_destroyed
|
||||
.um()
|
||||
.safe_write(exit)?;
|
||||
}
|
||||
|
||||
// Uses Flushable trait - Option<T> impl handles None case
|
||||
self.price_percentiles.safe_write(exit)?;
|
||||
}
|
||||
|
||||
state.commit(height)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
use brk_grouper::Filter;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64};
|
||||
use vecdb::{EagerVec, PcoVec};
|
||||
|
||||
use crate::grouped::{
|
||||
ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
|
||||
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
|
||||
PricePercentiles,
|
||||
};
|
||||
|
||||
/// Common vectors shared between UTXO and Address cohorts.
|
||||
///
|
||||
/// This struct contains all the computed vectors for a single cohort. The fields are
|
||||
/// organized into logical groups matching the initialization order in `forced_import`.
|
||||
///
|
||||
/// ## Field Groups
|
||||
/// - **Supply & UTXO count**: Basic supply metrics (always computed)
|
||||
/// - **Activity**: Sent amounts, satblocks/satdays destroyed
|
||||
/// - **Realized**: Realized cap, profit/loss, value created/destroyed, SOPR
|
||||
/// - **Unrealized**: Unrealized profit/loss, supply in profit/loss
|
||||
/// - **Price**: Min/max price paid, price percentiles
|
||||
/// - **Relative metrics**: Ratios relative to market cap, realized cap, etc.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct Vecs {
|
||||
#[traversable(skip)]
|
||||
pub filter: Filter,
|
||||
|
||||
// ==================== SUPPLY & UTXO COUNT ====================
|
||||
// Always computed - core supply metrics
|
||||
pub height_to_supply: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub height_to_supply_value: ComputedHeightValueVecs,
|
||||
pub indexes_to_supply: ComputedValueVecsFromDateIndex,
|
||||
pub height_to_utxo_count: EagerVec<PcoVec<Height, StoredU64>>,
|
||||
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
|
||||
pub height_to_supply_half_value: ComputedHeightValueVecs,
|
||||
pub indexes_to_supply_half: ComputedValueVecsFromDateIndex,
|
||||
|
||||
// ==================== ACTIVITY ====================
|
||||
// Always computed - transaction activity metrics
|
||||
pub height_to_sent: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub indexes_to_sent: ComputedValueVecsFromHeight,
|
||||
pub height_to_satblocks_destroyed: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub height_to_satdays_destroyed: EagerVec<PcoVec<Height, Sats>>,
|
||||
pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
|
||||
pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
|
||||
|
||||
// ==================== REALIZED CAP & PRICE ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_realized_cap: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub indexes_to_realized_price: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub indexes_to_realized_price_extra: Option<ComputedRatioVecsFromDateIndex>,
|
||||
pub indexes_to_realized_cap_rel_to_own_market_cap: Option<ComputedVecsFromHeight<StoredF32>>,
|
||||
pub indexes_to_realized_cap_30d_delta: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
|
||||
// ==================== REALIZED PROFIT & LOSS ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_realized_profit: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_realized_profit: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub height_to_realized_loss: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_realized_loss: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub indexes_to_neg_realized_loss: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub indexes_to_net_realized_pnl: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub indexes_to_realized_value: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub indexes_to_realized_profit_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
|
||||
pub indexes_to_realized_loss_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
|
||||
pub indexes_to_net_realized_pnl_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
|
||||
pub height_to_total_realized_pnl: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_total_realized_pnl: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
pub dateindex_to_realized_profit_to_loss_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
|
||||
// ==================== VALUE CREATED & DESTROYED ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_value_created: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub height_to_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub height_to_adjusted_value_created: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub height_to_adjusted_value_destroyed: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_adjusted_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
|
||||
// ==================== SOPR ====================
|
||||
// Spent Output Profit Ratio - conditional on compute_dollars
|
||||
pub dateindex_to_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_adjusted_sopr: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_adjusted_sopr_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
pub dateindex_to_adjusted_sopr_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF64>>>,
|
||||
|
||||
// ==================== SELL SIDE RISK ====================
|
||||
// Conditional on compute_dollars
|
||||
pub dateindex_to_sell_side_risk_ratio: Option<EagerVec<PcoVec<DateIndex, StoredF32>>>,
|
||||
pub dateindex_to_sell_side_risk_ratio_7d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF32>>>,
|
||||
pub dateindex_to_sell_side_risk_ratio_30d_ema: Option<EagerVec<PcoVec<DateIndex, StoredF32>>>,
|
||||
|
||||
// ==================== SUPPLY IN PROFIT/LOSS ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_supply_in_profit: Option<EagerVec<PcoVec<Height, Sats>>>,
|
||||
pub indexes_to_supply_in_profit: Option<ComputedValueVecsFromDateIndex>,
|
||||
pub height_to_supply_in_loss: Option<EagerVec<PcoVec<Height, Sats>>>,
|
||||
pub indexes_to_supply_in_loss: Option<ComputedValueVecsFromDateIndex>,
|
||||
pub dateindex_to_supply_in_profit: Option<EagerVec<PcoVec<DateIndex, Sats>>>,
|
||||
pub dateindex_to_supply_in_loss: Option<EagerVec<PcoVec<DateIndex, Sats>>>,
|
||||
pub height_to_supply_in_profit_value: Option<ComputedHeightValueVecs>,
|
||||
pub height_to_supply_in_loss_value: Option<ComputedHeightValueVecs>,
|
||||
|
||||
// ==================== UNREALIZED PROFIT & LOSS ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_unrealized_profit: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_unrealized_profit: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
pub height_to_unrealized_loss: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_unrealized_loss: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
pub dateindex_to_unrealized_profit: Option<EagerVec<PcoVec<DateIndex, Dollars>>>,
|
||||
pub dateindex_to_unrealized_loss: Option<EagerVec<PcoVec<DateIndex, Dollars>>>,
|
||||
pub height_to_neg_unrealized_loss: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_neg_unrealized_loss: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
pub height_to_net_unrealized_pnl: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_net_unrealized_pnl: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
pub height_to_total_unrealized_pnl: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_total_unrealized_pnl: Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
|
||||
// ==================== PRICE PAID ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_min_price_paid: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_min_price_paid: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub height_to_max_price_paid: Option<EagerVec<PcoVec<Height, Dollars>>>,
|
||||
pub indexes_to_max_price_paid: Option<ComputedVecsFromHeight<Dollars>>,
|
||||
pub price_percentiles: Option<PricePercentiles>,
|
||||
|
||||
// ==================== RELATIVE METRICS: UNREALIZED vs MARKET CAP ====================
|
||||
// Conditional on compute_dollars
|
||||
pub height_to_unrealized_profit_rel_to_market_cap: Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_unrealized_loss_rel_to_market_cap: Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_neg_unrealized_loss_rel_to_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_net_unrealized_pnl_rel_to_market_cap: Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub indexes_to_unrealized_profit_rel_to_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_unrealized_loss_rel_to_market_cap: Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_neg_unrealized_loss_rel_to_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_net_unrealized_pnl_rel_to_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
|
||||
// ==================== RELATIVE METRICS: UNREALIZED vs OWN MARKET CAP ====================
|
||||
// Conditional on compute_dollars && extended && compute_rel_to_all
|
||||
pub height_to_unrealized_profit_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_neg_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_net_unrealized_pnl_rel_to_own_market_cap:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub indexes_to_unrealized_profit_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_neg_unrealized_loss_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_net_unrealized_pnl_rel_to_own_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
|
||||
// ==================== RELATIVE METRICS: UNREALIZED vs OWN TOTAL UNREALIZED ====================
|
||||
// Conditional on compute_dollars && extended
|
||||
pub height_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub height_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
|
||||
Option<EagerVec<PcoVec<Height, StoredF32>>>,
|
||||
pub indexes_to_unrealized_profit_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_neg_unrealized_loss_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_net_unrealized_pnl_rel_to_own_total_unrealized_pnl:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
|
||||
// ==================== RELATIVE METRICS: SUPPLY vs CIRCULATING/OWN ====================
|
||||
// Conditional on compute_dollars
|
||||
pub indexes_to_supply_rel_to_circulating_supply: Option<ComputedVecsFromHeight<StoredF64>>,
|
||||
pub height_to_supply_in_profit_rel_to_own_supply: Option<EagerVec<PcoVec<Height, StoredF64>>>,
|
||||
pub height_to_supply_in_loss_rel_to_own_supply: Option<EagerVec<PcoVec<Height, StoredF64>>>,
|
||||
pub indexes_to_supply_in_profit_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
|
||||
pub indexes_to_supply_in_loss_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
|
||||
pub height_to_supply_in_profit_rel_to_circulating_supply:
|
||||
Option<EagerVec<PcoVec<Height, StoredF64>>>,
|
||||
pub height_to_supply_in_loss_rel_to_circulating_supply:
|
||||
Option<EagerVec<PcoVec<Height, StoredF64>>>,
|
||||
pub indexes_to_supply_in_profit_rel_to_circulating_supply:
|
||||
Option<ComputedVecsFromDateIndex<StoredF64>>,
|
||||
pub indexes_to_supply_in_loss_rel_to_circulating_supply:
|
||||
Option<ComputedVecsFromDateIndex<StoredF64>>,
|
||||
|
||||
// ==================== NET REALIZED PNL DELTAS ====================
|
||||
// Conditional on compute_dollars
|
||||
pub indexes_to_net_realized_pnl_cumulative_30d_delta:
|
||||
Option<ComputedVecsFromDateIndex<Dollars>>,
|
||||
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_realized_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
pub indexes_to_net_realized_pnl_cumulative_30d_delta_rel_to_market_cap:
|
||||
Option<ComputedVecsFromDateIndex<StoredF32>>,
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Traits for consistent state flushing and importing.
|
||||
//!
|
||||
//! These traits ensure all stateful components follow the same patterns
|
||||
//! for checkpoint/resume operations, preventing bugs where new fields
|
||||
//! are forgotten during flush operations.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::Height;
|
||||
use vecdb::Exit;
|
||||
|
||||
/// Trait for components that can be flushed to disk.
|
||||
///
|
||||
/// This is for simple flush operations that don't require height tracking.
|
||||
pub trait Flushable {
|
||||
/// Safely flush data to disk.
|
||||
fn safe_flush(&mut self, exit: &Exit) -> Result<()>;
|
||||
|
||||
/// Write to mmap without fsync. Data visible to readers immediately but not durable.
|
||||
fn safe_write(&mut self, exit: &Exit) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Trait for stateful components that track data indexed by height.
|
||||
///
|
||||
/// This ensures consistent patterns for:
|
||||
/// - Flushing state at checkpoints
|
||||
/// - Importing state when resuming from a checkpoint
|
||||
/// - Resetting state when starting from scratch
|
||||
pub trait HeightFlushable {
|
||||
/// Flush state to disk at the given height checkpoint.
|
||||
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()>;
|
||||
|
||||
/// Import state from the most recent checkpoint at or before the given height.
|
||||
/// Returns the actual height that was imported.
|
||||
fn import_at_or_before(&mut self, height: Height) -> Result<Height>;
|
||||
|
||||
/// Reset state for starting from scratch.
|
||||
fn reset(&mut self) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Blanket implementation for Option<T> where T: Flushable
|
||||
impl<T: Flushable> Flushable for Option<T> {
|
||||
fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.safe_flush(exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_write(&mut self, exit: &Exit) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.safe_write(exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Blanket implementation for Option<T> where T: HeightFlushable
|
||||
impl<T: HeightFlushable> HeightFlushable for Option<T> {
|
||||
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.flush_at_height(height, exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.import_at_or_before(height)
|
||||
} else {
|
||||
Ok(height)
|
||||
}
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.reset()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,53 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RangeMap<I, T>(BTreeMap<I, T>);
|
||||
|
||||
impl<I, T> RangeMap<I, T>
|
||||
where
|
||||
I: VecIndex,
|
||||
T: VecIndex,
|
||||
{
|
||||
pub fn get(&self, key: I) -> Option<&T> {
|
||||
self.0.range(..=key).next_back().map(|(&min, value)| {
|
||||
if min > key {
|
||||
unreachable!()
|
||||
}
|
||||
value
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> From<&BytesVec<I, T>> for RangeMap<T, I>
|
||||
where
|
||||
I: VecIndex,
|
||||
T: VecIndex + BytesVecValue,
|
||||
{
|
||||
#[inline]
|
||||
fn from(vec: &BytesVec<I, T>) -> Self {
|
||||
Self(
|
||||
vec.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (v, I::from(i)))
|
||||
.collect::<BTreeMap<_, _>>(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> From<&PcoVec<I, T>> for RangeMap<T, I>
|
||||
where
|
||||
I: VecIndex,
|
||||
T: VecIndex + PcoVecValue,
|
||||
{
|
||||
#[inline]
|
||||
fn from(vec: &PcoVec<I, T>) -> Self {
|
||||
Self(
|
||||
vec.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (v, I::from(i)))
|
||||
.collect::<BTreeMap<_, _>>(),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
use brk_grouper::{ByAddressType, ByAnyAddress};
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{OutputType, StoredU64, TxIndex};
|
||||
use vecdb::{BoxedVecIterator, GenericStoredVec, Reader, VecIndex};
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
pub struct IndexerReaders {
|
||||
pub txinindex_to_outpoint: Reader,
|
||||
pub txindex_to_first_txoutindex: Reader,
|
||||
pub txoutindex_to_value: Reader,
|
||||
pub txoutindex_to_outputtype: Reader,
|
||||
pub txoutindex_to_typeindex: Reader,
|
||||
}
|
||||
|
||||
impl IndexerReaders {
|
||||
pub fn new(indexer: &Indexer) -> Self {
|
||||
Self {
|
||||
txinindex_to_outpoint: indexer.vecs.txinindex_to_outpoint.create_reader(),
|
||||
txindex_to_first_txoutindex: indexer.vecs.txindex_to_first_txoutindex.create_reader(),
|
||||
txoutindex_to_value: indexer.vecs.txoutindex_to_value.create_reader(),
|
||||
txoutindex_to_outputtype: indexer.vecs.txoutindex_to_outputtype.create_reader(),
|
||||
txoutindex_to_typeindex: indexer.vecs.txoutindex_to_typeindex.create_reader(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VecsReaders {
|
||||
pub addresstypeindex_to_anyaddressindex: ByAddressType<Reader>,
|
||||
pub anyaddressindex_to_anyaddressdata: ByAnyAddress<Reader>,
|
||||
}
|
||||
|
||||
impl VecsReaders {
|
||||
pub fn new(vecs: &Vecs) -> Self {
|
||||
Self {
|
||||
addresstypeindex_to_anyaddressindex: ByAddressType {
|
||||
p2pk33: vecs.any_address_indexes.p2pk33.create_reader(),
|
||||
p2pk65: vecs.any_address_indexes.p2pk65.create_reader(),
|
||||
p2pkh: vecs.any_address_indexes.p2pkh.create_reader(),
|
||||
p2sh: vecs.any_address_indexes.p2sh.create_reader(),
|
||||
p2tr: vecs.any_address_indexes.p2tr.create_reader(),
|
||||
p2wpkh: vecs.any_address_indexes.p2wpkh.create_reader(),
|
||||
p2wsh: vecs.any_address_indexes.p2wsh.create_reader(),
|
||||
p2a: vecs.any_address_indexes.p2a.create_reader(),
|
||||
},
|
||||
anyaddressindex_to_anyaddressdata: ByAnyAddress {
|
||||
loaded: vecs.addresses_data.loaded.create_reader(),
|
||||
empty: vecs.addresses_data.empty.create_reader(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_anyaddressindex_reader(&self, address_type: OutputType) -> &Reader {
|
||||
self.addresstypeindex_to_anyaddressindex
|
||||
.get_unwrap(address_type)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_txoutindex_to_txindex<'a>(
|
||||
block_first_txindex: TxIndex,
|
||||
block_tx_count: u64,
|
||||
txindex_to_output_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
|
||||
) -> Vec<TxIndex> {
|
||||
let block_first_txindex = block_first_txindex.to_usize();
|
||||
|
||||
let counts: Vec<_> = (0..block_tx_count as usize)
|
||||
.map(|tx_offset| {
|
||||
let txindex = TxIndex::from(block_first_txindex + tx_offset);
|
||||
u64::from(txindex_to_output_count.get_unwrap(txindex))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total: u64 = counts.iter().sum();
|
||||
let mut vec = Vec::with_capacity(total as usize);
|
||||
|
||||
for (tx_offset, &output_count) in counts.iter().enumerate() {
|
||||
let txindex = TxIndex::from(block_first_txindex + tx_offset);
|
||||
for _ in 0..output_count {
|
||||
vec.push(txindex);
|
||||
}
|
||||
}
|
||||
|
||||
vec
|
||||
}
|
||||
|
||||
pub fn build_txinindex_to_txindex<'a>(
|
||||
block_first_txindex: TxIndex,
|
||||
block_tx_count: u64,
|
||||
txindex_to_input_count: &mut BoxedVecIterator<'a, TxIndex, StoredU64>,
|
||||
) -> Vec<TxIndex> {
|
||||
let block_first_txindex = block_first_txindex.to_usize();
|
||||
|
||||
let counts: Vec<_> = (0..block_tx_count as usize)
|
||||
.map(|tx_offset| {
|
||||
let txindex = TxIndex::from(block_first_txindex + tx_offset);
|
||||
u64::from(txindex_to_input_count.get_unwrap(txindex))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total: u64 = counts.iter().sum();
|
||||
let mut vec = Vec::with_capacity(total as usize);
|
||||
|
||||
for (tx_offset, &input_count) in counts.iter().enumerate() {
|
||||
let txindex = TxIndex::from(block_first_txindex + tx_offset);
|
||||
for _ in 0..input_count {
|
||||
vec.push(txindex);
|
||||
}
|
||||
}
|
||||
|
||||
vec
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
|
||||
use vecdb::{Exit, IterableVec};
|
||||
|
||||
use crate::{Indexes, indexes, price};
|
||||
|
||||
pub trait DynCohortVecs: Send + Sync {
|
||||
fn min_height_vecs_len(&self) -> usize;
|
||||
fn reset_state_starting_height(&mut self);
|
||||
|
||||
fn import_state(&mut self, starting_height: Height) -> Result<Height>;
|
||||
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
|
||||
|
||||
fn truncate_push(&mut self, height: Height) -> Result<()>;
|
||||
|
||||
fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
) -> Result<()>;
|
||||
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()>;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
pub trait CohortVecs: DynCohortVecs {
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>;
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{ByAddressType, Filtered};
|
||||
use brk_types::{
|
||||
CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex,
|
||||
};
|
||||
use vecdb::VecIndex;
|
||||
|
||||
use crate::utils::OptionExt;
|
||||
|
||||
use super::{
|
||||
address_cohorts,
|
||||
addresstype::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec},
|
||||
withaddressdatasource::WithAddressDataSource,
|
||||
};
|
||||
|
||||
impl AddressTypeToVec<(TypeIndex, Sats)> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn process_received(
|
||||
self,
|
||||
vecs: &mut address_cohorts::Vecs,
|
||||
addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexMap<
|
||||
WithAddressDataSource<LoadedAddressData>,
|
||||
>,
|
||||
addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexMap<
|
||||
WithAddressDataSource<EmptyAddressData>,
|
||||
>,
|
||||
price: Option<Dollars>,
|
||||
addresstype_to_addr_count: &mut ByAddressType<u64>,
|
||||
addresstype_to_empty_addr_count: &mut ByAddressType<u64>,
|
||||
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexMap<
|
||||
WithAddressDataSource<LoadedAddressData>,
|
||||
>,
|
||||
) {
|
||||
self.unwrap().into_iter().for_each(|(_type, vec)| {
|
||||
vec.into_iter().for_each(|(type_index, value)| {
|
||||
let mut is_new = false;
|
||||
let mut from_any_empty = false;
|
||||
|
||||
let addressdata_withsource = addresstype_to_typeindex_to_loadedaddressdata
|
||||
.get_mut(_type)
|
||||
.unwrap()
|
||||
.entry(type_index)
|
||||
.or_insert_with(|| {
|
||||
addresstype_to_typeindex_to_emptyaddressdata
|
||||
.get_mut(_type)
|
||||
.unwrap()
|
||||
.remove(&type_index)
|
||||
.map(|ad| {
|
||||
from_any_empty = true;
|
||||
ad.into()
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
let addressdata =
|
||||
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
|
||||
.remove_for_type(_type, &type_index);
|
||||
is_new = addressdata.is_new();
|
||||
from_any_empty = addressdata.is_from_emptyaddressdata();
|
||||
addressdata
|
||||
})
|
||||
});
|
||||
|
||||
if is_new || from_any_empty {
|
||||
(*addresstype_to_addr_count.get_mut(_type).unwrap()) += 1;
|
||||
if from_any_empty {
|
||||
(*addresstype_to_empty_addr_count.get_mut(_type).unwrap()) -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
let addressdata = addressdata_withsource.deref_mut();
|
||||
|
||||
let prev_amount = addressdata.balance();
|
||||
|
||||
let amount = prev_amount + value;
|
||||
|
||||
let filters_differ = vecs.amount_range.get(amount).filter()
|
||||
!= vecs.amount_range.get(prev_amount).filter();
|
||||
|
||||
if is_new || from_any_empty || filters_differ {
|
||||
if !is_new && !from_any_empty {
|
||||
vecs.amount_range
|
||||
.get_mut(prev_amount)
|
||||
.state
|
||||
.um()
|
||||
.subtract(addressdata);
|
||||
}
|
||||
|
||||
addressdata.receive(value, price);
|
||||
|
||||
vecs.amount_range
|
||||
.get_mut(amount)
|
||||
.state
|
||||
.um()
|
||||
.add(addressdata);
|
||||
} else {
|
||||
vecs.amount_range
|
||||
.get_mut(amount)
|
||||
.state
|
||||
.um()
|
||||
.receive(addressdata, value, price);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn process_sent(
|
||||
self,
|
||||
vecs: &mut address_cohorts::Vecs,
|
||||
addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexMap<
|
||||
WithAddressDataSource<LoadedAddressData>,
|
||||
>,
|
||||
addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexMap<
|
||||
WithAddressDataSource<EmptyAddressData>,
|
||||
>,
|
||||
price: Option<Dollars>,
|
||||
addresstype_to_addr_count: &mut ByAddressType<u64>,
|
||||
addresstype_to_empty_addr_count: &mut ByAddressType<u64>,
|
||||
height_to_price_close_vec: Option<&Vec<brk_types::Close<Dollars>>>,
|
||||
height_to_timestamp_fixed_vec: &[Timestamp],
|
||||
height: Height,
|
||||
timestamp: Timestamp,
|
||||
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexMap<
|
||||
WithAddressDataSource<LoadedAddressData>,
|
||||
>,
|
||||
) -> Result<()> {
|
||||
self.0.into_iter().try_for_each(|(prev_height, v)| {
|
||||
let prev_price = height_to_price_close_vec
|
||||
.as_ref()
|
||||
.map(|v| **v.get(prev_height.to_usize()).unwrap());
|
||||
|
||||
let prev_timestamp = *height_to_timestamp_fixed_vec
|
||||
.get(prev_height.to_usize())
|
||||
.unwrap();
|
||||
|
||||
let blocks_old = height.to_usize() - prev_height.to_usize();
|
||||
|
||||
let days_old = timestamp.difference_in_days_between_float(prev_timestamp);
|
||||
|
||||
let older_than_hour = timestamp
|
||||
.checked_sub(prev_timestamp)
|
||||
.unwrap()
|
||||
.is_more_than_hour();
|
||||
|
||||
v.unwrap().into_iter().try_for_each(|(_type, vec)| {
|
||||
vec.into_iter().try_for_each(|(type_index, value)| {
|
||||
let typeindex_to_loadedaddressdata =
|
||||
addresstype_to_typeindex_to_loadedaddressdata.get_mut_unwrap(_type);
|
||||
|
||||
let addressdata_withsource = typeindex_to_loadedaddressdata
|
||||
.entry(type_index)
|
||||
.or_insert_with(|| {
|
||||
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
|
||||
.remove_for_type(_type, &type_index)
|
||||
});
|
||||
|
||||
let addressdata = addressdata_withsource.deref_mut();
|
||||
|
||||
let prev_amount = addressdata.balance();
|
||||
|
||||
let amount = prev_amount.checked_sub(value).unwrap();
|
||||
|
||||
let will_be_empty = addressdata.has_1_utxos();
|
||||
|
||||
let filters_differ = vecs.amount_range.get(amount).filter()
|
||||
!= vecs.amount_range.get(prev_amount).filter();
|
||||
|
||||
if will_be_empty || filters_differ {
|
||||
vecs.amount_range
|
||||
.get_mut(prev_amount)
|
||||
.state
|
||||
.um()
|
||||
.subtract(addressdata);
|
||||
|
||||
addressdata.send(value, prev_price)?;
|
||||
|
||||
if will_be_empty {
|
||||
if amount.is_not_zero() {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
(*addresstype_to_addr_count.get_mut(_type).unwrap()) -= 1;
|
||||
(*addresstype_to_empty_addr_count.get_mut(_type).unwrap()) += 1;
|
||||
|
||||
let addressdata =
|
||||
typeindex_to_loadedaddressdata.remove(&type_index).unwrap();
|
||||
|
||||
addresstype_to_typeindex_to_emptyaddressdata
|
||||
.get_mut(_type)
|
||||
.unwrap()
|
||||
.insert(type_index, addressdata.into());
|
||||
} else {
|
||||
vecs.amount_range
|
||||
.get_mut(amount)
|
||||
.state
|
||||
.um()
|
||||
.add(addressdata);
|
||||
}
|
||||
} else {
|
||||
vecs.amount_range.get_mut(amount).state.um().send(
|
||||
addressdata,
|
||||
value,
|
||||
price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old,
|
||||
older_than_hour,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
use std::{ops::Deref, path::Path};
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{CohortContext, Filter, Filtered, StateLevel};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version};
|
||||
use vecdb::{Database, Exit, IterableVec};
|
||||
|
||||
use crate::{
|
||||
Indexes, PriceToAmount, UTXOCohortState,
|
||||
grouped::{PERCENTILES, PERCENTILES_LEN},
|
||||
indexes, price,
|
||||
stateful::{
|
||||
common,
|
||||
r#trait::{CohortVecs, DynCohortVecs},
|
||||
},
|
||||
utils::OptionExt,
|
||||
};
|
||||
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct Vecs {
|
||||
state_starting_height: Option<Height>,
|
||||
|
||||
#[traversable(skip)]
|
||||
pub state: Option<UTXOCohortState>,
|
||||
|
||||
/// For aggregate cohorts (all, sth, lth) that only need price_to_amount for percentiles
|
||||
#[traversable(skip)]
|
||||
pub price_to_amount: Option<PriceToAmount>,
|
||||
|
||||
#[traversable(flatten)]
|
||||
pub inner: common::Vecs,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
filter: Filter,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: &Path,
|
||||
state_level: StateLevel,
|
||||
) -> Result<Self> {
|
||||
let compute_dollars = price.is_some();
|
||||
|
||||
let full_name = filter.to_full_name(CohortContext::Utxo);
|
||||
|
||||
Ok(Self {
|
||||
state_starting_height: None,
|
||||
|
||||
state: if state_level.is_full() {
|
||||
Some(UTXOCohortState::new(
|
||||
states_path,
|
||||
&full_name,
|
||||
compute_dollars,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
|
||||
price_to_amount: if state_level.is_price_only() && compute_dollars {
|
||||
Some(PriceToAmount::create(states_path, &full_name))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
|
||||
inner: common::Vecs::forced_import(
|
||||
db,
|
||||
filter,
|
||||
CohortContext::Utxo,
|
||||
version,
|
||||
indexes,
|
||||
price,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl DynCohortVecs for Vecs {
|
||||
fn min_height_vecs_len(&self) -> usize {
|
||||
self.inner.min_height_vecs_len()
|
||||
}
|
||||
|
||||
fn reset_state_starting_height(&mut self) {
|
||||
self.state_starting_height = Some(Height::ZERO);
|
||||
}
|
||||
|
||||
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
|
||||
let starting_height = self
|
||||
.inner
|
||||
.import_state(starting_height, self.state.um())?;
|
||||
|
||||
self.state_starting_height = Some(starting_height);
|
||||
|
||||
Ok(starting_height)
|
||||
}
|
||||
|
||||
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
|
||||
self.inner.validate_computed_versions(base_version)
|
||||
}
|
||||
|
||||
fn truncate_push(&mut self, height: Height) -> Result<()> {
|
||||
if self.state_starting_height.unwrap() > height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.inner
|
||||
.truncate_push(height, self.state.u())
|
||||
}
|
||||
|
||||
fn compute_then_truncate_push_unrealized_states(
|
||||
&mut self,
|
||||
height: Height,
|
||||
height_price: Option<Dollars>,
|
||||
dateindex: Option<DateIndex>,
|
||||
date_price: Option<Option<Dollars>>,
|
||||
) -> Result<()> {
|
||||
self.inner.compute_then_truncate_push_unrealized_states(
|
||||
height,
|
||||
height_price,
|
||||
dateindex,
|
||||
date_price,
|
||||
self.state.um(),
|
||||
)
|
||||
}
|
||||
|
||||
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.inner
|
||||
.safe_flush_stateful_vecs(height, exit, self.state.um())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.inner
|
||||
.compute_rest_part1(indexes, price, starting_indexes, exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl CohortVecs for Vecs {
|
||||
fn compute_from_stateful(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
others: &[&Self],
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.inner.compute_from_stateful(
|
||||
starting_indexes,
|
||||
&others.iter().map(|v| &v.inner).collect::<Vec<_>>(),
|
||||
exit,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.inner.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
/// Compute percentile prices for aggregate cohorts that have standalone price_to_amount.
|
||||
/// Returns NaN array if price_to_amount is None or empty.
|
||||
pub fn compute_percentile_prices_from_standalone(
|
||||
&self,
|
||||
supply: Sats,
|
||||
) -> [Dollars; PERCENTILES_LEN] {
|
||||
let mut result = [Dollars::NAN; PERCENTILES_LEN];
|
||||
|
||||
let price_to_amount = match self.price_to_amount.as_ref() {
|
||||
Some(p) => p,
|
||||
None => return result,
|
||||
};
|
||||
|
||||
if price_to_amount.is_empty() || supply == Sats::ZERO {
|
||||
return result;
|
||||
}
|
||||
|
||||
let total = supply;
|
||||
let targets = PERCENTILES.map(|p| total * p / 100);
|
||||
|
||||
let mut accumulated = Sats::ZERO;
|
||||
let mut pct_idx = 0;
|
||||
|
||||
for (&price, &sats) in price_to_amount.iter() {
|
||||
accumulated += sats;
|
||||
|
||||
while pct_idx < PERCENTILES_LEN && accumulated >= targets[pct_idx] {
|
||||
result[pct_idx] = price;
|
||||
pct_idx += 1;
|
||||
}
|
||||
|
||||
if pct_idx >= PERCENTILES_LEN {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Vecs {
|
||||
type Target = common::Vecs;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl Filtered for Vecs {
|
||||
fn filter(&self) -> &Filter {
|
||||
&self.inner.filter
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,697 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_grouper::{
|
||||
AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount,
|
||||
ByMaxAge, ByMinAge, BySpendableType, ByTerm, Filter, Filtered, StateLevel, Term, TimeFilter,
|
||||
UTXOGroups,
|
||||
};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{
|
||||
Bitcoin, CheckedSub, DateIndex, Dollars, HalvingEpoch, Height, ONE_DAY_IN_SEC, OutputType,
|
||||
Sats, Timestamp, Version,
|
||||
};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
use rustc_hash::FxHashMap;
|
||||
use vecdb::{Database, Exit, IterableVec, VecIndex};
|
||||
|
||||
use crate::{
|
||||
Indexes, indexes, price,
|
||||
stateful::{Flushable, HeightFlushable, r#trait::DynCohortVecs},
|
||||
states::{BlockState, Transacted},
|
||||
utils::OptionExt,
|
||||
};
|
||||
|
||||
use super::{r#trait::CohortVecs, utxo_cohort};
|
||||
|
||||
const VERSION: Version = Version::new(0);
|
||||
|
||||
#[derive(Clone, Deref, DerefMut, Traversable)]
|
||||
pub struct Vecs(UTXOGroups<utxo_cohort::Vecs>);
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
db: &Database,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
states_path: &Path,
|
||||
) -> Result<Self> {
|
||||
let v = version + VERSION + Version::ZERO;
|
||||
|
||||
// Helper to create a cohort - booleans are now derived from filter
|
||||
let create = |filter: Filter, state_level: StateLevel| -> Result<utxo_cohort::Vecs> {
|
||||
utxo_cohort::Vecs::forced_import(
|
||||
db,
|
||||
filter,
|
||||
v,
|
||||
indexes,
|
||||
price,
|
||||
states_path,
|
||||
state_level,
|
||||
)
|
||||
};
|
||||
|
||||
let full = |f: Filter| create(f, StateLevel::Full);
|
||||
let none = |f: Filter| create(f, StateLevel::None);
|
||||
|
||||
Ok(Self(UTXOGroups {
|
||||
// Special case: all uses Version::ONE
|
||||
all: utxo_cohort::Vecs::forced_import(
|
||||
db,
|
||||
Filter::All,
|
||||
version + VERSION + Version::ONE,
|
||||
indexes,
|
||||
price,
|
||||
states_path,
|
||||
StateLevel::PriceOnly,
|
||||
)?,
|
||||
|
||||
term: ByTerm {
|
||||
short: create(Filter::Term(Term::Sth), StateLevel::PriceOnly)?,
|
||||
long: create(Filter::Term(Term::Lth), StateLevel::PriceOnly)?,
|
||||
},
|
||||
|
||||
epoch: ByEpoch {
|
||||
_0: full(Filter::Epoch(HalvingEpoch::new(0)))?,
|
||||
_1: full(Filter::Epoch(HalvingEpoch::new(1)))?,
|
||||
_2: full(Filter::Epoch(HalvingEpoch::new(2)))?,
|
||||
_3: full(Filter::Epoch(HalvingEpoch::new(3)))?,
|
||||
_4: full(Filter::Epoch(HalvingEpoch::new(4)))?,
|
||||
},
|
||||
|
||||
type_: BySpendableType {
|
||||
p2pk65: full(Filter::Type(OutputType::P2PK65))?,
|
||||
p2pk33: full(Filter::Type(OutputType::P2PK33))?,
|
||||
p2pkh: full(Filter::Type(OutputType::P2PKH))?,
|
||||
p2sh: full(Filter::Type(OutputType::P2SH))?,
|
||||
p2wpkh: full(Filter::Type(OutputType::P2WPKH))?,
|
||||
p2wsh: full(Filter::Type(OutputType::P2WSH))?,
|
||||
p2tr: full(Filter::Type(OutputType::P2TR))?,
|
||||
p2a: full(Filter::Type(OutputType::P2A))?,
|
||||
p2ms: full(Filter::Type(OutputType::P2MS))?,
|
||||
empty: full(Filter::Type(OutputType::Empty))?,
|
||||
unknown: full(Filter::Type(OutputType::Unknown))?,
|
||||
},
|
||||
|
||||
max_age: ByMaxAge {
|
||||
_1w: none(Filter::Time(TimeFilter::LowerThan(7)))?,
|
||||
_1m: none(Filter::Time(TimeFilter::LowerThan(30)))?,
|
||||
_2m: none(Filter::Time(TimeFilter::LowerThan(2 * 30)))?,
|
||||
_3m: none(Filter::Time(TimeFilter::LowerThan(3 * 30)))?,
|
||||
_4m: none(Filter::Time(TimeFilter::LowerThan(4 * 30)))?,
|
||||
_5m: none(Filter::Time(TimeFilter::LowerThan(5 * 30)))?,
|
||||
_6m: none(Filter::Time(TimeFilter::LowerThan(6 * 30)))?,
|
||||
_1y: none(Filter::Time(TimeFilter::LowerThan(365)))?,
|
||||
_2y: none(Filter::Time(TimeFilter::LowerThan(2 * 365)))?,
|
||||
_3y: none(Filter::Time(TimeFilter::LowerThan(3 * 365)))?,
|
||||
_4y: none(Filter::Time(TimeFilter::LowerThan(4 * 365)))?,
|
||||
_5y: none(Filter::Time(TimeFilter::LowerThan(5 * 365)))?,
|
||||
_6y: none(Filter::Time(TimeFilter::LowerThan(6 * 365)))?,
|
||||
_7y: none(Filter::Time(TimeFilter::LowerThan(7 * 365)))?,
|
||||
_8y: none(Filter::Time(TimeFilter::LowerThan(8 * 365)))?,
|
||||
_10y: none(Filter::Time(TimeFilter::LowerThan(10 * 365)))?,
|
||||
_12y: none(Filter::Time(TimeFilter::LowerThan(12 * 365)))?,
|
||||
_15y: none(Filter::Time(TimeFilter::LowerThan(15 * 365)))?,
|
||||
},
|
||||
|
||||
min_age: ByMinAge {
|
||||
_1d: none(Filter::Time(TimeFilter::GreaterOrEqual(1)))?,
|
||||
_1w: none(Filter::Time(TimeFilter::GreaterOrEqual(7)))?,
|
||||
_1m: none(Filter::Time(TimeFilter::GreaterOrEqual(30)))?,
|
||||
_2m: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 30)))?,
|
||||
_3m: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 30)))?,
|
||||
_4m: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 30)))?,
|
||||
_5m: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 30)))?,
|
||||
_6m: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 30)))?,
|
||||
_1y: none(Filter::Time(TimeFilter::GreaterOrEqual(365)))?,
|
||||
_2y: none(Filter::Time(TimeFilter::GreaterOrEqual(2 * 365)))?,
|
||||
_3y: none(Filter::Time(TimeFilter::GreaterOrEqual(3 * 365)))?,
|
||||
_4y: none(Filter::Time(TimeFilter::GreaterOrEqual(4 * 365)))?,
|
||||
_5y: none(Filter::Time(TimeFilter::GreaterOrEqual(5 * 365)))?,
|
||||
_6y: none(Filter::Time(TimeFilter::GreaterOrEqual(6 * 365)))?,
|
||||
_7y: none(Filter::Time(TimeFilter::GreaterOrEqual(7 * 365)))?,
|
||||
_8y: none(Filter::Time(TimeFilter::GreaterOrEqual(8 * 365)))?,
|
||||
_10y: none(Filter::Time(TimeFilter::GreaterOrEqual(10 * 365)))?,
|
||||
_12y: none(Filter::Time(TimeFilter::GreaterOrEqual(12 * 365)))?,
|
||||
},
|
||||
|
||||
age_range: ByAgeRange {
|
||||
up_to_1d: full(Filter::Time(TimeFilter::Range(0..1)))?,
|
||||
_1d_to_1w: full(Filter::Time(TimeFilter::Range(1..7)))?,
|
||||
_1w_to_1m: full(Filter::Time(TimeFilter::Range(7..30)))?,
|
||||
_1m_to_2m: full(Filter::Time(TimeFilter::Range(30..60)))?,
|
||||
_2m_to_3m: full(Filter::Time(TimeFilter::Range(60..90)))?,
|
||||
_3m_to_4m: full(Filter::Time(TimeFilter::Range(90..120)))?,
|
||||
_4m_to_5m: full(Filter::Time(TimeFilter::Range(120..150)))?,
|
||||
_5m_to_6m: full(Filter::Time(TimeFilter::Range(150..180)))?,
|
||||
_6m_to_1y: full(Filter::Time(TimeFilter::Range(180..365)))?,
|
||||
_1y_to_2y: full(Filter::Time(TimeFilter::Range(365..730)))?,
|
||||
_2y_to_3y: full(Filter::Time(TimeFilter::Range(730..1095)))?,
|
||||
_3y_to_4y: full(Filter::Time(TimeFilter::Range(1095..1460)))?,
|
||||
_4y_to_5y: full(Filter::Time(TimeFilter::Range(1460..1825)))?,
|
||||
_5y_to_6y: full(Filter::Time(TimeFilter::Range(1825..2190)))?,
|
||||
_6y_to_7y: full(Filter::Time(TimeFilter::Range(2190..2555)))?,
|
||||
_7y_to_8y: full(Filter::Time(TimeFilter::Range(2555..2920)))?,
|
||||
_8y_to_10y: full(Filter::Time(TimeFilter::Range(2920..3650)))?,
|
||||
_10y_to_12y: full(Filter::Time(TimeFilter::Range(3650..4380)))?,
|
||||
_12y_to_15y: full(Filter::Time(TimeFilter::Range(4380..5475)))?,
|
||||
from_15y: full(Filter::Time(TimeFilter::GreaterOrEqual(15 * 365)))?,
|
||||
},
|
||||
|
||||
amount_range: ByAmountRange {
|
||||
_0sats: full(Filter::Amount(AmountFilter::LowerThan(Sats::_1)))?,
|
||||
_1sat_to_10sats: full(Filter::Amount(AmountFilter::Range(Sats::_1..Sats::_10)))?,
|
||||
_10sats_to_100sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10..Sats::_100,
|
||||
)))?,
|
||||
_100sats_to_1k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100..Sats::_1K,
|
||||
)))?,
|
||||
_1k_sats_to_10k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1K..Sats::_10K,
|
||||
)))?,
|
||||
_10k_sats_to_100k_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10K..Sats::_100K,
|
||||
)))?,
|
||||
_100k_sats_to_1m_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100K..Sats::_1M,
|
||||
)))?,
|
||||
_1m_sats_to_10m_sats: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1M..Sats::_10M,
|
||||
)))?,
|
||||
_10m_sats_to_1btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10M..Sats::_1BTC,
|
||||
)))?,
|
||||
_1btc_to_10btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1BTC..Sats::_10BTC,
|
||||
)))?,
|
||||
_10btc_to_100btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10BTC..Sats::_100BTC,
|
||||
)))?,
|
||||
_100btc_to_1k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_100BTC..Sats::_1K_BTC,
|
||||
)))?,
|
||||
_1k_btc_to_10k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_1K_BTC..Sats::_10K_BTC,
|
||||
)))?,
|
||||
_10k_btc_to_100k_btc: full(Filter::Amount(AmountFilter::Range(
|
||||
Sats::_10K_BTC..Sats::_100K_BTC,
|
||||
)))?,
|
||||
_100k_btc_or_more: full(Filter::Amount(AmountFilter::GreaterOrEqual(
|
||||
Sats::_100K_BTC,
|
||||
)))?,
|
||||
},
|
||||
|
||||
lt_amount: ByLowerThanAmount {
|
||||
_10sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10)))?,
|
||||
_100sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100)))?,
|
||||
_1k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K)))?,
|
||||
_10k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K)))?,
|
||||
_100k_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K)))?,
|
||||
_1m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1M)))?,
|
||||
_10m_sats: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10M)))?,
|
||||
_1btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1BTC)))?,
|
||||
_10btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10BTC)))?,
|
||||
_100btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100BTC)))?,
|
||||
_1k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_1K_BTC)))?,
|
||||
_10k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_10K_BTC)))?,
|
||||
_100k_btc: none(Filter::Amount(AmountFilter::LowerThan(Sats::_100K_BTC)))?,
|
||||
},
|
||||
|
||||
ge_amount: ByGreatEqualAmount {
|
||||
_1sat: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1)))?,
|
||||
_10sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10)))?,
|
||||
_100sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100)))?,
|
||||
_1k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K)))?,
|
||||
_10k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K)))?,
|
||||
_100k_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100K)))?,
|
||||
_1m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1M)))?,
|
||||
_10m_sats: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10M)))?,
|
||||
_1btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1BTC)))?,
|
||||
_10btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10BTC)))?,
|
||||
_100btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_100BTC)))?,
|
||||
_1k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_1K_BTC)))?,
|
||||
_10k_btc: none(Filter::Amount(AmountFilter::GreaterOrEqual(Sats::_10K_BTC)))?,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
|
||||
if chain_state.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let prev_timestamp = chain_state.last().unwrap().timestamp;
|
||||
|
||||
// Only blocks whose age % ONE_DAY >= threshold can cross a day boundary.
|
||||
// Saves 1 subtraction + 2 divisions per block vs computing days_old directly.
|
||||
let elapsed = (*timestamp).saturating_sub(*prev_timestamp);
|
||||
let threshold = ONE_DAY_IN_SEC.saturating_sub(elapsed);
|
||||
|
||||
// Extract all mutable references upfront to avoid borrow checker issues
|
||||
// Use a single destructuring to get non-overlapping mutable borrows
|
||||
let UTXOGroups {
|
||||
all,
|
||||
term,
|
||||
age_range,
|
||||
..
|
||||
} = &mut self.0;
|
||||
|
||||
let mut vecs = age_range
|
||||
.iter_mut()
|
||||
.map(|v| (v.filter().clone(), &mut v.state))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Collect aggregate cohorts' filter and p2a for age transitions
|
||||
let mut aggregate_p2a: Vec<(Filter, Option<&mut crate::PriceToAmount>)> = vec![
|
||||
(all.filter().clone(), all.price_to_amount.as_mut()),
|
||||
(
|
||||
term.short.filter().clone(),
|
||||
term.short.price_to_amount.as_mut(),
|
||||
),
|
||||
(
|
||||
term.long.filter().clone(),
|
||||
term.long.price_to_amount.as_mut(),
|
||||
),
|
||||
];
|
||||
|
||||
chain_state
|
||||
.iter()
|
||||
.filter(|block_state| {
|
||||
let age = (*prev_timestamp).saturating_sub(*block_state.timestamp);
|
||||
age % ONE_DAY_IN_SEC >= threshold
|
||||
})
|
||||
.for_each(|block_state| {
|
||||
let prev_days_old =
|
||||
prev_timestamp.difference_in_days_between(block_state.timestamp);
|
||||
let days_old = timestamp.difference_in_days_between(block_state.timestamp);
|
||||
|
||||
if prev_days_old == days_old {
|
||||
return;
|
||||
}
|
||||
|
||||
vecs.iter_mut().for_each(|(filter, state)| {
|
||||
let is = filter.contains_time(days_old);
|
||||
let was = filter.contains_time(prev_days_old);
|
||||
|
||||
if is && !was {
|
||||
state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.increment(&block_state.supply, block_state.price);
|
||||
} else if was && !is {
|
||||
state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.decrement(&block_state.supply, block_state.price);
|
||||
}
|
||||
});
|
||||
|
||||
// Handle age transitions for aggregate cohorts' price_to_amount
|
||||
// Check which cohorts the UTXO was in vs is now in, and increment/decrement accordingly
|
||||
// Only process if there's remaining supply (like CohortState::increment/decrement do)
|
||||
if let Some(price) = block_state.price
|
||||
&& block_state.supply.value > Sats::ZERO
|
||||
{
|
||||
aggregate_p2a.iter_mut().for_each(|(filter, p2a)| {
|
||||
let is = filter.contains_time(days_old);
|
||||
let was = filter.contains_time(prev_days_old);
|
||||
|
||||
if is && !was {
|
||||
p2a.um().increment(price, &block_state.supply);
|
||||
} else if was && !is {
|
||||
p2a.um().decrement(price, &block_state.supply);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn send(
|
||||
&mut self,
|
||||
height_to_sent: FxHashMap<Height, Transacted>,
|
||||
chain_state: &mut [BlockState],
|
||||
) {
|
||||
// Extract all mutable references upfront to avoid borrow checker issues
|
||||
let UTXOGroups {
|
||||
all,
|
||||
term,
|
||||
age_range,
|
||||
epoch,
|
||||
type_,
|
||||
amount_range,
|
||||
..
|
||||
} = &mut self.0;
|
||||
|
||||
let mut time_based_vecs = age_range
|
||||
.iter_mut()
|
||||
.chain(epoch.iter_mut())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Collect aggregate cohorts' filter and p2a for iteration
|
||||
let mut aggregate_p2a: Vec<(Filter, Option<&mut crate::PriceToAmount>)> = vec![
|
||||
(all.filter().clone(), all.price_to_amount.as_mut()),
|
||||
(
|
||||
term.short.filter().clone(),
|
||||
term.short.price_to_amount.as_mut(),
|
||||
),
|
||||
(
|
||||
term.long.filter().clone(),
|
||||
term.long.price_to_amount.as_mut(),
|
||||
),
|
||||
];
|
||||
|
||||
let last_block = chain_state.last().unwrap();
|
||||
let last_timestamp = last_block.timestamp;
|
||||
let current_price = last_block.price;
|
||||
|
||||
let chain_state_len = chain_state.len();
|
||||
|
||||
height_to_sent.into_iter().for_each(|(height, sent)| {
|
||||
chain_state[height.to_usize()].supply -= &sent.spendable_supply;
|
||||
|
||||
let block_state = chain_state.get(height.to_usize()).unwrap();
|
||||
|
||||
let prev_price = block_state.price;
|
||||
|
||||
let blocks_old = chain_state_len - 1 - height.to_usize();
|
||||
|
||||
let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
|
||||
let days_old_float =
|
||||
last_timestamp.difference_in_days_between_float(block_state.timestamp);
|
||||
|
||||
let older_than_hour = last_timestamp
|
||||
.checked_sub(block_state.timestamp)
|
||||
.unwrap()
|
||||
.is_more_than_hour();
|
||||
|
||||
time_based_vecs
|
||||
.iter_mut()
|
||||
.filter(|v| match v.filter() {
|
||||
Filter::Time(TimeFilter::GreaterOrEqual(from)) => *from <= days_old,
|
||||
Filter::Time(TimeFilter::LowerThan(to)) => *to > days_old,
|
||||
Filter::Time(TimeFilter::Range(range)) => range.contains(&days_old),
|
||||
Filter::Epoch(epoch) => *epoch == HalvingEpoch::from(height),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
.for_each(|vecs| {
|
||||
vecs.state.um().send(
|
||||
&sent.spendable_supply,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_float,
|
||||
older_than_hour,
|
||||
);
|
||||
});
|
||||
|
||||
sent.by_type
|
||||
.spendable
|
||||
.iter_typed()
|
||||
.for_each(|(output_type, supply_state)| {
|
||||
type_.get_mut(output_type).state.um().send(
|
||||
supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_float,
|
||||
older_than_hour,
|
||||
)
|
||||
});
|
||||
|
||||
sent.by_size_group
|
||||
.iter_typed()
|
||||
.for_each(|(group, supply_state)| {
|
||||
amount_range.get_mut(group).state.um().send(
|
||||
supply_state,
|
||||
current_price,
|
||||
prev_price,
|
||||
blocks_old,
|
||||
days_old_float,
|
||||
older_than_hour,
|
||||
);
|
||||
});
|
||||
|
||||
// Update aggregate cohorts' price_to_amount using filter.contains_time()
|
||||
if let Some(prev_price) = prev_price {
|
||||
let supply_state = &sent.spendable_supply;
|
||||
if supply_state.value.is_not_zero() {
|
||||
aggregate_p2a
|
||||
.iter_mut()
|
||||
.filter(|(f, _)| f.contains_time(days_old))
|
||||
.map(|(_, p2a)| p2a)
|
||||
.for_each(|p2a| {
|
||||
p2a.um().decrement(prev_price, supply_state);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
|
||||
let supply_state = received.spendable_supply;
|
||||
|
||||
[
|
||||
&mut self.0.age_range.up_to_1d,
|
||||
self.0.epoch.mut_vec_from_height(height),
|
||||
]
|
||||
.into_iter()
|
||||
.for_each(|v| {
|
||||
v.state.um().receive(&supply_state, price);
|
||||
});
|
||||
|
||||
// Update aggregate cohorts' price_to_amount
|
||||
// New UTXOs have days_old = 0, so use filter.contains_time(0) to check applicability
|
||||
if let Some(price) = price
|
||||
&& supply_state.value.is_not_zero()
|
||||
{
|
||||
self.0
|
||||
.iter_aggregate_mut()
|
||||
.filter(|v| v.filter().contains_time(0))
|
||||
.for_each(|v| {
|
||||
v.price_to_amount
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.increment(price, &supply_state);
|
||||
});
|
||||
}
|
||||
|
||||
self.type_.iter_mut().for_each(|vecs| {
|
||||
let output_type = match vecs.filter() {
|
||||
Filter::Type(output_type) => *output_type,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
vecs.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.receive(received.by_type.get(output_type), price)
|
||||
});
|
||||
|
||||
received
|
||||
.by_size_group
|
||||
.iter_typed()
|
||||
.for_each(|(group, supply_state)| {
|
||||
self.amount_range
|
||||
.get_mut(group)
|
||||
.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.receive(supply_state, price);
|
||||
});
|
||||
}
|
||||
|
||||
pub fn compute_overlapping_vecs(
|
||||
&mut self,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let by_date_range = &self.0.age_range;
|
||||
let by_size_range = &self.0.amount_range;
|
||||
|
||||
[(&mut self.0.all, by_date_range.iter().collect::<Vec<_>>())]
|
||||
.into_par_iter()
|
||||
.chain(self.0.min_age.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_date_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.max_age.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_date_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.term.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_date_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.ge_amount.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.chain(self.0.lt_amount.par_iter_mut().map(|vecs| {
|
||||
let filter = vecs.filter().clone();
|
||||
(
|
||||
vecs,
|
||||
by_size_range
|
||||
.iter()
|
||||
.filter(|other| filter.includes(other.filter()))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}))
|
||||
.try_for_each(|(vecs, stateful)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.par_iter_mut()
|
||||
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexes: &indexes::Vecs,
|
||||
price: Option<&price::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
height_to_supply: &impl IterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl IterableVec<DateIndex, Bitcoin>,
|
||||
height_to_market_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_market_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
height_to_realized_cap: Option<&impl IterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.par_iter_mut().try_for_each(|v| {
|
||||
v.compute_rest_part2(
|
||||
indexes,
|
||||
price,
|
||||
starting_indexes,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_market_cap,
|
||||
dateindex_to_market_cap,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
// Flush stateful cohorts
|
||||
self.par_iter_separate_mut()
|
||||
.try_for_each(|v| v.safe_flush_stateful_vecs(height, exit))?;
|
||||
|
||||
// Flush aggregate cohorts' price_to_amount and price_percentiles
|
||||
// Using traits ensures we can't forget to flush any field
|
||||
self.0.par_iter_aggregate_mut().try_for_each(|v| {
|
||||
v.price_to_amount.flush_at_height(height, exit)?;
|
||||
v.inner.price_percentiles.safe_write(exit)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Reset aggregate cohorts' price_to_amount when starting from scratch
|
||||
pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> {
|
||||
self.0
|
||||
.iter_aggregate_mut()
|
||||
.try_for_each(|v| v.price_to_amount.reset())
|
||||
}
|
||||
|
||||
/// Import aggregate cohorts' price_to_amount from disk when resuming from a checkpoint.
|
||||
/// Returns the height to start processing from (checkpoint_height + 1), matching the
|
||||
/// behavior of `common::import_state` for separate cohorts.
|
||||
///
|
||||
/// Note: We don't check inner.min_height_vecs_len() for aggregate cohorts because their
|
||||
/// inner vecs (height_to_supply, etc.) are computed post-hoc by compute_overlapping_vecs,
|
||||
/// not maintained during the main processing loop.
|
||||
pub fn import_aggregate_price_to_amount(&mut self, height: Height) -> Result<Height> {
|
||||
// Match separate vecs behavior: decrement height to get prev_height
|
||||
let Some(mut prev_height) = height.decremented() else {
|
||||
// height is 0, return ZERO (caller will handle this)
|
||||
return Ok(Height::ZERO);
|
||||
};
|
||||
|
||||
for v in self.0.iter_aggregate_mut() {
|
||||
// Using HeightFlushable trait - if price_to_amount is None, returns height unchanged
|
||||
prev_height = prev_height.min(v.price_to_amount.import_at_or_before(prev_height)?);
|
||||
}
|
||||
// Return prev_height + 1, matching separate vecs behavior
|
||||
Ok(prev_height.incremented())
|
||||
}
|
||||
|
||||
/// Compute and push percentiles for aggregate cohorts (all, sth, lth).
|
||||
/// Must be called after receive()/send() when price_to_amount is up to date.
|
||||
pub fn truncate_push_aggregate_percentiles(&mut self, height: Height) -> Result<()> {
|
||||
let age_range_data: Vec<_> = self
|
||||
.0
|
||||
.age_range
|
||||
.iter()
|
||||
.map(|sub| (sub.filter().clone(), sub.state.u().supply.value))
|
||||
.collect();
|
||||
|
||||
let results: Vec<_> = self
|
||||
.0
|
||||
.par_iter_aggregate()
|
||||
.map(|v| {
|
||||
if v.price_to_amount.is_none() {
|
||||
panic!();
|
||||
}
|
||||
let filter = v.filter().clone();
|
||||
let supply = age_range_data
|
||||
.iter()
|
||||
.filter(|(sub_filter, _)| filter.includes(sub_filter))
|
||||
.map(|(_, value)| *value)
|
||||
.fold(Sats::ZERO, |acc, v| acc + v);
|
||||
let percentiles = v.compute_percentile_prices_from_standalone(supply);
|
||||
(filter, percentiles)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Push results sequentially (requires &mut)
|
||||
for (filter, percentiles) in results {
|
||||
let v = self
|
||||
.0
|
||||
.iter_aggregate_mut()
|
||||
.find(|v| v.filter() == &filter)
|
||||
.unwrap();
|
||||
|
||||
if let Some(pp) = v.inner.price_percentiles.as_mut() {
|
||||
pp.truncate_push(height, &percentiles)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
use brk_types::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum WithAddressDataSource<T> {
|
||||
New(T),
|
||||
FromLoadedAddressDataVec((LoadedAddressIndex, T)),
|
||||
FromEmptyAddressDataVec((EmptyAddressIndex, T)),
|
||||
}
|
||||
|
||||
impl<T> WithAddressDataSource<T> {
|
||||
pub fn is_new(&self) -> bool {
|
||||
matches!(self, Self::New(_))
|
||||
}
|
||||
|
||||
pub fn is_from_emptyaddressdata(&self) -> bool {
|
||||
matches!(self, Self::FromEmptyAddressDataVec(_))
|
||||
}
|
||||
|
||||
pub fn deref_mut(&mut self) -> &mut T {
|
||||
match self {
|
||||
Self::New(v) => v,
|
||||
Self::FromLoadedAddressDataVec((_, v)) => v,
|
||||
Self::FromEmptyAddressDataVec((_, v)) => v,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
|
||||
#[inline]
|
||||
fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
|
||||
match value {
|
||||
WithAddressDataSource::New(v) => Self::New(v.into()),
|
||||
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
|
||||
Self::FromLoadedAddressDataVec((i, v.into()))
|
||||
}
|
||||
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
|
||||
Self::FromEmptyAddressDataVec((i, v.into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
|
||||
#[inline]
|
||||
fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
|
||||
match value {
|
||||
WithAddressDataSource::New(v) => Self::New(v.into()),
|
||||
WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
|
||||
Self::FromLoadedAddressDataVec((i, v.into()))
|
||||
}
|
||||
WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
|
||||
Self::FromEmptyAddressDataVec((i, v.into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Traits for consistent state flushing and importing.
|
||||
//!
|
||||
//! These traits ensure all stateful components follow the same patterns
|
||||
//! for checkpoint/resume operations, preventing bugs where new fields
|
||||
//! are forgotten during flush operations.
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::Height;
|
||||
use vecdb::Exit;
|
||||
|
||||
/// Trait for components that can be flushed to disk.
|
||||
///
|
||||
/// This is for simple flush operations that don't require height tracking.
|
||||
pub trait Flushable {
|
||||
/// Safely flush data to disk.
|
||||
fn safe_flush(&mut self, exit: &Exit) -> Result<()>;
|
||||
|
||||
/// Write to mmap without fsync. Data visible to readers immediately but not durable.
|
||||
fn safe_write(&mut self, exit: &Exit) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Trait for stateful components that track data indexed by height.
|
||||
///
|
||||
/// This ensures consistent patterns for:
|
||||
/// - Flushing state at checkpoints
|
||||
/// - Importing state when resuming from a checkpoint
|
||||
/// - Resetting state when starting from scratch
|
||||
pub trait HeightFlushable {
|
||||
/// Flush state to disk at the given height checkpoint.
|
||||
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()>;
|
||||
|
||||
/// Import state from the most recent checkpoint at or before the given height.
|
||||
/// Returns the actual height that was imported.
|
||||
fn import_at_or_before(&mut self, height: Height) -> Result<Height>;
|
||||
|
||||
/// Reset state for starting from scratch.
|
||||
fn reset(&mut self) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Blanket implementation for Option<T> where T: Flushable
|
||||
impl<T: Flushable> Flushable for Option<T> {
|
||||
fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.safe_flush(exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_write(&mut self, exit: &Exit) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.safe_write(exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Blanket implementation for Option<T> where T: HeightFlushable
|
||||
impl<T: HeightFlushable> HeightFlushable for Option<T> {
|
||||
fn flush_at_height(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.flush_at_height(height, exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn import_at_or_before(&mut self, height: Height) -> Result<Height> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.import_at_or_before(height)
|
||||
} else {
|
||||
Ok(height)
|
||||
}
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> Result<()> {
|
||||
if let Some(inner) = self.as_mut() {
|
||||
inner.reset()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
mod block;
|
||||
mod cohorts;
|
||||
mod flushable;
|
||||
mod price_to_amount;
|
||||
mod realized;
|
||||
mod supply;
|
||||
@@ -8,6 +9,7 @@ mod unrealized;
|
||||
|
||||
pub use block::*;
|
||||
pub use cohorts::*;
|
||||
pub use flushable::*;
|
||||
pub use price_to_amount::*;
|
||||
pub use realized::*;
|
||||
pub use supply::*;
|
||||
|
||||
@@ -11,7 +11,9 @@ use pco::standalone::{simple_decompress, simpler_compress};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vecdb::{Bytes, Exit};
|
||||
|
||||
use crate::{stateful::HeightFlushable, states::SupplyState, utils::OptionExt};
|
||||
use crate::{states::SupplyState, utils::OptionExt};
|
||||
|
||||
use super::HeightFlushable;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PriceToAmount {
|
||||
|
||||
@@ -23,6 +23,7 @@ brk_traversable = { workspace = true }
|
||||
fjall = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
rlimit = "0.10.2"
|
||||
rustc-hash = { workspace = true }
|
||||
vecdb = { workspace = true }
|
||||
|
||||
|
||||
@@ -30,6 +30,14 @@ pub struct Indexer {
|
||||
|
||||
impl Indexer {
|
||||
pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
|
||||
info!("Increasing number of open files limit...");
|
||||
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
|
||||
rlimit::setrlimit(
|
||||
rlimit::Resource::NOFILE,
|
||||
no_file_limit.0.max(10_000),
|
||||
no_file_limit.1,
|
||||
)?;
|
||||
|
||||
info!("Importing indexer...");
|
||||
|
||||
let path = outputs_dir.join("indexed");
|
||||
|
||||
@@ -25,6 +25,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3);
|
||||
pub fn open_database(path: &Path) -> fjall::Result<Database> {
|
||||
Database::builder(path.join("fjall"))
|
||||
.cache_size(3 * 1024 * 1024 * 1024)
|
||||
.max_cached_files(Some(1024))
|
||||
.open()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
use crate::{EmptyAddressIndex, LoadedAddressIndex};
|
||||
|
||||
/// Source of address data update (where the data came from).
|
||||
#[derive(Clone)]
|
||||
pub enum AddressDataSource<T> {
|
||||
/// Brand new address, not in any storage yet.
|
||||
New(T),
|
||||
/// From empty address storage.
|
||||
FromEmpty((EmptyAddressIndex, T)),
|
||||
/// From loaded address storage.
|
||||
FromLoaded((LoadedAddressIndex, T)),
|
||||
}
|
||||
@@ -5,6 +5,7 @@ pub use vecdb::{CheckedSub, Exit, PrintableIndex, Version};
|
||||
mod address;
|
||||
mod addressbytes;
|
||||
mod addresschainstats;
|
||||
mod addressdata_source;
|
||||
mod addresshash;
|
||||
mod addressindexoutpoint;
|
||||
mod addressindextxindex;
|
||||
@@ -103,6 +104,7 @@ mod yearindex;
|
||||
pub use address::*;
|
||||
pub use addressbytes::*;
|
||||
pub use addresschainstats::*;
|
||||
pub use addressdata_source::*;
|
||||
pub use addresshash::*;
|
||||
pub use addressindexoutpoint::*;
|
||||
pub use addressindextxindex::*;
|
||||
|
||||
Reference in New Issue
Block a user