global: massive columnar rework part 2

This commit is contained in:
nym21
2026-08-11 18:15:26 +02:00
parent 4abbc87ce5
commit 09b50b6ede
361 changed files with 50653 additions and 16467 deletions
+14262 -1919
View File
File diff suppressed because it is too large Load Diff
-82
View File
@@ -1,82 +0,0 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use crate::Filter;
use super::{AmountRange, OverAmount, UnderAmount};
#[derive(Default, Clone, Traversable)]
pub struct AddrGroups<T> {
pub over_amount: OverAmount<T>,
pub amount_range: AmountRange<T>,
pub under_amount: UnderAmount<T>,
}
impl<T> AddrGroups<T> {
pub fn new<F>(mut create: F) -> Self
where
F: FnMut(Filter, &'static str) -> T,
{
Self {
over_amount: OverAmount::new(&mut create),
amount_range: AmountRange::new(&mut create),
under_amount: UnderAmount::new(&mut create),
}
}
pub fn try_new<F, E>(create: &F) -> Result<Self, E>
where
F: Fn(Filter, &'static str) -> Result<T, E>,
{
Ok(Self {
over_amount: OverAmount::try_new(create)?,
amount_range: AmountRange::try_new(create)?,
under_amount: UnderAmount::try_new(create)?,
})
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
self.over_amount
.iter()
.chain(self.amount_range.iter())
.chain(self.under_amount.iter())
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.over_amount
.iter_mut()
.chain(self.amount_range.iter_mut())
.chain(self.under_amount.iter_mut())
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
self.over_amount
.par_iter_mut()
.chain(self.amount_range.par_iter_mut())
.chain(self.under_amount.par_iter_mut())
}
pub fn iter_separate(&self) -> impl Iterator<Item = &T> {
self.amount_range.iter()
}
pub fn iter_separate_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.amount_range.iter_mut()
}
pub fn par_iter_separate_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
self.amount_range.par_iter_mut()
}
pub fn iter_overlapping_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.under_amount
.iter_mut()
.chain(self.over_amount.iter_mut())
}
}
+49 -8
View File
@@ -3,6 +3,7 @@ use std::ops::Range;
use brk_traversable::Traversable;
use brk_types::Age;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{ColumnId, VecValue, Version};
@@ -122,7 +123,7 @@ pub const LTH_AGE_RANGE_IDS: [AgeRangeId; LTH_AGE_RANGE_COUNT] = [
impl ColumnId for AgeRangeId {
type Row<T>
= [T; AGE_RANGE_COUNT]
= AgeRange<T>
where
T: VecValue;
@@ -136,35 +137,49 @@ impl ColumnId for AgeRangeId {
#[inline]
fn get<T: VecValue>(self, row: &Self::Row<T>) -> &T {
&row[self as usize]
self.select(row)
}
#[inline]
fn get_mut<T: VecValue>(self, row: &mut Self::Row<T>) -> &mut T {
&mut row[self as usize]
self.select_mut(row)
}
#[inline]
fn from_fn<T, F>(mut f: F) -> Self::Row<T>
fn from_fn<T, F>(f: F) -> Self::Row<T>
where
T: VecValue,
F: FnMut(Self) -> T,
{
std::array::from_fn(|index| f(AGE_RANGE_IDS[index]))
AgeRange::from_fn(f)
}
#[inline]
fn map<T, U, F>(row: Self::Row<T>, f: F) -> Self::Row<U>
fn map<T, U, F>(row: Self::Row<T>, mut f: F) -> Self::Row<U>
where
T: VecValue,
U: VecValue,
F: FnMut(T) -> U,
{
row.map(f)
AgeRange::from_fn(|column| f(column.get(&row).clone()))
}
}
impl AgeRangeId {
pub fn matching(filter: &Filter) -> Option<Self> {
Self::ALL
.iter()
.copied()
.find(|column| column.filter() == filter)
}
pub fn included_by(filter: &Filter) -> impl Iterator<Item = Self> + '_ {
Self::ALL
.iter()
.copied()
.filter(|column| filter.includes(column.filter()))
}
#[inline]
pub fn bounds(self) -> &'static Range<usize> {
self.select(&AGE_RANGE_BOUNDS)
@@ -367,7 +382,7 @@ pub const AGE_RANGE_NAMES: AgeRange<CohortName> = AgeRange {
over_15y: CohortName::new("over_15y_old", "15y+", "15+ Years Old"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct AgeRange<T> {
pub under_1h: T,
pub _1h_to_1d: T,
@@ -394,6 +409,32 @@ pub struct AgeRange<T> {
pub over_15y: T,
}
impl_column_row_formattable!(AgeRange {
under_1h,
_1h_to_1d,
_1d_to_1w,
_1w_to_1m,
_1m_to_2m,
_2m_to_3m,
_3m_to_4m,
_4m_to_5m,
_5m_to_6m,
_6m_to_9m,
_9m_to_1y,
_1y_to_18m,
_18m_to_2y,
_2y_to_3y,
_3y_to_4y,
_4y_to_5y,
_5y_to_6y,
_6y_to_7y,
_7y_to_8y,
_8y_to_10y,
_10y_to_12y,
_12y_to_15y,
over_15y,
});
impl<T> AgeRange<T> {
pub fn from_fn(mut create: impl FnMut(AgeRangeId) -> T) -> Self {
Self {
+106
View File
@@ -0,0 +1,106 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::ColumnId;
use crate::{
AmountRange, AmountRangeId, Filter, OVER_AMOUNT_FILTERS, OverAmount, OverAmountId,
UNDER_AMOUNT_FILTERS, UnderAmount, UnderAmountId,
};
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct Amount<T> {
pub range: AmountRange<T>,
pub under: UnderAmount<T>,
pub over: OverAmount<T>,
}
impl<T> Amount<T> {
pub fn new(mut create: impl FnMut(Filter, &'static str) -> T) -> Self {
Self {
range: AmountRange::new(&mut create),
under: UnderAmount::new(&mut create),
over: OverAmount::new(create),
}
}
pub fn try_new<E>(
mut create: impl FnMut(Filter, &'static str) -> Result<T, E>,
) -> Result<Self, E> {
Ok(Self {
range: AmountRange::try_new(&mut create)?,
under: UnderAmount::try_new(&mut create)?,
over: OverAmount::try_new(create)?,
})
}
pub fn get(&self, filter: &Filter) -> Option<&T> {
AmountRangeId::matching(filter)
.map(|id| id.select(&self.range))
.or_else(|| {
UnderAmountId::ALL
.iter()
.copied()
.find(|id| id.select(&UNDER_AMOUNT_FILTERS) == filter)
.map(|id| id.select(&self.under))
})
.or_else(|| {
OverAmountId::ALL
.iter()
.copied()
.find(|id| id.select(&OVER_AMOUNT_FILTERS) == filter)
.map(|id| id.select(&self.over))
})
}
pub fn map_named<U>(&self, mut map: impl FnMut(&Filter, &'static str, &T) -> U) -> Amount<U> {
Amount {
range: AmountRange::new(|filter, name| {
map(
&filter,
name,
self.get(&filter).expect("exact amount range"),
)
}),
under: UnderAmount::new(|filter, name| {
map(
&filter,
name,
self.get(&filter).expect("under-amount threshold"),
)
}),
over: OverAmount::new(|filter, name| {
map(
&filter,
name,
self.get(&filter).expect("over-amount threshold"),
)
}),
}
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
self.range
.iter()
.chain(self.under.iter())
.chain(self.over.iter())
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.range
.iter_mut()
.chain(self.under.iter_mut())
.chain(self.over.iter_mut())
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
self.range
.par_iter_mut()
.chain(self.under.par_iter_mut())
.chain(self.over.par_iter_mut())
}
}
+47
View File
@@ -0,0 +1,47 @@
use brk_types::Sats;
/// Bucket index for amount ranges. Use for cheap comparisons and direct lookups.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AmountBucket(u8);
impl AmountBucket {
/// Returns both buckets when they differ.
#[inline(always)]
pub fn transition_to(self, other: Self) -> Option<(Self, Self)> {
(self != other).then_some((self, other))
}
#[inline(always)]
pub fn index(self) -> u8 {
self.0
}
}
impl From<Sats> for AmountBucket {
#[inline(always)]
fn from(value: Sats) -> Self {
Self(match value {
v if v < Sats::_1 => 0,
v if v < Sats::_10 => 1,
v if v < Sats::_100 => 2,
v if v < Sats::_1K => 3,
v if v < Sats::_10K => 4,
v if v < Sats::_100K => 5,
v if v < Sats::_1M => 6,
v if v < Sats::_10M => 7,
v if v < Sats::_1BTC => 8,
v if v < Sats::_10BTC => 9,
v if v < Sats::_100BTC => 10,
v if v < Sats::_1K_BTC => 11,
v if v < Sats::_10K_BTC => 12,
v if v < Sats::_100K_BTC => 13,
_ => 14,
})
}
}
/// Checks whether two amounts belong to different buckets.
#[inline(always)]
pub fn amounts_in_different_buckets(a: Sats, b: Sats) -> bool {
AmountBucket::from(a) != AmountBucket::from(b)
}
+47 -55
View File
@@ -3,60 +3,11 @@ use std::ops::{Add, AddAssign, Range};
use brk_traversable::Traversable;
use brk_types::Sats;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::ColumnId;
use super::{AmountFilter, CohortName, Filter};
/// Bucket index for amount ranges. Use for cheap comparisons and direct lookups.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AmountBucket(u8);
impl AmountBucket {
/// Returns (self, other) if buckets differ, None if same.
/// Use with `AmountRange::get_mut_by_bucket` to avoid recomputing.
#[inline(always)]
pub fn transition_to(self, other: Self) -> Option<(Self, Self)> {
if self != other {
Some((self, other))
} else {
None
}
}
#[inline(always)]
pub fn index(self) -> u8 {
self.0
}
}
impl From<Sats> for AmountBucket {
#[inline(always)]
fn from(value: Sats) -> Self {
Self(match value {
v if v < Sats::_1 => 0,
v if v < Sats::_10 => 1,
v if v < Sats::_100 => 2,
v if v < Sats::_1K => 3,
v if v < Sats::_10K => 4,
v if v < Sats::_100K => 5,
v if v < Sats::_1M => 6,
v if v < Sats::_10M => 7,
v if v < Sats::_1BTC => 8,
v if v < Sats::_10BTC => 9,
v if v < Sats::_100BTC => 10,
v if v < Sats::_1K_BTC => 11,
v if v < Sats::_10K_BTC => 12,
v if v < Sats::_100K_BTC => 13,
_ => 14,
})
}
}
/// Check if two amounts are in different buckets. O(1).
#[inline(always)]
pub fn amounts_in_different_buckets(a: Sats, b: Sats) -> bool {
AmountBucket::from(a) != AmountBucket::from(b)
}
use super::{AmountBucket, AmountFilter, CohortName, Filter};
/// Amount range bounds
pub const AMOUNT_RANGE_BOUNDS: AmountRange<Range<Sats>> = AmountRange {
@@ -131,7 +82,7 @@ pub const AMOUNT_RANGE_FILTERS: AmountRange<Filter> = AmountRange {
over_100k_btc: Filter::Amount(AmountFilter::Range(AMOUNT_RANGE_BOUNDS.over_100k_btc)),
};
#[derive(Debug, Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct AmountRange<T> {
pub _0sats: T,
pub _1sat_to_10sats: T,
@@ -150,6 +101,47 @@ pub struct AmountRange<T> {
pub over_100k_btc: T,
}
define_column_id!(
AmountRangeId for AmountRange, version = 1 {
Zero => _0sats,
From1SatTo10Sats => _1sat_to_10sats,
From10SatsTo100Sats => _10sats_to_100sats,
From100SatsTo1KSats => _100sats_to_1k_sats,
From1KSatsTo10KSats => _1k_sats_to_10k_sats,
From10KSatsTo100KSats => _10k_sats_to_100k_sats,
From100KSatsTo1MSats => _100k_sats_to_1m_sats,
From1MSatsTo10MSats => _1m_sats_to_10m_sats,
From10MSatsTo1Btc => _10m_sats_to_1btc,
From1BtcTo10Btc => _1btc_to_10btc,
From10BtcTo100Btc => _10btc_to_100btc,
From100BtcTo1KBtc => _100btc_to_1k_btc,
From1KBtcTo10KBtc => _1k_btc_to_10k_btc,
From10KBtcTo100KBtc => _10k_btc_to_100k_btc,
Over100kBtc => over_100k_btc,
}
);
impl AmountRangeId {
#[inline]
pub fn filter(self) -> &'static Filter {
self.select(&AMOUNT_RANGE_FILTERS)
}
pub fn matching(filter: &Filter) -> Option<Self> {
Self::ALL
.iter()
.copied()
.find(|column| column.filter() == filter)
}
pub fn included_by(filter: &Filter) -> impl Iterator<Item = Self> + '_ {
Self::ALL
.iter()
.copied()
.filter(|column| filter.includes(column.filter()))
}
}
impl AmountRange<CohortName> {
pub const fn names() -> &'static Self {
&AMOUNT_RANGE_NAMES
@@ -230,7 +222,7 @@ impl<T> AmountRange<T> {
#[inline(always)]
pub fn get(&self, value: Sats) -> &T {
match AmountBucket::from(value).0 {
match AmountBucket::from(value).index() {
0 => &self._0sats,
1 => &self._1sat_to_10sats,
2 => &self._10sats_to_100sats,
@@ -258,7 +250,7 @@ impl<T> AmountRange<T> {
/// Use with `AmountBucket::transition_to` to avoid recomputing bucket.
#[inline(always)]
pub fn get_mut_by_bucket(&mut self, bucket: AmountBucket) -> &mut T {
match bucket.0 {
match bucket.index() {
0 => &mut self._0sats,
1 => &mut self._1sat_to_10sats,
2 => &mut self._10sats_to_100sats,
+105
View File
@@ -0,0 +1,105 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::ColumnId;
use crate::{
AGE_RANGE_FILTERS, AgeRange, AgeRangeId, Filter, OVER_AGE_FILTERS, OverAge, OverAgeId,
UNDER_AGE_FILTERS, UnderAge, UnderAgeId,
};
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct ByAge<T> {
pub range: AgeRange<T>,
pub under: UnderAge<T>,
pub over: OverAge<T>,
}
impl<T> ByAge<T> {
pub fn new(mut create: impl FnMut(Filter, &'static str) -> T) -> Self {
Self {
range: AgeRange::new(&mut create),
under: UnderAge::new(&mut create),
over: OverAge::new(create),
}
}
pub fn try_new<E>(
mut create: impl FnMut(Filter, &'static str) -> Result<T, E>,
) -> Result<Self, E> {
Ok(Self {
range: AgeRange::try_new(&mut create)?,
under: UnderAge::try_new(&mut create)?,
over: OverAge::try_new(create)?,
})
}
pub fn get(&self, filter: &Filter) -> Option<&T> {
AgeRangeId::ALL
.iter()
.copied()
.find(|id| id.select(&AGE_RANGE_FILTERS) == filter)
.map(|id| id.select(&self.range))
.or_else(|| {
UnderAgeId::ALL
.iter()
.copied()
.find(|id| id.select(&UNDER_AGE_FILTERS) == filter)
.map(|id| id.select(&self.under))
})
.or_else(|| {
OverAgeId::ALL
.iter()
.copied()
.find(|id| id.select(&OVER_AGE_FILTERS) == filter)
.map(|id| id.select(&self.over))
})
}
pub fn map_named<U>(&self, mut map: impl FnMut(&Filter, &'static str, &T) -> U) -> ByAge<U> {
ByAge {
range: AgeRange::new(|filter, name| {
map(&filter, name, self.get(&filter).expect("exact age range"))
}),
under: UnderAge::new(|filter, name| {
map(
&filter,
name,
self.get(&filter).expect("under-age threshold"),
)
}),
over: OverAge::new(|filter, name| {
map(
&filter,
name,
self.get(&filter).expect("over-age threshold"),
)
}),
}
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
self.range
.iter()
.chain(self.under.iter())
.chain(self.over.iter())
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.range
.iter_mut()
.chain(self.under.iter_mut())
.chain(self.over.iter_mut())
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
self.range
.par_iter_mut()
.chain(self.under.par_iter_mut())
.chain(self.over.par_iter_mut())
}
}
+9 -1
View File
@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use schemars::JsonSchema;
use serde::Serialize;
use super::{CohortName, Filter};
@@ -36,12 +37,19 @@ pub const ENTRY_NAMES: ByEntry<CohortName> = ByEntry {
premium: CohortName::new("rookie", "Rookie", "Rookie Coins"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct ByEntry<T> {
pub discount: T,
pub premium: T,
}
define_column_id!(
EntryId for ByEntry, version = 1 {
Discount => discount,
Premium => premium,
}
);
impl ByEntry<CohortName> {
pub const fn names() -> &'static Self {
&ENTRY_NAMES
+12 -1
View File
@@ -1,6 +1,7 @@
use brk_traversable::Traversable;
use brk_types::{Halving, Height};
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use schemars::JsonSchema;
use serde::Serialize;
use super::{CohortName, Filter};
@@ -32,7 +33,7 @@ pub const EPOCH_NAMES: ByEpoch<CohortName> = ByEpoch {
_4: CohortName::new("epoch_4", "4", "Epoch 4"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct ByEpoch<T> {
pub _0: T,
pub _1: T,
@@ -41,6 +42,16 @@ pub struct ByEpoch<T> {
pub _4: T,
}
define_column_id!(
EpochId for ByEpoch, version = 1 {
_0 => _0,
_1 => _1,
_2 => _2,
_3 => _3,
_4 => _4,
}
);
impl ByEpoch<CohortName> {
pub const fn names() -> &'static Self {
&EPOCH_NAMES
+9 -1
View File
@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use schemars::JsonSchema;
use serde::Serialize;
use super::{CohortName, Filter, Term};
@@ -22,12 +23,19 @@ pub const TERM_NAMES: ByTerm<CohortName> = ByTerm {
long: CohortName::new("lth", "LTH", "Long Term Holders"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Copy, Traversable, Serialize, JsonSchema)]
pub struct ByTerm<T> {
pub short: T,
pub long: T,
}
define_column_id!(
TermId for ByTerm, version = 1 {
Short => short,
Long => long,
}
);
impl ByTerm<CohortName> {
pub const fn names() -> &'static Self {
&TERM_NAMES
+25 -1
View File
@@ -1,6 +1,7 @@
use brk_traversable::Traversable;
use brk_types::{Timestamp, Year};
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use schemars::JsonSchema;
use serde::Serialize;
use super::{CohortName, Filter};
@@ -71,7 +72,7 @@ pub const CLASS_NAMES: Class<CohortName> = Class {
_2026: CohortName::new("class_2026", "2026", "Class 2026"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct Class<T> {
pub _2009: T,
pub _2010: T,
@@ -93,6 +94,29 @@ pub struct Class<T> {
pub _2026: T,
}
define_column_id!(
ClassId for Class, version = 1 {
_2009 => _2009,
_2010 => _2010,
_2011 => _2011,
_2012 => _2012,
_2013 => _2013,
_2014 => _2014,
_2015 => _2015,
_2016 => _2016,
_2017 => _2017,
_2018 => _2018,
_2019 => _2019,
_2020 => _2020,
_2021 => _2021,
_2022 => _2022,
_2023 => _2023,
_2024 => _2024,
_2025 => _2025,
_2026 => _2026,
}
);
impl Class<CohortName> {
pub const fn names() -> &'static Self {
&CLASS_NAMES
+157
View File
@@ -0,0 +1,157 @@
macro_rules! define_column_id {
(
$id:ident for $row:ident, version = $version:literal {
$($variant:ident => $field:ident),+ $(,)?
}
) => {
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(u8)]
pub enum $id {
$($variant),+
}
impl $id {
#[inline]
pub fn select<T>(self, row: &$row<T>) -> &T {
match self {
$(Self::$variant => &row.$field),+
}
}
#[inline]
pub fn select_mut<T>(self, row: &mut $row<T>) -> &mut T {
match self {
$(Self::$variant => &mut row.$field),+
}
}
}
impl<T> $row<T> {
pub fn from_fn(mut f: impl FnMut($id) -> T) -> Self {
Self {
$($field: f($id::$variant)),+
}
}
}
impl vecdb::ColumnId for $id {
type Row<T>
= $row<T>
where
T: vecdb::VecValue;
const VERSION: vecdb::Version = vecdb::Version::new($version);
const ALL: &'static [Self] = &[$(Self::$variant),+];
#[inline]
fn index(self) -> usize {
self as usize
}
#[inline]
fn get<T: vecdb::VecValue>(self, row: &Self::Row<T>) -> &T {
self.select(row)
}
#[inline]
fn get_mut<T: vecdb::VecValue>(self, row: &mut Self::Row<T>) -> &mut T {
self.select_mut(row)
}
#[inline]
fn from_fn<T, F>(f: F) -> Self::Row<T>
where
T: vecdb::VecValue,
F: FnMut(Self) -> T,
{
$row::from_fn(f)
}
#[inline]
fn map<T, U, F>(row: Self::Row<T>, mut f: F) -> Self::Row<U>
where
T: vecdb::VecValue,
U: vecdb::VecValue,
F: FnMut(T) -> U,
{
$row {
$($field: f(row.$field)),+
}
}
}
impl<T: vecdb::Formattable> vecdb::Formattable for $row<T> {
fn write_to(&self, output: &mut Vec<u8>) {
output.push(b'{');
let mut first = true;
$(
if !first {
output.push(b',');
}
first = false;
output.extend_from_slice(concat!("\"", stringify!($field), "\":").as_bytes());
vecdb::Formattable::fmt_json(&self.$field, output);
)+
let _ = first;
output.push(b'}');
}
fn fmt_csv(&self, output: &mut String) -> std::fmt::Result {
let mut json = Vec::new();
vecdb::Formattable::write_to(self, &mut json);
let json = std::str::from_utf8(&json).map_err(|_| std::fmt::Error)?;
output.push('"');
for character in json.chars() {
if character == '"' {
output.push('"');
}
output.push(character);
}
output.push('"');
Ok(())
}
}
};
}
macro_rules! impl_column_row_formattable {
(
$row:ident {
$($field:ident),+ $(,)?
}
) => {
impl<T: vecdb::Formattable> vecdb::Formattable for $row<T> {
fn write_to(&self, output: &mut Vec<u8>) {
output.push(b'{');
let mut first = true;
$(
if !first {
output.push(b',');
}
first = false;
output.extend_from_slice(concat!("\"", stringify!($field), "\":").as_bytes());
vecdb::Formattable::fmt_json(&self.$field, output);
)+
let _ = first;
output.push(b'}');
}
fn fmt_csv(&self, output: &mut String) -> std::fmt::Result {
let mut json = Vec::new();
vecdb::Formattable::write_to(self, &mut json);
let json = std::str::from_utf8(&json).map_err(|_| std::fmt::Error)?;
output.push('"');
for character in json.chars() {
if character == '"' {
output.push('"');
}
output.push(character);
}
output.push('"');
Ok(())
}
}
};
}
+17 -2
View File
@@ -1,10 +1,15 @@
#![doc = include_str!("../README.md")]
mod addr;
#[macro_use]
mod column_id;
mod age_range;
mod amount;
mod amount_bucket;
mod amount_filter;
mod amount_range;
mod by_addr_type;
mod by_age;
mod by_any_addr;
mod by_entry;
mod by_epoch;
@@ -28,14 +33,20 @@ mod under_age;
mod under_amount;
mod unspendable_type;
mod utxo;
mod utxo_aggregate;
mod utxo_all_and_sth;
mod utxo_groups_without_amount;
mod utxo_groups_without_amount_or_type;
pub use brk_types::{Age, Term};
pub use addr::*;
pub use age_range::*;
pub use amount::*;
pub use amount_bucket::*;
pub use amount_filter::*;
pub use amount_range::*;
pub use by_addr_type::*;
pub use by_age::*;
pub use by_any_addr::*;
pub use by_entry::*;
pub use by_epoch::*;
@@ -59,3 +70,7 @@ pub use under_age::*;
pub use under_amount::*;
pub use unspendable_type::*;
pub use utxo::*;
pub use utxo_aggregate::*;
pub use utxo_all_and_sth::*;
pub use utxo_groups_without_amount::*;
pub use utxo_groups_without_amount_or_type::*;
+18 -3
View File
@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::CohortName;
@@ -29,7 +30,7 @@ impl Loss<CohortName> {
/// 9 "at least X% loss" aggregate thresholds.
///
/// Each is a suffix sum over the profitability ranges, from most loss-making up.
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct Loss<T> {
pub all: T,
pub _10pct: T,
@@ -42,6 +43,20 @@ pub struct Loss<T> {
pub _80pct: T,
}
define_column_id!(
LossId for Loss, version = 1 {
All => all,
Over10Pct => _10pct,
Over20Pct => _20pct,
Over30Pct => _30pct,
Over40Pct => _40pct,
Over50Pct => _50pct,
Over60Pct => _60pct,
Over70Pct => _70pct,
Over80Pct => _80pct,
}
);
impl<T> Loss<T> {
pub fn new<F>(mut create: F) -> Self
where
@@ -79,7 +94,7 @@ impl<T> Loss<T> {
})
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
pub fn iter(&self) -> impl DoubleEndedIterator<Item = &T> + ExactSizeIterator {
[
&self.all,
&self._10pct,
@@ -94,7 +109,7 @@ impl<T> Loss<T> {
.into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
pub fn iter_mut(&mut self) -> impl DoubleEndedIterator<Item = &mut T> + ExactSizeIterator {
[
&mut self.all,
&mut self._10pct,
+27 -1
View File
@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::{
@@ -80,7 +81,7 @@ pub const OVER_AGE_NAMES: OverAge<CohortName> = OverAge {
_12y: CohortName::new("over_12y_old", "12y+", "Over 12 Years Old"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct OverAge<T> {
pub _1d: T,
pub _1w: T,
@@ -104,6 +105,31 @@ pub struct OverAge<T> {
pub _12y: T,
}
define_column_id!(
OverAgeId for OverAge, version = 1 {
Over1D => _1d,
Over1W => _1w,
Over1M => _1m,
Over2M => _2m,
Over3M => _3m,
Over4M => _4m,
Over5M => _5m,
Over6M => _6m,
Over9M => _9m,
Over1Y => _1y,
Over18M => _18m,
Over2Y => _2y,
Over3Y => _3y,
Over4Y => _4y,
Over5Y => _5y,
Over6Y => _6y,
Over7Y => _7y,
Over8Y => _8y,
Over10Y => _10y,
Over12Y => _12y,
}
);
impl OverAge<CohortName> {
pub const fn names() -> &'static Self {
&OVER_AGE_NAMES
+20 -1
View File
@@ -1,6 +1,7 @@
use brk_traversable::Traversable;
use brk_types::Sats;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::{AmountFilter, CohortName, Filter};
@@ -70,7 +71,7 @@ pub const OVER_AMOUNT_FILTERS: OverAmount<Filter> = OverAmount {
)),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct OverAmount<T> {
pub _1sat: T,
pub _10sats: T,
@@ -87,6 +88,24 @@ pub struct OverAmount<T> {
pub _10k_btc: T,
}
define_column_id!(
OverAmountId for OverAmount, version = 1 {
Over1Sat => _1sat,
Over10Sats => _10sats,
Over100Sats => _100sats,
Over1KSats => _1k_sats,
Over10KSats => _10k_sats,
Over100KSats => _100k_sats,
Over1MSats => _1m_sats,
Over10MSats => _10m_sats,
Over1Btc => _1btc,
Over10Btc => _10btc,
Over100Btc => _100btc,
Over1KBtc => _1k_btc,
Over10KBtc => _10k_btc,
}
);
impl OverAmount<CohortName> {
pub const fn names() -> &'static Self {
&OVER_AMOUNT_NAMES
+23 -3
View File
@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::CohortName;
@@ -50,7 +51,7 @@ impl Profit<CohortName> {
/// 14 "at least X% profit" aggregate thresholds.
///
/// Each is a prefix sum over the profitability ranges, from most profitable down.
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct Profit<T> {
pub all: T,
pub _10pct: T,
@@ -68,6 +69,25 @@ pub struct Profit<T> {
pub _500pct: T,
}
define_column_id!(
ProfitId for Profit, version = 1 {
All => all,
Over10Pct => _10pct,
Over20Pct => _20pct,
Over30Pct => _30pct,
Over40Pct => _40pct,
Over50Pct => _50pct,
Over60Pct => _60pct,
Over70Pct => _70pct,
Over80Pct => _80pct,
Over90Pct => _90pct,
Over100Pct => _100pct,
Over200Pct => _200pct,
Over300Pct => _300pct,
Over500Pct => _500pct,
}
);
impl<T> Profit<T> {
pub fn new<F>(mut create: F) -> Self
where
@@ -115,7 +135,7 @@ impl<T> Profit<T> {
})
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
pub fn iter(&self) -> impl DoubleEndedIterator<Item = &T> + ExactSizeIterator {
[
&self.all,
&self._10pct,
@@ -135,7 +155,7 @@ impl<T> Profit<T> {
.into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
pub fn iter_mut(&mut self) -> impl DoubleEndedIterator<Item = &mut T> + ExactSizeIterator {
[
&mut self.all,
&mut self._10pct,
+167 -62
View File
@@ -1,43 +1,41 @@
use std::{fmt, ops::AddAssign};
use brk_traversable::Traversable;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{ColumnId, Formattable, VecValue, Version};
use crate::{
LOSS_COUNT, Loss, PROFIT_COUNT, PROFITABILITY_RANGE_COUNT, Profit, ProfitabilityRange,
LOSS_COUNT, Loss, LossId, PROFIT_COUNT, PROFITABILITY_RANGE_COUNT, Profit, ProfitId,
ProfitabilityRange, ProfitabilityRangeId,
};
pub const PROFITABILITY_COUNT: usize = PROFITABILITY_RANGE_COUNT + PROFIT_COUNT + LOSS_COUNT;
#[derive(Debug, Clone, Serialize, JsonSchema)]
#[derive(Debug, Clone, Traversable, Serialize, JsonSchema)]
pub struct ProfitabilityRow<T> {
pub range: [T; PROFITABILITY_RANGE_COUNT],
pub profit: [T; PROFIT_COUNT],
pub loss: [T; LOSS_COUNT],
pub range: ProfitabilityRange<T>,
pub profit: Profit<T>,
pub loss: Loss<T>,
}
impl<T> ProfitabilityRow<T>
where
T: AddAssign + Copy + Default,
{
pub fn from_ranges(range: [T; PROFITABILITY_RANGE_COUNT]) -> Self {
let (profit_ranges, loss_ranges) = range.split_at(PROFIT_COUNT + 1);
let mut profit = [T::default(); PROFIT_COUNT];
let mut total = profit_ranges[0];
for (threshold, &value) in profit.iter_mut().rev().zip(&profit_ranges[1..]) {
pub fn from_ranges(range: ProfitabilityRange<T>) -> Self {
let mut profit = Profit::default();
let mut profit_ranges = range.iter().take(PROFIT_COUNT + 1);
let mut total = *profit_ranges.next().expect("profitability profit range");
for (threshold, &value) in profit.iter_mut().rev().zip(profit_ranges) {
total += value;
*threshold = total;
}
let mut loss = [T::default(); LOSS_COUNT];
let mut total = loss_ranges[loss_ranges.len() - 1];
for (threshold, &value) in loss
.iter_mut()
.rev()
.zip(loss_ranges[..loss_ranges.len() - 1].iter().rev())
{
let mut loss = Loss::default();
let mut loss_ranges = range.iter().skip(PROFIT_COUNT + 1).rev();
let mut total = *loss_ranges.next().expect("profitability loss range");
for (threshold, &value) in loss.iter_mut().rev().zip(loss_ranges) {
total += value;
*threshold = total;
}
@@ -53,11 +51,11 @@ where
impl<T: Formattable> Formattable for ProfitabilityRow<T> {
fn write_to(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(b"{\"range\":");
self.range.write_to(buf);
write_array(self.range.iter(), buf);
buf.extend_from_slice(b",\"profit\":");
self.profit.write_to(buf);
write_array(self.profit.iter(), buf);
buf.extend_from_slice(b",\"loss\":");
self.loss.write_to(buf);
write_array(self.loss.iter(), buf);
buf.push(b'}');
}
@@ -78,6 +76,17 @@ impl<T: Formattable> Formattable for ProfitabilityRow<T> {
}
}
fn write_array<'a, T: Formattable + 'a>(values: impl Iterator<Item = &'a T>, buf: &mut Vec<u8>) {
buf.push(b'[');
for (index, value) in values.enumerate() {
if index > 0 {
buf.push(b',');
}
value.write_to(buf);
}
buf.push(b']');
}
/// Every profitability range and aggregate threshold in column storage order.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(u8)]
@@ -184,6 +193,14 @@ pub const PROFITABILITY_IDS: [ProfitabilityId; PROFITABILITY_COUNT] = [
];
impl ProfitabilityId {
pub fn series<T>(mut create: impl FnMut(Self, &'static str) -> T) -> ProfitabilityRow<T> {
ProfitabilityRow {
range: Self::range_series(&mut create),
profit: Self::profit_series(&mut create),
loss: Self::loss_series(create),
}
}
pub fn range_ids() -> &'static [Self] {
&PROFITABILITY_IDS[..PROFITABILITY_RANGE_COUNT]
}
@@ -196,6 +213,18 @@ impl ProfitabilityId {
&PROFITABILITY_IDS[PROFITABILITY_RANGE_COUNT + PROFIT_COUNT..]
}
pub fn ranges(self) -> &'static [ProfitabilityRangeId] {
match self.group() {
ProfitabilityGroupId::Range(id) => &ProfitabilityRangeId::ALL[id.index()..=id.index()],
ProfitabilityGroupId::Profit(id) => {
&ProfitabilityRangeId::ALL[..PROFIT_COUNT + 1 - id.index()]
}
ProfitabilityGroupId::Loss(id) => {
&ProfitabilityRangeId::ALL[PROFIT_COUNT + 1 + id.index()..]
}
}
}
pub fn range_series<T>(
mut create: impl FnMut(Self, &'static str) -> T,
) -> ProfitabilityRange<T> {
@@ -348,13 +377,84 @@ impl ProfitabilityId {
}
}
enum ProfitabilityGroupId {
Range(ProfitabilityRangeId),
Profit(ProfitId),
Loss(LossId),
}
impl ProfitabilityId {
const fn group(self) -> ProfitabilityGroupId {
use ProfitabilityGroupId::{Loss, Profit, Range};
match self {
Self::RangeOver1000PctInProfit => Range(ProfitabilityRangeId::Over1000PctInProfit),
Self::Range500To1000PctInProfit => {
Range(ProfitabilityRangeId::From500PctTo1000PctInProfit)
}
Self::Range300To500PctInProfit => {
Range(ProfitabilityRangeId::From300PctTo500PctInProfit)
}
Self::Range200To300PctInProfit => {
Range(ProfitabilityRangeId::From200PctTo300PctInProfit)
}
Self::Range100To200PctInProfit => {
Range(ProfitabilityRangeId::From100PctTo200PctInProfit)
}
Self::Range90To100PctInProfit => Range(ProfitabilityRangeId::From90PctTo100PctInProfit),
Self::Range80To90PctInProfit => Range(ProfitabilityRangeId::From80PctTo90PctInProfit),
Self::Range70To80PctInProfit => Range(ProfitabilityRangeId::From70PctTo80PctInProfit),
Self::Range60To70PctInProfit => Range(ProfitabilityRangeId::From60PctTo70PctInProfit),
Self::Range50To60PctInProfit => Range(ProfitabilityRangeId::From50PctTo60PctInProfit),
Self::Range40To50PctInProfit => Range(ProfitabilityRangeId::From40PctTo50PctInProfit),
Self::Range30To40PctInProfit => Range(ProfitabilityRangeId::From30PctTo40PctInProfit),
Self::Range20To30PctInProfit => Range(ProfitabilityRangeId::From20PctTo30PctInProfit),
Self::Range10To20PctInProfit => Range(ProfitabilityRangeId::From10PctTo20PctInProfit),
Self::Range0To10PctInProfit => Range(ProfitabilityRangeId::From0PctTo10PctInProfit),
Self::Range0To10PctInLoss => Range(ProfitabilityRangeId::From0PctTo10PctInLoss),
Self::Range10To20PctInLoss => Range(ProfitabilityRangeId::From10PctTo20PctInLoss),
Self::Range20To30PctInLoss => Range(ProfitabilityRangeId::From20PctTo30PctInLoss),
Self::Range30To40PctInLoss => Range(ProfitabilityRangeId::From30PctTo40PctInLoss),
Self::Range40To50PctInLoss => Range(ProfitabilityRangeId::From40PctTo50PctInLoss),
Self::Range50To60PctInLoss => Range(ProfitabilityRangeId::From50PctTo60PctInLoss),
Self::Range60To70PctInLoss => Range(ProfitabilityRangeId::From60PctTo70PctInLoss),
Self::Range70To80PctInLoss => Range(ProfitabilityRangeId::From70PctTo80PctInLoss),
Self::Range80To90PctInLoss => Range(ProfitabilityRangeId::From80PctTo90PctInLoss),
Self::Range90To100PctInLoss => Range(ProfitabilityRangeId::From90PctTo100PctInLoss),
Self::Profit => Profit(ProfitId::All),
Self::ProfitOver10Pct => Profit(ProfitId::Over10Pct),
Self::ProfitOver20Pct => Profit(ProfitId::Over20Pct),
Self::ProfitOver30Pct => Profit(ProfitId::Over30Pct),
Self::ProfitOver40Pct => Profit(ProfitId::Over40Pct),
Self::ProfitOver50Pct => Profit(ProfitId::Over50Pct),
Self::ProfitOver60Pct => Profit(ProfitId::Over60Pct),
Self::ProfitOver70Pct => Profit(ProfitId::Over70Pct),
Self::ProfitOver80Pct => Profit(ProfitId::Over80Pct),
Self::ProfitOver90Pct => Profit(ProfitId::Over90Pct),
Self::ProfitOver100Pct => Profit(ProfitId::Over100Pct),
Self::ProfitOver200Pct => Profit(ProfitId::Over200Pct),
Self::ProfitOver300Pct => Profit(ProfitId::Over300Pct),
Self::ProfitOver500Pct => Profit(ProfitId::Over500Pct),
Self::Loss => Loss(LossId::All),
Self::LossOver10Pct => Loss(LossId::Over10Pct),
Self::LossOver20Pct => Loss(LossId::Over20Pct),
Self::LossOver30Pct => Loss(LossId::Over30Pct),
Self::LossOver40Pct => Loss(LossId::Over40Pct),
Self::LossOver50Pct => Loss(LossId::Over50Pct),
Self::LossOver60Pct => Loss(LossId::Over60Pct),
Self::LossOver70Pct => Loss(LossId::Over70Pct),
Self::LossOver80Pct => Loss(LossId::Over80Pct),
}
}
}
impl ColumnId for ProfitabilityId {
type Row<T>
= ProfitabilityRow<T>
where
T: VecValue;
const VERSION: Version = Version::ONE;
const VERSION: Version = Version::TWO;
const ALL: &'static [Self] = &PROFITABILITY_IDS;
#[inline]
@@ -364,25 +464,19 @@ impl ColumnId for ProfitabilityId {
#[inline]
fn get<T: VecValue>(self, row: &Self::Row<T>) -> &T {
let index = self as usize;
if index < PROFITABILITY_RANGE_COUNT {
&row.range[index]
} else if index < PROFITABILITY_RANGE_COUNT + PROFIT_COUNT {
&row.profit[index - PROFITABILITY_RANGE_COUNT]
} else {
&row.loss[index - PROFITABILITY_RANGE_COUNT - PROFIT_COUNT]
match self.group() {
ProfitabilityGroupId::Range(id) => id.get(&row.range),
ProfitabilityGroupId::Profit(id) => id.get(&row.profit),
ProfitabilityGroupId::Loss(id) => id.get(&row.loss),
}
}
#[inline]
fn get_mut<T: VecValue>(self, row: &mut Self::Row<T>) -> &mut T {
let index = self as usize;
if index < PROFITABILITY_RANGE_COUNT {
&mut row.range[index]
} else if index < PROFITABILITY_RANGE_COUNT + PROFIT_COUNT {
&mut row.profit[index - PROFITABILITY_RANGE_COUNT]
} else {
&mut row.loss[index - PROFITABILITY_RANGE_COUNT - PROFIT_COUNT]
match self.group() {
ProfitabilityGroupId::Range(id) => id.get_mut(&mut row.range),
ProfitabilityGroupId::Profit(id) => id.get_mut(&mut row.profit),
ProfitabilityGroupId::Loss(id) => id.get_mut(&mut row.loss),
}
}
@@ -392,15 +486,7 @@ impl ColumnId for ProfitabilityId {
T: VecValue,
F: FnMut(Self) -> T,
{
ProfitabilityRow {
range: std::array::from_fn(|index| f(PROFITABILITY_IDS[index])),
profit: std::array::from_fn(|index| {
f(PROFITABILITY_IDS[PROFITABILITY_RANGE_COUNT + index])
}),
loss: std::array::from_fn(|index| {
f(PROFITABILITY_IDS[PROFITABILITY_RANGE_COUNT + PROFIT_COUNT + index])
}),
}
Self::series(|id, _| f(id))
}
#[inline]
@@ -411,9 +497,9 @@ impl ColumnId for ProfitabilityId {
F: FnMut(T) -> U,
{
ProfitabilityRow {
range: row.range.map(&mut f),
profit: row.profit.map(&mut f),
loss: row.loss.map(f),
range: ProfitabilityRangeId::map(row.range, &mut f),
profit: ProfitId::map(row.profit, &mut f),
loss: LossId::map(row.loss, f),
}
}
}
@@ -456,24 +542,43 @@ mod tests {
#[test]
fn rows_expand_ranges_into_profit_prefixes_and_loss_suffixes() {
let ranges = std::array::from_fn(|index| index + 1);
let row = ProfitabilityRow::from_ranges(ranges);
let ranges = ProfitabilityRangeId::from_fn(|id| id.index() + 1);
let row = ProfitabilityRow::from_ranges(ranges.clone());
assert_eq!(
row.profit[0],
ranges[..PROFIT_COUNT + 1].iter().copied().sum::<usize>()
);
assert_eq!(row.profit[PROFIT_COUNT - 1], ranges[0] + ranges[1]);
assert_eq!(
row.loss[0],
ranges[PROFIT_COUNT + 1..].iter().copied().sum::<usize>()
row.profit.all,
ranges.iter().take(PROFIT_COUNT + 1).copied().sum::<usize>()
);
assert_eq!(
row.loss[LOSS_COUNT - 1],
ranges[PROFITABILITY_RANGE_COUNT - 2..]
.iter()
.copied()
.sum::<usize>()
row.profit._500pct,
ranges.over_1000pct_in_profit + ranges._500pct_to_1000pct_in_profit
);
assert_eq!(
row.loss.all,
ranges.iter().skip(PROFIT_COUNT + 1).copied().sum::<usize>()
);
assert_eq!(
row.loss._80pct,
ranges.iter().rev().take(2).copied().sum::<usize>()
);
}
#[test]
fn aggregate_ids_select_their_exact_ranges() {
for &id in ProfitabilityId::range_ids() {
assert_eq!(id.ranges().len(), 1);
}
for (threshold, &id) in ProfitabilityId::profit_ids().iter().enumerate() {
assert_eq!(
id.ranges(),
&ProfitabilityRangeId::ALL[..PROFIT_COUNT + 1 - threshold]
);
}
for (threshold, &id) in ProfitabilityId::loss_ids().iter().enumerate() {
assert_eq!(
id.ranges(),
&ProfitabilityRangeId::ALL[PROFIT_COUNT + 1 + threshold..]
);
}
}
}
+44 -5
View File
@@ -1,9 +1,10 @@
use brk_traversable::Traversable;
use brk_types::Cents;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::CohortName;
use super::{CohortName, PROFIT_COUNT};
/// Number of profitability range boundaries (24 boundaries → 25 buckets).
pub const PROFITABILITY_BOUNDARY_COUNT: usize = 24;
@@ -220,7 +221,7 @@ impl ProfitabilityRange<CohortName> {
///
/// During the k-way merge (ascending price order), the cursor starts at bucket 0
/// (over_1000pct_in_profit, lowest cost basis) and advances as price crosses each boundary.
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct ProfitabilityRange<T> {
pub over_1000pct_in_profit: T,
pub _500pct_to_1000pct_in_profit: T,
@@ -249,6 +250,42 @@ pub struct ProfitabilityRange<T> {
pub _90pct_to_100pct_in_loss: T,
}
define_column_id!(
ProfitabilityRangeId for ProfitabilityRange, version = 1 {
Over1000PctInProfit => over_1000pct_in_profit,
From500PctTo1000PctInProfit => _500pct_to_1000pct_in_profit,
From300PctTo500PctInProfit => _300pct_to_500pct_in_profit,
From200PctTo300PctInProfit => _200pct_to_300pct_in_profit,
From100PctTo200PctInProfit => _100pct_to_200pct_in_profit,
From90PctTo100PctInProfit => _90pct_to_100pct_in_profit,
From80PctTo90PctInProfit => _80pct_to_90pct_in_profit,
From70PctTo80PctInProfit => _70pct_to_80pct_in_profit,
From60PctTo70PctInProfit => _60pct_to_70pct_in_profit,
From50PctTo60PctInProfit => _50pct_to_60pct_in_profit,
From40PctTo50PctInProfit => _40pct_to_50pct_in_profit,
From30PctTo40PctInProfit => _30pct_to_40pct_in_profit,
From20PctTo30PctInProfit => _20pct_to_30pct_in_profit,
From10PctTo20PctInProfit => _10pct_to_20pct_in_profit,
From0PctTo10PctInProfit => _0pct_to_10pct_in_profit,
From0PctTo10PctInLoss => _0pct_to_10pct_in_loss,
From10PctTo20PctInLoss => _10pct_to_20pct_in_loss,
From20PctTo30PctInLoss => _20pct_to_30pct_in_loss,
From30PctTo40PctInLoss => _30pct_to_40pct_in_loss,
From40PctTo50PctInLoss => _40pct_to_50pct_in_loss,
From50PctTo60PctInLoss => _50pct_to_60pct_in_loss,
From60PctTo70PctInLoss => _60pct_to_70pct_in_loss,
From70PctTo80PctInLoss => _70pct_to_80pct_in_loss,
From80PctTo90PctInLoss => _80pct_to_90pct_in_loss,
From90PctTo100PctInLoss => _90pct_to_100pct_in_loss,
}
);
impl ProfitabilityRangeId {
pub const fn is_profit(self) -> bool {
(self as usize) < PROFIT_COUNT + 1
}
}
/// Number of profitability range buckets.
pub const PROFITABILITY_RANGE_COUNT: usize = 25;
@@ -321,7 +358,7 @@ impl<T> ProfitabilityRange<T> {
})
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
pub fn iter(&self) -> impl DoubleEndedIterator<Item = &T> + ExactSizeIterator {
[
&self.over_1000pct_in_profit,
&self._500pct_to_1000pct_in_profit,
@@ -352,12 +389,14 @@ impl<T> ProfitabilityRange<T> {
.into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
pub fn iter_mut(&mut self) -> impl DoubleEndedIterator<Item = &mut T> + ExactSizeIterator {
self.iter_mut_with_is_profit().map(|(_, v)| v)
}
/// Iterate mutably, yielding `(is_profit, &mut T)` for each range.
pub fn iter_mut_with_is_profit(&mut self) -> impl Iterator<Item = (bool, &mut T)> {
pub fn iter_mut_with_is_profit(
&mut self,
) -> impl DoubleEndedIterator<Item = (bool, &mut T)> + ExactSizeIterator {
[
(true, &mut self.over_1000pct_in_profit),
(true, &mut self._500pct_to_1000pct_in_profit),
+86 -7
View File
@@ -3,6 +3,7 @@ use std::ops::{Add, AddAssign};
use brk_traversable::Traversable;
use brk_types::OutputType;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{ColumnId, VecValue, Version};
@@ -41,6 +42,40 @@ pub const SPENDABLE_TYPE_IDS: [SpendableTypeId; SPENDABLE_TYPE_COUNT] = [
];
impl SpendableTypeId {
#[inline]
pub fn select<T>(self, row: &SpendableType<T>) -> &T {
match self {
Self::P2PK65 => &row.p2pk65,
Self::P2PK33 => &row.p2pk33,
Self::P2PKH => &row.p2pkh,
Self::P2MS => &row.p2ms,
Self::P2SH => &row.p2sh,
Self::P2WPKH => &row.p2wpkh,
Self::P2WSH => &row.p2wsh,
Self::P2TR => &row.p2tr,
Self::P2A => &row.p2a,
Self::Unknown => &row.unknown,
Self::Empty => &row.empty,
}
}
#[inline]
pub fn select_mut<T>(self, row: &mut SpendableType<T>) -> &mut T {
match self {
Self::P2PK65 => &mut row.p2pk65,
Self::P2PK33 => &mut row.p2pk33,
Self::P2PKH => &mut row.p2pkh,
Self::P2MS => &mut row.p2ms,
Self::P2SH => &mut row.p2sh,
Self::P2WPKH => &mut row.p2wpkh,
Self::P2WSH => &mut row.p2wsh,
Self::P2TR => &mut row.p2tr,
Self::P2A => &mut row.p2a,
Self::Unknown => &mut row.unknown,
Self::Empty => &mut row.empty,
}
}
pub const fn from_output_type(value: OutputType) -> Option<Self> {
match value {
OutputType::P2PK65 => Some(Self::P2PK65),
@@ -77,7 +112,7 @@ impl SpendableTypeId {
impl ColumnId for SpendableTypeId {
type Row<T>
= [T; SPENDABLE_TYPE_COUNT]
= SpendableType<T>
where
T: VecValue;
@@ -91,12 +126,12 @@ impl ColumnId for SpendableTypeId {
#[inline]
fn get<T: VecValue>(self, row: &Self::Row<T>) -> &T {
&row[self as usize]
self.select(row)
}
#[inline]
fn get_mut<T: VecValue>(self, row: &mut Self::Row<T>) -> &mut T {
&mut row[self as usize]
self.select_mut(row)
}
#[inline]
@@ -105,17 +140,29 @@ impl ColumnId for SpendableTypeId {
T: VecValue,
F: FnMut(Self) -> T,
{
std::array::from_fn(|index| f(SPENDABLE_TYPE_IDS[index]))
SpendableType {
p2pk65: f(Self::P2PK65),
p2pk33: f(Self::P2PK33),
p2pkh: f(Self::P2PKH),
p2ms: f(Self::P2MS),
p2sh: f(Self::P2SH),
p2wpkh: f(Self::P2WPKH),
p2wsh: f(Self::P2WSH),
p2tr: f(Self::P2TR),
p2a: f(Self::P2A),
unknown: f(Self::Unknown),
empty: f(Self::Empty),
}
}
#[inline]
fn map<T, U, F>(row: Self::Row<T>, f: F) -> Self::Row<U>
fn map<T, U, F>(row: Self::Row<T>, mut f: F) -> Self::Row<U>
where
T: VecValue,
U: VecValue,
F: FnMut(T) -> U,
{
row.map(f)
Self::from_fn(|column| f(column.get(&row).clone()))
}
}
@@ -164,7 +211,7 @@ pub const SPENDABLE_TYPE_NAMES: SpendableType<CohortName> = SpendableType {
empty: CohortName::new("empty_outputs", "Empty", "Empty Output"),
};
#[derive(Default, Clone, Debug, Traversable, Serialize)]
#[derive(Default, Clone, Debug, Traversable, Serialize, JsonSchema)]
pub struct SpendableType<T> {
pub p2pk65: T,
pub p2pk33: T,
@@ -179,6 +226,38 @@ pub struct SpendableType<T> {
pub empty: T,
}
impl<T> SpendableType<T> {
pub fn from_fn(mut f: impl FnMut(SpendableTypeId) -> T) -> Self {
Self {
p2pk65: f(SpendableTypeId::P2PK65),
p2pk33: f(SpendableTypeId::P2PK33),
p2pkh: f(SpendableTypeId::P2PKH),
p2ms: f(SpendableTypeId::P2MS),
p2sh: f(SpendableTypeId::P2SH),
p2wpkh: f(SpendableTypeId::P2WPKH),
p2wsh: f(SpendableTypeId::P2WSH),
p2tr: f(SpendableTypeId::P2TR),
p2a: f(SpendableTypeId::P2A),
unknown: f(SpendableTypeId::Unknown),
empty: f(SpendableTypeId::Empty),
}
}
}
impl_column_row_formattable!(SpendableType {
p2pk65,
p2pk33,
p2pkh,
p2ms,
p2sh,
p2wpkh,
p2wsh,
p2tr,
p2a,
unknown,
empty,
});
impl SpendableType<CohortName> {
pub const fn names() -> &'static Self {
&SPENDABLE_TYPE_NAMES
+27 -1
View File
@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::{
@@ -80,7 +81,7 @@ pub const UNDER_AGE_NAMES: UnderAge<CohortName> = UnderAge {
_15y: CohortName::new("under_15y_old", "<15y", "Under 15 Years Old"),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct UnderAge<T> {
pub _1w: T,
pub _1m: T,
@@ -104,6 +105,31 @@ pub struct UnderAge<T> {
pub _15y: T,
}
define_column_id!(
UnderAgeId for UnderAge, version = 1 {
Under1W => _1w,
Under1M => _1m,
Under2M => _2m,
Under3M => _3m,
Under4M => _4m,
Under5M => _5m,
Under6M => _6m,
Under9M => _9m,
Under1Y => _1y,
Under18M => _18m,
Under2Y => _2y,
Under3Y => _3y,
Under4Y => _4y,
Under5Y => _5y,
Under6Y => _6y,
Under7Y => _7y,
Under8Y => _8y,
Under10Y => _10y,
Under12Y => _12y,
Under15Y => _15y,
}
);
impl UnderAge<CohortName> {
pub const fn names() -> &'static Self {
&UNDER_AGE_NAMES
+20 -1
View File
@@ -1,6 +1,7 @@
use brk_traversable::Traversable;
use brk_types::Sats;
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use super::{AmountFilter, CohortName, Filter};
@@ -56,7 +57,7 @@ pub const UNDER_AMOUNT_FILTERS: UnderAmount<Filter> = UnderAmount {
_100k_btc: Filter::Amount(AmountFilter::LowerThan(UNDER_AMOUNT_THRESHOLDS._100k_btc)),
};
#[derive(Default, Clone, Traversable, Serialize)]
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct UnderAmount<T> {
pub _10sats: T,
pub _100sats: T,
@@ -73,6 +74,24 @@ pub struct UnderAmount<T> {
pub _100k_btc: T,
}
define_column_id!(
UnderAmountId for UnderAmount, version = 1 {
Under10Sats => _10sats,
Under100Sats => _100sats,
Under1KSats => _1k_sats,
Under10KSats => _10k_sats,
Under100KSats => _100k_sats,
Under1MSats => _1m_sats,
Under10MSats => _10m_sats,
Under1Btc => _1btc,
Under10Btc => _10btc,
Under100Btc => _100btc,
Under1KBtc => _1k_btc,
Under10KBtc => _10k_btc,
Under100KBtc => _100k_btc,
}
);
impl UnderAmount<CohortName> {
pub const fn names() -> &'static Self {
&UNDER_AMOUNT_NAMES
+108 -49
View File
@@ -2,51 +2,113 @@ use brk_traversable::Traversable;
use rayon::prelude::*;
use crate::{
AgeRange, AmountRange, ByEntry, ByEpoch, ByTerm, Class, CohortName, Filter, OverAge,
OverAmount, SpendableType, TERM_NAMES, UnderAge, UnderAmount,
Amount, ByAge, ByEntry, ByEpoch, ByTerm, CLASS_FILTERS, CLASS_NAMES, Class, ClassId,
ENTRY_FILTERS, ENTRY_NAMES, EPOCH_FILTERS, EPOCH_NAMES, EntryId, EpochId, Filter,
SPENDABLE_TYPE_FILTERS, SPENDABLE_TYPE_NAMES, SpendableType, TERM_FILTERS, TERM_NAMES,
};
/// Canonical name for the aggregate cohort containing every UTXO.
pub const UTXO_ALL_NAME: CohortName = CohortName::new("all", "All", "All UTXOs");
/// Canonical names for the aggregate UTXO cohorts.
pub const UTXO_AGGREGATE_NAMES: [CohortName; 3] =
[UTXO_ALL_NAME, TERM_NAMES.short, TERM_NAMES.long];
use vecdb::ColumnId;
#[derive(Default, Clone, Traversable)]
pub struct UTXOGroups<T> {
pub all: T,
pub age_range: AgeRange<T>,
pub age: ByAge<T>,
pub epoch: ByEpoch<T>,
pub class: Class<T>,
pub entry: ByEntry<T>,
pub over_age: OverAge<T>,
pub over_amount: OverAmount<T>,
pub amount_range: AmountRange<T>,
pub utxo_amount: Amount<T>,
pub term: ByTerm<T>,
#[traversable(rename = "type")]
pub type_: SpendableType<T>,
pub under_age: UnderAge<T>,
pub under_amount: UnderAmount<T>,
}
impl<T> UTXOGroups<T> {
pub fn get(&self, filter: &Filter) -> Option<&T> {
match filter {
Filter::All => Some(&self.all),
Filter::Term(term) => match term {
crate::Term::Sth => Some(&self.term.short),
crate::Term::Lth => Some(&self.term.long),
},
Filter::Time(_) => self.age.get(filter),
Filter::Amount(_) => self.utxo_amount.get(filter),
Filter::Epoch(_) => EpochId::ALL
.iter()
.copied()
.find(|id| id.select(&EPOCH_FILTERS) == filter)
.map(|id| id.select(&self.epoch)),
Filter::Class(_) => ClassId::ALL
.iter()
.copied()
.find(|id| id.select(&CLASS_FILTERS) == filter)
.map(|id| id.select(&self.class)),
Filter::Entry(_) => EntryId::ALL
.iter()
.copied()
.find(|id| id.select(&ENTRY_FILTERS) == filter)
.map(|id| id.select(&self.entry)),
Filter::Type(output_type) => Some(self.type_.get(*output_type)),
}
}
pub fn map_named<U>(
&self,
mut map: impl FnMut(&Filter, &'static str, &T) -> U,
) -> UTXOGroups<U> {
UTXOGroups {
all: map(&Filter::All, "", &self.all),
age: self.age.map_named(&mut map),
epoch: ByEpoch::from_fn(|id| {
map(
id.select(&EPOCH_FILTERS),
id.select(&EPOCH_NAMES).id,
id.select(&self.epoch),
)
}),
class: Class::from_fn(|id| {
map(
id.select(&CLASS_FILTERS),
id.select(&CLASS_NAMES).id,
id.select(&self.class),
)
}),
entry: ByEntry::from_fn(|id| {
map(
id.select(&ENTRY_FILTERS),
id.select(&ENTRY_NAMES).id,
id.select(&self.entry),
)
}),
utxo_amount: self.utxo_amount.map_named(&mut map),
term: ByTerm::from_fn(|id| {
map(
id.select(&TERM_FILTERS),
id.select(&TERM_NAMES).id,
id.select(&self.term),
)
}),
type_: SpendableType::from_fn(|id| {
map(
id.select(&SPENDABLE_TYPE_FILTERS),
id.select(&SPENDABLE_TYPE_NAMES).id,
id.select(&self.type_),
)
}),
}
}
pub fn new<F>(mut create: F) -> Self
where
F: FnMut(Filter, &'static str) -> T,
{
Self {
all: create(Filter::All, ""),
age_range: AgeRange::new(&mut create),
age: ByAge::new(&mut create),
epoch: ByEpoch::new(&mut create),
class: Class::new(&mut create),
entry: ByEntry::new(&mut create),
over_age: OverAge::new(&mut create),
over_amount: OverAmount::new(&mut create),
amount_range: AmountRange::new(&mut create),
utxo_amount: Amount::new(&mut create),
term: ByTerm::new(&mut create),
type_: SpendableType::new(&mut create),
under_age: UnderAge::new(&mut create),
under_amount: UnderAmount::new(&mut create),
}
}
@@ -54,15 +116,13 @@ impl<T> UTXOGroups<T> {
[&self.all]
.into_iter()
.chain(self.term.iter())
.chain(self.under_age.iter())
.chain(self.over_age.iter())
.chain(self.over_amount.iter())
.chain(self.age_range.iter())
.chain(self.age.under.iter())
.chain(self.age.over.iter())
.chain(self.utxo_amount.iter())
.chain(self.age.range.iter())
.chain(self.epoch.iter())
.chain(self.class.iter())
.chain(self.entry.iter())
.chain(self.amount_range.iter())
.chain(self.under_amount.iter())
.chain(self.type_.iter())
}
@@ -70,15 +130,13 @@ impl<T> UTXOGroups<T> {
[&mut self.all]
.into_iter()
.chain(self.term.iter_mut())
.chain(self.under_age.iter_mut())
.chain(self.over_age.iter_mut())
.chain(self.over_amount.iter_mut())
.chain(self.age_range.iter_mut())
.chain(self.age.under.iter_mut())
.chain(self.age.over.iter_mut())
.chain(self.utxo_amount.iter_mut())
.chain(self.age.range.iter_mut())
.chain(self.epoch.iter_mut())
.chain(self.class.iter_mut())
.chain(self.entry.iter_mut())
.chain(self.amount_range.iter_mut())
.chain(self.under_amount.iter_mut())
.chain(self.type_.iter_mut())
}
@@ -89,35 +147,35 @@ impl<T> UTXOGroups<T> {
[&mut self.all]
.into_par_iter()
.chain(self.term.par_iter_mut())
.chain(self.under_age.par_iter_mut())
.chain(self.over_age.par_iter_mut())
.chain(self.over_amount.par_iter_mut())
.chain(self.age_range.par_iter_mut())
.chain(self.age.under.par_iter_mut())
.chain(self.age.over.par_iter_mut())
.chain(self.utxo_amount.par_iter_mut())
.chain(self.age.range.par_iter_mut())
.chain(self.epoch.par_iter_mut())
.chain(self.class.par_iter_mut())
.chain(self.entry.par_iter_mut())
.chain(self.amount_range.par_iter_mut())
.chain(self.under_amount.par_iter_mut())
.chain(self.type_.par_iter_mut())
}
pub fn iter_separate(&self) -> impl Iterator<Item = &T> {
self.age_range
self.age
.range
.iter()
.chain(self.epoch.iter())
.chain(self.class.iter())
.chain(self.entry.iter())
.chain(self.amount_range.iter())
.chain(self.utxo_amount.range.iter())
.chain(self.type_.iter())
}
pub fn iter_separate_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.age_range
self.age
.range
.iter_mut()
.chain(self.epoch.iter_mut())
.chain(self.class.iter_mut())
.chain(self.entry.iter_mut())
.chain(self.amount_range.iter_mut())
.chain(self.utxo_amount.range.iter_mut())
.chain(self.type_.iter_mut())
}
@@ -125,12 +183,13 @@ impl<T> UTXOGroups<T> {
where
T: Send + Sync,
{
self.age_range
self.age
.range
.par_iter_mut()
.chain(self.epoch.par_iter_mut())
.chain(self.class.par_iter_mut())
.chain(self.entry.par_iter_mut())
.chain(self.amount_range.par_iter_mut())
.chain(self.utxo_amount.range.par_iter_mut())
.chain(self.type_.par_iter_mut())
}
@@ -138,10 +197,10 @@ impl<T> UTXOGroups<T> {
[&mut self.all]
.into_iter()
.chain(self.term.iter_mut())
.chain(self.under_age.iter_mut())
.chain(self.over_age.iter_mut())
.chain(self.under_amount.iter_mut())
.chain(self.over_amount.iter_mut())
.chain(self.age.under.iter_mut())
.chain(self.age.over.iter_mut())
.chain(self.utxo_amount.under.iter_mut())
.chain(self.utxo_amount.over.iter_mut())
}
/// Iterator over aggregate cohorts (all, sth, lth) that compute values from sub-cohorts.
+98
View File
@@ -0,0 +1,98 @@
use brk_traversable::Traversable;
use schemars::JsonSchema;
use serde::Serialize;
use crate::{CohortName, Filter, TERM_FILTERS, TERM_NAMES, TermId};
/// Canonical name for the aggregate cohort containing every UTXO.
pub const UTXO_ALL_NAME: CohortName = CohortName::new("all", "All", "All UTXOs");
pub const UTXO_AGGREGATE_FILTERS: UTXOAggregate<Filter> = UTXOAggregate {
all: Filter::All,
sth: TERM_FILTERS.short,
lth: TERM_FILTERS.long,
};
/// Canonical names for the aggregate UTXO cohorts.
pub const UTXO_AGGREGATE_NAMES: UTXOAggregate<CohortName> = UTXOAggregate {
all: UTXO_ALL_NAME,
sth: TERM_NAMES.short,
lth: TERM_NAMES.long,
};
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct UTXOAggregate<T> {
pub all: T,
pub sth: T,
pub lth: T,
}
define_column_id!(
UTXOAggregateId for UTXOAggregate, version = 1 {
All => all,
Sth => sth,
Lth => lth,
}
);
impl UTXOAggregateId {
pub const fn cohort_name(self) -> CohortName {
match self {
Self::All => UTXO_AGGREGATE_NAMES.all,
Self::Sth => UTXO_AGGREGATE_NAMES.sth,
Self::Lth => UTXO_AGGREGATE_NAMES.lth,
}
}
pub const fn term(self) -> Option<TermId> {
match self {
Self::All => None,
Self::Sth => Some(TermId::Short),
Self::Lth => Some(TermId::Long),
}
}
}
impl<T> UTXOAggregate<T> {
pub fn map<U>(&self, mut f: impl FnMut(&T) -> U) -> UTXOAggregate<U> {
UTXOAggregate {
all: f(&self.all),
sth: f(&self.sth),
lth: f(&self.lth),
}
}
pub fn try_from_fn<E>(mut f: impl FnMut(UTXOAggregateId) -> Result<T, E>) -> Result<Self, E> {
Ok(Self {
all: f(UTXOAggregateId::All)?,
sth: f(UTXOAggregateId::Sth)?,
lth: f(UTXOAggregateId::Lth)?,
})
}
pub fn get(&self, filter: &Filter) -> Option<&T> {
match filter {
Filter::All => Some(&self.all),
Filter::Term(crate::Term::Sth) => Some(&self.sth),
Filter::Term(crate::Term::Lth) => Some(&self.lth),
_ => None,
}
}
pub fn get_mut(&mut self, filter: &Filter) -> Option<&mut T> {
match filter {
Filter::All => Some(&mut self.all),
Filter::Term(crate::Term::Sth) => Some(&mut self.sth),
Filter::Term(crate::Term::Lth) => Some(&mut self.lth),
_ => None,
}
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
[&self.all, &self.sth, &self.lth].into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
[&mut self.all, &mut self.sth, &mut self.lth].into_iter()
}
}
+36
View File
@@ -0,0 +1,36 @@
use brk_traversable::Traversable;
use schemars::JsonSchema;
use serde::Serialize;
use crate::Filter;
#[derive(Debug, Default, Clone, Traversable, Serialize, JsonSchema)]
pub struct UTXOAllAndSth<T> {
pub all: T,
pub sth: T,
}
define_column_id!(
UTXOAllAndSthId for UTXOAllAndSth, version = 1 {
All => all,
Sth => sth,
}
);
impl<T> UTXOAllAndSth<T> {
pub fn get(&self, filter: &Filter) -> Option<&T> {
match filter {
Filter::All => Some(&self.all),
Filter::Term(crate::Term::Sth) => Some(&self.sth),
_ => None,
}
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
[&self.all, &self.sth].into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
[&mut self.all, &mut self.sth].into_iter()
}
}
@@ -0,0 +1,110 @@
use brk_traversable::Traversable;
use vecdb::ColumnId;
use crate::{
ByAge, ByEntry, ByEpoch, ByTerm, CLASS_FILTERS, CLASS_NAMES, Class, ClassId, ENTRY_FILTERS,
ENTRY_NAMES, EPOCH_FILTERS, EPOCH_NAMES, EntryId, EpochId, Filter, SPENDABLE_TYPE_FILTERS,
SPENDABLE_TYPE_NAMES, SpendableType, TERM_FILTERS, TERM_NAMES,
};
#[derive(Default, Clone, Traversable)]
pub struct UTXOGroupsWithoutAmount<T> {
pub all: T,
pub age: ByAge<T>,
pub epoch: ByEpoch<T>,
pub class: Class<T>,
pub entry: ByEntry<T>,
pub term: ByTerm<T>,
#[traversable(rename = "type")]
pub type_: SpendableType<T>,
}
impl<T> UTXOGroupsWithoutAmount<T> {
pub fn new<F>(mut create: F) -> Self
where
F: FnMut(Filter, &'static str) -> T,
{
Self {
all: create(Filter::All, ""),
age: ByAge::new(&mut create),
epoch: ByEpoch::new(&mut create),
class: Class::new(&mut create),
entry: ByEntry::new(&mut create),
term: ByTerm::new(&mut create),
type_: SpendableType::new(&mut create),
}
}
pub fn get(&self, filter: &Filter) -> Option<&T> {
match filter {
Filter::All => Some(&self.all),
Filter::Term(term) => match term {
crate::Term::Sth => Some(&self.term.short),
crate::Term::Lth => Some(&self.term.long),
},
Filter::Time(_) => self.age.get(filter),
Filter::Epoch(_) => EpochId::ALL
.iter()
.copied()
.find(|id| id.select(&EPOCH_FILTERS) == filter)
.map(|id| id.select(&self.epoch)),
Filter::Class(_) => ClassId::ALL
.iter()
.copied()
.find(|id| id.select(&CLASS_FILTERS) == filter)
.map(|id| id.select(&self.class)),
Filter::Entry(_) => EntryId::ALL
.iter()
.copied()
.find(|id| id.select(&ENTRY_FILTERS) == filter)
.map(|id| id.select(&self.entry)),
Filter::Type(output_type) => Some(self.type_.get(*output_type)),
Filter::Amount(_) => None,
}
}
pub fn map_named<U>(
&self,
mut map: impl FnMut(&Filter, &'static str, &T) -> U,
) -> UTXOGroupsWithoutAmount<U> {
UTXOGroupsWithoutAmount {
all: map(&Filter::All, "", &self.all),
age: self.age.map_named(&mut map),
epoch: ByEpoch::from_fn(|id| {
map(
id.select(&EPOCH_FILTERS),
id.select(&EPOCH_NAMES).id,
id.select(&self.epoch),
)
}),
class: Class::from_fn(|id| {
map(
id.select(&CLASS_FILTERS),
id.select(&CLASS_NAMES).id,
id.select(&self.class),
)
}),
entry: ByEntry::from_fn(|id| {
map(
id.select(&ENTRY_FILTERS),
id.select(&ENTRY_NAMES).id,
id.select(&self.entry),
)
}),
term: ByTerm::from_fn(|id| {
map(
id.select(&TERM_FILTERS),
id.select(&TERM_NAMES).id,
id.select(&self.term),
)
}),
type_: SpendableType::from_fn(|id| {
map(
id.select(&SPENDABLE_TYPE_FILTERS),
id.select(&SPENDABLE_TYPE_NAMES).id,
id.select(&self.type_),
)
}),
}
}
}
@@ -0,0 +1,105 @@
use brk_traversable::Traversable;
use vecdb::ColumnId;
use crate::{
ByAge, ByEntry, ByEpoch, ByTerm, CLASS_FILTERS, CLASS_NAMES, Class, ClassId, ENTRY_FILTERS,
ENTRY_NAMES, EPOCH_FILTERS, EPOCH_NAMES, EntryId, EpochId, Filter, TERM_FILTERS, TERM_NAMES,
};
#[derive(Default, Clone, Traversable)]
pub struct UTXOGroupsWithoutAmountOrType<T> {
pub all: T,
pub age: ByAge<T>,
pub epoch: ByEpoch<T>,
pub class: Class<T>,
pub entry: ByEntry<T>,
pub term: ByTerm<T>,
}
impl<T> UTXOGroupsWithoutAmountOrType<T> {
pub fn new<F>(mut create: F) -> Self
where
F: FnMut(Filter, &'static str) -> T,
{
Self::new_with(&mut create)
}
pub(crate) fn new_with<F>(create: &mut F) -> Self
where
F: FnMut(Filter, &'static str) -> T,
{
Self {
all: create(Filter::All, ""),
age: ByAge::new(&mut *create),
epoch: ByEpoch::new(&mut *create),
class: Class::new(&mut *create),
entry: ByEntry::new(&mut *create),
term: ByTerm::new(&mut *create),
}
}
pub fn get(&self, filter: &Filter) -> Option<&T> {
match filter {
Filter::All => Some(&self.all),
Filter::Term(term) => match term {
crate::Term::Sth => Some(&self.term.short),
crate::Term::Lth => Some(&self.term.long),
},
Filter::Time(_) => self.age.get(filter),
Filter::Epoch(_) => EpochId::ALL
.iter()
.copied()
.find(|id| id.select(&EPOCH_FILTERS) == filter)
.map(|id| id.select(&self.epoch)),
Filter::Class(_) => ClassId::ALL
.iter()
.copied()
.find(|id| id.select(&CLASS_FILTERS) == filter)
.map(|id| id.select(&self.class)),
Filter::Entry(_) => EntryId::ALL
.iter()
.copied()
.find(|id| id.select(&ENTRY_FILTERS) == filter)
.map(|id| id.select(&self.entry)),
Filter::Amount(_) | Filter::Type(_) => None,
}
}
pub fn map_named<U>(
&self,
mut map: impl FnMut(&Filter, &'static str, &T) -> U,
) -> UTXOGroupsWithoutAmountOrType<U> {
UTXOGroupsWithoutAmountOrType {
all: map(&Filter::All, "", &self.all),
age: self.age.map_named(&mut map),
epoch: ByEpoch::from_fn(|id| {
map(
id.select(&EPOCH_FILTERS),
id.select(&EPOCH_NAMES).id,
id.select(&self.epoch),
)
}),
class: Class::from_fn(|id| {
map(
id.select(&CLASS_FILTERS),
id.select(&CLASS_NAMES).id,
id.select(&self.class),
)
}),
entry: ByEntry::from_fn(|id| {
map(
id.select(&ENTRY_FILTERS),
id.select(&ENTRY_NAMES).id,
id.select(&self.entry),
)
}),
term: ByTerm::from_fn(|id| {
map(
id.select(&TERM_FILTERS),
id.select(&TERM_NAMES).id,
id.select(&self.term),
)
}),
}
}
}
@@ -0,0 +1,21 @@
/// Per-block activity counts, reset after every block.
#[derive(Debug, Default, Clone)]
pub struct BlockActivityCounts {
pub reactivated: u32,
pub sending: u32,
pub receiving: u32,
pub bidirectional: u32,
}
impl BlockActivityCounts {
#[inline]
pub(crate) fn reset(&mut self) {
*self = Self::default();
}
#[inline(always)]
pub(crate) fn active(&self) -> u32 {
debug_assert!(self.bidirectional <= self.sending.min(self.receiving));
self.sending + self.receiving - self.bidirectional
}
}
@@ -0,0 +1,56 @@
use brk_cohort::{AddrTypeId, ByAddrType};
use brk_types::{StoredU32, StoredU64};
use derive_more::{Deref, DerefMut};
use vecdb::ColumnId;
use super::BlockActivityCounts;
/// Activity counts accumulated during block processing for each address type.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToActivityCounts(pub ByAddrType<BlockActivityCounts>);
impl AddrTypeToActivityCounts {
pub(crate) fn reset(&mut self) {
self.0.values_mut().for_each(BlockActivityCounts::reset);
}
pub(crate) fn active(&self) -> u32 {
self.0.values().map(BlockActivityCounts::active).sum()
}
#[inline(always)]
pub(super) fn row(
&self,
value: impl Fn(&BlockActivityCounts) -> u32,
) -> <AddrTypeId as ColumnId>::Row<StoredU64> {
AddrTypeId::from_fn(|column| {
StoredU64::from(StoredU32::from(value(column.select(&self.0))))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn active_count_sums_distinct_addresses_across_types() {
let counts = AddrTypeToActivityCounts(ByAddrType {
p2pkh: BlockActivityCounts {
sending: 5,
receiving: 4,
bidirectional: 2,
..BlockActivityCounts::default()
},
p2tr: BlockActivityCounts {
sending: 3,
receiving: 2,
bidirectional: 1,
..BlockActivityCounts::default()
},
..ByAddrType::default()
});
assert_eq!(counts.active(), 11);
}
}
@@ -0,0 +1,7 @@
mod block_counts;
mod by_type;
mod vecs;
pub use block_counts::BlockActivityCounts;
pub use by_type::AddrTypeToActivityCounts;
pub use vecs::AddrActivityVecs;
@@ -1,12 +1,9 @@
//! Per-block address activity, split by address type.
use brk_cohort::{AddrTypeId, ByAddrType};
use brk_cohort::AddrTypeId;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{StoredU32, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, ColumnId, Database, ReadOnlyClone, Rw, StorageMode};
use vecdb::{AnyStoredVec, AnyVec, Database, ReadOnlyClone, Rw, StorageMode};
use crate::{
indexes,
@@ -16,68 +13,20 @@ use crate::{
},
};
/// Per-block activity counts, reset after every block.
#[derive(Debug, Default, Clone)]
pub struct BlockActivityCounts {
pub reactivated: u32,
pub sending: u32,
pub receiving: u32,
pub bidirectional: u32,
}
use super::{AddrTypeToActivityCounts, BlockActivityCounts};
impl BlockActivityCounts {
#[inline]
pub(crate) fn reset(&mut self) {
*self = Self::default();
}
#[inline(always)]
fn active(&self) -> u32 {
debug_assert!(self.bidirectional <= self.sending.min(self.receiving));
self.sending + self.receiving - self.bidirectional
}
}
/// Activity counts accumulated during block processing for each address type.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToActivityCounts(pub ByAddrType<BlockActivityCounts>);
impl AddrTypeToActivityCounts {
pub(crate) fn reset(&mut self) {
self.0.values_mut().for_each(BlockActivityCounts::reset);
}
pub(crate) fn active(&self) -> u32 {
self.0.values().map(BlockActivityCounts::active).sum()
}
#[inline(always)]
fn row(
&self,
value: impl Fn(&BlockActivityCounts) -> u32,
) -> <AddrTypeId as ColumnId>::Row<StoredU64> {
AddrTypeId::from_fn(|column| {
StoredU64::from(StoredU32::from(value(column.select(&self.0))))
})
}
}
#[derive(Clone, Traversable)]
pub struct ActivityCountVecs {
pub reactivated: LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>,
pub sending: LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>,
pub receiving: LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>,
pub bidirectional: LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>,
pub active: LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>,
}
/// Five metric-first cumulative matrices with cohort-first public views.
#[derive(Deref, DerefMut, Traversable)]
#[derive(Traversable)]
pub struct AddrActivityVecs<M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub series: WithAddrTypes<ActivityCountVecs>,
pub reactivated:
WithAddrTypes<LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>>,
pub sending:
WithAddrTypes<LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>>,
pub receiving:
WithAddrTypes<LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>>,
pub bidirectional:
WithAddrTypes<LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>>,
pub active:
WithAddrTypes<LazyPerBlockCumulativeAverage<StoredU32, StoredU64, StoredU64ToStoredU32>>,
#[traversable(hidden)]
cumulative_reactivated: ColumnarPerBlockCumulativeRolling<StoredU64, AddrTypeId, (), M>,
@@ -166,26 +115,12 @@ impl AddrActivityVecs {
cached_starts,
);
let by_addr_type = AddrTypeId::series(|column, _| ActivityCountVecs {
reactivated: column.select(&reactivated.by_addr_type).clone(),
sending: column.select(&sending.by_addr_type).clone(),
receiving: column.select(&receiving.by_addr_type).clone(),
bidirectional: column.select(&bidirectional.by_addr_type).clone(),
active: column.select(&active.by_addr_type).clone(),
});
let series = WithAddrTypes {
all: ActivityCountVecs {
reactivated: reactivated.all,
sending: sending.all,
receiving: receiving.all,
bidirectional: bidirectional.all,
active: active.all,
},
by_addr_type,
};
Ok(Self {
series,
reactivated,
sending,
receiving,
bidirectional,
active,
cumulative_reactivated,
cumulative_sending,
cumulative_receiving,
@@ -243,29 +178,3 @@ impl AddrActivityVecs {
.push_block(counts.row(BlockActivityCounts::active));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn active_count_sums_distinct_addresses_across_types() {
let counts = AddrTypeToActivityCounts(ByAddrType {
p2pkh: BlockActivityCounts {
sending: 5,
receiving: 4,
bidirectional: 2,
..BlockActivityCounts::default()
},
p2tr: BlockActivityCounts {
sending: 3,
receiving: 2,
bidirectional: 1,
..BlockActivityCounts::default()
},
..ByAddrType::default()
});
assert_eq!(counts.active(), 11);
}
}
@@ -2,11 +2,10 @@ use brk_cohort::{AddrTypeId, ByAddrType};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, Sats, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, CachedBoxedVec, Database, Exit, ReadOnlyClone, ReadableVec, Rw, StorageMode,
WritableVec,
AnyStoredVec, CachedBoxedVec, Database, Exit, ReadOnlyClone, ReadableCloneableVec, ReadableVec,
Rw, StorageMode, WritableVec,
};
use crate::{
@@ -17,24 +16,10 @@ use crate::{
},
};
/// Average amount held per UTXO and per funded address.
///
/// `utxo = supply / utxo_count`, `addr = supply / funded_addr_count`.
#[derive(Clone, Traversable)]
pub struct AvgAmountMetrics<V> {
pub utxo: V,
pub addr: V,
}
#[derive(Deref, DerefMut, Traversable)]
#[derive(Traversable)]
pub struct AvgAmountVecs<M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub series: WithAddrTypes<
AvgAmountMetrics<LazyColumnSpotValuePerBlock<AddrTypeId>>,
AvgAmountMetrics<LazySpotValuePerBlock>,
>,
pub utxo: WithAddrTypes<LazyColumnSpotValuePerBlock<AddrTypeId>, LazySpotValuePerBlock>,
pub addr: WithAddrTypes<LazyColumnSpotValuePerBlock<AddrTypeId>, LazySpotValuePerBlock>,
#[traversable(hidden)]
utxo_source: ColumnarPerBlock<Sats, AddrTypeId, (), M>,
#[traversable(hidden)]
@@ -48,8 +33,8 @@ impl AvgAmountVecs {
indexes: &indexes::Vecs,
spot_price: &CachedBoxedVec<Height, Cents>,
all_chain: &AllChainCache,
utxo_count: &(impl vecdb::ReadableCloneableVec<Height, StoredU64> + 'static),
funded_addr_count: &(impl vecdb::ReadableCloneableVec<Height, StoredU64> + 'static),
utxo_count: &(impl ReadableCloneableVec<Height, StoredU64> + 'static),
funded_addr_count: &(impl ReadableCloneableVec<Height, StoredU64> + 'static),
) -> Result<Self> {
let avg_utxo = all_chain.with_supply(
"avg_utxo_amount_sats_source",
@@ -63,50 +48,54 @@ impl AvgAmountVecs {
funded_addr_count,
|_, count, supply| supply / count,
);
let all = AvgAmountMetrics {
utxo: LazySpotValuePerBlock::from_sats_source(
let utxo_source =
ColumnarPerBlock::forced_import(db, "avg_utxo_amount_sats_by_type", version, |_| ())?;
let addr_source =
ColumnarPerBlock::forced_import(db, "avg_addr_amount_sats_by_type", version, |_| ())?;
let utxo_columns = utxo_source.height.read_only_clone();
let addr_columns = addr_source.height.read_only_clone();
let utxo = WithAddrTypes {
all: LazySpotValuePerBlock::from_sats_source(
"avg_utxo_amount",
version,
avg_utxo,
indexes,
spot_price,
),
addr: LazySpotValuePerBlock::from_sats_source(
by_addr_type: AddrTypeId::series(|column, type_name| {
LazyColumnSpotValuePerBlock::new(
&format!("{type_name}_avg_utxo_amount"),
version,
&utxo_columns,
column,
indexes,
spot_price,
)
}),
};
let addr = WithAddrTypes {
all: LazySpotValuePerBlock::from_sats_source(
"avg_addr_amount",
version,
avg_addr,
indexes,
spot_price,
),
by_addr_type: AddrTypeId::series(|column, type_name| {
LazyColumnSpotValuePerBlock::new(
&format!("{type_name}_avg_addr_amount"),
version,
&addr_columns,
column,
indexes,
spot_price,
)
}),
};
let utxo_source =
ColumnarPerBlock::forced_import(db, "avg_utxo_amount_sats_by_type", version, |_| ())?;
let addr_source =
ColumnarPerBlock::forced_import(db, "avg_addr_amount_sats_by_type", version, |_| ())?;
let utxo = utxo_source.height.read_only_clone();
let addr = addr_source.height.read_only_clone();
let by_addr_type = AddrTypeId::series(|column, type_name| AvgAmountMetrics {
utxo: LazyColumnSpotValuePerBlock::new(
&format!("{type_name}_avg_utxo_amount"),
version,
&utxo,
column,
indexes,
spot_price,
),
addr: LazyColumnSpotValuePerBlock::new(
&format!("{type_name}_avg_addr_amount"),
version,
&addr,
column,
indexes,
spot_price,
),
});
Ok(Self {
series: WithAddrTypes { all, by_addr_type },
utxo,
addr,
utxo_source,
addr_source,
})
@@ -9,10 +9,11 @@ use crate::{
use super::AddrCountsVecs;
type AddrDelta = LazyRollingDeltasFromHeight<StoredU64, StoredI64, PartsPerMillionSigned64>;
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct DeltaVecs(#[traversable(flatten)] pub WithAddrTypes<AddrDelta>);
pub struct DeltaVecs(
#[traversable(flatten)]
pub WithAddrTypes<LazyRollingDeltasFromHeight<StoredU64, StoredI64, PartsPerMillionSigned64>>,
);
impl DeltaVecs {
pub(crate) fn new(
@@ -0,0 +1,83 @@
use brk_cohort::{AmountRange, CohortContext};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{PartsPerMillionSigned64, StoredI64, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Rw, StorageMode};
use crate::{
distribution::metrics::ColumnarAmount,
indexes,
internal::{CachedWindowStartVec, LazyPerBlockWithDeltas, Windows},
};
use super::{AddrCountsVecs, AddrTypeToAddrCount};
#[derive(Traversable)]
pub struct FundedAddrCountsVecs<M: StorageMode = Rw> {
#[traversable(flatten)]
pub counts: AddrCountsVecs<M>,
pub balance: ColumnarAmount<
StoredU64,
LazyPerBlockWithDeltas<StoredU64, StoredI64, PartsPerMillionSigned64>,
M,
>,
}
impl FundedAddrCountsVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
) -> Result<Self> {
Ok(Self {
counts: AddrCountsVecs::forced_import(db, "addr_count", version, indexes)?,
balance: ColumnarAmount::forced_import(
db,
"addrs_addr_count_by_balance_range",
CohortContext::Addr,
"addr_count",
version + Version::ONE,
|name, source| {
LazyPerBlockWithDeltas::from_boxed_height_source(
name,
version + Version::ONE,
source,
Version::TWO,
indexes,
cached_starts,
)
},
)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.counts.min_stateful_len().min(self.balance.len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.counts
.par_iter_height_mut()
.chain(rayon::iter::once(self.balance.stored_mut()))
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.counts.reset_height()?;
self.balance.reset()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_counts(&mut self, counts: &AddrTypeToAddrCount) {
self.counts.push_counts(counts);
}
#[inline(always)]
pub(crate) fn push_balance(&mut self, counts: AmountRange<StoredU64>) {
self.balance.push(counts);
}
}
@@ -1,7 +1,15 @@
mod all_vecs;
mod delta_vecs;
mod funded_total_vecs;
mod funded_vecs;
mod new_vecs;
mod state;
mod total_vecs;
pub use all_vecs::AddrCountsVecs;
pub use delta_vecs::DeltaVecs;
pub use funded_total_vecs::AddrCountFundedTotalVecs;
pub use funded_vecs::FundedAddrCountsVecs;
pub use new_vecs::NewAddrCountVecs;
pub use state::AddrTypeToAddrCount;
pub use total_vecs::TotalAddrCountVecs;
@@ -12,7 +12,7 @@ use rayon::prelude::*;
use rustc_hash::FxHashMap;
use vecdb::{
AnyStoredVec, AnyVec, BytesVec, Database, ImportOptions, ImportableVec, ReadableVec, Rw, Stamp,
StorageMode, WritableVec,
StorageMode, VecIndex, WritableVec,
};
use super::super::AddrTypeToTypeIndexMap;
@@ -159,7 +159,7 @@ impl AnyAddrIndexesVecs {
}
/// Process updates for a single address type's BytesVec, merging two maps.
fn process_single_type_merged<I: vecdb::VecIndex>(
fn process_single_type_merged<I: VecIndex>(
vec: &mut BytesVec<I, AnyAddrIndex>,
map1: FxHashMap<TypeIndex, AnyAddrIndex>,
map2: FxHashMap<TypeIndex, AnyAddrIndex>,
@@ -1,25 +1,26 @@
mod activity;
mod avg_amount;
mod count;
mod data;
mod delta;
mod exposed;
mod indexes;
mod new_addr_count;
mod reused;
mod state;
mod supply;
mod total_addr_count;
mod type_map;
mod vecs;
pub use activity::{AddrActivityVecs, AddrTypeToActivityCounts};
pub use count::{AddrCountsVecs, AddrTypeToAddrCount};
pub use avg_amount::AvgAmountVecs;
pub use count::{
AddrCountsVecs, AddrTypeToAddrCount, DeltaVecs, FundedAddrCountsVecs, NewAddrCountVecs,
TotalAddrCountVecs,
};
pub use data::AddrsDataVecs;
pub use delta::DeltaVecs;
pub use exposed::{ExposedAddrState, ExposedAddrVecs};
pub use indexes::AnyAddrIndexesVecs;
pub use new_addr_count::NewAddrCountVecs;
pub use reused::{ReusedAddrState, ReusedAddrVecs};
pub use state::{AddrMetricsState, AddrReceivePreState, AddrSendPreState};
pub use supply::AddrTypeToSupply;
pub use total_addr_count::TotalAddrCountVecs;
pub use type_map::{AddrTypeToTypeIndexMap, AddrTypeToVec, HeightToAddrTypeToVec};
pub use vecs::AddrVecs;
@@ -59,43 +59,12 @@ use super::state::AddrTypeToAddrEventCount;
/// `active_reused_addr_share` is the per-block ratio
/// `reused / active * 100` as a percentage in `[0, 100]` (or `0.0` for
/// empty blocks). The denominator (distinct active addrs per block)
/// lives on `ActivityCountVecs::active` (`addrs.activity.all.active`),
/// lives at `addrs.activity.active.all`,
/// derived from `sending + receiving - bidirectional`. Both fields
/// expose lazy 24h/1w/1m/1y rolling *averages* of the per-block values.
/// Sums and cumulatives of distinct-address counts would be misleading
/// because the same address can appear in multiple blocks, so the
/// cumulative count remains an internal source for the lazy views.
#[derive(Clone, Traversable)]
pub struct AddrEventShares {
pub all: LazyPercentCumulativeRolling<PartsPerMillion32>,
#[traversable(flatten)]
pub by_addr_type: ByAddrType<LazyPercentCumulativeRolling<PartsPerMillion32>>,
}
impl AddrEventShares {
fn new(
name: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
all: LazyPercentCumulativeRolling<PartsPerMillion32>,
numerators: &ByAddrType<LazyColumnPerBlockCumulativeRolling<StoredU64, AddrTypeId>>,
denominators: &ByAddrType<CachedBlockCountReader>,
) -> Self {
let by_addr_type = AddrTypeId::series(|column, type_name| {
LazyPercentCumulativeRolling::from_cached_block_count(
&format!("{type_name}_{name}"),
version,
&column.select(numerators).cumulative.height,
column.select(denominators).clone(),
cached_starts,
indexes,
)
});
Self { all, by_addr_type }
}
}
#[derive(Traversable)]
pub struct AddrEventsVecs<M: StorageMode = Rw> {
pub output_to_reused_addr_count: ColumnarPerBlockCumulativeRolling<
@@ -107,7 +76,7 @@ pub struct AddrEventsVecs<M: StorageMode = Rw> {
>,
M,
>,
pub output_to_reused_addr_share: AddrEventShares,
pub output_to_reused_addr_share: WithAddrTypes<LazyPercentCumulativeRolling<PartsPerMillion32>>,
pub spendable_output_to_reused_addr_share: LazyPercentCumulativeRolling<PartsPerMillion32>,
pub input_from_reused_addr_count: ColumnarPerBlockCumulativeRolling<
StoredU64,
@@ -118,12 +87,34 @@ pub struct AddrEventsVecs<M: StorageMode = Rw> {
>,
M,
>,
pub input_from_reused_addr_share: AddrEventShares,
pub input_from_reused_addr_share:
WithAddrTypes<LazyPercentCumulativeRolling<PartsPerMillion32>>,
pub active_reused_addr_count: CountPerBlockRollingAverage<M>,
pub active_reused_addr_share: PerBlockRollingAverage<StoredF32, StoredF32, M>,
}
impl AddrEventsVecs {
fn event_shares(
name: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
all: LazyPercentCumulativeRolling<PartsPerMillion32>,
numerators: &ByAddrType<LazyColumnPerBlockCumulativeRolling<StoredU64, AddrTypeId>>,
denominators: &ByAddrType<CachedBlockCountReader>,
) -> WithAddrTypes<LazyPercentCumulativeRolling<PartsPerMillion32>> {
let by_addr_type = AddrTypeId::series(|column, type_name| {
LazyPercentCumulativeRolling::from_cached_block_count(
&format!("{type_name}_{name}"),
version,
&column.select(numerators).cumulative.height,
column.select(denominators).clone(),
cached_starts,
indexes,
)
});
WithAddrTypes { all, by_addr_type }
}
pub(crate) fn forced_import(
db: &Database,
name: &str,
@@ -153,7 +144,7 @@ impl AddrEventsVecs {
let output_to_reused_addr_count = import_count(&format!("output_to_{name}_addr_count"))?;
let output_share_name = format!("output_to_{name}_addr_share");
let output_denominators = outputs_by_type.output_count.cached_addr_type_counts();
let output_to_reused_addr_share = AddrEventShares::new(
let output_to_reused_addr_share = Self::event_shares(
&output_share_name,
version,
indexes,
@@ -185,7 +176,7 @@ impl AddrEventsVecs {
let input_from_reused_addr_count = import_count(&format!("input_from_{name}_addr_count"))?;
let input_share_name = format!("input_from_{name}_addr_share");
let input_denominators = inputs_by_type.input_count.cached_addr_type_counts();
let input_from_reused_addr_share = AddrEventShares::new(
let input_from_reused_addr_share = Self::event_shares(
&input_share_name,
version,
indexes,
@@ -271,8 +262,8 @@ impl AddrEventsVecs {
.push_block(StoredU32::from(active_reused_addr_count));
// Stored as a percentage in [0, 100] to match the rest of the
// codebase (Unit.percentage on the website expects 0..100). The
// `active_addr_count` denominator lives on `ActivityCountVecs`
// (`addrs.activity.all.active`), passed in here so we can
// `active_addr_count` denominator lives at
// `addrs.activity.active.all`, passed in here so we can
// compute the per-block ratio inline.
let share = if active_addr_count > 0 {
100.0 * (active_reused_addr_count as f32 / active_addr_count as f32)
@@ -1,180 +0,0 @@
use brk_types::{FundedAddrData, Height, OutputType, Sats};
use crate::distribution::{block::TrackingStatus, vecs::AddrMetricsVecs};
use super::{AddrTypeToActivityCounts, AddrTypeToAddrCount, ExposedAddrState, ReusedAddrState};
/// Bundle of per-block runtime state for the full address-metrics pipeline.
/// Feeds `process_received` / `process_sent` and is pushed to [`AddrMetricsVecs`]
/// once per block.
///
/// Recovery: [`From<(&AddrMetricsVecs, Height)>`] reads the prior block from
/// disk to seed all persistent running totals. Per-block counters (activity,
/// and event counts inside each [`ReusedAddrState`]) default to zero and are
/// cleared at the top of each block via [`Self::reset_per_block`].
#[derive(Debug, Default)]
pub struct AddrMetricsState {
pub funded: AddrTypeToAddrCount,
pub empty: AddrTypeToAddrCount,
pub activity: AddrTypeToActivityCounts,
pub reused: ReusedAddrState,
pub respent: ReusedAddrState,
pub exposed: ExposedAddrState,
}
/// Snapshot of [`FundedAddrData`] taken BEFORE a receive mutates it.
/// Feeds delta-based updates in [`AddrMetricsState::on_receive_applied`].
#[derive(Debug)]
pub struct AddrReceivePreState {
pub was_funded: bool,
pub was_reused: bool,
pub was_respent: bool,
pub was_pubkey_exposed: bool,
pub prev_funded_txo_count: u32,
pub exposed_contribution: Sats,
pub reused_contribution: Sats,
pub respent_contribution: Sats,
}
impl AddrReceivePreState {
#[inline]
pub fn capture(addr_data: &FundedAddrData, output_type: OutputType) -> Self {
Self {
was_funded: addr_data.is_funded(),
was_reused: addr_data.is_reused(),
was_respent: addr_data.is_respent(),
was_pubkey_exposed: addr_data.is_pubkey_exposed(output_type),
prev_funded_txo_count: addr_data.funded_txo_count,
exposed_contribution: addr_data.exposed_supply_contribution(output_type),
reused_contribution: addr_data.reused_supply_contribution(),
respent_contribution: addr_data.respent_supply_contribution(),
}
}
}
/// Snapshot of [`FundedAddrData`] taken BEFORE a spend mutates it.
/// Feeds delta-based updates in [`AddrMetricsState::on_send_applied`].
#[derive(Debug)]
pub struct AddrSendPreState {
pub was_reused: bool,
pub was_respent: bool,
pub was_pubkey_exposed: bool,
pub exposed_contribution: Sats,
pub reused_contribution: Sats,
pub respent_contribution: Sats,
}
impl AddrSendPreState {
#[inline]
pub fn capture(addr_data: &FundedAddrData, output_type: OutputType) -> Self {
Self {
was_reused: addr_data.is_reused(),
was_respent: addr_data.is_respent(),
was_pubkey_exposed: addr_data.is_pubkey_exposed(output_type),
exposed_contribution: addr_data.exposed_supply_contribution(output_type),
reused_contribution: addr_data.reused_supply_contribution(),
respent_contribution: addr_data.respent_supply_contribution(),
}
}
}
impl AddrMetricsState {
#[inline]
pub(crate) fn reset_per_block(&mut self) {
self.activity.reset();
self.reused.reset_per_block();
self.respent.reset_per_block();
}
/// Apply all state updates for a received output, AFTER the cohort and
/// `addr_data` have been mutated. `pre` is the snapshot captured before
/// the mutation, `addr_data` is the post-receive view.
#[inline]
pub(crate) fn on_receive_applied(
&mut self,
output_type: OutputType,
status: TrackingStatus,
addr_data: &FundedAddrData,
pre: &AddrReceivePreState,
output_count: u32,
) {
let activity = self.activity.get_mut_unwrap(output_type);
activity.receiving += 1;
match status {
TrackingStatus::New => {
*self.funded.get_mut_unwrap(output_type) += 1;
}
TrackingStatus::WasEmpty => {
activity.reactivated += 1;
*self.funded.get_mut_unwrap(output_type) += 1;
*self.empty.get_mut_unwrap(output_type) -= 1;
}
TrackingStatus::Tracked => {}
}
self.reused
.on_receive_as_reused(output_type, addr_data, pre, output_count);
self.respent
.on_receive_as_respent(output_type, addr_data, pre, output_count);
self.exposed.on_receive(output_type, addr_data, pre, status);
}
/// Apply all state updates for a spent UTXO, AFTER the cohort and
/// `addr_data` have been mutated. `pre` is the snapshot captured before
/// the mutation. `is_first_encounter` / `also_received` come from the
/// caller's per-block seen/received tracking. `will_be_empty` is from
/// the pre-mutation `addr_data.has_1_utxos()`.
#[inline]
pub(crate) fn on_send_applied(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrSendPreState,
is_first_encounter: bool,
also_received: bool,
will_be_empty: bool,
) {
if is_first_encounter {
let activity = self.activity.get_mut_unwrap(output_type);
activity.sending += 1;
if also_received {
activity.bidirectional += 1;
}
}
if will_be_empty {
*self.funded.get_mut_unwrap(output_type) -= 1;
*self.empty.get_mut_unwrap(output_type) += 1;
}
self.reused.on_send_as_reused(
output_type,
addr_data,
pre,
is_first_encounter,
also_received,
will_be_empty,
);
self.respent.on_send_as_respent(
output_type,
addr_data,
pre,
is_first_encounter,
also_received,
will_be_empty,
);
self.exposed
.on_send(output_type, addr_data, pre, will_be_empty);
}
}
impl From<(&AddrMetricsVecs, Height)> for AddrMetricsState {
#[inline]
fn from((vecs, starting_height): (&AddrMetricsVecs, Height)) -> Self {
Self {
funded: AddrTypeToAddrCount::from((&vecs.funded, starting_height)),
empty: AddrTypeToAddrCount::from((&vecs.empty, starting_height)),
activity: AddrTypeToActivityCounts::default(),
reused: ReusedAddrState::from((&vecs.reused, starting_height)),
respent: ReusedAddrState::from((&vecs.respent, starting_height)),
exposed: ExposedAddrState::from((&vecs.exposed, starting_height)),
}
}
}
@@ -0,0 +1,112 @@
use brk_types::{FundedAddrData, Height, OutputType};
use crate::distribution::block::TrackingStatus;
use super::super::{
AddrTypeToActivityCounts, AddrTypeToAddrCount, AddrVecs, ExposedAddrState, ReusedAddrState,
};
use super::{AddrReceivePreState, AddrSendPreState};
/// Runtime state for the address metrics pipeline.
#[derive(Debug, Default)]
pub struct AddrMetricsState {
pub funded: AddrTypeToAddrCount,
pub empty: AddrTypeToAddrCount,
pub activity: AddrTypeToActivityCounts,
pub reused: ReusedAddrState,
pub respent: ReusedAddrState,
pub exposed: ExposedAddrState,
}
impl AddrMetricsState {
#[inline]
pub(crate) fn reset_per_block(&mut self) {
self.activity.reset();
self.reused.reset_per_block();
self.respent.reset_per_block();
}
#[inline]
pub(crate) fn on_receive_applied(
&mut self,
output_type: OutputType,
status: TrackingStatus,
addr_data: &FundedAddrData,
pre: &AddrReceivePreState,
output_count: u32,
) {
let activity = self.activity.get_mut_unwrap(output_type);
activity.receiving += 1;
match status {
TrackingStatus::New => {
*self.funded.get_mut_unwrap(output_type) += 1;
}
TrackingStatus::WasEmpty => {
activity.reactivated += 1;
*self.funded.get_mut_unwrap(output_type) += 1;
*self.empty.get_mut_unwrap(output_type) -= 1;
}
TrackingStatus::Tracked => {}
}
self.reused
.on_receive_as_reused(output_type, addr_data, pre, output_count);
self.respent
.on_receive_as_respent(output_type, addr_data, pre, output_count);
self.exposed.on_receive(output_type, addr_data, pre, status);
}
#[inline]
pub(crate) fn on_send_applied(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrSendPreState,
is_first_encounter: bool,
also_received: bool,
will_be_empty: bool,
) {
if is_first_encounter {
let activity = self.activity.get_mut_unwrap(output_type);
activity.sending += 1;
if also_received {
activity.bidirectional += 1;
}
}
if will_be_empty {
*self.funded.get_mut_unwrap(output_type) -= 1;
*self.empty.get_mut_unwrap(output_type) += 1;
}
self.reused.on_send_as_reused(
output_type,
addr_data,
pre,
is_first_encounter,
also_received,
will_be_empty,
);
self.respent.on_send_as_respent(
output_type,
addr_data,
pre,
is_first_encounter,
also_received,
will_be_empty,
);
self.exposed
.on_send(output_type, addr_data, pre, will_be_empty);
}
}
impl From<(&AddrVecs, Height)> for AddrMetricsState {
#[inline]
fn from((vecs, starting_height): (&AddrVecs, Height)) -> Self {
Self {
funded: AddrTypeToAddrCount::from((&vecs.funded.counts, starting_height)),
empty: AddrTypeToAddrCount::from((&vecs.empty, starting_height)),
activity: AddrTypeToActivityCounts::default(),
reused: ReusedAddrState::from((&vecs.reused, starting_height)),
respent: ReusedAddrState::from((&vecs.respent, starting_height)),
exposed: ExposedAddrState::from((&vecs.exposed, starting_height)),
}
}
}
@@ -0,0 +1,7 @@
mod metrics;
mod receive;
mod send;
pub use metrics::AddrMetricsState;
pub use receive::AddrReceivePreState;
pub use send::AddrSendPreState;
@@ -0,0 +1,30 @@
use brk_types::{FundedAddrData, OutputType, Sats};
/// Snapshot of [`FundedAddrData`] taken before a receive mutates it.
#[derive(Debug)]
pub struct AddrReceivePreState {
pub was_funded: bool,
pub was_reused: bool,
pub was_respent: bool,
pub was_pubkey_exposed: bool,
pub prev_funded_txo_count: u32,
pub exposed_contribution: Sats,
pub reused_contribution: Sats,
pub respent_contribution: Sats,
}
impl AddrReceivePreState {
#[inline]
pub fn capture(addr_data: &FundedAddrData, output_type: OutputType) -> Self {
Self {
was_funded: addr_data.is_funded(),
was_reused: addr_data.is_reused(),
was_respent: addr_data.is_respent(),
was_pubkey_exposed: addr_data.is_pubkey_exposed(output_type),
prev_funded_txo_count: addr_data.funded_txo_count,
exposed_contribution: addr_data.exposed_supply_contribution(output_type),
reused_contribution: addr_data.reused_supply_contribution(),
respent_contribution: addr_data.respent_supply_contribution(),
}
}
}
@@ -0,0 +1,26 @@
use brk_types::{FundedAddrData, OutputType, Sats};
/// Snapshot of [`FundedAddrData`] taken before a spend mutates it.
#[derive(Debug)]
pub struct AddrSendPreState {
pub was_reused: bool,
pub was_respent: bool,
pub was_pubkey_exposed: bool,
pub exposed_contribution: Sats,
pub reused_contribution: Sats,
pub respent_contribution: Sats,
}
impl AddrSendPreState {
#[inline]
pub fn capture(addr_data: &FundedAddrData, output_type: OutputType) -> Self {
Self {
was_reused: addr_data.is_reused(),
was_respent: addr_data.is_respent(),
was_pubkey_exposed: addr_data.is_pubkey_exposed(output_type),
exposed_contribution: addr_data.exposed_supply_contribution(output_type),
reused_contribution: addr_data.reused_supply_contribution(),
respent_contribution: addr_data.respent_supply_contribution(),
}
}
}
@@ -0,0 +1,88 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, LazyVec, Rw, StorageMode};
use super::{
AddrActivityVecs, AddrCountsVecs, AddrMetricsState, AvgAmountVecs, DeltaVecs, ExposedAddrVecs,
FundedAddrCountsVecs, NewAddrCountVecs, ReusedAddrVecs, TotalAddrCountVecs,
};
#[derive(Traversable)]
pub struct AddrVecs<M: StorageMode = Rw> {
pub funded: FundedAddrCountsVecs<M>,
pub empty: AddrCountsVecs<M>,
pub activity: AddrActivityVecs<M>,
pub total: TotalAddrCountVecs<M>,
pub new: NewAddrCountVecs,
pub reused: ReusedAddrVecs<M>,
pub respent: ReusedAddrVecs<M>,
pub exposed: ExposedAddrVecs<M>,
pub delta: DeltaVecs,
pub avg_amount: AvgAmountVecs<M>,
#[traversable(wrap = "indexes", rename = "funded")]
pub funded_index: LazyVec<FundedAddrIndex, FundedAddrIndex, FundedAddrIndex, FundedAddrData>,
#[traversable(wrap = "indexes", rename = "empty")]
pub empty_index: LazyVec<EmptyAddrIndex, EmptyAddrIndex, EmptyAddrIndex, EmptyAddrData>,
}
impl AddrVecs {
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.funded.reset_height()?;
self.empty.reset_height()?;
self.activity.reset_height()?;
self.total.reset_height()?;
self.reused.reset_height()?;
self.respent.reset_height()?;
self.exposed.reset_height()?;
self.avg_amount.reset_height()?;
Ok(())
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.funded
.min_stateful_len()
.min(self.empty.min_stateful_len())
.min(self.activity.min_stateful_len())
.min(self.reused.min_stateful_len())
.min(self.respent.min_stateful_len())
.min(self.exposed.min_stateful_len())
}
pub(crate) fn par_iter_stateful_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.funded
.par_iter_height_mut()
.chain(self.empty.par_iter_height_mut())
.chain(self.activity.par_iter_height_mut())
.chain(self.reused.par_iter_stateful_height_mut())
.chain(self.respent.par_iter_stateful_height_mut())
.chain(self.exposed.par_iter_stateful_height_mut())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.funded
.par_iter_height_mut()
.chain(self.empty.par_iter_height_mut())
.chain(self.activity.par_iter_height_mut())
.chain(self.total.par_iter_height_mut())
.chain(self.reused.par_iter_height_mut())
.chain(self.respent.par_iter_height_mut())
.chain(self.exposed.par_iter_height_mut())
.chain(self.avg_amount.par_iter_height_mut())
}
#[inline(always)]
pub(crate) fn push_height(&mut self, state: &AddrMetricsState, active_addr_count: u32) {
self.funded.push_counts(&state.funded);
self.empty.push_counts(&state.empty);
self.activity.push_height(&state.activity);
self.exposed.push_height(&state.exposed);
self.reused.push_height(&state.reused, active_addr_count);
self.respent.push_height(&state.respent, active_addr_count);
}
}
@@ -1,4 +1,4 @@
use brk_types::{Cents, Height, Sats, Version};
use brk_types::{Cents, Height, PartsPerMillionSigned64, Sats, Version};
use vecdb::{
BinaryTransform, CachedBoxedVec, ReadableCloneableVec, ReadableVec, TypedVec, VecValue,
};
@@ -100,8 +100,8 @@ impl AllChainCache {
version: Version,
realized_cap: &(impl ReadableCloneableVec<Height, Cents> + 'static),
window_starts: CachedBoxedVec<Height, Height>,
) -> impl TypedVec<I = Height, T = brk_types::PartsPerMillionSigned64>
+ ReadableVec<Height, brk_types::PartsPerMillionSigned64>
) -> impl TypedVec<I = Height, T = PartsPerMillionSigned64>
+ ReadableVec<Height, PartsPerMillionSigned64>
+ Clone
+ 'static {
let caps = self.with_market_cap(
@@ -125,7 +125,7 @@ impl AllChainCache {
(f64::from(current) - f64::from(previous)) / f64::from(previous)
}
};
brk_types::PartsPerMillionSigned64::from(
PartsPerMillionSigned64::from(
growth(current.market, previous.market)
- growth(current.realized, previous.realized),
)
@@ -137,7 +137,10 @@ impl AllChainCache {
#[cfg(test)]
mod tests {
use brk_types::PartsPerMillionSigned64;
use vecdb::{AnyStoredVec, CachedVec, Database, EagerVec, ImportableVec, PcoVec, WritableVec};
use vecdb::{
AnyStoredVec, CachedVec, Database, EagerVec, ImportableVec, PcoVec, ReadOnlyClone,
WritableVec,
};
use super::*;
@@ -179,7 +182,7 @@ mod tests {
realized.write().unwrap();
starts.write().unwrap();
let supply_cache = AllSupplyCache::new(&supply);
let supply_cache = AllSupplyCache::new(supply.read_only_clone());
let price_cache = CachedVec::wrap(price);
let cache = AllChainCache::new(&supply_cache, &price_cache.read_only_cached_boxed_clone());
+3 -3
View File
@@ -7,7 +7,7 @@ use smallvec::SmallVec;
use crate::distribution::{
addr::{AddrTypeToTypeIndexMap, AddrsDataVecs, AnyAddrIndexesVecs},
compute::VecsReaders,
compute::AddrReaders,
};
use super::super::cohort::{WithAddrDataSource, update_tx_counts};
@@ -52,7 +52,7 @@ impl BlockAddress {
fn load(
self,
first_addr_indexes: &ByAddrType<TypeIndex>,
vr: &VecsReaders,
vr: &AddrReaders,
any_addr_indexes: &AnyAddrIndexesVecs,
addrs_data: &AddrsDataVecs,
) -> WithAddrDataSource<FundedAddrData> {
@@ -124,7 +124,7 @@ impl AddrCache {
&mut self,
addresses: impl Iterator<Item = (OutputType, TypeIndex)>,
first_addr_indexes: &ByAddrType<TypeIndex>,
vr: &VecsReaders,
vr: &AddrReaders,
any_addr_indexes: &AnyAddrIndexesVecs,
addrs_data: &AddrsDataVecs,
) {
@@ -3,8 +3,8 @@ use brk_types::{Cents, Sats, TypeIndex};
use rustc_hash::FxHashMap;
use crate::distribution::{
AddrStates,
addr::{AddrMetricsState, AddrReceivePreState, AddrTypeToVec},
cohorts::AddrCohorts,
};
use super::super::cache::{AddrLookup, TrackingStatus};
@@ -18,7 +18,7 @@ struct AggregatedReceive {
pub(crate) fn process_received(
received_data: AddrTypeToVec<(TypeIndex, Sats)>,
cohorts: &mut AddrCohorts,
cohorts: &mut AddrStates,
lookup: &mut AddrLookup<'_>,
price: Cents,
state: &mut AddrMetricsState,
@@ -52,9 +52,6 @@ pub(crate) fn process_received(
cohorts
.amount_range
.get_mut_by_bucket(AmountBucket::from(recv.total_value))
.state
.as_mut()
.unwrap()
.add(addr_data);
} else {
let prev_balance = addr_data.balance();
@@ -63,12 +60,7 @@ pub(crate) fn process_received(
let new_bucket = AmountBucket::from(new_balance);
if let Some((old_bucket, new_bucket)) = prev_bucket.transition_to(new_bucket) {
let cohort_state = cohorts
.amount_range
.get_mut_by_bucket(old_bucket)
.state
.as_mut()
.unwrap();
let cohort_state = cohorts.amount_range.get_mut_by_bucket(old_bucket);
if cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64 {
panic!(
@@ -90,17 +82,11 @@ pub(crate) fn process_received(
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.add(addr_data);
} else {
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.receive_outputs(addr_data, recv.total_value, price, recv.output_count);
}
}
@@ -4,8 +4,8 @@ use brk_types::{Cents, Sats, TypeIndex};
use vecdb::VecIndex;
use crate::distribution::{
AddrStates,
addr::{AddrMetricsState, AddrSendPreState, HeightToAddrTypeToVec},
cohorts::AddrCohorts,
};
use super::{super::cache::AddrLookup, transfer_address_cache::TransferAddressCache};
@@ -13,7 +13,7 @@ use super::{super::cache::AddrLookup, transfer_address_cache::TransferAddressCac
/// Process sent UTXOs for address cohort membership and empty-address transitions.
pub(crate) fn process_sent(
sent_data: HeightToAddrTypeToVec<(TypeIndex, Sats)>,
cohorts: &mut AddrCohorts,
cohorts: &mut AddrStates,
lookup: &mut AddrLookup<'_>,
current_price: Cents,
state: &mut AddrMetricsState,
@@ -36,12 +36,7 @@ pub(crate) fn process_sent(
let will_be_empty = addr_data.has_1_utxos();
let prev_bucket = AmountBucket::from(prev_balance);
let cohort_state = cohorts
.amount_range
.get_mut_by_bucket(prev_bucket)
.state
.as_mut()
.unwrap();
let cohort_state = cohorts.amount_range.get_mut_by_bucket(prev_bucket);
// Mutates addr_data.spent_txo_count (+= 1). on_send_applied reads the post-spend view.
cohort_state.send(addr_data, value, current_price, prev_price)?;
@@ -65,9 +60,6 @@ pub(crate) fn process_sent(
cohorts
.amount_range
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
.add(addr_data);
}
}
@@ -1,5 +1,5 @@
use brk_cohort::ByAddrType;
use brk_types::{Sats, TypeIndex};
use brk_types::{OutputType, Sats, TypeIndex};
use rustc_hash::FxHashSet;
use crate::distribution::addr::AddrTypeToVec;
@@ -24,7 +24,7 @@ impl TransferAddressCache {
pub(super) fn sets_for(
&mut self,
output_type: brk_types::OutputType,
output_type: OutputType,
) -> (Option<&FxHashSet<TypeIndex>>, &mut FxHashSet<TypeIndex>) {
(
self.received.get(output_type),
@@ -1,3 +1,5 @@
use std::ops::{Deref, DerefMut};
use brk_types::{EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex};
/// Address data wrapped with its source location for flush operations.
@@ -14,7 +16,7 @@ pub enum WithAddrDataSource<T> {
FromEmpty(EmptyAddrIndex, T),
}
impl<T> std::ops::Deref for WithAddrDataSource<T> {
impl<T> Deref for WithAddrDataSource<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
@@ -24,7 +26,7 @@ impl<T> std::ops::Deref for WithAddrDataSource<T> {
}
}
impl<T> std::ops::DerefMut for WithAddrDataSource<T> {
impl<T> DerefMut for WithAddrDataSource<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
Self::New(v) | Self::FromFunded(_, v) | Self::FromEmpty(_, v) => v,
@@ -1,177 +0,0 @@
use std::path::Path;
use brk_cohort::{
AddrGroups, AmountRange, CohortContext, Filter, Filtered, OverAmount, UnderAmount,
};
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, Version};
use derive_more::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, CachedBoxedVec, Database, Exit, Rw, StorageMode};
use crate::{
distribution::{
DynCohortVecs,
metrics::{AllSupplyCache, ImportConfig},
},
indexes,
internal::{CachedWindowStartVec, Windows},
price,
};
use super::{super::traits::CohortVecs, vecs::AddrCohortVecs};
const VERSION: Version = Version::new(0);
/// All Addr cohorts organized by filter type.
#[derive(Deref, DerefMut, Traversable)]
pub struct AddrCohorts<M: StorageMode = Rw>(AddrGroups<AddrCohortVecs<M>>);
impl AddrCohorts {
/// Import all Addr cohorts from database.
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
states_path: &Path,
cached_starts: &Windows<&CachedWindowStartVec>,
spot_price: &CachedBoxedVec<Height, Cents>,
all_supply: &AllSupplyCache,
) -> Result<Self> {
let v = version + VERSION;
// Helper to create a cohort - only amount_range cohorts have state
let create =
|filter: Filter, name: &'static str, has_state: bool| -> Result<AddrCohortVecs> {
let sp = if has_state { Some(states_path) } else { None };
let full_name = CohortContext::Addr.full_name(&filter, name);
let cfg = ImportConfig {
db,
filter: &filter,
full_name: &full_name,
version: v,
indexes,
cached_starts,
spot_price,
};
AddrCohortVecs::forced_import(&cfg, sp, all_supply)
};
let full = |f: Filter, name: &'static str| create(f, name, true);
let none = |f: Filter, name: &'static str| create(f, name, false);
Ok(Self(AddrGroups {
amount_range: AmountRange::try_new(&full)?,
under_amount: UnderAmount::try_new(&none)?,
over_amount: OverAmount::try_new(&none)?,
}))
}
/// Apply a function to each aggregate cohort with its source cohorts (in parallel).
fn for_each_aggregate<F>(&mut self, f: F) -> Result<()>
where
F: Fn(&mut AddrCohortVecs, Vec<&AddrCohortVecs>) -> Result<()> + Sync,
{
let by_amount_range = &self.0.amount_range;
let pairs: Vec<_> = self
.0
.over_amount
.iter_mut()
.chain(self.0.under_amount.iter_mut())
.map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
by_amount_range
.iter()
.filter(|other| filter.includes(other.filter()))
.collect(),
)
})
.collect();
pairs
.into_par_iter()
.try_for_each(|(vecs, sources)| f(vecs, sources))
}
/// Compute overlapping cohorts from component amount_range cohorts.
pub(crate) fn compute_overlapping_vecs(
&mut self,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.for_each_aggregate(|vecs, sources| {
vecs.compute_from_stateful(starting_lengths, &sources, exit)
})
}
/// First phase of post-processing: compute index transforms.
pub(crate) fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(prices, starting_lengths, exit))?;
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub(crate) fn par_iter_vecs_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
// Collect all vecs from all cohorts
self.0
.iter_mut()
.flat_map(|v| v.par_iter_vecs_mut().collect::<Vec<_>>())
.collect::<Vec<_>>()
.into_par_iter()
}
/// Commit all states to disk (separate from vec writes for parallelization).
pub(crate) fn commit_all_states(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.write_state(height, cleanup))
}
/// Get minimum height from all separate cohorts' height-indexed vectors.
pub(crate) fn min_stateful_len(&self) -> Height {
self.iter_separate()
.map(|v| Height::from(v.min_stateful_len()))
.min()
.unwrap_or_default()
}
/// Import state for all separate cohorts at or before given height.
/// Returns true if all imports succeeded and returned the expected height.
pub(crate) fn import_separate_states(&mut self, height: Height) -> bool {
self.par_iter_separate_mut()
.map(|v| v.import_state(height).unwrap_or_default())
.all(|h| h == height)
}
/// Reset state heights for all separate cohorts.
pub(crate) fn reset_separate_state_heights(&mut self) {
self.par_iter_separate_mut().for_each(|v| {
v.reset_state_starting_height();
});
}
/// Reset cost_basis_data for all separate cohorts (called during fresh start).
pub(crate) fn reset_separate_cost_basis_data(&mut self) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.reset_cost_basis_data_if_needed())
}
/// Validate computed versions for all separate cohorts.
pub(crate) fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.validate_computed_versions(base_version))
}
}
@@ -1,103 +0,0 @@
use brk_cohort::Filter;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use vecdb::{AnyStoredVec, Exit, Rw, StorageMode};
use crate::{
distribution::metrics::{
ActivityMinimal, AllSupplyCache, ImportConfig, OutputsUnspent, RealizedBase, SupplyBase,
},
price,
};
/// Address-balance metrics: holdings plus economically meaningful flows.
///
/// Address cohorts intentionally omit spent-output counts, realized price,
/// MVRV, and NUPL.
#[derive(Traversable)]
pub struct AddrCohortMetrics<M: StorageMode = Rw> {
#[traversable(skip)]
pub filter: Filter,
pub supply: Box<SupplyBase<M>>,
pub outputs: Box<OutputsUnspent<M>>,
pub activity: Box<ActivityMinimal<M>>,
pub realized: Box<RealizedBase<M>>,
}
impl AddrCohortMetrics {
pub(super) fn forced_import(cfg: &ImportConfig, all_supply: &AllSupplyCache) -> Result<Self> {
Ok(Self {
filter: cfg.filter.clone(),
supply: Box::new(SupplyBase::forced_import(cfg, all_supply)?),
outputs: Box::new(OutputsUnspent::forced_import(cfg)?),
activity: Box::new(ActivityMinimal::forced_import(cfg)?),
realized: Box::new(RealizedBase::forced_import(cfg)?),
})
}
pub(super) fn min_stateful_len(&self) -> usize {
self.supply
.min_len()
.min(self.outputs.min_len())
.min(self.activity.min_len())
.min(self.realized.min_stateful_len())
}
pub(super) fn collect_all_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs = Vec::new();
vecs.extend(self.supply.collect_vecs_mut());
vecs.extend(self.outputs.collect_vecs_mut());
vecs.extend(self.activity.collect_vecs_mut());
vecs.extend(self.realized.collect_vecs_mut());
vecs
}
pub(super) fn compute_from_sources(
&mut self,
starting_lengths: &Lengths,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.supply.compute_from_stateful(
starting_lengths,
&others.iter().map(|v| v.supply.as_ref()).collect::<Vec<_>>(),
exit,
)?;
self.outputs.compute_from_stateful(
starting_lengths,
&others
.iter()
.map(|v| v.outputs.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
self.activity.compute_from_stateful(
starting_lengths,
&others
.iter()
.map(|v| v.activity.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
self.realized.compute_from_stateful(
starting_lengths,
&others
.iter()
.map(|v| v.realized.as_ref())
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
pub(super) fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.activity
.compute_rest_part1(prices, starting_lengths, exit)
}
}
@@ -1,5 +0,0 @@
mod groups;
mod metrics;
mod vecs;
pub use groups::*;
@@ -1,211 +0,0 @@
use std::path::Path;
use brk_cohort::{Filter, Filtered};
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, PartsPerMillionSigned64, StoredI64, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVec};
use crate::{distribution::state::AddrCohortState, internal::PerBlockWithDeltas, price};
use crate::distribution::metrics::{AllSupplyCache, ImportConfig};
use super::super::traits::{CohortVecs, DynCohortVecs};
use super::metrics::AddrCohortMetrics;
#[derive(Traversable)]
pub struct AddrCohortVecs<M: StorageMode = Rw> {
starting_height: Option<Height>,
#[traversable(skip)]
pub state: Option<Box<AddrCohortState>>,
#[traversable(flatten)]
pub metrics: AddrCohortMetrics<M>,
pub addr_count: PerBlockWithDeltas<StoredU64, StoredI64, PartsPerMillionSigned64, M>,
}
impl AddrCohortVecs {
pub(crate) fn forced_import(
cfg: &ImportConfig,
states_path: Option<&Path>,
all_supply: &AllSupplyCache,
) -> Result<Self> {
let addr_count = PerBlockWithDeltas::forced_import(
cfg.db,
&cfg.name("addr_count"),
cfg.version,
Version::TWO,
cfg.indexes,
cfg.cached_starts,
)?;
Ok(Self {
starting_height: None,
state: states_path.map(|path| Box::new(AddrCohortState::new(path, cfg.full_name))),
metrics: AddrCohortMetrics::forced_import(cfg, all_supply)?,
addr_count,
})
}
pub(crate) fn reset_starting_height(&mut self) {
self.starting_height = Some(Height::ZERO);
}
pub(crate) fn par_iter_vecs_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = Vec::new();
vecs.push(&mut self.addr_count.height as &mut dyn AnyStoredVec);
vecs.extend(self.metrics.collect_all_vecs_mut());
vecs.into_par_iter()
}
pub(crate) fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.inner.write(height, cleanup)?;
}
Ok(())
}
}
impl Filtered for AddrCohortVecs {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for AddrCohortVecs {
fn min_stateful_len(&self) -> usize {
self.addr_count
.height
.len()
.min(self.metrics.min_stateful_len())
}
fn reset_state_starting_height(&mut self) {
self.reset_starting_height();
if let Some(state) = self.state.as_mut() {
state.reset();
}
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
if let Some(state) = self.state.as_mut() {
if let Some(mut prev_height) = starting_height.decremented() {
prev_height = state.inner.import_at_or_before(prev_height)?;
state.inner.supply.value = self
.metrics
.supply
.total
.sats
.height
.collect_one(prev_height)
.unwrap();
state.inner.supply.utxo_count = *self
.metrics
.outputs
.unspent_count
.height
.collect_one(prev_height)
.unwrap();
state.addr_count = *self.addr_count.height.collect_one(prev_height).unwrap();
state.inner.restore_realized_cap();
let result = prev_height.incremented();
self.starting_height = Some(result);
Ok(result)
} else {
self.starting_height = Some(Height::ZERO);
Ok(Height::ZERO)
}
} else {
self.starting_height = Some(starting_height);
Ok(starting_height)
}
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
use vecdb::WritableVec;
self.addr_count
.height
.validate_computed_version_or_reset(base_version)?;
Ok(())
}
fn push_state(&mut self, height: Height) {
if self.starting_height.is_some_and(|h| h > height) {
return;
}
if let Some(state) = self.state.as_ref() {
self.addr_count.height.push(state.addr_count.into());
self.metrics.supply.push_state(&state.inner);
self.metrics.outputs.push_state(&state.inner);
self.metrics.activity.push_state(&state.inner);
self.metrics.realized.push_state(&state.inner);
}
}
fn push_unrealized_state(&mut self, _height_price: Cents) {}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(prices, starting_lengths, exit)
}
fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.inner.write(height, cleanup)?;
}
Ok(())
}
fn reset_cost_basis_data_if_needed(&mut self) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.inner.reset_cost_basis_data_if_needed()?;
}
Ok(())
}
fn reset_single_iteration_values(&mut self) {
if let Some(state) = self.state.as_mut() {
state.inner.reset_single_iteration_values();
}
}
}
impl CohortVecs for AddrCohortVecs {
fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.addr_count.height.compute_sum_of_others(
starting_lengths.height,
others
.iter()
.map(|v| &v.addr_count.height)
.collect::<Vec<_>>()
.as_slice(),
exit,
)?;
self.metrics.compute_from_sources(
starting_lengths,
&others.iter().map(|v| &v.metrics).collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
}
@@ -1,7 +0,0 @@
mod addr;
mod traits;
mod utxo;
pub use addr::AddrCohorts;
pub use traits::DynCohortVecs;
pub use utxo::UTXOCohorts;
@@ -1,60 +0,0 @@
use brk_error::Result;
use brk_indexer::Lengths;
use brk_types::{Cents, Height, Version};
use vecdb::Exit;
use crate::price;
/// Dynamic dispatch trait for cohort vectors.
///
/// This trait enables heterogeneous cohort processing via trait objects.
pub trait DynCohortVecs: Send + Sync {
/// Get minimum length across height-indexed vectors written in block loop.
fn min_stateful_len(&self) -> usize;
/// Reset the starting height for state tracking.
fn reset_state_starting_height(&mut self);
/// Import state from checkpoint at or before the given height.
fn import_state(&mut self, starting_height: Height) -> Result<Height>;
/// Validate that computed vectors have correct versions.
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
/// Push state to height-indexed vectors.
/// Height is used for the state_starting_height guard check.
fn push_state(&mut self, height: Height);
/// Compute and push unrealized profit/loss states.
fn push_unrealized_state(&mut self, height_price: Cents);
/// First phase of post-processing computations.
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>;
/// Write state checkpoint to disk.
fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()>;
/// Reset cost basis data (called during fresh start).
fn reset_cost_basis_data_if_needed(&mut self) -> Result<()>;
/// Reset per-block iteration values.
fn reset_single_iteration_values(&mut self);
}
/// Static dispatch trait for cohort vectors with additional methods.
///
/// Used by address cohorts where all cohorts share the same concrete type.
pub trait CohortVecs: DynCohortVecs {
/// Compute aggregate cohort from component cohorts.
fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&Self],
exit: &Exit,
) -> Result<()>;
}
File diff suppressed because it is too large Load Diff
@@ -1,12 +0,0 @@
mod fenwick;
mod groups;
mod percentiles;
mod receive;
mod send;
mod tick_tock;
mod vecs;
/// Rounding precision for UTXO cost basis prices (5 significant digits in dollars).
const COST_BASIS_PRICE_DIGITS: i32 = 5;
pub use groups::*;
@@ -1,285 +0,0 @@
use std::{
cmp::Reverse,
collections::{BTreeMap, BinaryHeap},
path::Path,
};
use brk_cohort::{
AgeRangeId, CohortContext, Filtered, PROFITABILITY_RANGE_COUNT, TERM_NAMES, UTXO_ALL_NAME,
};
use brk_error::Result;
use brk_types::{Cents, CentsCompact, Date, Dollars, PartsPerMillion32, Sats, UrpdRaw};
use rayon::prelude::*;
use vecdb::ColumnId;
use crate::distribution::metrics::{CostBasis, ProfitabilityMetrics};
use super::{
fenwick::{PercentileResult, ProfitabilityRangeResult},
groups::UTXOCohorts,
};
use super::COST_BASIS_PRICE_DIGITS;
impl UTXOCohorts {
/// Compute and push percentiles + profitability for aggregate cohorts.
///
/// Percentiles and profitability are computed per-block from the Fenwick tree.
/// Disk distributions are written only at day boundaries via K-way merge.
pub(crate) fn push_aggregate_percentiles(
&mut self,
spot_price: Cents,
date_opt: Option<Date>,
states_path: &Path,
) -> Result<()> {
if self.caches.fenwick.is_initialized() {
self.push_fenwick_results(spot_price);
}
// Disk distributions only at day boundaries
if let Some(date) = date_opt {
self.write_disk_distributions(date, states_path)?;
}
Ok(())
}
/// Iterate over the current in-memory age-cohort URPD entries.
///
/// Prices use the same rounding as the persisted daily distributions, so
/// consumers can avoid writing and immediately rereading the current day.
pub(crate) fn age_range_urpd_entries(
&self,
) -> impl Iterator<Item = (AgeRangeId, CentsCompact, Sats)> + '_ {
AgeRangeId::ALL.iter().copied().flat_map(move |id| {
let cohort = id.select(&self.age_range);
cohort.state.iter().flat_map(move |state| {
state
.cost_basis_map()
.iter()
.map(move |(&price, &sats)| (id, rounded_urpd_price(price), sats))
})
})
}
/// Push all Fenwick-derived per-block results: percentiles, density, profitability.
fn push_fenwick_results(&mut self, spot_price: Cents) {
let (all_d, sth_d, lth_d) = self.caches.fenwick.density(spot_price);
let all = self.caches.fenwick.percentiles_all();
push_cost_basis(&all, all_d, &mut self.all.metrics.cost_basis);
let sth = self.caches.fenwick.percentiles_sth();
push_cost_basis(&sth, sth_d, &mut self.sth.metrics.cost_basis);
let lth = self.caches.fenwick.percentiles_lth();
push_cost_basis(&lth, lth_d, &mut self.lth.metrics.cost_basis);
let prof = self.caches.fenwick.profitability(spot_price);
push_profitability(&prof, &mut self.profitability);
}
/// K-way merge only for writing daily cost basis distributions to disk.
fn write_disk_distributions(&mut self, date: Date, states_path: &Path) -> Result<()> {
let sth_filter = self.sth.metrics.filter.clone();
AgeRangeId::ALL
.iter()
.map(|&id| (id, id.select(&self.age_range)))
.collect::<Vec<_>>()
.into_par_iter()
.try_for_each(|(id, sub)| -> Result<()> {
let Some(state) = sub.state.as_ref() else {
return Ok(());
};
let mut merged: Vec<(CentsCompact, Sats)> = Vec::new();
for (&price, &sats) in state.cost_basis_map().iter() {
let rounded = rounded_urpd_price(price);
if let Some(last) = merged.last_mut()
&& last.0 == rounded
{
last.1 += sats;
} else {
merged.push((rounded, sats));
}
}
let full = CohortContext::Utxo.prefixed(id.name().id);
UrpdRaw::write(states_path, &full, date, merged.into_iter())
})?;
let maps: Vec<_> = self
.age_range
.iter()
.filter_map(|sub| {
let state = sub.state.as_ref()?;
let map = state.cost_basis_map();
if map.is_empty() {
return None;
}
let is_sth = sth_filter.includes(sub.filter());
Some((map, is_sth))
})
.collect();
if maps.is_empty() {
return Ok(());
}
let cap = maps.iter().map(|(m, _)| m.len()).max().unwrap_or(0);
let mut targets = AllSthLth {
all: MergeTarget::new(cap),
sth: MergeTarget::new(cap),
lth: MergeTarget::new(cap),
};
merge_k_way(&maps, &mut targets);
[
(UTXO_ALL_NAME.id, targets.all.merged),
(TERM_NAMES.short.id, targets.sth.merged),
(TERM_NAMES.long.id, targets.lth.merged),
]
.into_par_iter()
.try_for_each(|(name, merged)| {
UrpdRaw::write(states_path, name, date, merged.into_iter())
})?;
Ok(())
}
}
#[inline]
fn rounded_urpd_price(price: CentsCompact) -> CentsCompact {
price.round_to_dollar(COST_BASIS_PRICE_DIGITS)
}
/// Push percentiles + density to cost basis vecs.
#[inline(always)]
fn push_cost_basis(
percentiles: &PercentileResult,
density: PartsPerMillion32,
cost_basis: &mut CostBasis,
) {
cost_basis.push_minmax(percentiles.min_price, percentiles.max_price);
cost_basis.push_percentiles(&percentiles.sat_prices, &percentiles.usd_prices);
cost_basis.push_density(density);
}
#[inline(always)]
fn raw_usd_to_dollars(raw: u128) -> Dollars {
Dollars::from(raw as f64 / 1e10)
}
fn push_profitability(
buckets: &[ProfitabilityRangeResult; PROFITABILITY_RANGE_COUNT],
metrics: &mut ProfitabilityMetrics,
) {
metrics.push_ranges(
(*buckets).map(|range| Sats::from(range.all_sats)),
(*buckets).map(|range| Sats::from(range.sth_sats)),
(*buckets).map(|range| raw_usd_to_dollars(range.all_usd)),
(*buckets).map(|range| raw_usd_to_dollars(range.sth_usd)),
);
}
// ---------------------------------------------------------------------------
// K-way merge (retained only for disk distribution writes)
// ---------------------------------------------------------------------------
struct AllSthLth<T> {
all: T,
sth: T,
lth: T,
}
impl<T> AllSthLth<T> {
fn term_mut(&mut self, is_sth: bool) -> &mut T {
if is_sth { &mut self.sth } else { &mut self.lth }
}
fn for_each_mut(&mut self, mut f: impl FnMut(&mut T)) {
f(&mut self.all);
f(&mut self.sth);
f(&mut self.lth);
}
}
/// Merge target that only collects rounded (price, sats) pairs for disk distribution.
struct MergeTarget {
price_sats: u64,
merged: Vec<(CentsCompact, Sats)>,
}
impl MergeTarget {
fn new(cap: usize) -> Self {
Self {
price_sats: 0,
merged: Vec::with_capacity(cap),
}
}
#[inline]
fn accumulate(&mut self, amount: u64) {
self.price_sats += amount;
}
fn finalize_price(&mut self, price: CentsCompact) {
if self.price_sats > 0 {
let rounded = rounded_urpd_price(price);
if let Some((lp, ls)) = self.merged.last_mut()
&& *lp == rounded
{
*ls += Sats::from(self.price_sats);
} else {
self.merged.push((rounded, Sats::from(self.price_sats)));
}
}
self.price_sats = 0;
}
}
/// K-way merge via BinaryHeap over BTreeMap iterators.
/// Only builds merged distribution for disk writes.
fn merge_k_way(
maps: &[(&BTreeMap<CentsCompact, Sats>, bool)],
targets: &mut AllSthLth<MergeTarget>,
) {
let mut iters: Vec<_> = maps
.iter()
.map(|(map, is_sth)| (map.iter().peekable(), *is_sth))
.collect();
let mut heap: BinaryHeap<Reverse<(CentsCompact, usize)>> =
BinaryHeap::with_capacity(iters.len());
for (i, (iter, _)) in iters.iter_mut().enumerate() {
if let Some(&(&price, _)) = iter.peek() {
heap.push(Reverse((price, i)));
}
}
let mut current_price: Option<CentsCompact> = None;
while let Some(Reverse((price, ci))) = heap.pop() {
let (ref mut iter, is_sth) = iters[ci];
let (_, &sats) = iter.next().unwrap();
let amount = u64::from(sats);
if let Some(prev) = current_price
&& prev != price
{
targets.for_each_mut(|t| t.finalize_price(prev));
}
current_price = Some(price);
targets.all.accumulate(amount);
targets.term_mut(is_sth).accumulate(amount);
if let Some(&(&next_price, _)) = iter.peek() {
heap.push(Reverse((next_price, ci)));
}
}
if let Some(price) = current_price {
targets.for_each_mut(|t| t.finalize_price(price));
}
}
@@ -1,78 +0,0 @@
use brk_cohort::{Filter, Filtered};
use brk_error::Result;
use brk_indexer::Lengths;
use brk_types::{Cents, Height, Version};
use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{cohorts::traits::DynCohortVecs, metrics::CoreCohortMetrics},
price,
};
use super::UTXOCohortVecs;
impl Filtered for UTXOCohortVecs<CoreCohortMetrics> {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for UTXOCohortVecs<CoreCohortMetrics> {
fn min_stateful_len(&self) -> usize {
self.metrics.min_stateful_len()
}
fn reset_state_starting_height(&mut self) {
self.reset_state_impl();
}
impl_import_state!();
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.metrics.validate_computed_versions(base_version)
}
fn push_state(&mut self, height: Height) {
if self.state_starting_height.is_some_and(|h| h > height) {
return;
}
if let Some(state) = self.state.as_ref() {
self.metrics.supply.push_state(state);
self.metrics.outputs.push_state(state);
self.metrics.activity.push_state(state);
self.metrics.realized.push_state(state);
}
}
fn push_unrealized_state(&mut self, height_price: Cents) {
if let Some(state) = self.state.as_mut() {
state.apply_pending();
let unrealized_state = state.compute_unrealized_state(height_price);
self.metrics.unrealized.push_state(&unrealized_state);
self.metrics.supply.push_profitability(&unrealized_state);
}
}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(prices, starting_lengths, exit)
}
fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.write_state_impl(height, cleanup)
}
fn reset_cost_basis_data_if_needed(&mut self) -> Result<()> {
self.reset_cost_basis_impl()
}
fn reset_single_iteration_values(&mut self) {
self.reset_iteration_impl();
}
}
@@ -1,71 +0,0 @@
use brk_cohort::{Filter, Filtered};
use brk_error::Result;
use brk_indexer::Lengths;
use brk_types::{Cents, Height, Version};
use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{cohorts::traits::DynCohortVecs, metrics::MinimalCohortMetrics},
price,
};
use super::UTXOCohortVecs;
impl Filtered for UTXOCohortVecs<MinimalCohortMetrics> {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for UTXOCohortVecs<MinimalCohortMetrics> {
fn min_stateful_len(&self) -> usize {
self.metrics.min_stateful_len()
}
fn reset_state_starting_height(&mut self) {
self.reset_state_impl();
}
impl_import_state!();
fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
Ok(())
}
fn push_state(&mut self, height: Height) {
if self.state_starting_height.is_some_and(|h| h > height) {
return;
}
if let Some(state) = self.state.as_ref() {
self.metrics.supply.push_state(state);
self.metrics.outputs.push_state(state);
self.metrics.activity.push_state(state);
self.metrics.realized.push_state(state);
}
}
fn push_unrealized_state(&mut self, _height_price: Cents) {}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(prices, starting_lengths, exit)
}
fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.write_state_impl(height, cleanup)
}
fn reset_cost_basis_data_if_needed(&mut self) -> Result<()> {
self.reset_cost_basis_impl()
}
fn reset_single_iteration_values(&mut self) {
self.reset_iteration_impl();
}
}
@@ -1,209 +0,0 @@
macro_rules! impl_import_state {
() => {
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
if let Some(state) = self.state.as_mut() {
if let Some(mut prev_height) = starting_height.decremented() {
prev_height = state.import_at_or_before(prev_height)?;
state.supply.value = self
.metrics
.supply
.total
.sats
.height
.collect_one(prev_height)
.unwrap();
state.supply.utxo_count = *self
.metrics
.outputs
.unspent_count
.height
.collect_one(prev_height)
.unwrap();
state.restore_realized_cap();
let result = prev_height.incremented();
self.state_starting_height = Some(result);
Ok(result)
} else {
self.state_starting_height = Some(Height::ZERO);
Ok(Height::ZERO)
}
} else {
self.state_starting_height = Some(starting_height);
Ok(starting_height)
}
}
};
}
mod core;
mod minimal;
mod r#type;
use brk_cohort::{Filter, Filtered};
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, Version};
use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{
cohorts::traits::DynCohortVecs,
metrics::{CohortMetricsBase, CohortMetricsState},
state::UTXOCohortState,
},
price,
};
#[derive(Traversable)]
pub struct UTXOCohortVecs<M: CohortMetricsState> {
#[traversable(skip)]
state_starting_height: Option<Height>,
#[traversable(skip)]
pub state: Option<Box<UTXOCohortState<M::Realized, M::CostBasis>>>,
#[traversable(flatten)]
pub metrics: M,
}
impl<M: CohortMetricsState> UTXOCohortVecs<M> {
pub(crate) fn new(
state: Option<Box<UTXOCohortState<M::Realized, M::CostBasis>>>,
metrics: M,
) -> Self {
Self {
state_starting_height: None,
state,
metrics,
}
}
fn reset_state_impl(&mut self) {
self.state_starting_height = Some(Height::ZERO);
if let Some(state) = self.state.as_mut() {
state.reset();
}
}
fn write_state_impl(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.write(height, cleanup)?;
}
Ok(())
}
fn reset_cost_basis_impl(&mut self) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.reset_cost_basis_data_if_needed()?;
}
Ok(())
}
fn reset_iteration_impl(&mut self) {
if let Some(state) = self.state.as_mut() {
state.reset_single_iteration_values();
}
}
}
// --- Blanket impl for CohortMetricsBase types (always use full RealizedState) ---
impl<M: CohortMetricsBase + Traversable> Filtered for UTXOCohortVecs<M> {
fn filter(&self) -> &Filter {
self.metrics.filter()
}
}
impl<M: CohortMetricsBase + Traversable> DynCohortVecs for UTXOCohortVecs<M> {
fn min_stateful_len(&self) -> usize {
self.metrics.min_stateful_len()
}
fn reset_state_starting_height(&mut self) {
self.reset_state_impl();
}
fn import_state(&mut self, starting_height: Height) -> Result<Height> {
if let Some(state) = self.state.as_mut() {
if let Some(mut prev_height) = starting_height.decremented() {
prev_height = state.import_at_or_before(prev_height)?;
state.supply.value = self
.metrics
.supply()
.total
.sats
.height
.collect_one(prev_height)
.unwrap();
state.supply.utxo_count = *self
.metrics
.outputs()
.unspent_count
.height
.collect_one(prev_height)
.unwrap();
state.restore_realized_cap();
let result = prev_height.incremented();
self.state_starting_height = Some(result);
Ok(result)
} else {
self.state_starting_height = Some(Height::ZERO);
Ok(Height::ZERO)
}
} else {
self.state_starting_height = Some(starting_height);
Ok(starting_height)
}
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.metrics.validate_computed_versions(base_version)
}
fn push_state(&mut self, height: Height) {
if self.state_starting_height.is_some_and(|h| h > height) {
return;
}
if let Some(state) = self.state.as_ref() {
self.metrics.push_state(state);
}
}
fn push_unrealized_state(&mut self, height_price: Cents) {
if let Some(state) = self.state.as_mut() {
self.metrics
.compute_and_push_unrealized(height_price, state);
}
}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(prices, starting_lengths, exit)?;
Ok(())
}
fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.write_state_impl(height, cleanup)
}
fn reset_cost_basis_data_if_needed(&mut self) -> Result<()> {
self.reset_cost_basis_impl()
}
fn reset_single_iteration_values(&mut self) {
self.reset_iteration_impl();
}
}
@@ -1,77 +0,0 @@
use brk_cohort::{Filter, Filtered};
use brk_error::Result;
use brk_indexer::Lengths;
use brk_types::{Cents, Height, Version};
use vecdb::{Exit, ReadableVec};
use crate::{
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, price,
};
use super::UTXOCohortVecs;
impl Filtered for UTXOCohortVecs<TypeCohortMetrics> {
fn filter(&self) -> &Filter {
&self.metrics.filter
}
}
impl DynCohortVecs for UTXOCohortVecs<TypeCohortMetrics> {
fn min_stateful_len(&self) -> usize {
self.metrics.min_stateful_len()
}
fn reset_state_starting_height(&mut self) {
self.reset_state_impl();
}
impl_import_state!();
fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
Ok(())
}
fn push_state(&mut self, height: Height) {
if self.state_starting_height.is_some_and(|h| h > height) {
return;
}
if let Some(state) = self.state.as_ref() {
self.metrics.supply.push_state(state);
self.metrics.outputs.push_state(state);
self.metrics.activity.push_state(state);
self.metrics.realized.push_state(state);
}
}
fn push_unrealized_state(&mut self, height_price: Cents) {
if let Some(state) = self.state.as_mut() {
state.apply_pending();
let unrealized_state = state.compute_unrealized_state(height_price);
self.metrics.unrealized.push_state(&unrealized_state);
self.metrics.supply.push_profitability(&unrealized_state);
}
}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.metrics
.compute_rest_part1(prices, starting_lengths, exit)
}
fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.write_state_impl(height, cleanup)
}
fn reset_cost_basis_data_if_needed(&mut self) -> Result<()> {
self.reset_cost_basis_impl()
}
fn reset_single_iteration_values(&mut self) {
self.reset_iteration_impl();
}
}
@@ -1,4 +1,4 @@
use brk_cohort::{ByAddrType, EntryPrice};
use brk_cohort::{ByAddrType, EntryPrice, Filter, Term};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{
@@ -10,7 +10,7 @@ use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, unlikely};
use crate::{
distribution::{
addr::AddrMetricsState,
addr::{AddrMetricsState, FundedAddrCountsVecs},
block::{
AddrCache, TransferAddressCache, process_inputs, process_outputs, process_received,
process_sent,
@@ -24,18 +24,21 @@ use crate::{
use super::{
super::{
RangeMap,
cohorts::{AddrCohorts, DynCohortVecs, UTXOCohorts},
metrics::CohortMetrics,
state::{AddrStates, UTXOStates},
vecs::Vecs,
},
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
AddrReaders, BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexToTxIndexBuf, PriceRangeMax,
TxInReaders, TxOutReaders, VecsReaders,
TxInReaders, TxOutReaders,
};
/// Process all blocks from starting_height to last_height.
#[allow(clippy::too_many_arguments)]
pub(crate) fn process_blocks(
vecs: &mut Vecs,
utxo_states: &mut UTXOStates,
addr_states: &mut AddrStates,
indexer: &Indexer,
indexes: &indexes::Vecs,
inputs: &inputs::Vecs,
@@ -101,9 +104,9 @@ pub(crate) fn process_blocks(
})
.collect();
debug!("creating VecsReaders");
let mut vr = VecsReaders::new(&vecs.any_addr_indexes, &vecs.addrs_data);
debug!("VecsReaders created");
debug!("creating AddrReaders");
let mut vr = AddrReaders::new(&vecs.any_addr_indexes, &vecs.addrs_data);
debug!("AddrReaders created");
// Extend tx_index_to_height RangeMap with new entries (incremental, O(new_blocks))
let target_len = indexer.vecs().transactions.first_tx_index.len();
@@ -201,15 +204,14 @@ pub(crate) fn process_blocks(
debug!("AddrCache created, entering main loop");
// Initialize Fenwick tree from imported BTreeMap state (one-time)
vecs.utxo_cohorts.init_fenwick_if_needed();
utxo_states.init_fenwick_if_needed(&Filter::Term(Term::Sth));
// Pre-truncate all stored vecs to starting_height (one-time).
// This eliminates per-push truncation checks inside the block loop.
{
let start = starting_height.to_usize();
vecs.utxo_cohorts
vecs.cohorts
.par_iter_vecs_mut()
.chain(vecs.addr_cohorts.par_iter_vecs_mut())
.chain(vecs.addrs.par_iter_height_mut())
.chain(rayon::iter::once(vecs.coinblocks_destroyed.stored_mut()))
.try_for_each(|v| v.any_truncate_if_needed_at(start))?;
@@ -263,10 +265,7 @@ pub(crate) fn process_blocks(
// Keep tick-tock concurrent with the block reads and address processing.
let (matured, (outputs_result, inputs_result)) = rayon::join(
|| {
vecs.utxo_cohorts
.tick_tock_next_block(chain_state, timestamp)
},
|| utxo_states.tick_tock_next_block(chain_state, timestamp),
|| {
// Collect both sides concurrently, then load their shared addresses once.
let (
@@ -389,7 +388,7 @@ pub(crate) fn process_blocks(
}
// Record maturation (sats crossing age boundaries)
vecs.utxo_cohorts.push_maturation(&matured);
vecs.cohorts.supply.push_maturation(&matured, block_price);
transfer_addresses.prepare(&outputs_result.received_data);
@@ -397,11 +396,9 @@ pub(crate) fn process_blocks(
let (_, addr_result) = rayon::join(
|| {
// UTXO cohorts receive/send
vecs.utxo_cohorts
.receive(transacted, height, timestamp, block_price, entry);
utxo_states.receive(transacted, height, timestamp, block_price, entry);
if let Some(min_h) =
vecs.utxo_cohorts
.send(height_to_sent, chain_state, ctx.price_range_max)
utxo_states.send(height_to_sent, chain_state, ctx.price_range_max)
{
min_supply_modified =
Some(min_supply_modified.map_or(min_h, |cur| cur.min(min_h)));
@@ -412,7 +409,7 @@ pub(crate) fn process_blocks(
process_received(
outputs_result.received_data,
&mut vecs.addr_cohorts,
addr_states,
&mut lookup,
block_price,
&mut state,
@@ -420,7 +417,7 @@ pub(crate) fn process_blocks(
process_sent(
inputs_result.sent_data,
&mut vecs.addr_cohorts,
addr_states,
&mut lookup,
block_price,
&mut state,
@@ -432,7 +429,7 @@ pub(crate) fn process_blocks(
addr_result?;
// Update Fenwick tree from pending deltas (must happen before push_cohort_states drains pending)
vecs.utxo_cohorts.update_fenwick_from_pending();
utxo_states.update_fenwick_from_pending();
let active_addr_count = state.activity.active();
vecs.addrs.push_height(&state, active_addr_count);
@@ -441,14 +438,20 @@ pub(crate) fn process_blocks(
let date_opt = is_last_of_day.then(|| Date::from(timestamp));
entry_anchor = push_cohort_states(
&mut vecs.utxo_cohorts,
&mut vecs.addr_cohorts,
&mut vecs.cohorts,
&mut vecs.addrs.funded,
utxo_states,
addr_states,
height,
block_price,
);
vecs.utxo_cohorts
.push_aggregate_percentiles(block_price, date_opt, &vecs.states_path)?;
vecs.cohorts.push_aggregate_percentiles(
utxo_states,
block_price,
date_opt,
&vecs.states_path,
)?;
// Periodic checkpoint flush
if height != last_height
@@ -471,12 +474,20 @@ pub(crate) fn process_blocks(
let _lock = exit.lock();
// Write to disk (pure I/O) - no changes saved for periodic flushes
write(vecs, height, chain_state, min_supply_modified, false)?;
write(
vecs,
utxo_states,
addr_states,
height,
chain_state,
min_supply_modified,
false,
)?;
min_supply_modified = None;
vecs.flush()?;
// Recreate readers
vr = VecsReaders::new(&vecs.any_addr_indexes, &vecs.addrs_data);
vr = AddrReaders::new(&vecs.any_addr_indexes, &vecs.addrs_data);
}
}
@@ -496,44 +507,43 @@ pub(crate) fn process_blocks(
)?;
// Write to disk (pure I/O) - save changes for rollback
write(vecs, last_height, chain_state, min_supply_modified, true)?;
write(
vecs,
utxo_states,
addr_states,
last_height,
chain_state,
min_supply_modified,
true,
)?;
Ok(())
}
/// Push cohort states to height-indexed vectors, then reset per-block values.
fn push_cohort_states(
utxo_cohorts: &mut UTXOCohorts,
addr_cohorts: &mut AddrCohorts,
cohorts: &mut CohortMetrics,
funded_addr_counts: &mut FundedAddrCountsVecs,
utxo_states: &mut UTXOStates,
addr_states: &mut AddrStates,
height: Height,
height_price: Cents,
) -> Cents {
// Phase 1: push + unrealized (no reset yet, states still needed for aggregation)
rayon::join(
|| {
utxo_cohorts.par_iter_separate_mut().for_each(|v| {
v.push_state(height);
v.push_unrealized_state(height_price);
})
},
|| {
addr_cohorts.par_iter_separate_mut().for_each(|v| {
v.push_state(height);
v.push_unrealized_state(height_price);
})
},
);
// Phase 1: finish state updates before metric-first sources read them.
utxo_states.apply_pending();
addr_states.push(cohorts, funded_addr_counts, height, height_price);
// Phase 2: aggregate age_range states → push to overlapping cohorts
let all_capitalized_price = utxo_cohorts.push_overlapping(height_price);
// Phase 2: push the typed supply matrices, then aggregate age-range states.
let unrealized_states = cohorts.push_supply_and_unrealized(utxo_states, height_price);
cohorts.push_outputs(utxo_states);
cohorts.push_activity(utxo_states, height_price);
cohorts.push_realized(utxo_states);
let all_capitalized_price =
cohorts.push_overlapping(utxo_states, height_price, &unrealized_states);
// Phase 3: reset per-block values
utxo_cohorts
.iter_separate_mut()
.for_each(|v| v.reset_single_iteration_values());
addr_cohorts
.iter_separate_mut()
.for_each(|v| v.reset_single_iteration_values());
utxo_states.reset_block();
addr_states.reset_block();
all_capitalized_price
}
@@ -1,110 +1,7 @@
use std::time::Instant;
use brk_types::{Cents, Height, Timestamp};
use tracing::debug;
use vecdb::VecIndex;
/// Sparse table for O(1) range maximum queries on prices.
/// Vec<Vec> per level for incremental O(new_blocks * log n) extension.
#[derive(Debug, Clone, Default)]
pub struct PriceRangeMax {
levels: Vec<Vec<Cents>>,
n: usize,
}
impl PriceRangeMax {
pub(crate) fn extend(&mut self, prices: &[Cents]) {
let new_n = prices.len();
if new_n <= self.n || new_n == 0 {
return;
}
let start = Instant::now();
let old_n = self.n;
let new_levels_count = (usize::BITS - new_n.leading_zeros()) as usize;
while self.levels.len() < new_levels_count {
self.levels.push(Vec::new());
}
self.levels[0].extend_from_slice(&prices[old_n..new_n]);
for k in 1..new_levels_count {
let half = 1 << (k - 1);
let new_end = if new_n >= (1 << k) {
new_n + 1 - (1 << k)
} else {
0
};
let old_end = self.levels[k].len();
if new_end > old_end {
let (prev_levels, curr_levels) = self.levels.split_at_mut(k);
let prev = &prev_levels[k - 1];
let curr = &mut curr_levels[0];
curr.reserve(new_end - old_end);
for i in old_end..new_end {
curr.push(prev[i].max(prev[i + half]));
}
}
}
self.n = new_n;
let elapsed = start.elapsed();
let total_entries: usize = self.levels.iter().map(|l| l.len()).sum();
debug!(
"PriceRangeMax extended: {} -> {} heights ({} new), {} levels, {:.2}MB, {:.2}ms",
old_n,
new_n,
new_n - old_n,
new_levels_count,
(total_entries * std::mem::size_of::<Cents>()) as f64 / 1_000_000.0,
elapsed.as_secs_f64() * 1000.0
);
}
pub(crate) fn truncate(&mut self, new_n: usize) {
if new_n >= self.n {
return;
}
if new_n == 0 {
self.levels.clear();
self.n = 0;
return;
}
let new_levels_count = (usize::BITS - new_n.leading_zeros()) as usize;
self.levels.truncate(new_levels_count);
for k in 0..new_levels_count {
let valid = if new_n >= (1 << k) {
new_n + 1 - (1 << k)
} else {
0
};
self.levels[k].truncate(valid);
}
self.n = new_n;
}
#[inline]
pub(crate) fn range_max(&self, l: usize, r: usize) -> Cents {
debug_assert!(l <= r && r < self.n);
let len = r - l + 1;
let k = (usize::BITS - len.leading_zeros() - 1) as usize;
let half = 1 << k;
let level = &self.levels[k];
unsafe {
let a = *level.get_unchecked(l);
let b = *level.get_unchecked(r + 1 - half);
a.max(b)
}
}
#[inline]
pub(crate) fn max_between(&self, from: Height, to: Height) -> Cents {
self.range_max(from.to_usize(), to.to_usize())
}
}
use super::PriceRangeMax;
pub struct ComputeContext<'a> {
pub starting_height: Height,
@@ -1,13 +1,15 @@
mod block_loop;
mod context;
mod price_range_max;
mod readers;
mod recover;
mod write;
pub(crate) use block_loop::process_blocks;
pub(crate) use context::{ComputeContext, PriceRangeMax};
pub(crate) use readers::{IndexToTxIndexBuf, TxInReaders, TxOutData, TxOutReaders, VecsReaders};
pub(crate) use recover::{StartMode, determine_start_mode, recover_state, reset_state};
pub(crate) use context::ComputeContext;
pub(crate) use price_range_max::PriceRangeMax;
pub(crate) use readers::{AddrReaders, IndexToTxIndexBuf, TxInReaders, TxOutData, TxOutReaders};
pub(crate) use recover::{StartMode, determine_start_mode, reset_state};
/// Flush checkpoint interval (every N blocks).
pub const FLUSH_INTERVAL: usize = 10_000;
@@ -0,0 +1,93 @@
use std::time::Instant;
use brk_types::{Cents, Height};
use tracing::debug;
use vecdb::VecIndex;
#[derive(Debug, Clone, Default)]
pub struct PriceRangeMax {
levels: Vec<Vec<Cents>>,
n: usize,
}
impl PriceRangeMax {
pub(crate) fn extend(&mut self, prices: &[Cents]) {
let new_n = prices.len();
if new_n <= self.n || new_n == 0 {
return;
}
let start = Instant::now();
let old_n = self.n;
let level_count = (usize::BITS - new_n.leading_zeros()) as usize;
while self.levels.len() < level_count {
self.levels.push(Vec::new());
}
self.levels[0].extend_from_slice(&prices[old_n..new_n]);
for level in 1..level_count {
let half = 1 << (level - 1);
let new_end = new_n.saturating_add(1).saturating_sub(1 << level);
let old_end = self.levels[level].len();
if new_end > old_end {
let (previous, current) = self.levels.split_at_mut(level);
let previous = &previous[level - 1];
let current = &mut current[0];
current.reserve(new_end - old_end);
for index in old_end..new_end {
current.push(previous[index].max(previous[index + half]));
}
}
}
self.n = new_n;
let entries: usize = self.levels.iter().map(Vec::len).sum();
debug!(
"PriceRangeMax extended: {} -> {} heights ({} new), {} levels, {:.2}MB, {:.2}ms",
old_n,
new_n,
new_n - old_n,
level_count,
(entries * std::mem::size_of::<Cents>()) as f64 / 1_000_000.0,
start.elapsed().as_secs_f64() * 1000.0
);
}
pub(crate) fn truncate(&mut self, new_n: usize) {
if new_n >= self.n {
return;
}
if new_n == 0 {
self.levels.clear();
self.n = 0;
return;
}
let level_count = (usize::BITS - new_n.leading_zeros()) as usize;
self.levels.truncate(level_count);
for level in 0..level_count {
let valid = new_n.saturating_add(1).saturating_sub(1 << level);
self.levels[level].truncate(valid);
}
self.n = new_n;
}
#[inline]
pub(crate) fn range_max(&self, start: usize, end: usize) -> Cents {
debug_assert!(start <= end && end < self.n);
let len = end - start + 1;
let level = (usize::BITS - len.leading_zeros() - 1) as usize;
let width = 1 << level;
let values = &self.levels[level];
unsafe {
let first = *values.get_unchecked(start);
let last = *values.get_unchecked(end + 1 - width);
first.max(last)
}
}
#[inline]
pub(crate) fn max_between(&self, from: Height, to: Height) -> Cents {
self.range_max(from.to_usize(), to.to_usize())
}
}
@@ -1,264 +0,0 @@
use brk_indexer::Indexer;
use brk_types::{
AnyAddrIndex, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, Height, OutPoint,
OutputType, P2AAddrIndex, P2PK33AddrIndex, P2PK65AddrIndex, P2PKHAddrIndex, P2SHAddrIndex,
P2TRAddrIndex, P2WPKHAddrIndex, P2WSHAddrIndex, Sats, StoredU64, TxInIndex, TxIndex, TypeIndex,
};
use vecdb::{BytesVecReader, PcoVec, ReadableVec, VecIndex};
use crate::distribution::{
RangeMap,
addr::{AddrsDataVecs, AnyAddrIndexesVecs},
};
/// Output data collected from separate vecs.
#[derive(Debug, Clone, Copy)]
pub struct TxOutData {
pub value: Sats,
pub output_type: OutputType,
pub type_index: TypeIndex,
}
/// Readers for txout vectors. Reuses internal buffers across blocks.
pub struct TxOutReaders<'a> {
indexer: &'a Indexer,
values_buf: Vec<Sats>,
output_types_buf: Vec<OutputType>,
type_indexes_buf: Vec<TypeIndex>,
txout_data_buf: Vec<TxOutData>,
}
impl<'a> TxOutReaders<'a> {
pub(crate) fn new(indexer: &'a Indexer) -> Self {
Self {
indexer,
values_buf: Vec::new(),
output_types_buf: Vec::new(),
type_indexes_buf: Vec::new(),
txout_data_buf: Vec::new(),
}
}
/// Collect output data for a block range using bulk reads with buffer reuse.
pub(crate) fn collect_block_outputs(
&mut self,
first_txout_index: usize,
output_count: usize,
) -> &[TxOutData] {
let end = first_txout_index + output_count;
self.indexer.vecs().outputs.value.collect_range_into_at(
first_txout_index,
end,
&mut self.values_buf,
);
self.indexer
.vecs()
.outputs
.output_type
.collect_range_into_at(first_txout_index, end, &mut self.output_types_buf);
self.indexer
.vecs()
.outputs
.type_index
.collect_range_into_at(first_txout_index, end, &mut self.type_indexes_buf);
self.txout_data_buf.clear();
self.txout_data_buf.extend(
self.values_buf
.iter()
.zip(&self.output_types_buf)
.zip(&self.type_indexes_buf)
.map(|((&value, &output_type), &type_index)| TxOutData {
value,
output_type,
type_index,
}),
);
&self.txout_data_buf
}
}
/// Readers for txin vectors. Reuses all buffers across blocks.
pub struct TxInReaders<'a> {
indexer: &'a Indexer,
input_values: &'a PcoVec<TxInIndex, Sats>,
tx_index_to_height: &'a mut RangeMap<TxIndex, Height>,
outpoints_buf: Vec<OutPoint>,
values_buf: Vec<Sats>,
prev_heights_buf: Vec<Height>,
output_types_buf: Vec<OutputType>,
type_indexes_buf: Vec<TypeIndex>,
}
impl<'a> TxInReaders<'a> {
pub(crate) fn new(
indexer: &'a Indexer,
input_values: &'a PcoVec<TxInIndex, Sats>,
tx_index_to_height: &'a mut RangeMap<TxIndex, Height>,
) -> Self {
Self {
indexer,
input_values,
tx_index_to_height,
outpoints_buf: Vec::new(),
values_buf: Vec::new(),
prev_heights_buf: Vec::new(),
output_types_buf: Vec::new(),
type_indexes_buf: Vec::new(),
}
}
/// Collect input data for a block range using bulk reads with buffer reuse.
pub(crate) fn collect_block_inputs(
&mut self,
first_txin_index: usize,
input_count: usize,
current_height: Height,
) -> (&[Sats], &[Height], &[OutputType], &[TypeIndex]) {
let end = first_txin_index + input_count;
self.input_values
.collect_range_into_at(first_txin_index, end, &mut self.values_buf);
self.indexer.vecs().inputs.outpoint.collect_range_into_at(
first_txin_index,
end,
&mut self.outpoints_buf,
);
self.indexer
.vecs()
.inputs
.output_type
.collect_range_into_at(first_txin_index, end, &mut self.output_types_buf);
self.indexer.vecs().inputs.type_index.collect_range_into_at(
first_txin_index,
end,
&mut self.type_indexes_buf,
);
self.prev_heights_buf.clear();
self.prev_heights_buf
.extend(self.outpoints_buf.iter().map(|outpoint| {
if outpoint.is_coinbase() {
current_height
} else {
self.tx_index_to_height
.get(outpoint.tx_index())
.unwrap_or(current_height)
}
}));
(
&self.values_buf,
&self.prev_heights_buf,
&self.output_types_buf,
&self.type_indexes_buf,
)
}
}
/// Cached readers for stateful vectors.
pub struct VecsReaders {
p2a: BytesVecReader<P2AAddrIndex, AnyAddrIndex>,
p2pk33: BytesVecReader<P2PK33AddrIndex, AnyAddrIndex>,
p2pk65: BytesVecReader<P2PK65AddrIndex, AnyAddrIndex>,
p2pkh: BytesVecReader<P2PKHAddrIndex, AnyAddrIndex>,
p2sh: BytesVecReader<P2SHAddrIndex, AnyAddrIndex>,
p2tr: BytesVecReader<P2TRAddrIndex, AnyAddrIndex>,
p2wpkh: BytesVecReader<P2WPKHAddrIndex, AnyAddrIndex>,
p2wsh: BytesVecReader<P2WSHAddrIndex, AnyAddrIndex>,
funded: BytesVecReader<FundedAddrIndex, FundedAddrData>,
empty: BytesVecReader<EmptyAddrIndex, EmptyAddrData>,
}
impl VecsReaders {
pub(crate) fn new(any_addr_indexes: &AnyAddrIndexesVecs, addrs_data: &AddrsDataVecs) -> Self {
Self {
p2a: any_addr_indexes.p2a.reader(),
p2pk33: any_addr_indexes.p2pk33.reader(),
p2pk65: any_addr_indexes.p2pk65.reader(),
p2pkh: any_addr_indexes.p2pkh.reader(),
p2sh: any_addr_indexes.p2sh.reader(),
p2tr: any_addr_indexes.p2tr.reader(),
p2wpkh: any_addr_indexes.p2wpkh.reader(),
p2wsh: any_addr_indexes.p2wsh.reader(),
funded: addrs_data.funded.reader(),
empty: addrs_data.empty.reader(),
}
}
/// Read the unified address index, including uncommitted updates after rollback.
pub(crate) fn any_addr_index(
&self,
vecs: &AnyAddrIndexesVecs,
addr_type: OutputType,
type_index: TypeIndex,
) -> AnyAddrIndex {
let index = match addr_type {
OutputType::P2A => vecs.p2a.get_with_reader(type_index.into(), &self.p2a),
OutputType::P2PK33 => vecs.p2pk33.get_with_reader(type_index.into(), &self.p2pk33),
OutputType::P2PK65 => vecs.p2pk65.get_with_reader(type_index.into(), &self.p2pk65),
OutputType::P2PKH => vecs.p2pkh.get_with_reader(type_index.into(), &self.p2pkh),
OutputType::P2SH => vecs.p2sh.get_with_reader(type_index.into(), &self.p2sh),
OutputType::P2TR => vecs.p2tr.get_with_reader(type_index.into(), &self.p2tr),
OutputType::P2WPKH => vecs.p2wpkh.get_with_reader(type_index.into(), &self.p2wpkh),
OutputType::P2WSH => vecs.p2wsh.get_with_reader(type_index.into(), &self.p2wsh),
_ => unreachable!("invalid address type: {addr_type:?}"),
};
index.unwrap()
}
#[inline]
pub(crate) fn funded_data(
&self,
vecs: &AddrsDataVecs,
index: FundedAddrIndex,
) -> FundedAddrData {
vecs.funded.get_with_reader(index, &self.funded).unwrap()
}
#[inline]
pub(crate) fn empty_data(&self, vecs: &AddrsDataVecs, index: EmptyAddrIndex) -> EmptyAddrData {
vecs.empty.get_with_reader(index, &self.empty).unwrap()
}
}
/// Reusable buffers for per-block tx_index mapping construction.
pub(crate) struct IndexToTxIndexBuf {
counts: Vec<StoredU64>,
result: Vec<TxIndex>,
}
impl IndexToTxIndexBuf {
pub(crate) fn new() -> Self {
Self {
counts: Vec::new(),
result: Vec::new(),
}
}
/// Build index -> tx_index mapping for a block, reusing internal buffers.
pub(crate) fn build(
&mut self,
block_first_tx_index: TxIndex,
block_tx_count: u64,
tx_index_to_count: &impl ReadableVec<TxIndex, StoredU64>,
) -> &[TxIndex] {
let first = block_first_tx_index.to_usize();
tx_index_to_count.collect_range_into_at(
first,
first + block_tx_count as usize,
&mut self.counts,
);
let total: u64 = self.counts.iter().map(|c| u64::from(*c)).sum();
self.result.clear();
self.result.reserve(total as usize);
for (offset, count) in self.counts.iter().enumerate() {
let tx_index = TxIndex::from(first + offset);
self.result
.extend(std::iter::repeat_n(tx_index, u64::from(*count) as usize));
}
&self.result
}
}
@@ -0,0 +1,73 @@
use brk_types::{
AnyAddrIndex, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, OutputType,
P2AAddrIndex, P2PK33AddrIndex, P2PK65AddrIndex, P2PKHAddrIndex, P2SHAddrIndex, P2TRAddrIndex,
P2WPKHAddrIndex, P2WSHAddrIndex, TypeIndex,
};
use vecdb::BytesVecReader;
use crate::distribution::addr::{AddrsDataVecs, AnyAddrIndexesVecs};
/// Cached readers for address indexes and stateful address data.
pub struct AddrReaders {
p2a: BytesVecReader<P2AAddrIndex, AnyAddrIndex>,
p2pk33: BytesVecReader<P2PK33AddrIndex, AnyAddrIndex>,
p2pk65: BytesVecReader<P2PK65AddrIndex, AnyAddrIndex>,
p2pkh: BytesVecReader<P2PKHAddrIndex, AnyAddrIndex>,
p2sh: BytesVecReader<P2SHAddrIndex, AnyAddrIndex>,
p2tr: BytesVecReader<P2TRAddrIndex, AnyAddrIndex>,
p2wpkh: BytesVecReader<P2WPKHAddrIndex, AnyAddrIndex>,
p2wsh: BytesVecReader<P2WSHAddrIndex, AnyAddrIndex>,
funded: BytesVecReader<FundedAddrIndex, FundedAddrData>,
empty: BytesVecReader<EmptyAddrIndex, EmptyAddrData>,
}
impl AddrReaders {
pub(crate) fn new(any_addr_indexes: &AnyAddrIndexesVecs, addrs_data: &AddrsDataVecs) -> Self {
Self {
p2a: any_addr_indexes.p2a.reader(),
p2pk33: any_addr_indexes.p2pk33.reader(),
p2pk65: any_addr_indexes.p2pk65.reader(),
p2pkh: any_addr_indexes.p2pkh.reader(),
p2sh: any_addr_indexes.p2sh.reader(),
p2tr: any_addr_indexes.p2tr.reader(),
p2wpkh: any_addr_indexes.p2wpkh.reader(),
p2wsh: any_addr_indexes.p2wsh.reader(),
funded: addrs_data.funded.reader(),
empty: addrs_data.empty.reader(),
}
}
pub(crate) fn any_addr_index(
&self,
vecs: &AnyAddrIndexesVecs,
addr_type: OutputType,
type_index: TypeIndex,
) -> AnyAddrIndex {
let index = match addr_type {
OutputType::P2A => vecs.p2a.get_with_reader(type_index.into(), &self.p2a),
OutputType::P2PK33 => vecs.p2pk33.get_with_reader(type_index.into(), &self.p2pk33),
OutputType::P2PK65 => vecs.p2pk65.get_with_reader(type_index.into(), &self.p2pk65),
OutputType::P2PKH => vecs.p2pkh.get_with_reader(type_index.into(), &self.p2pkh),
OutputType::P2SH => vecs.p2sh.get_with_reader(type_index.into(), &self.p2sh),
OutputType::P2TR => vecs.p2tr.get_with_reader(type_index.into(), &self.p2tr),
OutputType::P2WPKH => vecs.p2wpkh.get_with_reader(type_index.into(), &self.p2wpkh),
OutputType::P2WSH => vecs.p2wsh.get_with_reader(type_index.into(), &self.p2wsh),
_ => unreachable!("invalid address type: {addr_type:?}"),
};
index.unwrap()
}
#[inline]
pub(crate) fn funded_data(
&self,
vecs: &AddrsDataVecs,
index: FundedAddrIndex,
) -> FundedAddrData {
vecs.funded.get_with_reader(index, &self.funded).unwrap()
}
#[inline]
pub(crate) fn empty_data(&self, vecs: &AddrsDataVecs, index: EmptyAddrIndex) -> EmptyAddrData {
vecs.empty.get_with_reader(index, &self.empty).unwrap()
}
}
@@ -0,0 +1,43 @@
use brk_types::{StoredU64, TxIndex};
use vecdb::{ReadableVec, VecIndex};
/// Reusable buffers for a block's index-to-transaction-index mapping.
pub(crate) struct IndexToTxIndexBuf {
counts: Vec<StoredU64>,
result: Vec<TxIndex>,
}
impl IndexToTxIndexBuf {
pub(crate) fn new() -> Self {
Self {
counts: Vec::new(),
result: Vec::new(),
}
}
pub(crate) fn build(
&mut self,
block_first_tx_index: TxIndex,
block_tx_count: u64,
tx_index_to_count: &impl ReadableVec<TxIndex, StoredU64>,
) -> &[TxIndex] {
let first = block_first_tx_index.to_usize();
tx_index_to_count.collect_range_into_at(
first,
first + block_tx_count as usize,
&mut self.counts,
);
let total: u64 = self.counts.iter().map(|count| u64::from(*count)).sum();
self.result.clear();
self.result.reserve(total as usize);
for (offset, count) in self.counts.iter().enumerate() {
let tx_index = TxIndex::from(first + offset);
self.result
.extend(std::iter::repeat_n(tx_index, u64::from(*count) as usize));
}
&self.result
}
}
@@ -0,0 +1,11 @@
mod addr;
mod index_to_tx_index;
mod tx_in;
mod tx_out;
mod tx_out_data;
pub(crate) use addr::AddrReaders;
pub(crate) use index_to_tx_index::IndexToTxIndexBuf;
pub(crate) use tx_in::TxInReaders;
pub(crate) use tx_out::TxOutReaders;
pub(crate) use tx_out_data::TxOutData;
@@ -0,0 +1,81 @@
use brk_indexer::Indexer;
use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TypeIndex};
use vecdb::{PcoVec, ReadableVec};
use crate::distribution::RangeMap;
/// Bulk txin reader with reusable buffers.
pub struct TxInReaders<'a> {
indexer: &'a Indexer,
input_values: &'a PcoVec<TxInIndex, Sats>,
tx_index_to_height: &'a mut RangeMap<TxIndex, Height>,
outpoints_buf: Vec<OutPoint>,
values_buf: Vec<Sats>,
prev_heights_buf: Vec<Height>,
output_types_buf: Vec<OutputType>,
type_indexes_buf: Vec<TypeIndex>,
}
impl<'a> TxInReaders<'a> {
pub(crate) fn new(
indexer: &'a Indexer,
input_values: &'a PcoVec<TxInIndex, Sats>,
tx_index_to_height: &'a mut RangeMap<TxIndex, Height>,
) -> Self {
Self {
indexer,
input_values,
tx_index_to_height,
outpoints_buf: Vec::new(),
values_buf: Vec::new(),
prev_heights_buf: Vec::new(),
output_types_buf: Vec::new(),
type_indexes_buf: Vec::new(),
}
}
pub(crate) fn collect_block_inputs(
&mut self,
first_txin_index: usize,
input_count: usize,
current_height: Height,
) -> (&[Sats], &[Height], &[OutputType], &[TypeIndex]) {
let end = first_txin_index + input_count;
self.input_values
.collect_range_into_at(first_txin_index, end, &mut self.values_buf);
self.indexer.vecs().inputs.outpoint.collect_range_into_at(
first_txin_index,
end,
&mut self.outpoints_buf,
);
self.indexer
.vecs()
.inputs
.output_type
.collect_range_into_at(first_txin_index, end, &mut self.output_types_buf);
self.indexer.vecs().inputs.type_index.collect_range_into_at(
first_txin_index,
end,
&mut self.type_indexes_buf,
);
self.prev_heights_buf.clear();
self.prev_heights_buf
.extend(self.outpoints_buf.iter().map(|outpoint| {
if outpoint.is_coinbase() {
current_height
} else {
self.tx_index_to_height
.get(outpoint.tx_index())
.unwrap_or(current_height)
}
}));
(
&self.values_buf,
&self.prev_heights_buf,
&self.output_types_buf,
&self.type_indexes_buf,
)
}
}
@@ -0,0 +1,63 @@
use brk_indexer::Indexer;
use brk_types::{OutputType, Sats, TypeIndex};
use vecdb::ReadableVec;
use super::TxOutData;
/// Bulk txout reader with reusable buffers.
pub struct TxOutReaders<'a> {
indexer: &'a Indexer,
values_buf: Vec<Sats>,
output_types_buf: Vec<OutputType>,
type_indexes_buf: Vec<TypeIndex>,
txout_data_buf: Vec<TxOutData>,
}
impl<'a> TxOutReaders<'a> {
pub(crate) fn new(indexer: &'a Indexer) -> Self {
Self {
indexer,
values_buf: Vec::new(),
output_types_buf: Vec::new(),
type_indexes_buf: Vec::new(),
txout_data_buf: Vec::new(),
}
}
pub(crate) fn collect_block_outputs(
&mut self,
first_txout_index: usize,
output_count: usize,
) -> &[TxOutData] {
let end = first_txout_index + output_count;
self.indexer.vecs().outputs.value.collect_range_into_at(
first_txout_index,
end,
&mut self.values_buf,
);
self.indexer
.vecs()
.outputs
.output_type
.collect_range_into_at(first_txout_index, end, &mut self.output_types_buf);
self.indexer
.vecs()
.outputs
.type_index
.collect_range_into_at(first_txout_index, end, &mut self.type_indexes_buf);
self.txout_data_buf.clear();
self.txout_data_buf.extend(
self.values_buf
.iter()
.zip(&self.output_types_buf)
.zip(&self.type_indexes_buf)
.map(|((&value, &output_type), &type_index)| TxOutData {
value,
output_type,
type_index,
}),
);
&self.txout_data_buf
}
}
@@ -0,0 +1,9 @@
use brk_types::{OutputType, Sats, TypeIndex};
/// Output data collected from separate vectors.
#[derive(Debug, Clone, Copy)]
pub struct TxOutData {
pub value: Sats,
pub output_type: OutputType,
pub type_index: TypeIndex,
}
@@ -3,117 +3,117 @@ use std::{cmp::Ordering, collections::BTreeSet};
use brk_error::Result;
use brk_types::Height;
use tracing::{debug, warn};
use vecdb::Stamp;
use vecdb::{Result as VecdbResult, Stamp};
use super::super::{
AddrsDataVecs,
addr::AnyAddrIndexesVecs,
cohorts::{AddrCohorts, UTXOCohorts},
AddrsDataVecs, AnyAddrIndexesVecs, Vecs,
state::{AddrStates, UTXOStates},
};
/// Result of state recovery.
pub struct RecoveredState {
pub(crate) struct RecoveredState {
/// Height to start processing from. Zero means fresh start.
pub starting_height: Height,
pub(crate) starting_height: Height,
}
/// Perform state recovery for resuming from checkpoint.
///
/// Rolls back state vectors and imports cohort states.
/// Validates that all rollbacks and imports are consistent.
/// Returns Height::ZERO if any validation fails (triggers fresh start).
pub(crate) fn recover_state(
height: Height,
chain_state_rollback: Option<vecdb::Result<Stamp>>,
any_addr_indexes: &mut AnyAddrIndexesVecs,
addrs_data: &mut AddrsDataVecs,
utxo_cohorts: &mut UTXOCohorts,
addr_cohorts: &mut AddrCohorts,
) -> Result<RecoveredState> {
// `None`: clean resume, already at the checkpoint, nothing to undo.
// `Some`: reorg, undo state past the resume point.
let consistent_height = match chain_state_rollback {
None => height,
Some(chain_state_rollback) => {
let stamp = Stamp::from(height);
impl Vecs {
/// Perform state recovery for resuming from checkpoint.
///
/// Rolls back state vectors and imports cohort states.
/// Validates that all rollbacks and imports are consistent.
/// Returns Height::ZERO if any validation fails (triggers fresh start).
pub(crate) fn recover_state(
&mut self,
height: Height,
chain_state_rollback: Option<VecdbResult<Stamp>>,
utxo_states: &mut UTXOStates,
addr_states: &mut AddrStates,
) -> Result<RecoveredState> {
// `None`: clean resume, already at the checkpoint, nothing to undo.
// `Some`: reorg, undo state past the resume point.
let consistent_height = match chain_state_rollback {
None => height,
Some(chain_state_rollback) => {
let stamp = Stamp::from(height);
// Rollback address state vectors
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
let addr_data_rollback = addrs_data.rollback_before(stamp);
// Rollback address state vectors
let addr_indexes_rollback = self.any_addr_indexes.rollback_before(stamp);
let addr_data_rollback = self.addrs_data.rollback_before(stamp);
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
addr_indexes_rollback,
addr_data_rollback,
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
addr_indexes_rollback,
addr_data_rollback,
);
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
warn!("Rollback consistency check failed: inconsistent heights");
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Rollback can land at an earlier height (multi-block change file), which is fine.
// But if it lands AHEAD of target, that means rollback failed (missing change files).
if consistent_height > height {
warn!(
"Rollback failed: still at {} but target was {}, falling back to fresh start",
consistent_height, height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
if consistent_height != height {
debug!(
"Rollback landed at {} instead of {}, will resume from there",
consistent_height, height
);
}
consistent_height
}
};
// Import UTXO cohort states - all must succeed
debug!(
"importing UTXO cohort states at height {}",
consistent_height
);
if !utxo_states.import(&self.cohorts, consistent_height)? {
warn!(
"UTXO cohort state import failed at height {}",
consistent_height
);
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
warn!("Rollback consistency check failed: inconsistent heights");
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Rollback can land at an earlier height (multi-block change file), which is fine.
// But if it lands AHEAD of target, that means rollback failed (missing change files).
if consistent_height > height {
warn!(
"Rollback failed: still at {} but target was {}, falling back to fresh start",
consistent_height, height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
if consistent_height != height {
debug!(
"Rollback landed at {} instead of {}, will resume from there",
consistent_height, height
);
}
consistent_height
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
};
debug!("UTXO cohort states imported");
// Import UTXO cohort states - all must succeed
debug!(
"importing UTXO cohort states at height {}",
consistent_height
);
if !utxo_cohorts.import_separate_states(consistent_height) {
warn!(
"UTXO cohort state import failed at height {}",
// Import address cohort states - all must succeed
debug!(
"importing addr cohort states at height {}",
consistent_height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
debug!("UTXO cohort states imported");
if !addr_states.import(&self.cohorts, &self.addrs.funded, consistent_height)? {
warn!(
"Addr cohort state import failed at height {}",
consistent_height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
debug!("addr cohort states imported");
// Import address cohort states - all must succeed
debug!(
"importing addr cohort states at height {}",
consistent_height
);
if !addr_cohorts.import_separate_states(consistent_height) {
warn!(
"Addr cohort state import failed at height {}",
consistent_height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
Ok(RecoveredState {
starting_height: consistent_height,
})
}
debug!("addr cohort states imported");
Ok(RecoveredState {
starting_height: consistent_height,
})
}
/// Reset all state for fresh start.
@@ -122,23 +122,16 @@ pub(crate) fn recover_state(
pub(crate) fn reset_state(
any_addr_indexes: &mut AnyAddrIndexesVecs,
addrs_data: &mut AddrsDataVecs,
utxo_cohorts: &mut UTXOCohorts,
addr_cohorts: &mut AddrCohorts,
utxo_states: &mut UTXOStates,
addr_states: &mut AddrStates,
) -> Result<RecoveredState> {
// Reset address state
any_addr_indexes.reset()?;
addrs_data.reset()?;
// Reset cohort state heights
utxo_cohorts.reset_separate_state_heights();
addr_cohorts.reset_separate_state_heights();
// Reset cost_basis_data for all cohorts
utxo_cohorts.reset_separate_cost_basis_data()?;
addr_cohorts.reset_separate_cost_basis_data()?;
// Reset in-memory caches (fenwick, tick_tock positions)
utxo_cohorts.reset_caches();
// Reset cohort state.
utxo_states.reset()?;
addr_states.reset()?;
Ok(RecoveredState {
starting_height: Height::ZERO,
@@ -175,7 +168,7 @@ pub enum StartMode {
/// Returns the consistent starting height if ALL rollbacks succeed and agree,
/// otherwise returns Height::ZERO (need fresh start).
fn rollback_states(
chain_state_rollback: vecdb::Result<Stamp>,
chain_state_rollback: VecdbResult<Stamp>,
addr_indexes_rollbacks: Result<Vec<Stamp>>,
addr_data_rollbacks: Result<[Stamp; 2]>,
) -> Height {
@@ -9,7 +9,7 @@ use vecdb::{AnyStoredVec, AnyVec, Stamp, VecIndex, WritableVec};
use crate::distribution::{
Vecs,
block::{WithAddrDataSource, process_empty_addrs, process_funded_addrs},
state::BlockState,
state::{AddrStates, BlockState, UTXOStates},
};
use super::super::addr::{AddrTypeToTypeIndexMap, AddrsDataVecs, AnyAddrIndexesVecs};
@@ -52,6 +52,8 @@ pub(crate) fn process_addr_updates(
/// Set `with_changes=true` near chain tip to enable rollback support.
pub(crate) fn write(
vecs: &mut Vecs,
utxo_states: &mut UTXOStates,
addr_states: &mut AddrStates,
height: Height,
chain_state: &[BlockState],
min_supply_modified: Option<Height>,
@@ -84,14 +86,13 @@ pub(crate) fn write(
]
.into_par_iter(),
)
.chain(vecs.utxo_cohorts.par_iter_vecs_mut())
.chain(vecs.addr_cohorts.par_iter_vecs_mut())
.chain(vecs.cohorts.par_iter_vecs_mut())
.try_for_each(|v| v.any_stamped_write_maybe_with_changes(stamp, with_changes))?;
// Commit states after vec writes
let cleanup = with_changes;
vecs.utxo_cohorts.commit_all_states(height, cleanup)?;
vecs.addr_cohorts.commit_all_states(height, cleanup)?;
utxo_states.write(height, cleanup)?;
addr_states.write(height, cleanup)?;
info!("Wrote in {:?}", i.elapsed());
@@ -1,108 +0,0 @@
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, StoredF64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode};
use crate::{
distribution::{
metrics::ImportConfig,
state::{CohortState, CostBasisOps, RealizedOps},
},
internal::{PerBlockCumulativeRolling, ValuePerBlockCumulativeRolling},
price,
};
use super::ActivityMinimal;
#[derive(Deref, DerefMut, Traversable)]
pub struct ActivityCore<M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub minimal: ActivityMinimal<M>,
pub coindays_destroyed: PerBlockCumulativeRolling<StoredF64, M>,
#[traversable(wrap = "transfer_volume", rename = "in_profit")]
pub transfer_volume_in_profit: ValuePerBlockCumulativeRolling<M>,
#[traversable(wrap = "transfer_volume", rename = "in_loss")]
pub transfer_volume_in_loss: ValuePerBlockCumulativeRolling<M>,
}
impl ActivityCore {
pub(crate) fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v1 = Version::ONE;
Ok(Self {
minimal: ActivityMinimal::forced_import(cfg)?,
coindays_destroyed: cfg.import("coindays_destroyed", v1)?,
transfer_volume_in_profit: cfg.import("transfer_volume_in_profit", v1)?,
transfer_volume_in_loss: cfg.import("transfer_volume_in_loss", v1)?,
})
}
pub(crate) fn min_len(&self) -> usize {
self.minimal
.min_len()
.min(self.coindays_destroyed.block.len())
.min(self.transfer_volume_in_profit.cumulative.sats.height.len())
.min(self.transfer_volume_in_loss.cumulative.sats.height.len())
}
#[inline(always)]
pub(crate) fn push_state(&mut self, state: &CohortState<impl RealizedOps, impl CostBasisOps>) {
self.minimal.push_state(state);
self.coindays_destroyed
.push_block(StoredF64::from(Bitcoin::from(state.satdays_destroyed)));
self.transfer_volume_in_profit
.push_block_sats(state.realized.sent_in_profit());
self.transfer_volume_in_loss
.push_block_sats(state.realized.sent_in_loss());
}
pub(crate) fn collect_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs = self.minimal.collect_vecs_mut();
vecs.push(self.coindays_destroyed.stored_mut());
vecs.push(&mut self.transfer_volume_in_profit.inner.cumulative.sats.height);
vecs.push(&mut self.transfer_volume_in_profit.inner.cumulative.cents.height);
vecs.push(&mut self.transfer_volume_in_loss.inner.cumulative.sats.height);
vecs.push(&mut self.transfer_volume_in_loss.inner.cumulative.cents.height);
vecs
}
pub(crate) fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
Ok(())
}
pub(crate) fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
let minimal_refs: Vec<&ActivityMinimal> = others.iter().map(|o| &o.minimal).collect();
self.minimal
.compute_from_stateful(starting_lengths, &minimal_refs, exit)?;
sum_others!(self, starting_lengths, others, exit; coindays_destroyed.cumulative.height);
sum_others!(self, starting_lengths, others, exit; transfer_volume_in_profit.cumulative.sats.height);
sum_others!(self, starting_lengths, others, exit; transfer_volume_in_loss.cumulative.sats.height);
Ok(())
}
pub(crate) fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.minimal
.compute_rest_part1(prices, starting_lengths, exit)?;
self.transfer_volume_in_profit
.compute_rest(starting_lengths.height, prices, exit)?;
self.transfer_volume_in_loss
.compute_rest(starting_lengths.height, prices, exit)?;
Ok(())
}
}
@@ -1,117 +0,0 @@
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{StoredF32, StoredF64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{AnyStoredVec, Exit, Rw, StorageMode};
use crate::internal::{ColumnarRollingWindows, Identity, LazyPerBlock};
use crate::{
distribution::{
metrics::ImportConfig,
state::{CohortState, CostBasisOps, RealizedOps},
},
price,
};
use super::ActivityCore;
#[derive(Deref, DerefMut, Traversable)]
pub struct ActivityFull<M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub inner: ActivityCore<M>,
pub coinyears_destroyed: LazyPerBlock<StoredF64, StoredF64>,
pub dormancy: ColumnarRollingWindows<StoredF32, M>,
}
impl ActivityFull {
pub(crate) fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v1 = Version::ONE;
let inner = ActivityCore::forced_import(cfg)?;
let coinyears_destroyed = LazyPerBlock::from_height_source::<Identity<StoredF64>, _>(
&cfg.name("coinyears_destroyed"),
cfg.version + v1,
inner.coindays_destroyed.sum._1y.height.clone(),
cfg.indexes,
);
let dormancy = ColumnarRollingWindows::forced_import(
cfg.db,
&cfg.name("dormancy"),
cfg.version + v1,
cfg.indexes,
)?;
Ok(Self {
inner,
coinyears_destroyed,
dormancy,
})
}
pub(crate) fn full_min_len(&self) -> usize {
self.inner.min_len()
}
#[inline(always)]
pub(crate) fn full_push_state(
&mut self,
state: &CohortState<impl RealizedOps, impl CostBasisOps>,
) {
self.inner.push_state(state);
}
pub(crate) fn collect_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs = self.inner.collect_vecs_mut();
vecs.push(self.dormancy.stored_mut());
vecs
}
pub(crate) fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&ActivityCore],
exit: &Exit,
) -> Result<()> {
self.inner
.compute_from_stateful(starting_lengths, others, exit)
}
pub(crate) fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.inner
.compute_rest_part1(prices, starting_lengths, exit)?;
let Self {
inner, dormancy, ..
} = self;
let cdd_sums = &inner.coindays_destroyed.sum;
let transfer_volume_sums = &inner.minimal.transfer_volume.sum.0;
dormancy.compute_columns2(
starting_lengths.height,
|window| &window.select(cdd_sums).height,
|window| &window.select(transfer_volume_sums).btc.height,
|_, rolling_cdd, rolling_btc| {
let btc = f64::from(rolling_btc);
if btc == 0.0 {
StoredF32::from(0.0f32)
} else {
StoredF32::from((f64::from(rolling_cdd) / btc) as f32)
}
},
exit,
)?;
Ok(())
}
}
@@ -1,78 +0,0 @@
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode};
use crate::{
distribution::{
metrics::ImportConfig,
state::{CohortState, CostBasisOps, RealizedOps},
},
internal::ValuePerBlockCumulativeRolling,
price,
};
#[derive(Traversable)]
pub struct ActivityMinimal<M: StorageMode = Rw> {
pub transfer_volume: ValuePerBlockCumulativeRolling<M>,
}
impl ActivityMinimal {
pub(crate) fn forced_import(cfg: &ImportConfig) -> Result<Self> {
let v1 = Version::ONE;
Ok(Self {
transfer_volume: cfg.import("transfer_volume", v1)?,
})
}
pub(crate) fn min_len(&self) -> usize {
self.transfer_volume.cumulative.sats.height.len()
}
#[inline(always)]
pub(crate) fn push_state(&mut self, state: &CohortState<impl RealizedOps, impl CostBasisOps>) {
self.transfer_volume.push_block_sats(state.sent);
}
pub(crate) fn collect_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let inner = &mut self.transfer_volume.inner;
vec![
&mut inner.cumulative.sats.height as &mut dyn AnyStoredVec,
&mut inner.cumulative.cents.height,
]
}
pub(crate) fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&Self],
exit: &Exit,
) -> Result<()> {
self.transfer_volume
.cumulative
.sats
.height
.compute_sum_of_others(
starting_lengths.height,
&others
.iter()
.map(|v| &v.transfer_volume.cumulative.sats.height)
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
pub(crate) fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.transfer_volume
.compute_rest(starting_lengths.height, prices, exit)?;
Ok(())
}
}
@@ -1,105 +1,3 @@
mod core;
mod full;
mod minimal;
mod vecs;
pub use self::core::ActivityCore;
pub use full::ActivityFull;
pub use minimal::ActivityMinimal;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_types::Version;
use vecdb::Exit;
use crate::{
distribution::state::{CohortState, CostBasisOps, RealizedOps},
price,
};
pub trait ActivityLike: Send + Sync {
fn as_core(&self) -> &ActivityCore;
fn as_core_mut(&mut self) -> &mut ActivityCore;
fn min_len(&self) -> usize;
fn push_state<R: RealizedOps>(&mut self, state: &CohortState<R, impl CostBasisOps>);
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>;
fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&ActivityCore],
exit: &Exit,
) -> Result<()>;
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>;
}
impl ActivityLike for ActivityCore {
fn as_core(&self) -> &ActivityCore {
self
}
fn as_core_mut(&mut self) -> &mut ActivityCore {
self
}
fn min_len(&self) -> usize {
self.min_len()
}
fn push_state<R: RealizedOps>(&mut self, state: &CohortState<R, impl CostBasisOps>) {
self.push_state(state);
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.validate_computed_versions(base_version)
}
fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&ActivityCore],
exit: &Exit,
) -> Result<()> {
self.compute_from_stateful(starting_lengths, others, exit)
}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.compute_rest_part1(prices, starting_lengths, exit)
}
}
impl ActivityLike for ActivityFull {
fn as_core(&self) -> &ActivityCore {
&self.inner
}
fn as_core_mut(&mut self) -> &mut ActivityCore {
&mut self.inner
}
fn min_len(&self) -> usize {
self.full_min_len()
}
fn push_state<R: RealizedOps>(&mut self, state: &CohortState<R, impl CostBasisOps>) {
self.full_push_state(state);
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.inner.validate_computed_versions(base_version)
}
fn compute_from_stateful(
&mut self,
starting_lengths: &Lengths,
others: &[&ActivityCore],
exit: &Exit,
) -> Result<()> {
self.compute_from_stateful(starting_lengths, others, exit)
}
fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.compute_rest_part1(prices, starting_lengths, exit)
}
}
pub use vecs::{ActivitySources, ActivityVecs};
@@ -0,0 +1,52 @@
use brk_cohort::UTXOGroupsWithoutAmountOrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{StoredF64, Version};
use vecdb::{Database, Rw, StorageMode};
use crate::{
distribution::metrics::{CumulativeUTXOColumnarMetricWithoutAmountOrType, utxo_metric_name},
indexes,
internal::{CachedWindowStartVec, LazyPerBlockCumulativeRolling, Windows},
};
#[derive(Traversable)]
pub struct CoindaysDestroyedByCohort<M: StorageMode = Rw> {
#[traversable(flatten)]
pub cohorts: UTXOGroupsWithoutAmountOrType<LazyPerBlockCumulativeRolling<StoredF64>>,
#[traversable(flatten)]
pub cumulative: CumulativeUTXOColumnarMetricWithoutAmountOrType<StoredF64, M>,
}
impl CoindaysDestroyedByCohort {
pub(super) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
) -> Result<Self> {
let cumulative = CumulativeUTXOColumnarMetricWithoutAmountOrType::forced_import(
db,
"coindays_destroyed_cumulative",
version,
)?;
let cohorts = UTXOGroupsWithoutAmountOrType::new(|filter, cohort_name| {
let name = utxo_metric_name(&filter, cohort_name, "coindays_destroyed");
let source = cumulative
.matrices
.additive_source(&filter, &format!("{name}_cumulative"), version)
.expect("supported coindays-destroyed cohort");
LazyPerBlockCumulativeRolling::from_boxed_cumulative_source(
&name,
version,
source,
cached_starts,
indexes,
)
});
Ok(Self {
cohorts,
cumulative,
})
}
}
@@ -0,0 +1,226 @@
use brk_cohort::{
AmountRange, Filter, UTXO_AGGREGATE_FILTERS, UTXO_AGGREGATE_NAMES, UTXOAggregate,
UTXOAggregateId,
};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, Sats, StoredF32, StoredF64, Version};
use vecdb::{AnyStoredVec, AnyVec, BinaryTransform, ColumnId, Database, Exit, Rw, StorageMode};
use crate::{
distribution::metrics::{UTXORows, utxo_metric_name},
indexes,
internal::{
CachedWindowStartVec, ColumnarRollingWindows, Identity, LazyPerBlock, SatsToCents, Windows,
},
};
use super::{
ActivitySources, CoindaysDestroyedByCohort, CoreCumulativeValueByCohort,
CumulativeValueByCohort,
};
#[derive(Traversable)]
pub struct ActivityVecs<M: StorageMode = Rw> {
pub transfer_volume: Box<CumulativeValueByCohort<M>>,
pub coindays_destroyed: Box<CoindaysDestroyedByCohort<M>>,
#[traversable(wrap = "transfer_volume", rename = "in_profit")]
pub transfer_volume_in_profit: Box<CoreCumulativeValueByCohort<M>>,
#[traversable(wrap = "transfer_volume", rename = "in_loss")]
pub transfer_volume_in_loss: Box<CoreCumulativeValueByCohort<M>>,
pub coinyears_destroyed: Box<UTXOAggregate<LazyPerBlock<StoredF64, StoredF64>>>,
pub dormancy: Box<UTXOAggregate<ColumnarRollingWindows<StoredF32, M>>>,
}
impl ActivityVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
) -> Result<Self> {
let aggregate_version = version;
let version = version + Version::ONE;
let transfer_volume = Box::new(CumulativeValueByCohort::forced_import(
db,
"transfer_volume",
version,
indexes,
cached_starts,
)?);
let coindays_destroyed = Box::new(CoindaysDestroyedByCohort::forced_import(
db,
version,
indexes,
cached_starts,
)?);
let transfer_volume_in_profit = Box::new(CoreCumulativeValueByCohort::forced_import(
db,
"transfer_volume_in_profit",
version,
indexes,
cached_starts,
)?);
let transfer_volume_in_loss = Box::new(CoreCumulativeValueByCohort::forced_import(
db,
"transfer_volume_in_loss",
version,
indexes,
cached_starts,
)?);
let coinyears_destroyed = Box::new(UTXOAggregate::from_fn(|id| {
let filter = id.select(&UTXO_AGGREGATE_FILTERS);
let name = Self::aggregate_metric_name(id, "coinyears_destroyed");
LazyPerBlock::from_height_source::<Identity<StoredF64>, _>(
&name,
Self::aggregate_version(aggregate_version, id),
coindays_destroyed
.cohorts
.get(filter)
.expect("aggregate coindays-destroyed source")
.sum
._1y
.height
.clone(),
indexes,
)
}));
let dormancy = Box::new(UTXOAggregate::try_from_fn(|id| {
ColumnarRollingWindows::forced_import(
db,
&Self::aggregate_metric_name(id, "dormancy"),
Self::aggregate_version(aggregate_version, id),
indexes,
)
})?);
Ok(Self {
transfer_volume,
coindays_destroyed,
transfer_volume_in_profit,
transfer_volume_in_loss,
coinyears_destroyed,
dormancy,
})
}
fn aggregate_version(version: Version, id: UTXOAggregateId) -> Version {
version
+ Version::ONE
+ if matches!(id, UTXOAggregateId::All) {
Version::ONE
} else {
Version::ZERO
}
}
fn aggregate_metric_name(id: UTXOAggregateId, metric: &str) -> String {
utxo_metric_name(
id.select(&UTXO_AGGREGATE_FILTERS),
id.select(&UTXO_AGGREGATE_NAMES).id,
metric,
)
}
pub(crate) fn sources(&self, filter: &Filter) -> Option<ActivitySources> {
Some(ActivitySources {
transfer_volume: self.transfer_volume.cohorts.get(filter)?.clone(),
})
}
#[inline(always)]
pub(crate) fn push(
&mut self,
height_price: Cents,
transfer_volume: UTXORows<Sats>,
coindays_destroyed: UTXORows<StoredF64>,
transfer_volume_in_profit: UTXORows<Sats>,
transfer_volume_in_loss: UTXORows<Sats>,
) {
let transfer_value = transfer_volume.map(|sats| SatsToCents::apply(*sats, height_price));
let profit_value =
transfer_volume_in_profit.map(|sats| SatsToCents::apply(*sats, height_price));
let loss_value =
transfer_volume_in_loss.map(|sats| SatsToCents::apply(*sats, height_price));
self.transfer_volume
.push_block(transfer_volume, transfer_value);
self.coindays_destroyed
.cumulative
.push_block(coindays_destroyed);
self.transfer_volume_in_profit
.push_block(transfer_volume_in_profit, profit_value);
self.transfer_volume_in_loss
.push_block(transfer_volume_in_loss, loss_value);
}
#[inline(always)]
pub(crate) fn push_addr_balance(
&mut self,
height_price: Cents,
transfer_volume: &AmountRange<Sats>,
) {
let cents = AmountRange::from_fn(|amount| {
SatsToCents::apply(*amount.select(transfer_volume), height_price)
});
self.transfer_volume
.push_addr_balance(transfer_volume, &cents);
}
pub(crate) fn min_len(&self) -> usize {
self.transfer_volume
.min_len()
.min(self.coindays_destroyed.cumulative.min_len())
.min(self.transfer_volume_in_profit.min_len())
.min(self.transfer_volume_in_loss.min_len())
.min(
self.dormancy
.iter()
.map(|value| value.height.len())
.min()
.unwrap_or_default(),
)
}
pub(crate) fn collect_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs = self.transfer_volume.collect_vecs_mut();
vecs.extend(self.coindays_destroyed.cumulative.collect_vecs_mut());
vecs.extend(self.transfer_volume_in_profit.collect_vecs_mut());
vecs.extend(self.transfer_volume_in_loss.collect_vecs_mut());
vecs.extend(self.dormancy.iter_mut().map(|value| value.stored_mut()));
vecs
}
pub(crate) fn compute_dormancy(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
for id in UTXOAggregateId::ALL {
let filter = id.select(&UTXO_AGGREGATE_FILTERS);
let coindays_destroyed = &self
.coindays_destroyed
.cohorts
.get(filter)
.expect("aggregate coindays-destroyed cohort")
.sum;
let transfer_volume = &self
.transfer_volume
.cohorts
.get(filter)
.expect("aggregate transfer-volume cohort")
.sum
.0;
id.select_mut(&mut self.dormancy).compute_columns2(
max_from,
|window| &window.select(coindays_destroyed).height,
|window| &window.select(transfer_volume).btc.height,
|_, rolling_coindays, rolling_btc| {
let btc = f64::from(rolling_btc);
if btc == 0.0 {
StoredF32::from(0.0f32)
} else {
StoredF32::from((f64::from(rolling_coindays) / btc) as f32)
}
},
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,67 @@
use brk_cohort::UTXOGroupsWithoutAmountOrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Cents, Sats, Version};
use vecdb::{AnyStoredVec, Database, Rw, StorageMode};
use crate::{
distribution::metrics::{
CumulativeUTXOValueColumnarMetricWithoutAmountOrType, UTXORows, utxo_metric_name,
},
indexes,
internal::{CachedWindowStartVec, LazyValuePerBlockCumulativeRolling, Windows},
};
#[derive(Traversable)]
pub struct CoreCumulativeValueByCohort<M: StorageMode = Rw> {
#[traversable(flatten)]
pub cohorts: UTXOGroupsWithoutAmountOrType<LazyValuePerBlockCumulativeRolling>,
pub cumulative: CumulativeUTXOValueColumnarMetricWithoutAmountOrType<M>,
}
impl CoreCumulativeValueByCohort {
pub(super) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
) -> Result<Self> {
let cumulative = CumulativeUTXOValueColumnarMetricWithoutAmountOrType::forced_import(
db,
&format!("{metric}_cumulative"),
version,
)?;
let cohorts = UTXOGroupsWithoutAmountOrType::new(|filter, cohort_name| {
let name = utxo_metric_name(&filter, cohort_name, metric);
let (sats, cents) = cumulative
.sources(&filter, &name, version)
.expect("supported core cumulative value cohort");
LazyValuePerBlockCumulativeRolling::from_boxed_cumulative_sources(
&name,
version,
sats,
cents,
indexes,
cached_starts,
)
});
Ok(Self {
cohorts,
cumulative,
})
}
#[inline(always)]
pub(super) fn push_block(&mut self, sats: UTXORows<Sats>, cents: UTXORows<Cents>) {
self.cumulative.push_block(sats, cents);
}
pub(super) fn min_len(&self) -> usize {
self.cumulative.min_len()
}
pub(super) fn collect_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
self.cumulative.collect_vecs_mut()
}
}
@@ -0,0 +1,98 @@
use brk_cohort::{AmountRange, CohortContext, UTXOGroups};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Cents, Sats, Version};
use vecdb::{AnyStoredVec, Database, Rw, StorageMode};
use crate::{
distribution::metrics::{
ColumnarAmountValue, CumulativeUTXOValueColumnarMetric, UTXORows, utxo_metric_name,
},
indexes,
internal::{CachedWindowStartVec, LazyValuePerBlockCumulativeRolling, Windows},
};
#[derive(Traversable)]
pub struct CumulativeValueByCohort<M: StorageMode = Rw> {
#[traversable(flatten)]
pub cohorts: UTXOGroups<LazyValuePerBlockCumulativeRolling>,
pub cumulative: CumulativeUTXOValueColumnarMetric<M>,
pub addr_balance: ColumnarAmountValue<LazyValuePerBlockCumulativeRolling, M>,
}
impl CumulativeValueByCohort {
pub(super) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
) -> Result<Self> {
let cumulative = CumulativeUTXOValueColumnarMetric::forced_import(
db,
&format!("{metric}_cumulative"),
version,
)?;
let cohorts = UTXOGroups::new(|filter, cohort_name| {
let name = utxo_metric_name(&filter, cohort_name, metric);
let (sats, cents) = cumulative
.sources(&filter, &name, version)
.expect("supported cumulative value cohort");
LazyValuePerBlockCumulativeRolling::from_boxed_cumulative_sources(
&name,
version,
sats,
cents,
indexes,
cached_starts,
)
});
let addr_version = version + Version::ONE;
let addr_balance = ColumnarAmountValue::forced_import(
db,
&format!("addrs_{metric}_cumulative_by_balance_range"),
CohortContext::Addr,
metric,
addr_version,
|name, sats, cents| {
LazyValuePerBlockCumulativeRolling::from_boxed_cumulative_sources(
name,
addr_version,
sats,
cents,
indexes,
cached_starts,
)
},
)?;
Ok(Self {
cohorts,
cumulative,
addr_balance,
})
}
#[inline(always)]
pub(super) fn push_block(&mut self, sats: UTXORows<Sats>, cents: UTXORows<Cents>) {
self.cumulative.push_block(sats, cents);
}
#[inline(always)]
pub(super) fn push_addr_balance(
&mut self,
sats: &AmountRange<Sats>,
cents: &AmountRange<Cents>,
) {
self.addr_balance.push_cumulative(sats, cents);
}
pub(super) fn min_len(&self) -> usize {
self.cumulative.min_len().min(self.addr_balance.len())
}
pub(super) fn collect_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs = self.cumulative.collect_vecs_mut();
vecs.extend(self.addr_balance.collect_vecs_mut());
vecs
}
}
@@ -0,0 +1,11 @@
mod coindays_destroyed;
mod collection;
mod core_cumulative_value;
mod cumulative_value;
mod sources;
pub(super) use coindays_destroyed::CoindaysDestroyedByCohort;
pub use collection::ActivityVecs;
pub(super) use core_cumulative_value::CoreCumulativeValueByCohort;
pub(super) use cumulative_value::CumulativeValueByCohort;
pub use sources::ActivitySources;
@@ -0,0 +1,6 @@
use crate::internal::LazyValuePerBlockCumulativeRolling;
#[derive(Clone)]
pub struct ActivitySources {
pub transfer_volume: LazyValuePerBlockCumulativeRolling,
}
@@ -0,0 +1,84 @@
use brk_cohort::{
ByTerm, TermId, UTXO_AGGREGATE_FILTERS, UTXO_AGGREGATE_NAMES, UTXOAggregate, UTXOAggregateId,
};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Version;
use derive_more::{Deref, DerefMut};
use vecdb::{
AnyStoredVec, AnyVec, ColumnId, Database, ReadableCloneableVec, ReadableColumnarVec, Rw,
StorageMode,
};
use crate::{
indexes,
internal::{ColumnarPerBlock, FiatType, LazyFiatPerBlock},
};
use super::super::utxo_metric_name;
#[derive(Deref, DerefMut, Traversable)]
pub struct AdditiveAggregateFiatPerBlock<C: FiatType, M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub values: ColumnarPerBlock<C, TermId, UTXOAggregate<LazyFiatPerBlock<C>>, M>,
}
impl<C: FiatType> AdditiveAggregateFiatPerBlock<C> {
pub(crate) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let values = ColumnarPerBlock::forced_import(
db,
&format!("{metric}_cents_by_term"),
version,
|source| {
let source = source.clone();
UTXOAggregate::from_fn(|aggregate| {
let name = utxo_metric_name(
aggregate.select(&UTXO_AGGREGATE_FILTERS),
aggregate.select(&UTXO_AGGREGATE_NAMES).id,
metric,
);
let cents = match aggregate {
UTXOAggregateId::All => source
.sum_columns(
&format!("{name}_cents"),
version,
TermId::ALL.iter().copied(),
)
.read_only_boxed_clone(),
UTXOAggregateId::Sth => source
.column(&format!("{name}_cents"), version, TermId::Short)
.read_only_boxed_clone(),
UTXOAggregateId::Lth => source
.column(&format!("{name}_cents"), version, TermId::Long)
.read_only_boxed_clone(),
};
LazyFiatPerBlock::from_boxed_cents_source(&name, version, cents, indexes)
})
},
)?;
Ok(Self { values })
}
#[inline(always)]
pub(crate) fn push(&mut self, row: UTXOAggregate<C>) {
self.values.push(ByTerm {
short: row.sth,
long: row.lth,
});
}
pub(crate) fn len(&self) -> usize {
self.values.height.len()
}
pub(crate) fn stored_mut(&mut self) -> &mut dyn AnyStoredVec {
self.values.stored_mut()
}
}
@@ -0,0 +1,3 @@
mod fiat;
pub use fiat::AdditiveAggregateFiatPerBlock;
@@ -0,0 +1,7 @@
mod aggregate;
mod utxo_raw;
pub use aggregate::AdditiveAggregateFiatPerBlock;
pub(crate) use utxo_raw::AdditiveUTXORawVec;
use super::utxo_metric_name;
@@ -0,0 +1,45 @@
use std::ops::AddAssign;
use brk_cohort::{ByTerm, TermId, UTXOAggregate};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, Version};
use vecdb::{
AnyStoredVec, AnyVec, BytesVec, BytesVecValue, ColumnarVec, Database, ImportableVec, Rw,
StorageMode, WritableVec,
};
#[derive(Traversable)]
pub struct AdditiveUTXORawVec<T, M: StorageMode = Rw>
where
T: BytesVecValue,
{
pub matrix: M::Stored<ColumnarVec<BytesVec<Height, T>, TermId>>,
}
impl<T> AdditiveUTXORawVec<T>
where
T: BytesVecValue + AddAssign + Copy,
{
pub(crate) fn forced_import(db: &Database, name: &str, version: Version) -> Result<Self> {
Ok(Self {
matrix: ImportableVec::forced_import(db, &format!("{name}_by_term"), version)?,
})
}
#[inline(always)]
pub(crate) fn push(&mut self, row: &UTXOAggregate<T>) {
self.matrix.push(ByTerm {
short: row.sth,
long: row.lth,
});
}
pub(crate) fn len(&self) -> usize {
self.matrix.len()
}
pub(crate) fn stored_mut(&mut self) -> &mut dyn AnyStoredVec {
&mut self.matrix
}
}
@@ -0,0 +1,99 @@
use brk_cohort::{
ByTerm, TermId, UTXO_AGGREGATE_FILTERS, UTXO_AGGREGATE_NAMES, UTXOAggregate, UTXOAggregateId,
};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Version;
use derive_more::{Deref, DerefMut};
use vecdb::{
AnyStoredVec, AnyVec, ColumnId, Database, ReadableCloneableVec, ReadableColumnarVec, Rw,
StorageMode,
};
use crate::{
indexes,
internal::{
CachedWindowStartVec, ColumnarPerBlockCumulativeRolling, FiatType,
LazyFiatPerBlockCumulativeWithSums, Windows,
},
};
use super::utxo_metric_name;
#[derive(Deref, DerefMut, Traversable)]
pub struct AdditiveAggregateFiatPerBlockCumulativeWithSums<C: FiatType, M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub values: ColumnarPerBlockCumulativeRolling<
C,
TermId,
UTXOAggregate<LazyFiatPerBlockCumulativeWithSums<C>>,
M,
>,
}
impl<C: FiatType> AdditiveAggregateFiatPerBlockCumulativeWithSums<C> {
pub(crate) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&CachedWindowStartVec>,
) -> Result<Self> {
let values = ColumnarPerBlockCumulativeRolling::forced_import(
db,
&format!("{metric}_cumulative_cents_by_term"),
version,
|source| {
let source = source.clone();
UTXOAggregate::from_fn(|id| {
let name = utxo_metric_name(
id.select(&UTXO_AGGREGATE_FILTERS),
id.select(&UTXO_AGGREGATE_NAMES).id,
metric,
);
let cumulative = match id {
UTXOAggregateId::All => source
.sum_columns(
&format!("{name}_cumulative_cents"),
version,
TermId::ALL.iter().copied(),
)
.read_only_boxed_clone(),
UTXOAggregateId::Sth => source
.column(&format!("{name}_cumulative_cents"), version, TermId::Short)
.read_only_boxed_clone(),
UTXOAggregateId::Lth => source
.column(&format!("{name}_cumulative_cents"), version, TermId::Long)
.read_only_boxed_clone(),
};
LazyFiatPerBlockCumulativeWithSums::from_boxed_cumulative_cents_source(
&name,
version,
cumulative,
indexes,
cached_starts,
)
})
},
)?;
Ok(Self { values })
}
#[inline(always)]
pub(crate) fn push_block(&mut self, row: UTXOAggregate<C>) {
self.values.push_block(ByTerm {
short: row.sth,
long: row.lth,
});
}
pub(crate) fn len(&self) -> usize {
self.values.cumulative.len()
}
pub(crate) fn stored_mut(&mut self) -> &mut dyn AnyStoredVec {
self.values.stored_mut()
}
}
@@ -0,0 +1,69 @@
use brk_cohort::{UTXO_AGGREGATE_FILTERS, UTXO_AGGREGATE_NAMES, UTXOAggregate, UTXOAggregateId};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Version;
use derive_more::{Deref, DerefMut};
use vecdb::{
AnyStoredVec, AnyVec, Database, ReadableCloneableVec, ReadableColumnarVec, Rw, StorageMode,
};
use crate::{
indexes,
internal::{ColumnarPerBlock, FiatType, LazyFiatPerBlock},
};
use super::utxo_metric_name;
#[derive(Deref, DerefMut, Traversable)]
pub struct AggregateFiatPerBlock<C: FiatType, M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub values: ColumnarPerBlock<C, UTXOAggregateId, UTXOAggregate<LazyFiatPerBlock<C>>, M>,
}
impl<C: FiatType> AggregateFiatPerBlock<C> {
pub(crate) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let values = ColumnarPerBlock::forced_import(
db,
&format!("{metric}_cents_by_aggregate"),
version,
|source| {
UTXOAggregate::from_fn(|id| {
let name = utxo_metric_name(
id.select(&UTXO_AGGREGATE_FILTERS),
id.select(&UTXO_AGGREGATE_NAMES).id,
metric,
);
LazyFiatPerBlock::from_boxed_cents_source(
&name,
version,
source
.column(&format!("{name}_cents"), version, id)
.read_only_boxed_clone(),
indexes,
)
})
},
)?;
Ok(Self { values })
}
#[inline(always)]
pub(crate) fn push(&mut self, row: UTXOAggregate<C>) {
self.values.push(row);
}
pub(crate) fn len(&self) -> usize {
self.values.height.len()
}
pub(crate) fn stored_mut(&mut self) -> &mut dyn AnyStoredVec {
self.values.stored_mut()
}
}
@@ -0,0 +1,11 @@
mod cumulative_fiat;
mod fiat;
mod percent;
mod price;
pub use cumulative_fiat::AdditiveAggregateFiatPerBlockCumulativeWithSums;
pub use fiat::AggregateFiatPerBlock;
pub use percent::AggregatePercentPerBlock;
pub use price::AggregatePriceWithRatioPerBlock;
use super::utxo_metric_name;
@@ -0,0 +1,74 @@
use brk_cohort::{UTXO_AGGREGATE_FILTERS, UTXO_AGGREGATE_NAMES, UTXOAggregate, UTXOAggregateId};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode, VecValue};
use crate::{
indexes,
internal::{ColumnarPerBlock, FixedRatio, LazyColumnPercentPerBlock},
};
use super::utxo_metric_name;
#[derive(Deref, DerefMut, Traversable)]
pub struct AggregatePercentPerBlock<B: FixedRatio, M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub values: ColumnarPerBlock<
B,
UTXOAggregateId,
UTXOAggregate<LazyColumnPercentPerBlock<B, UTXOAggregateId>>,
M,
>,
}
impl<B: FixedRatio> AggregatePercentPerBlock<B> {
pub(crate) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let values = ColumnarPerBlock::forced_import(
db,
&format!("{metric}_{}_by_aggregate", B::SUFFIX),
version,
|source| {
UTXOAggregate::from_fn(|id| {
let name = utxo_metric_name(
id.select(&UTXO_AGGREGATE_FILTERS),
id.select(&UTXO_AGGREGATE_NAMES).id,
metric,
);
LazyColumnPercentPerBlock::new(&name, version, source, id, indexes)
})
},
)?;
Ok(Self { values })
}
pub(crate) fn compute_columns2<'a, A, C, V1, V2>(
&mut self,
max_from: Height,
source1: impl Fn(UTXOAggregateId) -> &'a V1,
source2: impl Fn(UTXOAggregateId) -> &'a V2,
transform: impl FnMut(UTXOAggregateId, A, C) -> B,
exit: &Exit,
) -> Result<()>
where
A: VecValue,
C: VecValue,
V1: ReadableVec<Height, A> + 'a,
V2: ReadableVec<Height, C> + 'a,
{
self.values
.compute_columns2(max_from, source1, source2, transform, exit)
}
pub(crate) fn stored_mut(&mut self) -> &mut dyn AnyStoredVec {
self.values.stored_mut()
}
}
@@ -0,0 +1,68 @@
use brk_cohort::{UTXO_AGGREGATE_FILTERS, UTXO_AGGREGATE_NAMES, UTXOAggregate, UTXOAggregateId};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{AnyStoredVec, AnyVec, CachedBoxedVec, Database, Rw, StorageMode};
use crate::{
indexes,
internal::{ColumnarPerBlock, LazyColumnPriceWithRatioPerBlock},
};
use super::utxo_metric_name;
#[derive(Deref, DerefMut, Traversable)]
pub struct AggregatePriceWithRatioPerBlock<M: StorageMode = Rw> {
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub values: ColumnarPerBlock<
Cents,
UTXOAggregateId,
UTXOAggregate<LazyColumnPriceWithRatioPerBlock<UTXOAggregateId>>,
M,
>,
}
impl AggregatePriceWithRatioPerBlock {
pub(crate) fn forced_import(
db: &Database,
metric: &str,
version: Version,
indexes: &indexes::Vecs,
spot_price: &CachedBoxedVec<Height, Cents>,
) -> Result<Self> {
let values = ColumnarPerBlock::forced_import(
db,
&format!("{metric}_cents_by_aggregate"),
version,
|source| {
UTXOAggregate::from_fn(|id| {
let name = utxo_metric_name(
id.select(&UTXO_AGGREGATE_FILTERS),
id.select(&UTXO_AGGREGATE_NAMES).id,
metric,
);
LazyColumnPriceWithRatioPerBlock::new(
&name, version, source, id, indexes, spot_price,
)
})
},
)?;
Ok(Self { values })
}
#[inline(always)]
pub(crate) fn push(&mut self, row: UTXOAggregate<Cents>) {
self.values.push(row);
}
pub(crate) fn len(&self) -> usize {
self.values.height.len()
}
pub(crate) fn stored_mut(&mut self) -> &mut dyn AnyStoredVec {
self.values.stored_mut()
}
}
@@ -1,151 +0,0 @@
use brk_cohort::Filter;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Cents, Height, Version};
use vecdb::{AnyStoredVec, Exit, ReadableVec, Rw, StorageMode};
use crate::{
distribution::AllChainCache,
distribution::metrics::{
ActivityFull, AdjustedSopr, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase,
RealizedFull, RelativeForAll, SupplyCore, UnrealizedFull,
},
price,
};
/// All-cohort metrics: extended realized + adjusted (as composable add-on),
/// extended cost basis, relative for-all (no rel_to_all).
/// Used by: the "all" cohort.
#[derive(Traversable)]
pub struct AllCohortMetrics<M: StorageMode = Rw> {
#[traversable(skip)]
pub filter: Filter,
pub supply: Box<SupplyCore<M>>,
pub outputs: Box<OutputsBase<M>>,
pub activity: Box<ActivityFull<M>>,
pub realized: Box<RealizedFull<M>>,
pub cost_basis: Box<CostBasis<M>>,
pub unrealized: Box<UnrealizedFull<M>>,
#[traversable(wrap = "realized/sopr", rename = "adjusted")]
pub asopr: Box<AdjustedSopr<M>>,
#[traversable(flatten)]
pub relative: Box<RelativeForAll<M>>,
}
impl CohortMetricsBase for AllCohortMetrics {
type ActivityVecs = ActivityFull;
type RealizedVecs = RealizedFull;
type UnrealizedVecs = UnrealizedFull;
impl_cohort_accessors!();
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.supply.validate_computed_versions(base_version)?;
self.activity.validate_computed_versions(base_version)?;
self.cost_basis.validate_computed_versions(base_version)?;
Ok(())
}
fn min_stateful_len(&self) -> usize {
// Only check per-block pushed vecs, not aggregated ones (supply, outputs,
// activity, realized core, unrealized core are summed from age_range).
self.realized
.min_stateful_len()
.min(self.unrealized.min_stateful_len())
.min(self.cost_basis.min_stateful_len())
}
fn collect_all_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = Vec::new();
vecs.extend(self.supply.collect_vecs_mut());
vecs.extend(self.outputs.collect_vecs_mut());
vecs.extend(self.activity.collect_vecs_mut());
vecs.extend(self.realized.collect_vecs_mut());
vecs.extend(self.cost_basis.collect_vecs_mut());
vecs.extend(self.unrealized.collect_vecs_mut());
vecs
}
}
impl AllCohortMetrics {
/// Import the "all" cohort metrics with a pre-imported supply.
///
/// Supply is imported first (before other cohorts) so it can be used as `all_supply`
/// reference for relative metric lazy vecs in other cohorts.
pub(crate) fn forced_import_with_supply(
cfg: &ImportConfig,
supply: SupplyCore,
all_chain: &AllChainCache,
) -> Result<Self> {
let realized = RealizedFull::forced_import(cfg, all_chain)?;
let unrealized = UnrealizedFull::forced_import(cfg, &realized.price.ppm)?;
let asopr = AdjustedSopr::forced_import(cfg)?;
let relative = RelativeForAll::forced_import(cfg, &unrealized, all_chain)?;
Ok(Self {
filter: cfg.filter.clone(),
supply: Box::new(supply),
outputs: Box::new(OutputsBase::forced_import(cfg)?),
activity: Box::new(ActivityFull::forced_import(cfg)?),
realized: Box::new(realized),
cost_basis: Box::new(CostBasis::forced_import(cfg)?),
unrealized: Box::new(unrealized),
asopr: Box::new(asopr),
relative: Box::new(relative),
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute_rest_part2(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
under_1h_value_created: &impl ReadableVec<Height, Cents>,
under_1h_value_destroyed: &impl ReadableVec<Height, Cents>,
exit: &Exit,
) -> Result<()> {
self.realized.compute_rest_part2(
prices,
starting_lengths,
&self.supply.total.btc.height,
&self.activity.transfer_volume,
exit,
)?;
self.asopr.compute_rest_part2(
starting_lengths,
&self.activity.transfer_volume.inner.cumulative.cents.height,
&self.realized.core.sopr.value_destroyed.cumulative.height,
under_1h_value_created,
under_1h_value_destroyed,
exit,
)?;
self.cost_basis.compute_prices(
starting_lengths,
&prices.spot.cents.height,
&self.unrealized.invested_capital.in_profit.cents.height,
&self.unrealized.invested_capital.in_loss.cents.height,
&self.supply.in_profit.sats.height,
&self.supply.in_loss.sats.height,
&self.unrealized.capitalized_cap_in_profit_raw,
&self.unrealized.capitalized_cap_in_loss_raw,
exit,
)?;
self.unrealized
.compute_sentiment(starting_lengths, &prices.spot.cents.height, exit)?;
self.relative.compute(
starting_lengths.height,
&self.supply,
&self.unrealized,
&self.realized,
exit,
)?;
Ok(())
}
}
@@ -1,76 +0,0 @@
use brk_cohort::Filter;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use vecdb::{AnyStoredVec, Exit, Rw, StorageMode};
use crate::{
distribution::metrics::{
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
UnrealizedCore,
},
price,
};
/// Basic cohort metrics: no extensions, used by age_range cohorts.
#[derive(Traversable)]
pub struct BasicCohortMetrics<M: StorageMode = Rw> {
#[traversable(skip)]
pub filter: Filter,
pub supply: Box<SupplyCore<M>>,
pub outputs: Box<OutputsBase<M>>,
pub activity: Box<ActivityCore<M>>,
pub realized: Box<RealizedCore<M>>,
pub unrealized: Box<UnrealizedCore<M>>,
}
impl CohortMetricsBase for BasicCohortMetrics {
type ActivityVecs = ActivityCore;
type RealizedVecs = RealizedCore;
type UnrealizedVecs = UnrealizedCore;
impl_cohort_accessors!();
fn collect_all_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = Vec::new();
vecs.extend(self.supply.collect_vecs_mut());
vecs.extend(self.outputs.collect_vecs_mut());
vecs.extend(self.activity.collect_vecs_mut());
vecs.extend(self.realized.collect_vecs_mut());
vecs.extend(self.unrealized.collect_vecs_mut());
vecs
}
}
impl BasicCohortMetrics {
pub(crate) fn forced_import(cfg: &ImportConfig, supply: SupplyCore) -> Result<Self> {
let realized = RealizedCore::forced_import(cfg)?;
let unrealized = UnrealizedCore::forced_import(cfg, &realized.price.ppm)?;
Ok(Self {
filter: cfg.filter.clone(),
supply: Box::new(supply),
outputs: Box::new(OutputsBase::forced_import(cfg)?),
activity: Box::new(ActivityCore::forced_import(cfg)?),
realized: Box::new(realized),
unrealized: Box::new(unrealized),
})
}
pub(crate) fn compute_rest_part2(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.realized.compute_rest_part2(
prices,
starting_lengths,
&self.supply.total.btc.height,
&self.activity.transfer_volume.sum._24h.cents.height,
exit,
)?;
Ok(())
}
}
@@ -1,146 +0,0 @@
use brk_cohort::Filter;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{AnyStoredVec, Exit, Rw, StorageMode};
use crate::{
distribution::metrics::{
ActivityCore, AllSupplyCache, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore,
SupplyCore, UnrealizedCore,
},
price,
};
#[derive(Traversable)]
pub struct CoreCohortMetrics<M: StorageMode = Rw> {
#[traversable(skip)]
pub filter: Filter,
pub supply: Box<SupplyCore<M>>,
pub outputs: Box<OutputsBase<M>>,
pub activity: Box<ActivityCore<M>>,
pub realized: Box<RealizedCore<M>>,
pub unrealized: Box<UnrealizedCore<M>>,
}
impl CoreCohortMetrics {
pub(crate) fn forced_import(cfg: &ImportConfig, all_supply: &AllSupplyCache) -> Result<Self> {
let supply = SupplyCore::forced_import(cfg, all_supply)?;
Self::forced_import_with_supply(cfg, supply)
}
pub(crate) fn forced_import_with_supply(
cfg: &ImportConfig,
supply: SupplyCore,
) -> Result<Self> {
let realized = RealizedCore::forced_import(cfg)?;
let unrealized = UnrealizedCore::forced_import(cfg, &realized.price.ppm)?;
Ok(Self {
filter: cfg.filter.clone(),
supply: Box::new(supply),
outputs: Box::new(OutputsBase::forced_import(cfg)?),
activity: Box::new(ActivityCore::forced_import(cfg)?),
realized: Box::new(realized),
unrealized: Box::new(unrealized),
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.supply
.min_len()
.min(self.outputs.min_len())
.min(self.activity.min_len())
.min(self.realized.min_stateful_len())
.min(self.unrealized.min_stateful_len())
}
pub(crate) fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.supply.validate_computed_versions(base_version)?;
self.activity.validate_computed_versions(base_version)?;
Ok(())
}
pub(crate) fn collect_all_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = Vec::new();
vecs.extend(self.supply.collect_vecs_mut());
vecs.extend(self.outputs.collect_vecs_mut());
vecs.extend(self.activity.collect_vecs_mut());
vecs.extend(self.realized.collect_vecs_mut());
vecs.extend(self.unrealized.collect_vecs_mut());
vecs
}
/// Aggregate Core-tier fields from CohortMetricsBase sources (e.g. age_range -> under_age/over_age).
pub(crate) fn compute_from_base_sources<T: CohortMetricsBase>(
&mut self,
starting_lengths: &Lengths,
others: &[&T],
exit: &Exit,
) -> Result<()> {
self.supply.compute_from_stateful(
starting_lengths,
&others.iter().map(|v| v.supply()).collect::<Vec<_>>(),
exit,
)?;
self.outputs.compute_from_stateful(
starting_lengths,
&others.iter().map(|v| v.outputs()).collect::<Vec<_>>(),
exit,
)?;
self.activity.compute_from_stateful(
starting_lengths,
&others.iter().map(|v| v.activity_core()).collect::<Vec<_>>(),
exit,
)?;
self.realized.compute_from_stateful(
starting_lengths,
&others.iter().map(|v| v.realized_core()).collect::<Vec<_>>(),
exit,
)?;
self.unrealized.compute_from_stateful(
starting_lengths,
&others
.iter()
.map(|v| v.unrealized_core())
.collect::<Vec<_>>(),
exit,
)?;
Ok(())
}
pub(crate) fn compute_rest_part1(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.activity
.compute_rest_part1(prices, starting_lengths, exit)?;
self.realized.compute_rest_part1(starting_lengths, exit)?;
self.unrealized.compute_rest(starting_lengths, exit)?;
Ok(())
}
pub(crate) fn compute_rest_part2(
&mut self,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
self.realized.compute_rest_part2(
prices,
starting_lengths,
&self.supply.total.btc.height,
&self.activity.transfer_volume.sum._24h.cents.height,
exit,
)?;
Ok(())
}
}

Some files were not shown because too many files have changed in this diff Show More