global: chain + cointime datasets

This commit is contained in:
nym21
2025-09-13 18:26:28 +02:00
parent 38d5c7dff6
commit 01aa425f81
13 changed files with 807 additions and 305 deletions
+94 -13
View File
@@ -5,9 +5,9 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Bitcoin, CheckedSub, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, FeeRate, HalvingEpoch,
Height, InputIndex, MonthIndex, OutputIndex, QuarterIndex, Sats, SemesterIndex, StoredBool,
StoredF32, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion, Version, WeekIndex,
Weight, YearIndex,
Height, InputIndex, MonthIndex, ONE_DAY_IN_SEC_F64, OutputIndex, QuarterIndex, Sats,
SemesterIndex, StoredBool, StoredF32, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex,
TxVersion, Version, WeekIndex, Weight, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit,
@@ -136,8 +136,11 @@ pub struct Vecs {
pub indexes_to_annualized_volume: ComputedVecsFromDateIndex<Sats>,
pub indexes_to_annualized_volume_btc: ComputedVecsFromDateIndex<Bitcoin>,
pub indexes_to_annualized_volume_usd: ComputedVecsFromDateIndex<Dollars>,
pub indexes_to_velocity_btc: ComputedVecsFromDateIndex<StoredF64>,
pub indexes_to_velocity_usd: ComputedVecsFromDateIndex<StoredF64>,
pub indexes_to_tx_btc_velocity: ComputedVecsFromDateIndex<StoredF64>,
pub indexes_to_tx_usd_velocity: ComputedVecsFromDateIndex<StoredF64>,
pub indexes_to_tx_per_sec: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_outputs_per_sec: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_inputs_per_sec: ComputedVecsFromDateIndex<StoredF32>,
}
impl Vecs {
@@ -1093,22 +1096,46 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_velocity_btc: ComputedVecsFromDateIndex::forced_import(
indexes_to_tx_btc_velocity: ComputedVecsFromDateIndex::forced_import(
&db,
"velocity_btc",
"tx_btc_velocity",
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_velocity_usd: ComputedVecsFromDateIndex::forced_import(
indexes_to_tx_usd_velocity: ComputedVecsFromDateIndex::forced_import(
&db,
"velocity_usd",
"tx_usd_velocity",
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_tx_per_sec: ComputedVecsFromDateIndex::forced_import(
&db,
"tx_per_sec",
Source::Compute,
version + Version::TWO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_outputs_per_sec: ComputedVecsFromDateIndex::forced_import(
&db,
"outputs_per_sec",
Source::Compute,
version + Version::TWO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_inputs_per_sec: ComputedVecsFromDateIndex::forced_import(
&db,
"inputs_per_sec",
Source::Compute,
version + Version::TWO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
txindex_to_is_coinbase,
inputindex_to_value,
@@ -2160,7 +2187,7 @@ impl Vecs {
Ok(())
})?;
self.indexes_to_velocity_btc
self.indexes_to_tx_btc_velocity
.compute_all(starting_indexes, exit, |v| {
v.compute_divide(
starting_indexes.dateindex,
@@ -2189,7 +2216,7 @@ impl Vecs {
Ok(())
})?;
self.indexes_to_velocity_usd
self.indexes_to_tx_usd_velocity
.compute_all(starting_indexes, exit, |v| {
v.compute_divide(
starting_indexes.dateindex,
@@ -2209,6 +2236,57 @@ impl Vecs {
})?;
}
self.indexes_to_tx_per_sec
.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
self.indexes_to_tx_count.dateindex.unwrap_sum(),
&indexes.dateindex_to_date,
|(i, tx_count, date, ..)| {
(
i,
(*tx_count as f64 / (date.completion() * ONE_DAY_IN_SEC_F64)).into(),
)
},
exit,
)?;
Ok(())
})?;
self.indexes_to_inputs_per_sec
.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
self.indexes_to_input_count.dateindex.unwrap_sum(),
&indexes.dateindex_to_date,
|(i, tx_count, date, ..)| {
(
i,
(*tx_count as f64 / (date.completion() * ONE_DAY_IN_SEC_F64)).into(),
)
},
exit,
)?;
Ok(())
})?;
self.indexes_to_outputs_per_sec
.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
self.indexes_to_output_count.dateindex.unwrap_sum(),
&indexes.dateindex_to_date,
|(i, tx_count, date, ..)| {
(
i,
(*tx_count as f64 / (date.completion() * ONE_DAY_IN_SEC_F64)).into(),
)
},
exit,
)?;
Ok(())
})?;
Ok(())
}
@@ -2326,8 +2404,11 @@ impl Vecs {
iter = Box::new(iter.chain(self.indexes_to_annualized_volume.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_annualized_volume_btc.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_annualized_volume_usd.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_velocity_btc.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_velocity_usd.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_tx_btc_velocity.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_tx_usd_velocity.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_tx_per_sec.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_outputs_per_sec.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_inputs_per_sec.iter_any_collectable()));
iter = Box::new(
iter.chain(
self.indexes_to_subsidy_usd_1y_sma
+118 -35
View File
@@ -1,9 +1,11 @@
use std::path::Path;
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF32, StoredF64, Version};
use vecdb::{AnyCollectableVec, Database, Exit, PAGE_SIZE, VecIterator};
use crate::grouped::ComputedVecsFromDateIndex;
use super::{
Indexes, chain,
grouped::{
@@ -13,8 +15,6 @@ use super::{
indexes, price, stateful,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
@@ -42,27 +42,32 @@ pub struct Vecs {
pub indexes_to_cointime_price: ComputedVecsFromHeight<Dollars>,
pub indexes_to_cointime_cap: ComputedVecsFromHeight<Dollars>,
pub indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex,
pub indexes_to_cointime_adj_inflation_rate: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_cointime_adj_tx_btc_velocity: ComputedVecsFromDateIndex<StoredF64>,
pub indexes_to_cointime_adj_tx_usd_velocity: ComputedVecsFromDateIndex<StoredF64>,
// pub indexes_to_thermo_cap_rel_to_investor_cap: ComputedValueVecsFromHeight,
}
impl Vecs {
pub fn forced_import(
parent: &Path,
version: Version,
parent_path: &Path,
parent_version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let db = Database::open(&parent.join("cointime"))?;
let db = Database::open(&parent_path.join("cointime"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let compute_dollars = price.is_some();
let version = parent_version + Version::ZERO;
let this = Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
&db,
"coinblocks_created",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
@@ -70,7 +75,7 @@ impl Vecs {
&db,
"coinblocks_stored",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
@@ -78,7 +83,7 @@ impl Vecs {
&db,
"liveliness",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -86,7 +91,7 @@ impl Vecs {
&db,
"vaultedness",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -94,7 +99,7 @@ impl Vecs {
&db,
"activity_to_vaultedness_ratio",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -102,7 +107,7 @@ impl Vecs {
&db,
"vaulted_supply",
Source::Compute,
version + VERSION + Version::ONE,
version + Version::ONE,
VecBuilderOptions::default().add_last(),
compute_dollars,
indexes,
@@ -111,7 +116,7 @@ impl Vecs {
&db,
"active_supply",
Source::Compute,
version + VERSION + Version::ONE,
version + Version::ONE,
VecBuilderOptions::default().add_last(),
compute_dollars,
indexes,
@@ -120,7 +125,7 @@ impl Vecs {
&db,
"thermo_cap",
Source::Compute,
version + VERSION + Version::ONE,
version + Version::ONE,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -128,7 +133,7 @@ impl Vecs {
&db,
"investor_cap",
Source::Compute,
version + VERSION + Version::ONE,
version + Version::ONE,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -136,7 +141,7 @@ impl Vecs {
&db,
"vaulted_cap",
Source::Compute,
version + VERSION + Version::ONE,
version + Version::ONE,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -144,7 +149,7 @@ impl Vecs {
&db,
"active_cap",
Source::Compute,
version + VERSION + Version::ONE,
version + Version::ONE,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -152,7 +157,7 @@ impl Vecs {
&db,
"vaulted_price",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -160,7 +165,7 @@ impl Vecs {
&db,
"vaulted_price",
Source::None,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
true,
)?,
@@ -168,7 +173,7 @@ impl Vecs {
&db,
"active_price",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -176,7 +181,7 @@ impl Vecs {
&db,
"active_price",
Source::None,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
true,
)?,
@@ -184,7 +189,7 @@ impl Vecs {
&db,
"true_market_mean",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -192,7 +197,7 @@ impl Vecs {
&db,
"true_market_mean",
Source::None,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
true,
)?,
@@ -200,7 +205,7 @@ impl Vecs {
&db,
"cointime_value_destroyed",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
@@ -208,7 +213,7 @@ impl Vecs {
&db,
"cointime_value_created",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
@@ -216,7 +221,7 @@ impl Vecs {
&db,
"cointime_value_stored",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
@@ -224,7 +229,7 @@ impl Vecs {
&db,
"cointime_price",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -232,7 +237,7 @@ impl Vecs {
&db,
"cointime_cap",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -240,10 +245,34 @@ impl Vecs {
&db,
"cointime_price",
Source::None,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
true,
)?,
indexes_to_cointime_adj_inflation_rate: ComputedVecsFromDateIndex::forced_import(
&db,
"cointime_adj_inflation_rate",
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_adj_tx_btc_velocity: ComputedVecsFromDateIndex::forced_import(
&db,
"cointime_adj_tx_btc_velocity",
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_adj_tx_usd_velocity: ComputedVecsFromDateIndex::forced_import(
&db,
"cointime_adj_tx_usd_velocity",
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
db,
};
@@ -392,6 +421,32 @@ impl Vecs {
},
)?;
self.indexes_to_cointime_adj_inflation_rate
.compute_all(starting_indexes, exit, |v| {
v.compute_multiply(
starting_indexes.dateindex,
self.indexes_to_activity_to_vaultedness_ratio
.dateindex
.unwrap_last(),
chain.indexes_to_inflation_rate.dateindex.as_ref().unwrap(),
exit,
)?;
Ok(())
})?;
self.indexes_to_cointime_adj_tx_btc_velocity
.compute_all(starting_indexes, exit, |v| {
v.compute_multiply(
starting_indexes.dateindex,
self.indexes_to_activity_to_vaultedness_ratio
.dateindex
.unwrap_last(),
chain.indexes_to_tx_btc_velocity.dateindex.as_ref().unwrap(),
exit,
)?;
Ok(())
})?;
if let Some(price) = price {
let realized_cap = stateful
.utxo_cohorts
@@ -603,6 +658,22 @@ impl Vecs {
exit,
Some(self.indexes_to_cointime_price.dateindex.unwrap_last()),
)?;
self.indexes_to_cointime_adj_tx_usd_velocity.compute_all(
starting_indexes,
exit,
|v| {
v.compute_multiply(
starting_indexes.dateindex,
self.indexes_to_activity_to_vaultedness_ratio
.dateindex
.unwrap_last(),
chain.indexes_to_tx_usd_velocity.dateindex.as_ref().unwrap(),
exit,
)?;
Ok(())
},
)?;
}
Ok(())
@@ -612,6 +683,24 @@ impl Vecs {
let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(std::iter::empty());
iter = Box::new(
iter.chain(
self.indexes_to_cointime_adj_inflation_rate
.iter_any_collectable(),
),
);
iter = Box::new(
iter.chain(
self.indexes_to_cointime_adj_tx_btc_velocity
.iter_any_collectable(),
),
);
iter = Box::new(
iter.chain(
self.indexes_to_cointime_adj_tx_usd_velocity
.iter_any_collectable(),
),
);
iter = Box::new(iter.chain(self.indexes_to_coinblocks_created.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_coinblocks_stored.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_liveliness.iter_any_collectable()));
@@ -622,20 +711,16 @@ impl Vecs {
.iter_any_collectable(),
),
);
iter = Box::new(iter.chain(self.indexes_to_vaulted_supply.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_active_supply.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_thermo_cap.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_investor_cap.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_vaulted_cap.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_active_cap.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_vaulted_price.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_vaulted_price_ratio.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_active_price.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_active_price_ratio.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_true_market_mean.iter_any_collectable()));
iter = Box::new(
iter.chain(
@@ -643,7 +728,6 @@ impl Vecs {
.iter_any_collectable(),
),
);
iter = Box::new(iter.chain(self.indexes_to_cointime_price.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_cointime_cap.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes_to_cointime_price_ratio.iter_any_collectable()));
@@ -660,7 +744,6 @@ impl Vecs {
),
);
iter = Box::new(iter.chain(self.indexes_to_cointime_value_stored.iter_any_collectable()));
iter
}
}
File diff suppressed because it is too large Load Diff
+24 -1
View File
@@ -1,8 +1,10 @@
use jiff::{Span, civil::Date as Date_, tz::TimeZone};
use jiff::{Span, Zoned, civil::Date as Date_, tz::TimeZone};
use serde::{Serialize, Serializer};
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::ONE_DAY_IN_SEC_F64;
use super::{DateIndex, Timestamp};
#[derive(
@@ -51,6 +53,21 @@ impl Date {
pub fn today() -> Self {
Self::from(Timestamp::now())
}
pub fn completion(&self) -> f64 {
let date = Date_::from(*self);
let now = Zoned::now().with_time_zone(TimeZone::UTC);
let today = now.date();
if date < today {
1.0
} else if date == today {
let rounded = jiff::Timestamp::from(*self);
now.timestamp().duration_since(rounded).as_secs_f64() / ONE_DAY_IN_SEC_F64
} else {
0.0
}
}
}
impl Default for Date {
@@ -71,6 +88,12 @@ impl From<Date> for Date_ {
}
}
impl From<Date> for jiff::Timestamp {
fn from(value: Date) -> Self {
Self::from(Timestamp::from(value))
}
}
impl From<Timestamp> for Date {
fn from(value: Timestamp) -> Self {
Self::from(Date_::from(
+38
View File
@@ -9,6 +9,8 @@ use serde::{Serialize, Serializer, ser::SerializeTuple};
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::StoredF64;
use super::{Cents, Dollars, Sats};
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)]
@@ -217,6 +219,15 @@ where
}
}
impl<T> From<Open<T>> for StoredF64
where
StoredF64: From<T>,
{
fn from(value: Open<T>) -> Self {
Self::from(value.0)
}
}
impl<T> From<Close<T>> for Open<T>
where
T: Copy,
@@ -315,6 +326,15 @@ where
}
}
impl<T> From<High<T>> for StoredF64
where
StoredF64: From<T>,
{
fn from(value: High<T>) -> Self {
Self::from(value.0)
}
}
impl<T> From<Close<T>> for High<T>
where
T: Copy,
@@ -413,6 +433,15 @@ where
}
}
impl<T> From<Low<T>> for StoredF64
where
StoredF64: From<T>,
{
fn from(value: Low<T>) -> Self {
Self::from(value.0)
}
}
impl<T> From<Close<T>> for Low<T>
where
T: Copy,
@@ -530,6 +559,15 @@ where
}
}
impl<T> From<Close<T>> for StoredF64
where
StoredF64: From<T>,
{
fn from(value: Close<T>) -> Self {
Self::from(value.0)
}
}
// impl<A, B> From<Close<A>> for Close<B>
// where
// B: From<A>,
@@ -153,6 +153,13 @@ impl Mul<usize> for StoredF32 {
}
}
impl Mul<StoredF32> for StoredF32 {
type Output = Self;
fn mul(self, rhs: Self) -> Self::Output {
Self(self.0 * rhs.0)
}
}
impl Mul<StoredF32> for usize {
type Output = StoredF32;
fn mul(self, rhs: StoredF32) -> Self::Output {
@@ -64,6 +64,20 @@ impl Mul<usize> for StoredF64 {
}
}
impl Mul<StoredF64> for StoredF64 {
type Output = Self;
fn mul(self, rhs: Self) -> Self::Output {
Self(self.0 * rhs.0)
}
}
impl Mul<Dollars> for StoredF64 {
type Output = Self;
fn mul(self, rhs: Dollars) -> Self::Output {
Self(self.0 * *rhs)
}
}
impl Div<usize> for StoredF64 {
type Output = Self;
fn div(self, rhs: usize) -> Self::Output {
+3 -11
View File
@@ -28,9 +28,9 @@ use super::Date;
)]
pub struct Timestamp(u32);
const ONE_HOUR_IN_SEC: u32 = 60 * 60;
const ONE_DAY_IN_SEC: u32 = 24 * 60 * 60;
const ONE_DAY_IN_SEC_F64: f64 = ONE_DAY_IN_SEC as f64;
pub const ONE_HOUR_IN_SEC: u32 = 60 * 60;
pub const ONE_DAY_IN_SEC: u32 = 24 * 60 * 60;
pub const ONE_DAY_IN_SEC_F64: f64 = ONE_DAY_IN_SEC as f64;
impl Timestamp {
pub const ZERO: Self = Self(0);
@@ -75,14 +75,6 @@ impl Timestamp {
pub fn now() -> Self {
Self::from(jiff::Timestamp::now())
}
pub fn day_completion(&self) -> f64 {
let rounded = jiff::Timestamp::from(Self::from(Date::from(*self)));
ONE_DAY_IN_SEC_F64
/ jiff::Timestamp::from(*self)
.duration_since(rounded)
.as_secs_f64()
}
}
impl From<u32> for Timestamp {