global: snapshot

This commit is contained in:
nym21
2026-02-13 13:54:09 +01:00
parent b779edc0d6
commit 80b2c636b0
53 changed files with 1819 additions and 1184 deletions

View File

@@ -249,8 +249,10 @@ impl UTXOCohorts {
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))?;
// 2. Compute net_sentiment.height for separate cohorts (greed - pain)
self.par_iter_separate_mut()
.try_for_each(|v| v.metrics.compute_net_sentiment_height(starting_indexes, exit))?;
self.par_iter_separate_mut().try_for_each(|v| {
v.metrics
.compute_net_sentiment_height(starting_indexes, exit)
})?;
// 3. Compute net_sentiment.height for aggregate cohorts (weighted average)
self.for_each_aggregate(|vecs, sources| {
@@ -260,8 +262,10 @@ impl UTXOCohorts {
})?;
// 4. Compute net_sentiment dateindex for ALL cohorts
self.par_iter_mut()
.try_for_each(|v| v.metrics.compute_net_sentiment_rest(indexes, starting_indexes, exit))
self.par_iter_mut().try_for_each(|v| {
v.metrics
.compute_net_sentiment_rest(indexes, starting_indexes, exit)
})
}
/// Second phase of post-processing: compute relative metrics.
@@ -468,7 +472,8 @@ impl UTXOCohorts {
// Collect merged entries during the merge (already in sorted order)
// Pre-allocate with max possible unique prices (actual count likely lower due to dedup)
let max_unique_prices = relevant.iter().map(|e| e.len()).max().unwrap_or(0);
let mut merged: Vec<(CentsUnsignedCompact, Sats)> = Vec::with_capacity(max_unique_prices);
let mut merged: Vec<(CentsUnsignedCompact, Sats)> =
Vec::with_capacity(max_unique_prices);
// Finalize a price point: compute percentiles and accumulate for merged vec
let mut finalize_price = |price: CentsUnsigned, sats: u64, usd: u128| {
@@ -489,7 +494,8 @@ impl UTXOCohorts {
}
// Round to nearest dollar with N significant digits for storage
let rounded: CentsUnsignedCompact = price.round_to_dollar(COST_BASIS_PRICE_DIGITS).into();
let rounded: CentsUnsignedCompact =
price.round_to_dollar(COST_BASIS_PRICE_DIGITS).into();
// Merge entries with same rounded price using last_mut
if let Some((last_price, last_sats)) = merged.last_mut()
@@ -562,7 +568,10 @@ impl UTXOCohorts {
let dir = states_path.join(format!("utxo_{cohort_name}_cost_basis/by_date"));
fs::create_dir_all(&dir)?;
let path = dir.join(date.to_string());
fs::write(path, CostBasisDistribution::serialize_iter(merged.into_iter())?)?;
fs::write(
path,
CostBasisDistribution::serialize_iter(merged.into_iter())?,
)?;
Ok(())
})

View File

@@ -67,7 +67,7 @@ impl Vecs {
let cents = CentsVecs::forced_import(db, version)?;
let usd = UsdVecs::forced_import(db, version, indexes)?;
let sats = SatsVecs::forced_import(db, version, indexes)?;
let oracle = OracleVecs::forced_import(db, version)?;
let oracle = OracleVecs::forced_import(db, version, indexes)?;
Ok(Self {
db: db.clone(),

View File

@@ -4,8 +4,8 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_oracle::{Config, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
use brk_types::{
CentsUnsigned, Close, DateIndex, Height, High, Low, OHLCCentsUnsigned, Open, OutputType, Sats,
TxIndex, TxOutIndex,
CentsUnsigned, Close, DateIndex, Height, High, Low, OHLCCentsUnsigned, OHLCDollars, Open,
OutputType, Sats, TxIndex, TxOutIndex,
};
use tracing::info;
use vecdb::{
@@ -26,6 +26,224 @@ impl Vecs {
) -> Result<()> {
self.compute_prices(indexer, starting_indexes, exit)?;
self.compute_daily_ohlc(indexes, starting_indexes, exit)?;
self.compute_split_and_ohlc(starting_indexes, exit)?;
Ok(())
}
fn compute_split_and_ohlc(
&mut self,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
// Destructure to allow simultaneous borrows of different fields
let Self {
price_cents,
ohlc_cents,
split,
ohlc,
ohlc_dollars,
} = self;
// Open: first-value aggregation
split.open.height.compute_transform(
starting_indexes.height,
&*price_cents,
|(h, price, ..)| (h, Open::new(price)),
exit,
)?;
split.open.compute_rest(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.dateindex,
&*ohlc_cents,
|(di, ohlc_val, ..)| (di, ohlc_val.open),
exit,
)?;
Ok(())
})?;
// High: max-value aggregation
split.high.height.compute_transform(
starting_indexes.height,
&*price_cents,
|(h, price, ..)| (h, High::new(price)),
exit,
)?;
split.high.compute_rest(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.dateindex,
&*ohlc_cents,
|(di, ohlc_val, ..)| (di, ohlc_val.high),
exit,
)?;
Ok(())
})?;
// Low: min-value aggregation
split.low.height.compute_transform(
starting_indexes.height,
&*price_cents,
|(h, price, ..)| (h, Low::new(price)),
exit,
)?;
split.low.compute_rest(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.dateindex,
&*ohlc_cents,
|(di, ohlc_val, ..)| (di, ohlc_val.low),
exit,
)?;
Ok(())
})?;
// Close: last-value aggregation
split.close.height.compute_transform(
starting_indexes.height,
&*price_cents,
|(h, price, ..)| (h, Close::new(price)),
exit,
)?;
split.close.compute_rest(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.dateindex,
&*ohlc_cents,
|(di, ohlc_val, ..)| (di, ohlc_val.close),
exit,
)?;
Ok(())
})?;
// Period OHLC aggregates - time based
ohlc.dateindex.compute_transform4(
starting_indexes.dateindex,
&split.open.dateindex,
&split.high.dateindex,
&split.low.dateindex,
&split.close.dateindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.week.compute_transform4(
starting_indexes.weekindex,
&*split.open.weekindex,
&*split.high.weekindex,
&*split.low.weekindex,
&*split.close.weekindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.month.compute_transform4(
starting_indexes.monthindex,
&*split.open.monthindex,
&*split.high.monthindex,
&*split.low.monthindex,
&*split.close.monthindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.quarter.compute_transform4(
starting_indexes.quarterindex,
&*split.open.quarterindex,
&*split.high.quarterindex,
&*split.low.quarterindex,
&*split.close.quarterindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.semester.compute_transform4(
starting_indexes.semesterindex,
&*split.open.semesterindex,
&*split.high.semesterindex,
&*split.low.semesterindex,
&*split.close.semesterindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.year.compute_transform4(
starting_indexes.yearindex,
&*split.open.yearindex,
&*split.high.yearindex,
&*split.low.yearindex,
&*split.close.yearindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.decade.compute_transform4(
starting_indexes.decadeindex,
&*split.open.decadeindex,
&*split.high.decadeindex,
&*split.low.decadeindex,
&*split.close.decadeindex,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
// Period OHLC aggregates - chain based
ohlc.height.compute_transform4(
starting_indexes.height,
&split.open.height,
&split.high.height,
&split.low.height,
&split.close.height,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
ohlc.difficultyepoch.compute_transform4(
starting_indexes.difficultyepoch,
&*split.open.difficultyepoch,
&*split.high.difficultyepoch,
&*split.low.difficultyepoch,
&*split.close.difficultyepoch,
|(i, open, high, low, close, _)| {
(i, OHLCCentsUnsigned { open, high, low, close })
},
exit,
)?;
// OHLC dollars - transform cents to dollars at every period level
macro_rules! cents_to_dollars {
($field:ident, $idx:expr) => {
ohlc_dollars.$field.compute_transform(
$idx,
&ohlc.$field,
|(i, c, ..)| (i, OHLCDollars::from(c)),
exit,
)?;
};
}
cents_to_dollars!(dateindex, starting_indexes.dateindex);
cents_to_dollars!(week, starting_indexes.weekindex);
cents_to_dollars!(month, starting_indexes.monthindex);
cents_to_dollars!(quarter, starting_indexes.quarterindex);
cents_to_dollars!(semester, starting_indexes.semesterindex);
cents_to_dollars!(year, starting_indexes.yearindex);
cents_to_dollars!(decade, starting_indexes.decadeindex);
cents_to_dollars!(height, starting_indexes.height);
cents_to_dollars!(difficultyepoch, starting_indexes.difficultyepoch);
Ok(())
}

View File

@@ -1,29 +1,53 @@
use brk_error::Result;
use brk_types::{DateIndex, OHLCCentsUnsigned, OHLCDollars, Version};
use vecdb::{BytesVec, Database, ImportableVec, IterableCloneableVec, LazyVecFrom1, PcoVec};
use brk_types::Version;
use vecdb::{BytesVec, Database, EagerVec, ImportableVec, PcoVec};
use super::Vecs;
use crate::indexes;
use crate::internal::{ComputedOHLC, LazyFromHeightAndDateOHLC};
impl Vecs {
pub fn forced_import(db: &Database, parent_version: Version) -> Result<Self> {
let version = parent_version + Version::new(10);
pub fn forced_import(
db: &Database,
parent_version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let version = parent_version + Version::new(11);
let price_cents = PcoVec::forced_import(db, "oracle_price_cents", version)?;
let ohlc_cents = BytesVec::forced_import(db, "oracle_ohlc_cents", version)?;
let ohlc_dollars = LazyVecFrom1::init(
"oracle_ohlc_dollars",
version,
ohlc_cents.boxed_clone(),
|di: DateIndex, iter| {
iter.get(di)
.map(|o: OHLCCentsUnsigned| OHLCDollars::from(o))
},
);
let split = ComputedOHLC::forced_import(db, "oracle_price", version, indexes)?;
let ohlc = LazyFromHeightAndDateOHLC {
dateindex: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
week: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
month: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
quarter: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
semester: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
year: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
decade: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
height: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
difficultyepoch: EagerVec::forced_import(db, "oracle_price_ohlc", version)?,
};
let ohlc_dollars = LazyFromHeightAndDateOHLC {
dateindex: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
week: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
month: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
quarter: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
semester: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
year: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
decade: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
height: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
difficultyepoch: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?,
};
Ok(Self {
price_cents,
ohlc_cents,
split,
ohlc,
ohlc_dollars,
})
}

View File

@@ -1,10 +1,14 @@
use brk_traversable::Traversable;
use brk_types::{CentsUnsigned, DateIndex, Height, OHLCCentsUnsigned, OHLCDollars};
use vecdb::{BytesVec, LazyVecFrom1, PcoVec};
use vecdb::{BytesVec, PcoVec};
use crate::internal::{ComputedOHLC, LazyFromHeightAndDateOHLC};
#[derive(Clone, Traversable)]
pub struct Vecs {
pub price_cents: PcoVec<Height, CentsUnsigned>,
pub ohlc_cents: BytesVec<DateIndex, OHLCCentsUnsigned>,
pub ohlc_dollars: LazyVecFrom1<DateIndex, OHLCDollars, DateIndex, OHLCCentsUnsigned>,
pub split: ComputedOHLC<CentsUnsigned>,
pub ohlc: LazyFromHeightAndDateOHLC<OHLCCentsUnsigned>,
pub ohlc_dollars: LazyFromHeightAndDateOHLC<OHLCDollars>,
}

View File

@@ -52,7 +52,7 @@ impl Vecs {
vec.compute_percentage_change(
starting_indexes.dateindex,
mcap_dateindex,
30,
365,
exit,
)?;
Ok(())
@@ -66,7 +66,7 @@ impl Vecs {
vec.compute_percentage_change(
starting_indexes.dateindex,
rcap_dateindex,
30,
365,
exit,
)?;
Ok(())

View File

@@ -7,11 +7,12 @@ use vecdb::{Database, IterableCloneableVec, LazyVecFrom2, PAGE_SIZE};
use super::Vecs;
use crate::{
distribution, indexes, price,
distribution, indexes,
internal::{
ComputedFromDateAverage, ComputedFromDateLast, DifferenceF32, DollarsIdentity,
LazyFromHeightLast, LazyValueFromHeightLast, SatsIdentity,
},
price,
};
const VERSION: Version = Version::ONE;
@@ -61,10 +62,18 @@ impl Vecs {
});
// Growth rates
let market_cap_growth_rate =
ComputedFromDateLast::forced_import(&db, "market_cap_growth_rate", version, indexes)?;
let realized_cap_growth_rate =
ComputedFromDateLast::forced_import(&db, "realized_cap_growth_rate", version, indexes)?;
let market_cap_growth_rate = ComputedFromDateLast::forced_import(
&db,
"market_cap_growth_rate",
version + Version::ONE,
indexes,
)?;
let realized_cap_growth_rate = ComputedFromDateLast::forced_import(
&db,
"realized_cap_growth_rate",
version + Version::ONE,
indexes,
)?;
let cap_growth_rate_diff = LazyVecFrom2::transformed::<DifferenceF32>(
"cap_growth_rate_diff",
version,