global: snapshot

This commit is contained in:
nym21
2026-02-12 22:52:57 +01:00
parent 3bc20a0a46
commit b779edc0d6
60 changed files with 8720 additions and 1504 deletions

View File

@@ -1 +0,0 @@
oracle

View File

@@ -6,7 +6,6 @@ use super::Vecs;
use crate::{indexes, ComputeIndexes};
impl Vecs {
#[allow(unused_variables)]
pub fn compute(
&mut self,
indexer: &Indexer,
@@ -18,18 +17,8 @@ impl Vecs {
self.sats.compute(starting_indexes, &self.usd, exit)?;
// Oracle price computation is slow and still WIP, only run in dev builds
// #[cfg(debug_assertions)]
// {
// use std::time::Instant;
// use tracing::info;
//
// info!("Computing oracle prices...");
// let i = Instant::now();
// self.oracle
// .compute(indexer, indexes, &self.cents, starting_indexes, exit)?;
// info!("Computed oracle prices in {:?}", i.elapsed());
// }
self.oracle
.compute(indexer, indexes, starting_indexes, exit)?;
let _lock = exit.lock();
self.db().compact()?;

View File

@@ -2,12 +2,12 @@ mod compute;
mod fetch;
pub mod cents;
// pub mod oracle;
pub mod oracle;
pub mod sats;
pub mod usd;
pub use cents::Vecs as CentsVecs;
// pub use oracle::Vecs as OracleVecs;
pub use oracle::Vecs as OracleVecs;
pub use sats::Vecs as SatsVecs;
pub use usd::Vecs as UsdVecs;
@@ -33,7 +33,7 @@ pub struct Vecs {
pub cents: CentsVecs,
pub usd: UsdVecs,
pub sats: SatsVecs,
// pub oracle: OracleVecs,
pub oracle: OracleVecs,
}
impl Vecs {
@@ -67,7 +67,7 @@ impl Vecs {
let cents = CentsVecs::forced_import(db, version)?;
let usd = UsdVecs::forced_import(db, version, indexes)?;
let sats = SatsVecs::forced_import(db, version, indexes)?;
// let oracle = OracleVecs::forced_import(db, version)?;
let oracle = OracleVecs::forced_import(db, version)?;
Ok(Self {
db: db.clone(),
@@ -75,7 +75,7 @@ impl Vecs {
cents,
usd,
sats,
// oracle,
oracle,
})
}

View File

@@ -0,0 +1,273 @@
use std::ops::Range;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_oracle::{Config, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
use brk_types::{
CentsUnsigned, Close, DateIndex, Height, High, Low, OHLCCentsUnsigned, Open, OutputType, Sats,
TxIndex, TxOutIndex,
};
use tracing::info;
use vecdb::{
AnyStoredVec, AnyVec, Exit, GenericStoredVec, IterableVec, TypedVecIterator, VecIndex,
VecIterator,
};
use super::Vecs;
use crate::{ComputeIndexes, indexes};
impl Vecs {
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.compute_prices(indexer, starting_indexes, exit)?;
self.compute_daily_ohlc(indexes, starting_indexes, exit)?;
Ok(())
}
fn compute_prices(
&mut self,
indexer: &Indexer,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version =
indexer.vecs.outputs.value.version() + indexer.vecs.outputs.outputtype.version();
self.price_cents
.validate_computed_version_or_reset(source_version)?;
let total_heights = indexer.vecs.blocks.timestamp.len();
if total_heights <= START_HEIGHT {
return Ok(());
}
// Reorg: truncate to starting_indexes
let truncate_to = self
.price_cents
.len()
.min(starting_indexes.height.to_usize());
self.price_cents.truncate_if_needed_at(truncate_to)?;
if self.price_cents.len() < START_HEIGHT {
for line in brk_oracle::PRICES.lines().skip(self.price_cents.len()) {
if self.price_cents.len() >= START_HEIGHT {
break;
}
let dollars: f64 = line.parse().unwrap_or(0.0);
let cents = (dollars * 100.0).round() as u64;
self.price_cents.push(CentsUnsigned::new(cents));
}
}
if self.price_cents.len() >= total_heights {
return Ok(());
}
let config = Config::default();
let committed = self.price_cents.len();
let prev_cents = self.price_cents
.iter()?
.get(Height::from(committed - 1))
.unwrap();
let seed_bin = cents_to_bin(prev_cents.inner() as f64);
let warmup = config.window_size.min(committed - START_HEIGHT);
let mut oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
Self::feed_blocks(o, indexer, (committed - warmup)..committed);
});
let num_new = total_heights - committed;
info!(
"Computing oracle prices: {} to {} ({warmup} warmup)",
committed, total_heights
);
let ref_bins = Self::feed_blocks(&mut oracle, indexer, committed..total_heights);
for (i, ref_bin) in ref_bins.into_iter().enumerate() {
self.price_cents.push(CentsUnsigned::new(bin_to_cents(ref_bin)));
let progress = ((i + 1) * 100 / num_new) as u8;
if i > 0 && progress > ((i * 100 / num_new) as u8) {
info!("Oracle price computation: {}%", progress);
}
}
{
let _lock = exit.lock();
self.price_cents.write()?;
}
info!(
"Oracle prices complete: {} committed",
self.price_cents.len()
);
Ok(())
}
/// Returns an Oracle seeded from the last committed price, with the last
/// window_size blocks already processed. Ready for additional blocks (e.g. mempool).
pub fn live_oracle(&self, indexer: &Indexer) -> Result<Oracle> {
let config = Config::default();
let height = indexer.vecs.blocks.timestamp.len();
let last_cents = self.price_cents
.iter()?
.get(Height::from(self.price_cents.len() - 1))
.unwrap();
let seed_bin = cents_to_bin(last_cents.inner() as f64);
let window_size = config.window_size;
let oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
Self::feed_blocks(o, indexer, height.saturating_sub(window_size)..height);
});
Ok(oracle)
}
/// Feed a range of blocks from the indexer into an Oracle (skipping coinbase),
/// returning per-block ref_bin values.
fn feed_blocks(oracle: &mut Oracle, indexer: &Indexer, range: Range<usize>) -> Vec<f64> {
let total_txs = indexer.vecs.transactions.height.len();
let total_outputs = indexer.vecs.outputs.value.len();
let mut first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter();
let mut first_txoutindex_iter = indexer.vecs.transactions.first_txoutindex.into_iter();
let mut out_first_iter = indexer.vecs.outputs.first_txoutindex.into_iter();
let mut value_iter = indexer.vecs.outputs.value.into_iter();
let mut outputtype_iter = indexer.vecs.outputs.outputtype.into_iter();
let mut block_outputs: Vec<(Sats, OutputType)> = Vec::new();
let mut ref_bins = Vec::with_capacity(range.len());
for h in range {
let first_txindex: TxIndex = first_txindex_iter.get_at_unwrap(h);
let next_first_txindex = first_txindex_iter
.get_at(h + 1)
.unwrap_or(TxIndex::from(total_txs));
let out_start = if first_txindex.to_usize() + 1 < next_first_txindex.to_usize() {
first_txoutindex_iter
.get_at_unwrap(first_txindex.to_usize() + 1)
.to_usize()
} else {
out_first_iter
.get_at(h + 1)
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let out_end = out_first_iter
.get_at(h + 1)
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
block_outputs.clear();
for i in out_start..out_end {
block_outputs.push((
value_iter.get_at_unwrap(i),
outputtype_iter.get_at_unwrap(i),
));
}
ref_bins.push(oracle.process_outputs(block_outputs.iter().copied()));
}
ref_bins
}
fn compute_daily_ohlc(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let last_dateindex = DateIndex::from(indexes.dateindex.date.len());
let start_dateindex = starting_indexes
.dateindex
.min(DateIndex::from(self.ohlc_cents.len()));
if start_dateindex >= last_dateindex {
return Ok(());
}
let last_height = Height::from(self.price_cents.len());
let mut height_to_price_iter = self.price_cents.iter()?;
let mut dateindex_to_first_height_iter = indexes.dateindex.first_height.iter();
let mut height_count_iter = indexes.dateindex.height_count.iter();
for dateindex_usize in start_dateindex.to_usize()..last_dateindex.to_usize() {
let dateindex = DateIndex::from(dateindex_usize);
let first_height = dateindex_to_first_height_iter.get_unwrap(dateindex);
let count = height_count_iter.get_unwrap(dateindex);
if *count == 0 || first_height >= last_height {
self.ohlc_cents
.truncate_push(dateindex, self.previous_ohlc(dateindex)?)?;
continue;
}
let count = *count as usize;
let mut open = None;
let mut high = CentsUnsigned::ZERO;
let mut low = CentsUnsigned::MAX;
let mut close = CentsUnsigned::ZERO;
for i in 0..count {
let height = first_height + Height::from(i);
if height >= last_height {
break;
}
if let Some(price) = height_to_price_iter.get(height) {
if price == CentsUnsigned::ZERO {
continue;
}
if open.is_none() {
open = Some(price);
}
if price > high {
high = price;
}
if price < low {
low = price;
}
close = price;
}
}
let ohlc = if let Some(open_price) = open {
OHLCCentsUnsigned {
open: Open::new(open_price),
high: High::new(high),
low: Low::new(low),
close: Close::new(close),
}
} else {
self.previous_ohlc(dateindex)?
};
self.ohlc_cents.truncate_push(dateindex, ohlc)?;
}
{
let _lock = exit.lock();
self.ohlc_cents.write()?;
}
Ok(())
}
fn previous_ohlc(&self, dateindex: DateIndex) -> Result<OHLCCentsUnsigned> {
Ok(if dateindex > DateIndex::from(0usize) {
self.ohlc_cents
.iter()?
.get(dateindex.decremented().unwrap())
.unwrap_or_default()
} else {
OHLCCentsUnsigned::default()
})
}
}

View File

@@ -0,0 +1,30 @@
use brk_error::Result;
use brk_types::{DateIndex, OHLCCentsUnsigned, OHLCDollars, Version};
use vecdb::{BytesVec, Database, ImportableVec, IterableCloneableVec, LazyVecFrom1, PcoVec};
use super::Vecs;
impl Vecs {
pub fn forced_import(db: &Database, parent_version: Version) -> Result<Self> {
let version = parent_version + Version::new(10);
let price_cents = PcoVec::forced_import(db, "oracle_price_cents", version)?;
let ohlc_cents = BytesVec::forced_import(db, "oracle_ohlc_cents", version)?;
let ohlc_dollars = LazyVecFrom1::init(
"oracle_ohlc_dollars",
version,
ohlc_cents.boxed_clone(),
|di: DateIndex, iter| {
iter.get(di)
.map(|o: OHLCCentsUnsigned| OHLCDollars::from(o))
},
);
Ok(Self {
price_cents,
ohlc_cents,
ohlc_dollars,
})
}
}

View File

@@ -0,0 +1,10 @@
use brk_traversable::Traversable;
use brk_types::{CentsUnsigned, DateIndex, Height, OHLCCentsUnsigned, OHLCDollars};
use vecdb::{BytesVec, LazyVecFrom1, PcoVec};
#[derive(Clone, Traversable)]
pub struct Vecs {
pub price_cents: PcoVec<Height, CentsUnsigned>,
pub ohlc_cents: BytesVec<DateIndex, OHLCCentsUnsigned>,
pub ohlc_dollars: LazyVecFrom1<DateIndex, OHLCDollars, DateIndex, OHLCCentsUnsigned>,
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,20 +0,0 @@
use brk_types::Version;
use super::Vecs;
use crate::{
distribution,
internal::{DollarsIdentity, LazyValueFromHeightLast, SatsIdentity},
};
impl Vecs {
pub fn import(version: Version, distribution: &distribution::Vecs) -> Self {
let supply_metrics = &distribution.utxo_cohorts.all.metrics.supply;
Self(LazyValueFromHeightLast::from_block_source::<
SatsIdentity,
DollarsIdentity,
>(
"circulating_supply", &supply_metrics.total, version)
)
}
}

View File

@@ -1,4 +0,0 @@
mod import;
mod vecs;
pub use vecs::Vecs;

View File

@@ -1,8 +0,0 @@
use brk_traversable::Traversable;
use derive_more::{Deref, DerefMut};
use crate::internal::LazyValueFromHeightLast;
/// Circulating supply - lazy references to distribution's actual supply (KISS)
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct Vecs(pub LazyValueFromHeightLast);

View File

@@ -2,7 +2,7 @@ use brk_error::Result;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, distribution, indexes, scripts, transactions, ComputeIndexes};
use crate::{ComputeIndexes, blocks, distribution, indexes, scripts, transactions};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -20,15 +20,60 @@ impl Vecs {
self.burned
.compute(indexes, scripts, blocks, starting_indexes, exit)?;
// 2. Compute inflation rate
self.inflation
.compute(blocks, distribution, starting_indexes, exit)?;
// 2. Compute inflation rate: daily_subsidy / circulating_supply * 365 * 100
let circulating_supply = &distribution.utxo_cohorts.all.metrics.supply.total.sats;
self.inflation.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
&blocks.rewards.subsidy.sats.dateindex.sum_cum.sum.0,
&circulating_supply.dateindex.0,
|(i, subsidy_1d_sum, supply, ..)| {
let inflation = if *supply > 0 {
365.0 * *subsidy_1d_sum as f64 / *supply as f64 * 100.0
} else {
0.0
};
(i, inflation.into())
},
exit,
)?;
Ok(())
})?;
// 3. Compute velocity
self.velocity
.compute(transactions, distribution, starting_indexes, exit)?;
// Note: circulating and market_cap are lazy - no compute needed
// 4. Compute cap growth rates
if let Some(market_cap) = self.market_cap.as_ref() {
let mcap_dateindex = &market_cap.dateindex.0;
self.market_cap_growth_rate
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage_change(
starting_indexes.dateindex,
mcap_dateindex,
30,
exit,
)?;
Ok(())
})?;
}
if let Some(realized) = distribution.utxo_cohorts.all.metrics.realized.as_ref() {
let rcap_dateindex = &realized.realized_cap.dateindex.0;
self.realized_cap_growth_rate
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage_change(
starting_indexes.dateindex,
rcap_dateindex,
30,
exit,
)?;
Ok(())
})?;
}
// Note: circulating, market_cap, cap_growth_rate_diff are lazy
let _lock = exit.lock();
self.db.compact()?;

View File

@@ -3,12 +3,18 @@ use std::path::Path;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{Database, PAGE_SIZE};
use vecdb::{Database, IterableCloneableVec, LazyVecFrom2, PAGE_SIZE};
use super::Vecs;
use crate::{distribution, indexes, price};
use crate::{
distribution, indexes, price,
internal::{
ComputedFromDateAverage, ComputedFromDateLast, DifferenceF32, DollarsIdentity,
LazyFromHeightLast, LazyValueFromHeightLast, SatsIdentity,
},
};
const VERSION: Version = Version::ZERO;
const VERSION: Version = Version::ONE;
impl Vecs {
pub fn forced_import(
@@ -24,21 +30,47 @@ impl Vecs {
let version = parent_version + VERSION;
let compute_dollars = price.is_some();
let supply_metrics = &distribution.utxo_cohorts.all.metrics.supply;
// Circulating supply - lazy refs to distribution
let circulating = super::circulating::Vecs::import(version, distribution);
let circulating = LazyValueFromHeightLast::from_block_source::<SatsIdentity, DollarsIdentity>(
"circulating_supply",
&supply_metrics.total,
version,
);
// Burned/unspendable supply - computed from scripts
let burned = super::burned::Vecs::forced_import(&db, version, indexes, price)?;
// Inflation rate
let inflation = super::inflation::Vecs::forced_import(&db, version, indexes)?;
let inflation =
ComputedFromDateAverage::forced_import(&db, "inflation_rate", version, indexes)?;
// Velocity
let velocity =
super::velocity::Vecs::forced_import(&db, version, indexes, compute_dollars)?;
// Market cap - lazy refs to supply in USD
let market_cap = super::market_cap::Vecs::import(version, distribution);
// Market cap - lazy identity from distribution supply in USD
let market_cap = supply_metrics.total.dollars.as_ref().map(|d| {
LazyFromHeightLast::from_lazy_binary_computed::<DollarsIdentity, _, _>(
"market_cap",
version,
d.height.boxed_clone(),
d,
)
});
// Growth rates
let market_cap_growth_rate =
ComputedFromDateLast::forced_import(&db, "market_cap_growth_rate", version, indexes)?;
let realized_cap_growth_rate =
ComputedFromDateLast::forced_import(&db, "realized_cap_growth_rate", version, indexes)?;
let cap_growth_rate_diff = LazyVecFrom2::transformed::<DifferenceF32>(
"cap_growth_rate_diff",
version,
market_cap_growth_rate.dateindex.boxed_clone(),
realized_cap_growth_rate.dateindex.boxed_clone(),
);
let this = Self {
db,
@@ -47,6 +79,9 @@ impl Vecs {
inflation,
velocity,
market_cap,
market_cap_growth_rate,
realized_cap_growth_rate,
cap_growth_rate_diff,
};
this.db.retain_regions(

View File

@@ -1,38 +0,0 @@
use brk_error::Result;
use vecdb::Exit;
use super::Vecs;
use crate::{ComputeIndexes, blocks, distribution};
impl Vecs {
pub fn compute(
&mut self,
blocks: &blocks::Vecs,
distribution: &distribution::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
// inflation = daily_subsidy / circulating_supply * 365 * 100
let circulating_supply = &distribution.utxo_cohorts.all.metrics.supply.total.sats;
self.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
&blocks.rewards.subsidy.sats.dateindex.sum_cum.sum.0,
&circulating_supply.dateindex.0,
|(i, subsidy_1d_sum, supply, ..)| {
let inflation = if *supply > 0 {
365.0 * *subsidy_1d_sum as f64 / *supply as f64 * 100.0
} else {
0.0
};
(i, inflation.into())
},
exit,
)?;
Ok(())
})?;
Ok(())
}
}

View File

@@ -1,17 +0,0 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::Database;
use super::Vecs;
use crate::{indexes, internal::ComputedFromDateAverage};
impl Vecs {
pub fn forced_import(db: &Database, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
Ok(Self(ComputedFromDateAverage::forced_import(
db,
"inflation_rate",
version,
indexes,
)?))
}
}

View File

@@ -1,10 +0,0 @@
use brk_traversable::Traversable;
use brk_types::StoredF32;
use derive_more::{Deref, DerefMut};
use crate::internal::ComputedFromDateAverage;
/// Inflation rate metrics
#[derive(Clone, Deref, DerefMut, Traversable)]
#[traversable(transparent)]
pub struct Vecs(pub ComputedFromDateAverage<StoredF32>);

View File

@@ -1,23 +0,0 @@
use brk_types::Version;
use vecdb::IterableCloneableVec;
use super::Vecs;
use crate::{
distribution,
internal::{DollarsIdentity, LazyFromHeightLast},
};
impl Vecs {
pub fn import(version: Version, distribution: &distribution::Vecs) -> Option<Self> {
let supply_metrics = &distribution.utxo_cohorts.all.metrics.supply;
supply_metrics.total.dollars.as_ref().map(|d| {
Self(LazyFromHeightLast::from_lazy_binary_computed::<DollarsIdentity, _, _>(
"market_cap",
version,
d.height.boxed_clone(),
d,
))
})
}
}

View File

@@ -1,4 +0,0 @@
mod import;
mod vecs;
pub use vecs::Vecs;

View File

@@ -1,8 +0,0 @@
use brk_traversable::Traversable;
use brk_types::Dollars;
use derive_more::{Deref, DerefMut};
use crate::internal::LazyFromHeightLast;
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct Vecs(pub LazyFromHeightLast<Dollars>);

View File

@@ -1,7 +1,4 @@
pub mod burned;
pub mod circulating;
pub mod inflation;
pub mod market_cap;
pub mod velocity;
mod compute;

View File

@@ -1,24 +1,24 @@
use brk_traversable::Traversable;
use vecdb::Database;
use brk_types::{DateIndex, Dollars, StoredF32};
use vecdb::{Database, LazyVecFrom2};
use super::{burned, circulating, inflation, market_cap, velocity};
use super::{burned, velocity};
use crate::internal::{
ComputedFromDateAverage, ComputedFromDateLast, LazyFromHeightLast, LazyValueFromHeightLast,
};
/// Supply metrics module
///
/// This module owns all supply-related metrics:
/// - circulating: Lazy references to distribution's actual circulating supply
/// - burned: Cumulative opreturn and unspendable supply
/// - inflation: Inflation rate derived from supply
/// - velocity: BTC and USD velocity metrics
/// - market_cap: Lazy references to supply in USD (circulating * price)
#[derive(Clone, Traversable)]
pub struct Vecs {
#[traversable(skip)]
pub(crate) db: Database,
pub circulating: circulating::Vecs,
pub circulating: LazyValueFromHeightLast,
pub burned: burned::Vecs,
pub inflation: inflation::Vecs,
pub inflation: ComputedFromDateAverage<StoredF32>,
pub velocity: velocity::Vecs,
pub market_cap: Option<market_cap::Vecs>,
pub market_cap: Option<LazyFromHeightLast<Dollars>>,
pub market_cap_growth_rate: ComputedFromDateLast<StoredF32>,
pub realized_cap_growth_rate: ComputedFromDateLast<StoredF32>,
pub cap_growth_rate_diff:
LazyVecFrom2<DateIndex, StoredF32, DateIndex, StoredF32, DateIndex, StoredF32>,
}