Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-26 15:49:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: MASSIVE snapshot

Browse Source
This commit is contained in:
nym21
2026-02-23 17:22:12 +01:00
parent be0d749f9c
commit 3b7aa8242a
703 changed files with 29130 additions and 30779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
crates/brk_computer/src/internal/multi/tx_derived/distribution.rs
View File

@@ -1,39 +1,45 @@
//! TxDerivedDistribution - computes TxIndex data to height Distribution + dateindex MinMaxAverage + lazy aggregations.
//!
//! Note: Percentiles are computed at height level only. DateIndex and coarser
//! periods only have average+min+max since computing percentiles across all
//! transactions per day would be expensive.
//! TxDerivedDistribution - computes TxIndex data to height Distribution + lazy time periods + epochs.
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{DateIndex, DifficultyEpoch, Height, TxIndex, Version};
use derive_more::{Deref, DerefMut};
use brk_types::{
Day1, Day3, DifficultyEpoch, HalvingEpoch, Height, Hour1, Hour12, Hour4, Minute1, Minute10,
Minute30, Minute5, Month1, Month3, Month6, TxIndex, Version, Week1, Year1, Year10,
};
use schemars::JsonSchema;
use vecdb::{CollectableVec, Database, Exit, IterableCloneableVec};
use vecdb::{Database, Exit, ReadableCloneableVec, ReadableVec, Rw, StorageMode};
use crate::{
ComputeIndexes, indexes,
internal::{
ComputedVecValue, Distribution, LazyDateDerivedSpread, LazySpread, MinMaxAverage,
NumericValue,
},
internal::{ComputedVecValue, Distribution, LazyDistribution, NumericValue},
};
#[derive(Clone, Deref, DerefMut, Traversable)]
#[derive(Traversable)]
#[traversable(merge)]
pub struct TxDerivedDistribution<T>
pub struct TxDerivedDistribution<T, M: StorageMode = Rw>
where
T: ComputedVecValue + PartialOrd + JsonSchema,
{
pub height: Distribution<Height, T>,
pub difficultyepoch: LazySpread<DifficultyEpoch, T, Height, DifficultyEpoch>,
pub dateindex: MinMaxAverage<DateIndex, T>,
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub dates: LazyDateDerivedSpread<T>,
pub height: Distribution<Height, T, M>,
pub minute1: LazyDistribution<Minute1, T, Height, Height>,
pub minute5: LazyDistribution<Minute5, T, Height, Height>,
pub minute10: LazyDistribution<Minute10, T, Height, Height>,
pub minute30: LazyDistribution<Minute30, T, Height, Height>,
pub hour1: LazyDistribution<Hour1, T, Height, Height>,
pub hour4: LazyDistribution<Hour4, T, Height, Height>,
pub hour12: LazyDistribution<Hour12, T, Height, Height>,
pub day1: LazyDistribution<Day1, T, Height, Height>,
pub day3: LazyDistribution<Day3, T, Height, Height>,
pub week1: LazyDistribution<Week1, T, Height, Height>,
pub month1: LazyDistribution<Month1, T, Height, Height>,
pub month3: LazyDistribution<Month3, T, Height, Height>,
pub month6: LazyDistribution<Month6, T, Height, Height>,
pub year1: LazyDistribution<Year1, T, Height, Height>,
pub year10: LazyDistribution<Year10, T, Height, Height>,
pub halvingepoch: LazyDistribution<HalvingEpoch, T, Height, HalvingEpoch>,
pub difficultyepoch: LazyDistribution<DifficultyEpoch, T, Height, DifficultyEpoch>,
}
const VERSION: Version = Version::ZERO;
@@ -42,49 +48,83 @@ impl<T> TxDerivedDistribution<T>
where
T: NumericValue + JsonSchema,
{
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let height = Distribution::forced_import(db, name, version + VERSION)?;
let dateindex = MinMaxAverage::forced_import(db, name, version + VERSION)?;
let v = version + VERSION;
let difficultyepoch =
LazySpread::<DifficultyEpoch, T, Height, DifficultyEpoch>::from_distribution(
name,
v,
height.boxed_average(),
height.boxed_min(),
height.boxed_max(),
indexes.difficultyepoch.identity.boxed_clone(),
);
macro_rules! period {
($idx:ident) => {
LazyDistribution::from_height_source(
name,
v,
height.boxed_average(),
indexes.$idx.first_height.read_only_boxed_clone(),
)
};
}
let dates = LazyDateDerivedSpread::from_sources(
name,
v,
dateindex.boxed_average(),
dateindex.boxed_min(),
dateindex.boxed_max(),
indexes,
);
macro_rules! epoch {
($idx:ident) => {
LazyDistribution::from_source(
name,
v,
height.boxed_average(),
indexes.$idx.identity.read_only_boxed_clone(),
)
};
}
let minute1 = period!(minute1);
let minute5 = period!(minute5);
let minute10 = period!(minute10);
let minute30 = period!(minute30);
let hour1 = period!(hour1);
let hour4 = period!(hour4);
let hour12 = period!(hour12);
let day1 = period!(day1);
let day3 = period!(day3);
let week1 = period!(week1);
let month1 = period!(month1);
let month3 = period!(month3);
let month6 = period!(month6);
let year1 = period!(year1);
let year10 = period!(year10);
let halvingepoch = epoch!(halvingepoch);
let difficultyepoch = epoch!(difficultyepoch);
Ok(Self {
height,
minute1,
minute5,
minute10,
minute30,
hour1,
hour4,
hour12,
day1,
day3,
week1,
month1,
month3,
month6,
year1,
year10,
halvingepoch,
difficultyepoch,
dateindex,
dates,
})
}
pub fn derive_from(
pub(crate) fn derive_from(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
txindex_source: &impl CollectableVec<TxIndex, T>,
txindex_source: &impl ReadableVec<TxIndex, T>,
exit: &Exit,
) -> Result<()> {
self.derive_from_with_skip(indexer, indexes, starting_indexes, txindex_source, exit, 0)
@@ -93,12 +133,12 @@ where
/// Derive from source, skipping first N transactions per block from all calculations.
///
/// Use `skip_count: 1` to exclude coinbase transactions from fee/feerate stats.
pub fn derive_from_with_skip(
pub(crate) fn derive_from_with_skip(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
txindex_source: &impl CollectableVec<TxIndex, T>,
txindex_source: &impl ReadableVec<TxIndex, T>,
exit: &Exit,
skip_count: usize,
) -> Result<()> {
@@ -111,14 +151,6 @@ where
skip_count,
)?;
self.dateindex.compute(
starting_indexes.dateindex,
&self.height.average().0,
&indexes.dateindex.first_height,
&indexes.dateindex.height_count,
exit,
)?;
Ok(())
}
}

143
crates/brk_computer/src/internal/multi/tx_derived/full.rs
View File

@@ -1,33 +1,46 @@
//! TxDerivedFull - aggregates from TxIndex to height Full + dateindex Stats + lazy date periods.
//! TxDerivedFull - aggregates from TxIndex to height Full + lazy time periods + epochs.
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{DateIndex, DifficultyEpoch, Height, TxIndex, Version};
use derive_more::{Deref, DerefMut};
use brk_types::{
Day1, Day3, DifficultyEpoch, HalvingEpoch, Height, Hour1, Hour12, Hour4, Minute1, Minute10,
Minute30, Minute5, Month1, Month3, Month6, TxIndex, Version, Week1, Year1, Year10,
};
use schemars::JsonSchema;
use vecdb::{CollectableVec, Database, Exit, IterableCloneableVec};
use vecdb::{Database, Exit, ReadableCloneableVec, ReadableVec, Rw, StorageMode};
use crate::{
indexes, ComputeIndexes,
internal::{ComputedVecValue, LazyDateDerivedFull, Full, LazyFull, NumericValue, Stats},
internal::{ComputedVecValue, Full, LazyFull, NumericValue},
};
/// Aggregates from TxIndex to height/dateindex with full stats.
#[derive(Clone, Deref, DerefMut, Traversable)]
/// Aggregates from TxIndex to height/time periods with full stats.
#[derive(Traversable)]
#[traversable(merge)]
pub struct TxDerivedFull<T>
pub struct TxDerivedFull<T, M: StorageMode = Rw>
where
T: ComputedVecValue + PartialOrd + JsonSchema,
{
pub height: Full<Height, T>,
pub height: Full<Height, T, M>,
pub minute1: LazyFull<Minute1, T, Height, Height>,
pub minute5: LazyFull<Minute5, T, Height, Height>,
pub minute10: LazyFull<Minute10, T, Height, Height>,
pub minute30: LazyFull<Minute30, T, Height, Height>,
pub hour1: LazyFull<Hour1, T, Height, Height>,
pub hour4: LazyFull<Hour4, T, Height, Height>,
pub hour12: LazyFull<Hour12, T, Height, Height>,
pub day1: LazyFull<Day1, T, Height, Height>,
pub day3: LazyFull<Day3, T, Height, Height>,
pub week1: LazyFull<Week1, T, Height, Height>,
pub month1: LazyFull<Month1, T, Height, Height>,
pub month3: LazyFull<Month3, T, Height, Height>,
pub month6: LazyFull<Month6, T, Height, Height>,
pub year1: LazyFull<Year1, T, Height, Height>,
pub year10: LazyFull<Year10, T, Height, Height>,
pub halvingepoch: LazyFull<HalvingEpoch, T, Height, HalvingEpoch>,
pub difficultyepoch: LazyFull<DifficultyEpoch, T, Height, DifficultyEpoch>,
pub dateindex: Stats<DateIndex, T>,
#[deref]
#[deref_mut]
#[traversable(flatten)]
pub dates: LazyDateDerivedFull<T>,
}
const VERSION: Version = Version::ONE;
@@ -36,53 +49,89 @@ impl<T> TxDerivedFull<T>
where
T: NumericValue + JsonSchema,
{
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let height = Full::forced_import(db, name, version + VERSION)?;
let dateindex = Stats::forced_import(db, name, version + VERSION)?;
let v = version + VERSION;
let difficultyepoch =
LazyFull::<DifficultyEpoch, T, Height, DifficultyEpoch>::from_stats_aggregate(
name,
v,
height.boxed_average(),
height.boxed_min(),
height.boxed_max(),
height.boxed_sum(),
height.boxed_cumulative(),
indexes.difficultyepoch.identity.boxed_clone(),
);
macro_rules! period {
($idx:ident) => {
LazyFull::from_height_source(
name,
v,
height.boxed_sum(),
height.boxed_cumulative(),
indexes.$idx.first_height.read_only_boxed_clone(),
)
};
}
let dates = LazyDateDerivedFull::from_sources(
name,
v,
dateindex.boxed_average(),
dateindex.boxed_min(),
dateindex.boxed_max(),
dateindex.boxed_sum(),
dateindex.boxed_cumulative(),
indexes,
);
macro_rules! epoch {
($idx:ident) => {
LazyFull::from_stats_aggregate(
name,
v,
height.boxed_average(),
height.boxed_min(),
height.boxed_max(),
height.boxed_sum(),
height.boxed_cumulative(),
height.boxed_average(),
indexes.$idx.identity.read_only_boxed_clone(),
)
};
}
let minute1 = period!(minute1);
let minute5 = period!(minute5);
let minute10 = period!(minute10);
let minute30 = period!(minute30);
let hour1 = period!(hour1);
let hour4 = period!(hour4);
let hour12 = period!(hour12);
let day1 = period!(day1);
let day3 = period!(day3);
let week1 = period!(week1);
let month1 = period!(month1);
let month3 = period!(month3);
let month6 = period!(month6);
let year1 = period!(year1);
let year10 = period!(year10);
let halvingepoch = epoch!(halvingepoch);
let difficultyepoch = epoch!(difficultyepoch);
Ok(Self {
height,
minute1,
minute5,
minute10,
minute30,
hour1,
hour4,
hour12,
day1,
day3,
week1,
month1,
month3,
month6,
year1,
year10,
halvingepoch,
difficultyepoch,
dateindex,
dates,
})
}
pub fn derive_from(
pub(crate) fn derive_from(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
txindex_source: &impl CollectableVec<TxIndex, T>,
txindex_source: &impl ReadableVec<TxIndex, T>,
exit: &Exit,
) -> Result<()> {
self.derive_from_with_skip(indexer, indexes, starting_indexes, txindex_source, exit, 0)
@@ -91,12 +140,12 @@ where
/// Derive from source, skipping first N transactions per block from all calculations.
///
/// Use `skip_count: 1` to exclude coinbase transactions from fee/feerate stats.
pub fn derive_from_with_skip(
pub(crate) fn derive_from_with_skip(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
txindex_source: &impl CollectableVec<TxIndex, T>,
txindex_source: &impl ReadableVec<TxIndex, T>,
exit: &Exit,
skip_count: usize,
) -> Result<()> {
@@ -109,14 +158,6 @@ where
skip_count,
)?;
self.dateindex.compute(
starting_indexes.dateindex,
&self.height.sum().0,
&indexes.dateindex.first_height,
&indexes.dateindex.height_count,
exit,
)?;
Ok(())
}
}

83
crates/brk_computer/src/internal/multi/tx_derived/lazy_full.rs
View File

@@ -2,13 +2,13 @@
use brk_traversable::Traversable;
use brk_types::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
Day1, Day3, DifficultyEpoch, HalvingEpoch, Height, Hour1, Hour12, Hour4, Minute1, Minute10,
Minute30, Minute5, Month1, Month3, Month6, Version, Week1, Year1, Year10,
};
use schemars::JsonSchema;
use vecdb::{IterableCloneableVec, UnaryTransform};
use vecdb::{ReadableCloneableVec, UnaryTransform};
use crate::internal::{ComputedVecValue, TxDerivedFull, LazyTransformFull, LazyTransformStats};
use crate::internal::{ComputedVecValue, TxDerivedFull, LazyTransformFull};
#[derive(Clone, Traversable)]
#[traversable(merge)]
@@ -18,14 +18,23 @@ where
S1T: ComputedVecValue,
{
pub height: LazyTransformFull<Height, T, S1T>,
pub difficultyepoch: LazyTransformStats<DifficultyEpoch, T, S1T>,
pub dateindex: LazyTransformStats<DateIndex, T, S1T>,
pub weekindex: LazyTransformStats<WeekIndex, T, S1T>,
pub monthindex: LazyTransformStats<MonthIndex, T, S1T>,
pub quarterindex: LazyTransformStats<QuarterIndex, T, S1T>,
pub semesterindex: LazyTransformStats<SemesterIndex, T, S1T>,
pub yearindex: LazyTransformStats<YearIndex, T, S1T>,
pub decadeindex: LazyTransformStats<DecadeIndex, T, S1T>,
pub minute1: LazyTransformFull<Minute1, T, S1T>,
pub minute5: LazyTransformFull<Minute5, T, S1T>,
pub minute10: LazyTransformFull<Minute10, T, S1T>,
pub minute30: LazyTransformFull<Minute30, T, S1T>,
pub hour1: LazyTransformFull<Hour1, T, S1T>,
pub hour4: LazyTransformFull<Hour4, T, S1T>,
pub hour12: LazyTransformFull<Hour12, T, S1T>,
pub day1: LazyTransformFull<Day1, T, S1T>,
pub day3: LazyTransformFull<Day3, T, S1T>,
pub week1: LazyTransformFull<Week1, T, S1T>,
pub month1: LazyTransformFull<Month1, T, S1T>,
pub month3: LazyTransformFull<Month3, T, S1T>,
pub month6: LazyTransformFull<Month6, T, S1T>,
pub year1: LazyTransformFull<Year1, T, S1T>,
pub year10: LazyTransformFull<Year10, T, S1T>,
pub halvingepoch: LazyTransformFull<HalvingEpoch, T, S1T>,
pub difficultyepoch: LazyTransformFull<DifficultyEpoch, T, S1T>,
}
const VERSION: Version = Version::ZERO;
@@ -35,7 +44,7 @@ where
T: ComputedVecValue + JsonSchema + 'static,
S1T: ComputedVecValue + JsonSchema,
{
pub fn from_computed<F: UnaryTransform<S1T, T>>(
pub(crate) fn from_computed<F: UnaryTransform<S1T, T>>(
name: &str,
version: Version,
source: &TxDerivedFull<S1T>,
@@ -44,32 +53,42 @@ where
macro_rules! period {
($p:ident) => {
LazyTransformStats::from_boxed::<F>(
name, v,
source.$p.average.boxed_clone(), source.$p.min.boxed_clone(),
source.$p.max.boxed_clone(), source.$p.sum.boxed_clone(),
source.$p.cumulative.boxed_clone(),
LazyTransformFull::from_boxed::<F>(
name,
v,
source.$p.average.read_only_boxed_clone(),
source.$p.min.read_only_boxed_clone(),
source.$p.max.read_only_boxed_clone(),
source.$p.percentiles.pct10.read_only_boxed_clone(),
source.$p.percentiles.pct25.read_only_boxed_clone(),
source.$p.percentiles.median.read_only_boxed_clone(),
source.$p.percentiles.pct75.read_only_boxed_clone(),
source.$p.percentiles.pct90.read_only_boxed_clone(),
source.$p.sum.read_only_boxed_clone(),
source.$p.cumulative.read_only_boxed_clone(),
)
};
}
Self {
height: LazyTransformFull::from_stats_aggregate::<F>(name, v, &source.height),
minute1: period!(minute1),
minute5: period!(minute5),
minute10: period!(minute10),
minute30: period!(minute30),
hour1: period!(hour1),
hour4: period!(hour4),
hour12: period!(hour12),
day1: period!(day1),
day3: period!(day3),
week1: period!(week1),
month1: period!(month1),
month3: period!(month3),
month6: period!(month6),
year1: period!(year1),
year10: period!(year10),
halvingepoch: period!(halvingepoch),
difficultyepoch: period!(difficultyepoch),
dateindex: LazyTransformStats::from_boxed::<F>(
name, v,
source.dateindex.boxed_average(),
source.dateindex.boxed_min(),
source.dateindex.boxed_max(),
source.dateindex.boxed_sum(),
source.dateindex.boxed_cumulative(),
),
weekindex: period!(weekindex),
monthindex: period!(monthindex),
quarterindex: period!(quarterindex),
semesterindex: period!(semesterindex),
yearindex: period!(yearindex),
decadeindex: period!(decadeindex),
}
}
}

71
crates/brk_computer/src/internal/multi/tx_derived/value_full.rs
View File

@@ -4,82 +4,67 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Sats, TxIndex, Version};
use vecdb::{CollectableVec, Database, Exit, IterableCloneableVec};
use vecdb::{Database, Exit, ReadableCloneableVec, ReadableVec, Rw, StorageMode};
use crate::{
ComputeIndexes, indexes,
internal::{TxDerivedFull, ValueDollarsFromTxFull, LazyTxDerivedFull, SatsToBitcoin},
price,
internal::{LazyTxDerivedFull, SatsToBitcoin, TxDerivedFull, ValueDollarsFromTxFull},
prices,
};
#[derive(Clone, Traversable)]
pub struct ValueTxDerivedFull {
pub sats: TxDerivedFull<Sats>,
pub bitcoin: LazyTxDerivedFull<Bitcoin, Sats>,
pub dollars: Option<ValueDollarsFromTxFull>,
#[derive(Traversable)]
pub struct ValueTxDerivedFull<M: StorageMode = Rw> {
pub sats: TxDerivedFull<Sats, M>,
pub btc: LazyTxDerivedFull<Bitcoin, Sats>,
pub usd: ValueDollarsFromTxFull<M>,
}
const VERSION: Version = Version::ZERO;
impl ValueTxDerivedFull {
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
indexer: &Indexer,
price: Option<&price::Vecs>,
sats_txindex: &impl IterableCloneableVec<TxIndex, Sats>,
prices: &prices::Vecs,
sats_txindex: &impl ReadableCloneableVec<TxIndex, Sats>,
) -> Result<Self> {
let v = version + VERSION;
let sats = TxDerivedFull::forced_import(db, name, v, indexes)?;
let bitcoin =
let btc =
LazyTxDerivedFull::from_computed::<SatsToBitcoin>(&format!("{name}_btc"), v, &sats);
let dollars = price
.map(|price| {
ValueDollarsFromTxFull::forced_import(
db,
&format!("{name}_usd"),
v,
indexes,
&sats.height,
price.usd.split.close.height.boxed_clone(),
sats_txindex.boxed_clone(),
indexer.vecs.transactions.height.boxed_clone(),
)
})
.transpose()?;
let usd = ValueDollarsFromTxFull::forced_import(
db,
&format!("{name}_usd"),
v,
indexes,
&sats.height,
prices.usd.price.read_only_boxed_clone(),
sats_txindex.read_only_boxed_clone(),
indexer.vecs.transactions.height.read_only_boxed_clone(),
)?;
Ok(Self {
sats,
bitcoin,
dollars,
btc,
usd,
})
}
pub fn derive_from(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
txindex_source: &impl CollectableVec<TxIndex, Sats>,
exit: &Exit,
) -> Result<()> {
self.derive_from_with_skip(indexer, indexes, starting_indexes, txindex_source, exit, 0)
}
/// Derive from source, skipping first N transactions per block from all calculations.
///
/// Use `skip_count: 1` to exclude coinbase transactions from fee/feerate stats.
pub fn derive_from_with_skip(
pub(crate) fn derive_from_with_skip(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
txindex_source: &impl CollectableVec<TxIndex, Sats>,
txindex_source: &impl ReadableVec<TxIndex, Sats>,
exit: &Exit,
skip_count: usize,
) -> Result<()> {
@@ -92,9 +77,7 @@ impl ValueTxDerivedFull {
skip_count,
)?;
if let Some(dollars) = self.dollars.as_mut() {
dollars.derive_from(indexer, indexes, starting_indexes, exit)?;
}
self.usd.derive_from(indexes, starting_indexes, exit)?;
Ok(())
}