global: utxos part 5

This commit is contained in:
nym21
2025-05-22 19:04:55 +02:00
parent 5893376279
commit e002a61a19
38 changed files with 3047 additions and 1623 deletions
+14 -14
View File
@@ -18,18 +18,18 @@ where
I: StoredIndex,
T: ComputedType,
{
first: Option<Box<EagerVec<I, T>>>,
average: Option<Box<EagerVec<I, T>>>,
sum: Option<Box<EagerVec<I, T>>>,
max: Option<Box<EagerVec<I, T>>>,
_90p: Option<Box<EagerVec<I, T>>>,
_75p: Option<Box<EagerVec<I, T>>>,
median: Option<Box<EagerVec<I, T>>>,
_25p: Option<Box<EagerVec<I, T>>>,
_10p: Option<Box<EagerVec<I, T>>>,
min: Option<Box<EagerVec<I, T>>>,
last: Option<Box<EagerVec<I, T>>>,
total: Option<Box<EagerVec<I, T>>>,
pub first: Option<Box<EagerVec<I, T>>>,
pub average: Option<Box<EagerVec<I, T>>>,
pub sum: Option<Box<EagerVec<I, T>>>,
pub max: Option<Box<EagerVec<I, T>>>,
pub _90p: Option<Box<EagerVec<I, T>>>,
pub _75p: Option<Box<EagerVec<I, T>>>,
pub median: Option<Box<EagerVec<I, T>>>,
pub _25p: Option<Box<EagerVec<I, T>>>,
pub _10p: Option<Box<EagerVec<I, T>>>,
pub min: Option<Box<EagerVec<I, T>>>,
pub last: Option<Box<EagerVec<I, T>>>,
pub total: Option<Box<EagerVec<I, T>>>,
}
const VERSION: Version = Version::ZERO;
@@ -537,7 +537,7 @@ where
Ok(())
}
fn starting_index(&self, max_from: I) -> I {
pub fn starting_index(&self, max_from: I) -> I {
max_from.min(I::from(
self.vecs().into_iter().map(|v| v.len()).min().unwrap(),
))
@@ -671,7 +671,7 @@ where
Ok(())
}
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
if let Some(first) = self.first.as_mut() {
first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
}
@@ -1,16 +1,17 @@
use std::path::Path;
use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, TxIndex, WeekIndex,
YearIndex,
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, TxIndex, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
AnyCollectableVec, AnyVec, CollectableVec, Compressed, EagerVec, Result, StoredIndex,
VecIterator, Version,
};
use crate::vecs::{Indexes, indexes};
use crate::vecs::{Indexes, fetched, indexes};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
@@ -77,37 +78,37 @@ where
})
}
#[allow(unused)]
pub fn compute_all<F>(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
) -> color_eyre::Result<()>
where
F: FnMut(
&mut EagerVec<TxIndex, T>,
&Indexer,
&indexes::Vecs,
&Indexes,
&Exit,
) -> Result<()>,
{
compute(
self.txindex.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
// #[allow(unused)]
// pub fn compute_all<F>(
// &mut self,
// indexer: &Indexer,
// indexes: &indexes::Vecs,
// starting_indexes: &Indexes,
// exit: &Exit,
// mut compute: F,
// ) -> color_eyre::Result<()>
// where
// F: FnMut(
// &mut EagerVec<TxIndex, T>,
// &Indexer,
// &indexes::Vecs,
// &Indexes,
// &Exit,
// ) -> Result<()>,
// {
// compute(
// self.txindex.as_mut().unwrap(),
// indexer,
// indexes,
// starting_indexes,
// exit,
// )?;
let txindex: Option<&StoredVec<TxIndex, T>> = None;
self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
// let txindex: Option<&StoredVec<TxIndex, T>> = None;
// self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
Ok(())
}
// Ok(())
// }
pub fn compute_rest(
&mut self,
@@ -116,7 +117,7 @@ where
starting_indexes: &Indexes,
exit: &Exit,
txindex: Option<&impl CollectableVec<TxIndex, T>>,
) -> color_eyre::Result<()> {
) -> Result<()> {
if let Some(txindex) = txindex {
self.height.compute(
starting_indexes.height,
@@ -137,6 +138,15 @@ where
)?;
}
self.compute_after_height(indexes, starting_indexes, exit)
}
fn compute_after_height(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.dateindex.from_aligned(
starting_indexes.dateindex,
&self.height,
@@ -216,3 +226,362 @@ where
.collect::<Vec<_>>()
}
}
impl ComputedVecsFromTxindex<Bitcoin> {
pub fn compute_rest_from_sats(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
sats: &ComputedVecsFromTxindex<Sats>,
txindex: Option<&impl CollectableVec<TxIndex, Bitcoin>>,
) -> Result<()> {
let txindex_version = if let Some(txindex) = txindex {
txindex.version()
} else {
self.txindex.as_ref().unwrap().as_ref().version()
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);
(starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len())
.map(Height::from)
.try_for_each(|height| -> Result<()> {
if let Some(first) = self.height.first.as_mut() {
first.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_first()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(average) = self.height.average.as_mut() {
average.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_average()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(sum) = self.height.sum.as_mut() {
sum.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_sum()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(max) = self.height.max.as_mut() {
max.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_max()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(_90p) = self.height._90p.as_mut() {
_90p.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_90p()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(_75p) = self.height._75p.as_mut() {
_75p.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_75p()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(median) = self.height.median.as_mut() {
median.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_median()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(_25p) = self.height._25p.as_mut() {
_25p.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_25p()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(_10p) = self.height._10p.as_mut() {
_10p.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_10p()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(min) = self.height.min.as_mut() {
min.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_min()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(last) = self.height.last.as_mut() {
last.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_last()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
if let Some(total) = self.height.total.as_mut() {
total.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_total()
.into_iter()
.unwrap_get_inner(height),
),
exit,
)?;
}
Ok(())
})?;
self.height.safe_flush(exit)?;
self.compute_after_height(indexes, starting_indexes, exit)
}
}
impl ComputedVecsFromTxindex<Dollars> {
pub fn compute_rest_from_bitcoin(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
bitcoin: &ComputedVecsFromTxindex<Bitcoin>,
txindex: Option<&impl CollectableVec<TxIndex, Dollars>>,
fetched: &fetched::Vecs,
) -> Result<()> {
let txindex_version = if let Some(txindex) = txindex {
txindex.version()
} else {
self.txindex.as_ref().unwrap().as_ref().version()
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);
let mut close_iter = fetched.chainindexes_to_close.height.into_iter();
(starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len())
.map(Height::from)
.try_for_each(|height| -> Result<()> {
let price = *close_iter.unwrap_get_inner(height);
if let Some(first) = self.height.first.as_mut() {
first.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_first()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(average) = self.height.average.as_mut() {
average.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_average()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(sum) = self.height.sum.as_mut() {
sum.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_sum()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(max) = self.height.max.as_mut() {
max.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_max()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(_90p) = self.height._90p.as_mut() {
_90p.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_90p()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(_75p) = self.height._75p.as_mut() {
_75p.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_75p()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(median) = self.height.median.as_mut() {
median.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_median()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(_25p) = self.height._25p.as_mut() {
_25p.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_25p()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(_10p) = self.height._10p.as_mut() {
_10p.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_10p()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(min) = self.height.min.as_mut() {
min.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_min()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(last) = self.height.last.as_mut() {
last.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_last()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
if let Some(total) = self.height.total.as_mut() {
total.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_total()
.into_iter()
.unwrap_get_inner(height),
exit,
)?;
}
Ok(())
})?;
self.height.safe_flush(exit)?;
self.compute_after_height(indexes, starting_indexes, exit)
}
}
@@ -180,6 +180,7 @@ impl ComputedValueVecsFromTxindex {
starting_indexes: &Indexes,
exit: &Exit,
txindex: Option<&impl CollectableVec<TxIndex, Sats>>,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<()> {
if let Some(txindex) = txindex {
self.sats
@@ -190,11 +191,12 @@ impl ComputedValueVecsFromTxindex {
.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
}
self.bitcoin.compute_rest(
self.bitcoin.compute_rest_from_sats(
indexer,
indexes,
starting_indexes,
exit,
&self.sats,
Some(&self.bitcoin_txindex),
)?;
@@ -203,12 +205,14 @@ impl ComputedValueVecsFromTxindex {
dollars_txindex.compute_if_necessary(starting_indexes.txindex, exit)?;
dollars.compute_rest(
dollars.compute_rest_from_bitcoin(
indexer,
indexes,
starting_indexes,
exit,
&self.bitcoin,
Some(dollars_txindex),
fetched.as_ref().unwrap(),
)?;
}
@@ -784,6 +784,7 @@ impl Vecs {
starting_indexes,
exit,
Some(&self.txindex_to_fee),
fetched,
)?;
self.indexes_to_feerate.compute_rest(
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,266 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, Result, VecIterator, Version,
};
use crate::{
states::CohortState,
vecs::{
Indexes, fetched,
grouped::{
ComputedValueVecsFromHeight, ComputedVecsFromHeight, StorableVecGeneatorOptions,
},
indexes,
},
};
const VERSION: Version = Version::ZERO;
pub struct Vecs {
starting_height: Height,
pub state: CohortState,
pub height_to_realized_cap: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_supply: EagerVec<Height, Sats>,
pub indexes_to_supply: ComputedValueVecsFromHeight,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
impl Vecs {
pub fn forced_import(
path: &Path,
cohort_name: Option<&str>,
_computation: Computation,
compressed: Compressed,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
fs::create_dir_all(path)?;
// let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}"));
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
Ok(Self {
starting_height: Height::ZERO,
state: CohortState::default(),
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("realized_cap"),
VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_cap"),
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply: EagerVec::forced_import(
path,
&suffix("supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_supply: ComputedValueVecsFromHeight::forced_import(
path,
&suffix("supply"),
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_utxo_count: EagerVec::forced_import(
path,
&suffix("utxo_count"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
&suffix("utxo_count"),
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
})
}
pub fn init(&mut self, starting_indexes: &Indexes) -> Height {
self.starting_height = [
self.height_to_supply.len(),
self.height_to_utxo_count.len(),
self.height_to_realized_cap
.as_ref()
.map_or(usize::MAX, |v| v.len()),
]
.into_iter()
.map(Height::from)
.min()
.unwrap()
.min(starting_indexes.height);
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
if let Some(prev_height) = self.starting_height.checked_sub(Height::new(1)) {
self.state.supply.value = self
.height_to_supply
.into_iter()
.unwrap_get_inner(prev_height);
self.state.supply.utxos = *self
.height_to_utxo_count
.into_iter()
.unwrap_get_inner(prev_height);
self.state.realized_cap = Some(
height_to_realized_cap
.into_iter()
.unwrap_get_inner(prev_height),
);
} else {
self.state.realized_cap = Some(Dollars::ZERO);
}
}
self.starting_height
}
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.height_to_supply
.validate_computed_version_or_reset_file(
base_version + self.height_to_supply.inner_version(),
)?;
self.height_to_utxo_count
.validate_computed_version_or_reset_file(
base_version + self.height_to_utxo_count.inner_version(),
)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() {
height_to_realized_cap.validate_computed_version_or_reset_file(
base_version + height_to_realized_cap.inner_version(),
)?;
}
Ok(())
}
pub fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.height_to_supply
.forced_push_at(height, self.state.supply.value, exit)?;
self.height_to_utxo_count.forced_push_at(
height,
StoredUsize::from(self.state.supply.utxos),
exit,
)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.forced_push_at(
height,
self.state.realized_cap.unwrap(),
exit,
)?;
}
Ok(())
}
pub fn safe_flush_height_vecs(&mut self, exit: &Exit) -> Result<()> {
self.height_to_supply.safe_flush(exit)?;
self.height_to_utxo_count.safe_flush(exit)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.safe_flush(exit)?;
}
Ok(())
}
pub fn compute_rest(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.indexes_to_supply.compute_rest(
indexer,
indexes,
fetched,
starting_indexes,
exit,
Some(&self.height_to_supply),
)?;
self.indexes_to_utxo_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.height_to_utxo_count),
)?;
if let Some(indexes_to_realized_cap) = self.indexes_to_realized_cap.as_mut() {
indexes_to_realized_cap.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_realized_cap.as_ref().unwrap()),
)?;
}
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![
&self.height_to_supply as &dyn AnyCollectableVec,
&self.height_to_utxo_count,
],
self.height_to_realized_cap
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_supply.vecs(),
self.indexes_to_utxo_count.vecs(),
self.indexes_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
impl Clone for Vecs {
fn clone(&self) -> Self {
Self {
starting_height: self.starting_height,
state: CohortState::default(),
height_to_realized_cap: self.height_to_realized_cap.clone(),
indexes_to_realized_cap: self.indexes_to_realized_cap.clone(),
height_to_supply: self.height_to_supply.clone(),
indexes_to_supply: self.indexes_to_supply.clone(),
height_to_utxo_count: self.height_to_utxo_count.clone(),
indexes_to_utxo_count: self.indexes_to_utxo_count.clone(),
}
}
}
File diff suppressed because it is too large Load Diff