Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 07:09:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-04-27 16:29:21 +02:00
parent 1e38c21f8e
commit 9ae0a57f22
22 changed files with 1643 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
crates/brk_computer/src/storage/vecs/blocks.rs
View File

@@ -7,7 +7,8 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, Version};
use brk_vec::{Compressed, StoredIndex, Version};
use color_eyre::eyre::ContextCompat;
use super::{
EagerVec, Indexes,
@@ -150,10 +151,18 @@ impl Vecs {
self.height_to_interval.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.mut_vec(),
|(height, timestamp, _, height_to_timestamp)| {
indexer_vecs.height_to_timestamp.vec(),
|(height, timestamp, _, height_to_timestamp_iter)| {
let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
let prev_timestamp = height_to_timestamp.double_unwrap_cached_get(prev_h);
let prev_timestamp = height_to_timestamp_iter
.get(prev_h.unwrap_to_usize())
.context("To work")
.inspect_err(|_| {
dbg!(prev_h);
})
.unwrap()
.1
.into_inner();
timestamp
.checked_sub(prev_timestamp)
.unwrap_or(Timestamp::ZERO)

394
crates/brk_computer/src/storage/vecs/grouped/builder.rs
View File

@@ -145,20 +145,22 @@ where
let total_vec = self.total.as_mut().unwrap();
source.iter_from(index, |(i, v, ..)| {
let prev = i
.unwrap_to_usize()
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec
.unwrap_cached_get(I::from(prev_i))
.unwrap_or(T::from(0_usize))
});
let value = v.clone() + prev;
total_vec.forced_push_at(i, value, exit)?;
Ok(())
})?;
source
.into_iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, v)| -> Result<()> {
let v = v.into_inner();
let prev = i
.unwrap_to_usize()
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec
.unwrap_cached_get(I::from(prev_i))
.unwrap_or(T::from(0_usize))
});
let value = v.clone() + prev;
total_vec.forced_push_at(i, value, exit)
})?;
self.safe_flush(exit)?;
@@ -178,114 +180,119 @@ where
{
let index = self.starting_index(max_from);
first_indexes.iter_from(index, |(i, first_index, first_indexes)| {
let last_index = last_indexes.double_unwrap_cached_get(i);
first_indexes
.into_iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, first_index)| -> Result<()> {
let first_index = first_index.into_inner();
if let Some(first) = self.first.as_mut() {
let v = source.double_unwrap_cached_get(first_index);
first.forced_push_at(index, v, exit)?;
}
let last_index = last_indexes.double_unwrap_cached_get(i);
if let Some(last) = self.last.as_mut() {
let v = source.double_unwrap_cached_get(last_index);
last.forced_push_at(index, v, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sorted = self.max.is_some()
|| self._90p.is_some()
|| self._75p.is_some()
|| self.median.is_some()
|| self._25p.is_some()
|| self._10p.is_some()
|| self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
if needs_values {
let mut values = source.collect_inclusive_range(first_index, last_index)?;
if needs_sorted {
values.sort_unstable();
if let Some(max) = self.max.as_mut() {
max.forced_push_at(
i,
values
.last()
.context("expect some")
.inspect_err(|_| {
dbg!(
&values,
max.path(),
first_indexes.path(),
first_index,
last_indexes.path(),
last_index,
source.len(),
source.path()
);
})
.unwrap()
.clone(),
exit,
)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.forced_push_at(i, Self::get_percentile(&values, 0.90), exit)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.forced_push_at(i, Self::get_percentile(&values, 0.75), exit)?;
}
if let Some(median) = self.median.as_mut() {
median.forced_push_at(i, Self::get_percentile(&values, 0.50), exit)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.forced_push_at(i, Self::get_percentile(&values, 0.25), exit)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.forced_push_at(i, Self::get_percentile(&values, 0.10), exit)?;
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
}
if let Some(first) = self.first.as_mut() {
let v = source.double_unwrap_cached_get(first_index);
first.forced_push_at(index, v, exit)?;
}
if needs_average_sum_or_total {
let len = values.len();
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(last) = self.last.as_mut() {
let v = source.double_unwrap_cached_get(last_index);
last.forced_push_at(index, v, exit)?;
}
if let Some(average) = self.average.as_mut() {
let avg = sum.clone() / len;
average.forced_push_at(i, avg, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sorted = self.max.is_some()
|| self._90p.is_some()
|| self._75p.is_some()
|| self.median.is_some()
|| self._25p.is_some()
|| self._10p.is_some()
|| self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
if needs_sum_or_total {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
if needs_values {
let mut values = source.collect_inclusive_range(first_index, last_index)?;
if needs_sorted {
values.sort_unstable();
if let Some(max) = self.max.as_mut() {
max.forced_push_at(
i,
values
.last()
.context("expect some")
.inspect_err(|_| {
dbg!(
&values,
max.path(),
first_indexes.path(),
first_index,
last_indexes.path(),
last_index,
source.len(),
source.path()
);
})
.unwrap()
.clone(),
exit,
)?;
}
if let Some(total_vec) = self.total.as_mut() {
let prev = i
.unwrap_to_usize()
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec.double_unwrap_cached_get(I::from(prev_i))
});
total_vec.forced_push_at(i, prev + sum, exit)?;
if let Some(_90p) = self._90p.as_mut() {
_90p.forced_push_at(i, Self::get_percentile(&values, 0.90), exit)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.forced_push_at(i, Self::get_percentile(&values, 0.75), exit)?;
}
if let Some(median) = self.median.as_mut() {
median.forced_push_at(i, Self::get_percentile(&values, 0.50), exit)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.forced_push_at(i, Self::get_percentile(&values, 0.25), exit)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.forced_push_at(i, Self::get_percentile(&values, 0.10), exit)?;
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
}
}
if needs_average_sum_or_total {
let len = values.len();
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(average) = self.average.as_mut() {
let avg = sum.clone() / len;
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_total {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let prev = i
.unwrap_to_usize()
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec.double_unwrap_cached_get(I::from(prev_i))
});
total_vec.forced_push_at(i, prev + sum, exit)?;
}
}
}
}
}
Ok(())
})?;
Ok(())
})?;
self.safe_flush(exit)?;
@@ -315,99 +322,104 @@ where
let index = self.starting_index(max_from);
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.double_unwrap_cached_get(i);
first_indexes
.into_iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, first_index, ..)| -> Result<()> {
let first_index = first_index.into_inner();
if let Some(first) = self.first.as_mut() {
let v = source
.first
.as_mut()
.unwrap()
.double_unwrap_cached_get(first_index);
first.forced_push_at(index, v, exit)?;
}
let last_index = last_indexes.double_unwrap_cached_get(i);
if let Some(last) = self.last.as_mut() {
let v = source
.last
.as_mut()
.unwrap()
.double_unwrap_cached_get(last_index);
last.forced_push_at(index, v, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sorted = self.max.is_some() || self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
if needs_values {
if needs_sorted {
if let Some(max) = self.max.as_mut() {
let mut values = source
.max
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
values.sort_unstable();
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
}
if let Some(min) = self.min.as_mut() {
let mut values = source
.min
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
values.sort_unstable();
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
}
if let Some(first) = self.first.as_mut() {
let v = source
.first
.as_mut()
.unwrap()
.double_unwrap_cached_get(first_index);
first.forced_push_at(index, v, exit)?;
}
if needs_average_sum_or_total {
if let Some(average) = self.average.as_mut() {
let values = source
.average
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
if let Some(last) = self.last.as_mut() {
let v = source
.last
.as_mut()
.unwrap()
.double_unwrap_cached_get(last_index);
last.forced_push_at(index, v, exit)?;
}
let len = values.len();
let total = values.into_iter().fold(T::from(0), |a, b| a + b);
// TODO: Multiply by count then divide by total
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = total / len;
average.forced_push_at(i, avg, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sorted = self.max.is_some() || self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
if needs_sum_or_total {
let values = source
.sum
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
if needs_values {
if needs_sorted {
if let Some(max) = self.max.as_mut() {
let mut values = source
.max
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
values.sort_unstable();
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let prev = i
.unwrap_to_usize()
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec.double_unwrap_cached_get(I::from(prev_i))
});
total_vec.forced_push_at(i, prev + sum, exit)?;
if let Some(min) = self.min.as_mut() {
let mut values = source
.min
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
values.sort_unstable();
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
}
}
if needs_average_sum_or_total {
if let Some(average) = self.average.as_mut() {
let values = source
.average
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
let len = values.len();
let total = values.into_iter().fold(T::from(0), |a, b| a + b);
// TODO: Multiply by count then divide by total
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = total / len;
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_total {
let values = source
.sum
.as_ref()
.unwrap()
.collect_inclusive_range(first_index, last_index)?;
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let prev = i
.unwrap_to_usize()
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec.double_unwrap_cached_get(I::from(prev_i))
});
total_vec.forced_push_at(i, prev + sum, exit)?;
}
}
}
}
}
Ok(())
})?;
Ok(())
})?;
self.safe_flush(exit)?;

179
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -36,7 +36,19 @@ pub struct Vecs {
pub height_to_difficultyepoch: EagerVec<Height, DifficultyEpoch>,
pub height_to_halvingepoch: EagerVec<Height, HalvingEpoch>,
pub height_to_height: EagerVec<Height, Height>,
pub height_to_last_emptyoutputindex: EagerVec<Height, EmptyOutputIndex>,
pub height_to_last_opreturnindex: EagerVec<Height, OpReturnIndex>,
pub height_to_last_p2aindex: EagerVec<Height, P2AIndex>,
pub height_to_last_p2msindex: EagerVec<Height, P2MSIndex>,
pub height_to_last_p2pk33index: EagerVec<Height, P2PK33Index>,
pub height_to_last_p2pk65index: EagerVec<Height, P2PK65Index>,
pub height_to_last_p2pkhindex: EagerVec<Height, P2PKHIndex>,
pub height_to_last_p2shindex: EagerVec<Height, P2SHIndex>,
pub height_to_last_p2trindex: EagerVec<Height, P2TRIndex>,
pub height_to_last_p2wpkhindex: EagerVec<Height, P2WPKHIndex>,
pub height_to_last_p2wshindex: EagerVec<Height, P2WSHIndex>,
pub height_to_last_txindex: EagerVec<Height, TxIndex>,
pub height_to_last_unknownoutputindex: EagerVec<Height, UnknownOutputIndex>,
pub height_to_timestamp_fixed: EagerVec<Height, Timestamp>,
pub inputindex_to_inputindex: EagerVec<InputIndex, InputIndex>,
pub monthindex_to_first_dateindex: EagerVec<MonthIndex, DateIndex>,
@@ -357,6 +369,66 @@ impl Vecs {
Version::ZERO,
compressed,
)?,
height_to_last_p2aindex: EagerVec::forced_import(
&path.join("height_to_last_p2aindex"),
Version::ZERO,
compressed,
)?,
height_to_last_p2msindex: EagerVec::forced_import(
&path.join("height_to_last_p2msindex"),
Version::ZERO,
compressed,
)?,
height_to_last_p2pk33index: EagerVec::forced_import(
&path.join("height_to_last_p2pk33index"),
Version::ZERO,
compressed,
)?,
height_to_last_p2pk65index: EagerVec::forced_import(
&path.join("height_to_last_p2pk65index"),
Version::ZERO,
compressed,
)?,
height_to_last_p2pkhindex: EagerVec::forced_import(
&path.join("height_to_last_p2pkhindex"),
Version::ZERO,
compressed,
)?,
height_to_last_p2shindex: EagerVec::forced_import(
&path.join("height_to_last_p2shindex"),
Version::ZERO,
compressed,
)?,
height_to_last_p2trindex: EagerVec::forced_import(
&path.join("height_to_last_p2trindex"),
Version::ZERO,
compressed,
)?,
height_to_last_p2wpkhindex: EagerVec::forced_import(
&path.join("height_to_last_p2wpkhindex"),
Version::ZERO,
compressed,
)?,
height_to_last_p2wshindex: EagerVec::forced_import(
&path.join("height_to_last_p2wshindex"),
Version::ZERO,
compressed,
)?,
height_to_last_opreturnindex: EagerVec::forced_import(
&path.join("height_to_last_opreturnindex"),
Version::ZERO,
compressed,
)?,
height_to_last_unknownoutputindex: EagerVec::forced_import(
&path.join("height_to_last_unknownoutputindex"),
Version::ZERO,
compressed,
)?,
height_to_last_emptyoutputindex: EagerVec::forced_import(
&path.join("height_to_last_emptyoutputindex"),
Version::ZERO,
compressed,
)?,
})
}
@@ -469,6 +541,101 @@ impl Vecs {
exit,
)?;
self.height_to_last_p2aindex.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2aindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2msindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2msindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2pk33index
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2pk33index.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2pk65index
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2pk65index.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2pkhindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2pkhindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2shindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2shindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2trindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2trindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2wpkhindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2wpkhindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_p2wshindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_p2wshindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_opreturnindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_opreturnindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_unknownoutputindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_unknownoutputindex.mut_vec(),
height_count,
exit,
)?;
self.height_to_last_emptyoutputindex
.compute_last_index_from_first(
starting_indexes.height,
indexer_vecs.height_to_first_emptyoutputindex.mut_vec(),
height_count,
exit,
)?;
// ---
// InputIndex
// ---
@@ -959,6 +1126,18 @@ impl Vecs {
self.p2aindex_to_p2aindex.any_vec(),
self.unknownoutputindex_to_unknownoutputindex.any_vec(),
self.outputindex_to_outputindex.any_vec(),
self.height_to_last_p2aindex.any_vec(),
self.height_to_last_p2msindex.any_vec(),
self.height_to_last_p2pk33index.any_vec(),
self.height_to_last_p2pk65index.any_vec(),
self.height_to_last_p2pkhindex.any_vec(),
self.height_to_last_p2shindex.any_vec(),
self.height_to_last_p2trindex.any_vec(),
self.height_to_last_p2wpkhindex.any_vec(),
self.height_to_last_p2wshindex.any_vec(),
self.height_to_last_opreturnindex.any_vec(),
self.height_to_last_unknownoutputindex.any_vec(),
self.height_to_last_emptyoutputindex.any_vec(),
]
}
}

13
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -7,7 +7,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{Compressed, Version};
use brk_vec::{Compressed, StoredIndex, Version};
use super::{
EagerVec, Indexes,
@@ -336,13 +336,18 @@ impl Vecs {
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.mut_vec(),
|(h, t, _, height_to_timestamp)| {
|(h, t, _, height_to_timestamp_iter)| {
let ohlc = fetcher
.get_height(
h,
t,
h.decremented()
.map(|prev_h| height_to_timestamp.double_unwrap_cached_get(prev_h)),
h.decremented().map(|prev_h| {
height_to_timestamp_iter
.get(prev_h.unwrap_to_usize())
.unwrap()
.1
.into_inner()
}),
)
.unwrap();
(h, ohlc)

20
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -21,7 +21,7 @@ pub struct Vecs {
pub blocks: blocks::Vecs,
pub indexes: indexes::Vecs,
pub mining: mining::Vecs,
// pub transactions: transactions::Vecs,
pub transactions: transactions::Vecs,
pub marketprice: Option<marketprice::Vecs>,
}
@@ -33,7 +33,7 @@ impl Vecs {
blocks: blocks::Vecs::forced_import(path, compressed)?,
indexes: indexes::Vecs::forced_import(path, compressed)?,
mining: mining::Vecs::forced_import(path, compressed)?,
// transactions: transactions::Vecs::forced_import(path, compressed, fetch)?,
transactions: transactions::Vecs::forced_import(path, compressed, fetch)?,
marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()),
})
}
@@ -63,13 +63,13 @@ impl Vecs {
)?;
}
// self.transactions.compute(
// indexer,
// &mut self.indexes,
// &starting_indexes,
// &mut self.marketprice.as_mut(),
// exit,
// )?;
self.transactions.compute(
indexer,
&mut self.indexes,
&starting_indexes,
&mut self.marketprice.as_mut(),
exit,
)?;
Ok(())
}
@@ -79,7 +79,7 @@ impl Vecs {
self.indexes.as_any_vecs(),
self.blocks.as_any_vecs(),
self.mining.as_any_vecs(),
// self.transactions.as_any_vecs(),
self.transactions.as_any_vecs(),
self.marketprice
.as_ref()
.map_or(vec![], |v| v.as_any_vecs()),

376
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -20,28 +20,40 @@ use super::{
#[derive(Clone)]
pub struct Vecs {
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredU64>,
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
// pub txindex_to_is_v2: LazyVec<Txindex, bool>,
// pub txindex_to_is_v3: LazyVec<Txindex, bool>,
pub indexes_to_coinbase: ComputedValueVecsFromHeight,
pub indexes_to_emptyoutput_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_fee: ComputedValueVecsFromTxindex,
pub indexes_to_feerate: ComputedVecsFromTxindex<Feerate>,
/// Value == 0 when Coinbase
pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_output_value: ComputedVecsFromTxindex<Sats>,
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2pk33_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2pk65_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2pkh_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2sh_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2tr_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU32>,
// pub txindex_to_is_v2: LazyVec<Txindex, bool>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU32>,
// pub txindex_to_is_v3: LazyVec<Txindex, bool>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredUsize>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU32>,
pub inputindex_to_value: EagerVec<InputIndex, Sats>,
pub txindex_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase: EagerVec<TxIndex, bool>,
pub txindex_to_output_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_vsize: EagerVec<TxIndex, StoredUsize>,
pub txindex_to_weight: EagerVec<TxIndex, Weight>,
/// Value == 0 when Coinbase
pub inputindex_to_value: EagerVec<InputIndex, Sats>,
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
pub indexes_to_coinbase: ComputedValueVecsFromHeight,
}
impl Vecs {
@@ -234,6 +246,162 @@ impl Vecs {
.add_average(),
compute_dollars,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
path,
"p2a_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
path,
"p2ms_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
path,
"p2pk33_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
path,
"p2pk65_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
path,
"p2pkh_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
path,
"p2sh_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
path,
"p2tr_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
path,
"p2wpkh_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
path,
"p2wsh_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
path,
"opreturn_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
path,
"unknownoutput_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
path,
"emptyoutput_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
})
}
@@ -356,10 +524,11 @@ impl Vecs {
exit,
)?;
let inputs_len = indexer_vecs.inputindex_to_outputindex.vec().len();
self.inputindex_to_value.compute_transform(
starting_indexes.inputindex,
indexer_vecs.inputindex_to_outputindex.mut_vec(),
|(inputindex, outputindex, slf, other)| {
|(inputindex, outputindex, slf, ..)| {
let value = if outputindex == OutputIndex::COINBASE {
Sats::ZERO
} else if let Some(value) = indexer_vecs
@@ -369,7 +538,7 @@ impl Vecs {
{
value
} else {
dbg!(inputindex, outputindex, slf.len(), other.len());
dbg!(inputindex, outputindex, slf.len(), inputs_len);
panic!()
};
(inputindex, value)
@@ -547,6 +716,181 @@ impl Vecs {
},
)?;
self.indexes_to_p2a_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2aindex.mut_vec(),
indexes.height_to_last_p2aindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2ms_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2msindex.mut_vec(),
indexes.height_to_last_p2msindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2pk33_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2pk33index.mut_vec(),
indexes.height_to_last_p2pk33index.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2pk65_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2pk65index.mut_vec(),
indexes.height_to_last_p2pk65index.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2pkh_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2pkhindex.mut_vec(),
indexes.height_to_last_p2pkhindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2sh_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2shindex.mut_vec(),
indexes.height_to_last_p2shindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2tr_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2trindex.mut_vec(),
indexes.height_to_last_p2trindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2wpkh_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2wpkhindex.mut_vec(),
indexes.height_to_last_p2wpkhindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_p2wsh_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_p2wshindex.mut_vec(),
indexes.height_to_last_p2wshindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_opreturn_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer.mut_vecs().height_to_first_opreturnindex.mut_vec(),
indexes.height_to_last_opreturnindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_unknownoutput_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer
.mut_vecs()
.height_to_first_unknownoutputindex
.mut_vec(),
indexes.height_to_last_unknownoutputindex.mut_vec(),
exit,
)
},
)?;
self.indexes_to_emptyoutput_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, indexes, starting_indexes, exit| {
v.compute_count_from_indexes(
starting_indexes.height,
indexer
.mut_vecs()
.height_to_first_emptyoutputindex
.mut_vec(),
indexes.height_to_last_emptyoutputindex.mut_vec(),
exit,
)
},
)?;
Ok(())
}
@@ -572,6 +916,18 @@ impl Vecs {
self.indexes_to_tx_weight.any_vecs(),
self.txindex_to_input_count.any_vecs(),
self.txindex_to_output_count.any_vecs(),
self.indexes_to_p2a_count.any_vecs(),
self.indexes_to_p2ms_count.any_vecs(),
self.indexes_to_p2pk33_count.any_vecs(),
self.indexes_to_p2pk65_count.any_vecs(),
self.indexes_to_p2pkh_count.any_vecs(),
self.indexes_to_p2sh_count.any_vecs(),
self.indexes_to_p2tr_count.any_vecs(),
self.indexes_to_p2wpkh_count.any_vecs(),
self.indexes_to_p2wsh_count.any_vecs(),
self.indexes_to_opreturn_count.any_vecs(),
self.indexes_to_unknownoutput_count.any_vecs(),
self.indexes_to_emptyoutput_count.any_vecs(),
]
.concat()
}

170
crates/brk_computer/src/storage/vecs/vec/eager.rs
View File

@@ -9,7 +9,8 @@ use std::{
use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, TxIndex};
use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Version,
};
use log::info;
@@ -181,24 +182,28 @@ where
pub fn compute_transform<A, B, F>(
&mut self,
max_from: A,
other: &mut StoredVec<A, B>,
other: &StoredVec<A, B>,
mut t: F,
exit: &Exit,
) -> Result<()>
where
A: StoredIndex,
B: StoredType,
F: FnMut((A, B, &mut Self, &mut dyn DynamicVec<I = A, T = B>)) -> (I, T),
F: FnMut((A, B, &mut Self, &'_ mut StoredVecIterator<'_, A, B>)) -> (I, T),
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.version() + other.version(),
)?;
let index = max_from.min(A::from(self.len()));
other.iter_from(index, |(a, b, other)| {
let (i, v) = t((a, b, self, other));
self.forced_push_at(i, v, exit)
})?;
let mut other_iter = other.iter();
other
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(a, b)| {
let (i, v) = t((a, b.into_inner(), self, &mut other_iter));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
@@ -222,13 +227,17 @@ where
.cached_get_last()?
.map_or_else(T::default, |v| v.into_inner()),
);
other.iter_from(index, |(v, i, ..)| {
if self.unwrap_cached_get(i).is_none_or(|old_v| old_v > v) {
self.forced_push_at(i, v, exit)
} else {
Ok(())
}
})?;
other
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(v, i)| {
let i = i.into_inner();
if self.unwrap_cached_get(i).is_none_or(|old_v| old_v > v) {
self.forced_push_at(i, v, exit)
} else {
Ok(())
}
})?;
self.safe_flush(exit)
}
@@ -249,12 +258,15 @@ where
)?;
let index = max_from.min(T::from(self.len()));
first_indexes.iter_from(index, |(value, first_index, ..)| {
let first_index = (first_index).to_usize()?;
let last_index = (last_indexes.double_unwrap_cached_get(value)).to_usize()?;
(first_index..=last_index)
.try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
})?;
first_indexes
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(value, first_index)| {
let first_index = (first_index).to_usize()?;
let last_index = (last_indexes.double_unwrap_cached_get(value)).to_usize()?;
(first_index..=last_index)
.try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
})?;
self.safe_flush(exit)
}
@@ -276,14 +288,17 @@ where
let index = max_from.min(I::from(self.len()));
let one = T::from(1);
let mut prev_index: Option<I> = None;
first_indexes.iter_from(index, |(index, v, ..)| {
if let Some(prev_index) = prev_index.take() {
let value = v.checked_sub(one).unwrap();
self.forced_push_at(prev_index, value, exit)?;
}
prev_index.replace(index);
Ok(())
})?;
first_indexes
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(index, v)| -> Result<()> {
if let Some(prev_index) = prev_index.take() {
let value = v.checked_sub(one).unwrap();
self.forced_push_at(prev_index, value, exit)?;
}
prev_index.replace(index);
Ok(())
})?;
if let Some(prev_index) = prev_index {
self.forced_push_at(
prev_index,
@@ -370,16 +385,19 @@ where
)?;
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.double_unwrap_cached_get(i);
let range = first_index.unwrap_to_usize()..=last_index.unwrap_to_usize();
let count = if let Some(filter) = filter.as_mut() {
range.into_iter().filter(|i| filter(T2::from(*i))).count()
} else {
range.count()
};
self.forced_push_at(i, T::from(T2::from(count)), exit)
})?;
first_indexes
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, first_index)| {
let last_index = last_indexes.double_unwrap_cached_get(i);
let range = first_index.unwrap_to_usize()..=last_index.unwrap_to_usize();
let count = if let Some(filter) = filter.as_mut() {
range.into_iter().filter(|i| filter(T2::from(*i))).count()
} else {
range.count()
};
self.forced_push_at(i, T::from(T2::from(count)), exit)
})?;
self.safe_flush(exit)
}
@@ -401,13 +419,16 @@ where
)?;
let index = max_from.min(I::from(self.len()));
self_to_other.iter_from(index, |(i, other, ..)| {
self.forced_push_at(
i,
T::from(other_to_self.double_unwrap_cached_get(other) == i),
exit,
)
})?;
self_to_other
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, other)| {
self.forced_push_at(
i,
T::from(other_to_self.double_unwrap_cached_get(other.into_inner()) == i),
exit,
)
})?;
self.safe_flush(exit)
}
@@ -429,15 +450,18 @@ where
)?;
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.double_unwrap_cached_get(i);
let range = first_index.unwrap_to_usize()..=last_index.unwrap_to_usize();
let mut sum = T::from(0_usize);
range.into_iter().for_each(|i| {
sum = sum.clone() + source.double_unwrap_cached_get(T2::from(i));
});
self.forced_push_at(i, sum, exit)
})?;
first_indexes
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, first_index)| {
let last_index = last_indexes.double_unwrap_cached_get(i);
let range = first_index.unwrap_to_usize()..=last_index.unwrap_to_usize();
let mut sum = T::from(0_usize);
range.into_iter().for_each(|i| {
sum = sum.clone() + source.double_unwrap_cached_get(T2::from(i));
});
self.forced_push_at(i, sum, exit)
})?;
self.safe_flush(exit)
}
@@ -458,10 +482,12 @@ where
)?;
let index = max_from.min(I::from(self.len()));
sats.iter_from(index, |(i, sats, ..)| {
let (i, v) = (i, Bitcoin::from(sats));
self.forced_push_at(i, v, exit)
})?;
sats.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, sats)| {
let (i, v) = (i, Bitcoin::from(sats.into_inner()));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
@@ -480,11 +506,14 @@ impl EagerVec<Height, Dollars> {
)?;
let index = max_from.min(Height::from(self.len()));
bitcoin.iter_from(index, |(i, bitcoin, ..)| {
let dollars = price.double_unwrap_cached_get(i);
let (i, v) = (i, *dollars * bitcoin);
self.forced_push_at(i, v, exit)
})?;
bitcoin
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, bitcoin)| {
let dollars = price.double_unwrap_cached_get(i);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
@@ -504,12 +533,15 @@ impl EagerVec<TxIndex, Dollars> {
)?;
let index = max_from.min(TxIndex::from(self.len()));
bitcoin.iter_from(index, |(i, bitcoin, ..)| {
let height = i_to_height.double_unwrap_cached_get(i);
let dollars = price.double_unwrap_cached_get(height);
let (i, v) = (i, *dollars * bitcoin);
self.forced_push_at(i, v, exit)
})?;
bitcoin
.iter()
.skip(index.unwrap_to_usize())
.try_for_each(|(i, bitcoin, ..)| {
let height = i_to_height.double_unwrap_cached_get(i);
let dollars = price.double_unwrap_cached_get(height);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}