indexer: speed

This commit is contained in:
nym21
2025-10-19 21:18:15 +02:00
parent 71078b5bdd
commit e9f6295014
27 changed files with 248 additions and 288 deletions

View File

@@ -5,8 +5,8 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Bitcoin, CheckedSub, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, FeeRate, HalvingEpoch,
Height, TxInIndex, MonthIndex, ONE_DAY_IN_SEC_F64, TxOutIndex, QuarterIndex, Sats,
SemesterIndex, StoredBool, StoredF32, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex,
Height, MonthIndex, ONE_DAY_IN_SEC_F64, QuarterIndex, Sats, SemesterIndex, StoredBool,
StoredF32, StoredF64, StoredU32, StoredU64, Timestamp, TxInIndex, TxIndex, TxOutIndex,
TxVersion, Version, WeekIndex, Weight, YearIndex,
};
use brk_traversable::Traversable;
@@ -98,8 +98,7 @@ pub struct Vecs {
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredU64>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU64>,
pub txinindex_to_value:
LazyVecFrom2<TxInIndex, Sats, TxInIndex, TxOutIndex, TxOutIndex, Sats>,
pub txinindex_to_value: LazyVecFrom2<TxInIndex, Sats, TxInIndex, TxOutIndex, TxOutIndex, Sats>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase: LazyVecFrom2<TxIndex, StoredBool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredU64>,
@@ -165,21 +164,21 @@ impl Vecs {
indexer.vecs.txinindex_to_txoutindex.boxed_clone(),
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index: TxInIndex, txinindex_to_txoutindex_iter, txoutindex_to_value_iter| {
txinindex_to_txoutindex_iter
.next_at(index.unwrap_to_usize())
.map(|(txinindex, txoutindex)| {
txinindex_to_txoutindex_iter.next_at(index.to_usize()).map(
|(txinindex, txoutindex)| {
let txoutindex = txoutindex.into_owned();
if txoutindex == TxOutIndex::COINBASE {
Sats::ZERO
} else if let Some((_, value)) =
txoutindex_to_value_iter.next_at(txoutindex.unwrap_to_usize())
txoutindex_to_value_iter.next_at(txoutindex.to_usize())
{
value.into_owned()
} else {
dbg!(txinindex, txoutindex);
panic!()
}
})
},
)
},
);
@@ -189,7 +188,7 @@ impl Vecs {
indexer.vecs.txindex_to_base_size.boxed_clone(),
indexer.vecs.txindex_to_total_size.boxed_clone(),
|index: TxIndex, txindex_to_base_size_iter, txindex_to_total_size_iter| {
let index = index.unwrap_to_usize();
let index = index.to_usize();
txindex_to_base_size_iter
.next_at(index)
.map(|(_, base_size)| {
@@ -213,7 +212,7 @@ impl Vecs {
version + Version::ZERO,
txindex_to_weight.boxed_clone(),
|index: TxIndex, iter| {
let index = index.unwrap_to_usize();
let index = index.to_usize();
iter.next_at(index).map(|(_, weight)| {
StoredU64::from(
bitcoin::Weight::from(weight.into_owned()).to_vbytes_ceil() as usize
@@ -229,11 +228,11 @@ impl Vecs {
indexer.vecs.height_to_first_txindex.boxed_clone(),
|index: TxIndex, txindex_to_height_iter, height_to_first_txindex_iter| {
txindex_to_height_iter
.next_at(index.unwrap_to_usize())
.next_at(index.to_usize())
.map(|(_, height)| {
let height = height.into_owned();
let txindex = height_to_first_txindex_iter
.next_at(height.unwrap_to_usize())
.next_at(height.to_usize())
.unwrap()
.1
.into_owned();
@@ -252,7 +251,7 @@ impl Vecs {
txindex_to_first_txinindex_iter,
txindex_to_input_count_iter,
txinindex_to_value_iter| {
let txindex = index.unwrap_to_usize();
let txindex = index.to_usize();
txindex_to_first_txinindex_iter
.next_at(txindex)
.map(|(_, first_index)| {
@@ -299,7 +298,7 @@ impl Vecs {
txindex_to_first_txoutindex_iter,
txindex_to_output_count_iter,
txoutindex_to_value_iter| {
let txindex = index.unwrap_to_usize();
let txindex = index.to_usize();
txindex_to_first_txoutindex_iter
.next_at(txindex)
.map(|(_, first_index)| {
@@ -1539,14 +1538,15 @@ impl Vecs {
|(height, txindex, ..)| {
let first_txoutindex = txindex_to_first_txoutindex_iter
.unwrap_get_inner(txindex)
.unwrap_to_usize();
.to_usize();
let output_count = txindex_to_output_count_iter.unwrap_get_inner(txindex);
let mut sats = Sats::ZERO;
(first_txoutindex..first_txoutindex + usize::from(output_count))
.for_each(|txoutindex| {
(first_txoutindex..first_txoutindex + usize::from(output_count)).for_each(
|txoutindex| {
sats += txoutindex_to_value_iter
.unwrap_get_inner(TxOutIndex::from(txoutindex));
});
},
);
(height, sats)
},
exit,
@@ -1624,8 +1624,7 @@ impl Vecs {
self.indexes_to_subsidy.sats.height.as_ref().unwrap(),
|(height, subsidy, ..)| {
let halving = HalvingEpoch::from(height);
let expected =
Sats::FIFTY_BTC / 2_usize.pow(halving.unwrap_to_usize() as u32);
let expected = Sats::FIFTY_BTC / 2_usize.pow(halving.to_usize() as u32);
(height, expected.checked_sub(subsidy).unwrap())
},
exit,

View File

@@ -105,7 +105,7 @@ impl Vecs {
.try_for_each(|(i, v)| -> Result<()> {
let d = v.into_owned();
if prev.is_none() {
let i = i.unwrap_to_usize();
let i = i.to_usize();
prev.replace(if i > 0 {
self.dateindex_to_price_ohlc_in_cents
.into_iter()
@@ -115,8 +115,7 @@ impl Vecs {
});
}
let ohlc = if i.unwrap_to_usize() + 100
>= self.dateindex_to_price_ohlc_in_cents.len()
let ohlc = if i.to_usize() + 100 >= self.dateindex_to_price_ohlc_in_cents.len()
&& let Ok(mut ohlc) = self.fetcher.get_date(d)
{
let prev_open = *prev.as_ref().unwrap().close;

View File

@@ -68,7 +68,7 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
source
@@ -95,7 +95,7 @@ where
),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
source
@@ -114,7 +114,7 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
S1I::inclusive_range_from(i, source.len())
@@ -133,7 +133,7 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
S1I::inclusive_range_from(i, source.len())
@@ -152,7 +152,7 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
let vec = S1I::inclusive_range_from(i, source.len())
@@ -182,7 +182,7 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
let vec = S1I::inclusive_range_from(i, source.len())
@@ -204,7 +204,7 @@ where
source_extra.cumulative.as_ref().unwrap().boxed_clone(),
len_source.clone(),
|i: I, source, len_source| {
if i.unwrap_to_usize() >= len_source.len() {
if i.to_usize() >= len_source.len() {
return None;
}
source

View File

@@ -249,7 +249,7 @@ impl ComputedVecsFromTxindex<Bitcoin> {
let starting_index = self.height.starting_index(starting_indexes.height);
(starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len())
(starting_index.to_usize()..indexer.vecs.height_to_weight.len())
.map(Height::from)
.try_for_each(|height| -> Result<()> {
if let Some(first) = self.height.first.as_mut() {
@@ -430,7 +430,7 @@ impl ComputedVecsFromTxindex<Dollars> {
let mut close_iter = price.chainindexes_to_price_close.height.into_iter();
(starting_index.unwrap_to_usize()..indexer.vecs.height_to_weight.len())
(starting_index.to_usize()..indexer.vecs.height_to_weight.len())
.map(Height::from)
.try_for_each(|height| -> Result<()> {
let price = *close_iter.unwrap_get_inner(height);

View File

@@ -384,8 +384,8 @@ impl ComputedRatioVecsFromDateIndex {
.min(starting_indexes.dateindex);
let mut sorted = self.ratio.dateindex.as_ref().unwrap().collect_range(
Some(min_ratio_date.unwrap_to_usize()),
Some(starting_dateindex.unwrap_to_usize()),
Some(min_ratio_date.to_usize()),
Some(starting_dateindex.to_usize()),
);
sorted.sort_unstable();

View File

@@ -477,8 +477,8 @@ impl ComputedStandardDeviationVecsFromDateIndex {
.min(starting_indexes.dateindex);
let mut sorted = source.collect_range(
Some(min_date.unwrap_to_usize()),
Some(starting_dateindex.unwrap_to_usize()),
Some(min_date.to_usize()),
Some(starting_dateindex.to_usize()),
);
sorted.sort_unstable();
@@ -551,8 +551,7 @@ impl ComputedStandardDeviationVecsFromDateIndex {
let avg = sma_iter.unwrap_get_inner(index);
let population =
index.checked_sub(min_date).unwrap().unwrap_to_usize() as f32 + 1.0;
let population = index.checked_sub(min_date).unwrap().to_usize() as f32 + 1.0;
let sd = StoredF32::from(
(sorted.iter().map(|v| (**v - *avg).powi(2)).sum::<f32>() / population)

View File

@@ -58,7 +58,7 @@ impl ComputedValueVecsFromTxindex {
version + VERSION,
source_vec.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s),
|txindex: TxIndex, iter| {
iter.next_at(txindex.unwrap_to_usize()).map(|(_, value)| {
iter.next_at(txindex.to_usize()).map(|(_, value)| {
let sats = value.into_owned();
Bitcoin::from(sats)
})
@@ -85,7 +85,7 @@ impl ComputedValueVecsFromTxindex {
txindex_to_btc_iter,
txindex_to_height_iter,
height_to_price_close_iter| {
let txindex = txindex.unwrap_to_usize();
let txindex = txindex.to_usize();
txindex_to_btc_iter.next_at(txindex).and_then(|(_, value)| {
let btc = value.into_owned();
txindex_to_height_iter
@@ -93,7 +93,7 @@ impl ComputedValueVecsFromTxindex {
.and_then(|(_, value)| {
let height = value.into_owned();
height_to_price_close_iter
.next_at(height.unwrap_to_usize())
.next_at(height.to_usize())
.map(|(_, close)| *close.into_owned() * btc)
})
})

View File

@@ -4,11 +4,11 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
TxInIndex, MonthIndex, OpReturnIndex, TxOutIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex,
P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex,
P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex,
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64,
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
MonthIndex, OpReturnIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex,
P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex,
P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, P2WPKHBytes, P2WSHAddressIndex,
P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64, Timestamp, TxInIndex, TxIndex,
TxOutIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_traversable::Traversable;
use vecdb::{
@@ -130,7 +130,7 @@ impl Vecs {
indexer.vecs.txindex_to_first_txinindex.boxed_clone(),
indexer.vecs.txinindex_to_txoutindex.boxed_clone(),
|index: TxIndex, txindex_to_first_txinindex_iter, txinindex_to_txoutindex_iter| {
let txindex = index.unwrap_to_usize();
let txindex = index.to_usize();
txindex_to_first_txinindex_iter
.next_at(txindex)
.map(|(_, start)| {
@@ -150,7 +150,7 @@ impl Vecs {
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index: TxIndex, txindex_to_first_txoutindex_iter, txoutindex_to_value_iter| {
let txindex = index.unwrap_to_usize();
let txindex = index.to_usize();
txindex_to_first_txoutindex_iter
.next_at(txindex)
.map(|(_, start)| {

View File

@@ -1589,7 +1589,7 @@ impl Vecs {
self.indexes_to_price_ath.dateindex.as_ref().unwrap(),
|(i, ath, slf)| {
if prev.is_none() {
let i = i.unwrap_to_usize();
let i = i.to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
} else {
@@ -1620,7 +1620,7 @@ impl Vecs {
.unwrap(),
|(i, days, slf)| {
if prev.is_none() {
let i = i.unwrap_to_usize();
let i = i.to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
} else {

View File

@@ -4,7 +4,7 @@ use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_store::AnyStore;
use brk_structs::{Address, AddressBytes, Height, TxOutIndex, OutputType, PoolId, Pools, pools};
use brk_structs::{Address, AddressBytes, Height, OutputType, PoolId, Pools, TxOutIndex, pools};
use brk_traversable::Traversable;
use rayon::prelude::*;
use vecdb::{
@@ -122,8 +122,7 @@ impl Vecs {
)?;
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.txindex_to_first_txoutindex.iter();
let mut txindex_to_first_txoutindex_iter = indexer.vecs.txindex_to_first_txoutindex.iter();
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_outputtype_iter = indexer.vecs.txoutindex_to_outputtype.iter();
let mut txoutindex_to_typeindex_iter = indexer.vecs.txoutindex_to_typeindex.iter();
@@ -147,7 +146,7 @@ impl Vecs {
let min = starting_indexes
.height
.unwrap_to_usize()
.to_usize()
.min(self.height_to_pool.len());
indexer
@@ -163,8 +162,7 @@ impl Vecs {
let pool = (*txoutindex..(*txoutindex + *outputcount))
.map(TxOutIndex::from)
.find_map(|txoutindex| {
let outputtype =
txoutindex_to_outputtype_iter.unwrap_get_inner(txoutindex);
let outputtype = txoutindex_to_outputtype_iter.unwrap_get_inner(txoutindex);
let typeindex = txoutindex_to_typeindex_iter.unwrap_get_inner(txoutindex);
match outputtype {

View File

@@ -352,7 +352,7 @@ impl Vecs {
self.indexes_to_blocks_mined.dateindex.unwrap_cumulative(),
|(i, sum, cumulative, slf)| {
if prev.is_none() {
let i = i.unwrap_to_usize();
let i = i.to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
} else {

View File

@@ -11,10 +11,10 @@ use brk_grouper::{ByAddressType, ByAnyAddress, Filtered};
use brk_indexer::Indexer;
use brk_structs::{
AnyAddressDataIndexEnum, AnyAddressIndex, CheckedSub, DateIndex, Dollars, EmptyAddressData,
EmptyAddressIndex, Height, TxInIndex, LoadedAddressData, LoadedAddressIndex, TxOutIndex,
OutputType, P2AAddressIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex,
P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, Sats, StoredU64,
Timestamp, TypeIndex, Version,
EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, OutputType, P2AAddressIndex,
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
P2WPKHAddressIndex, P2WSHAddressIndex, Sats, StoredU64, Timestamp, TxInIndex, TxOutIndex,
TypeIndex, Version,
};
use brk_traversable::Traversable;
use log::info;
@@ -201,7 +201,7 @@ impl Vecs {
.unwrap()
.boxed_clone(),
|height: Height, iter| {
iter.next_at(height.unwrap_to_usize()).map(|(_, value)| {
iter.next_at(height.to_usize()).map(|(_, value)| {
let d: Dollars = value.into_owned();
d
})
@@ -874,7 +874,7 @@ impl Vecs {
.unwrap_or_default(),
);
(height.unwrap_to_usize()..height_to_date_fixed.len())
(height.to_usize()..height_to_date_fixed.len())
.map(Height::from)
.try_for_each(|_height| -> Result<()> {
height = _height;
@@ -895,10 +895,10 @@ impl Vecs {
.map(|i| *i.unwrap_get_inner(height));
let first_txoutindex = height_to_first_txoutindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
.to_usize();
let first_txinindex = height_to_first_txinindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
.to_usize();
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
@@ -1268,7 +1268,7 @@ impl Vecs {
)
})?;
if height != last_height && height != Height::ZERO && height.unwrap_to_usize() % 10_000 == 0 {
if height != last_height && height != Height::ZERO && height.to_usize() % 10_000 == 0 {
let _lock = exit.lock();
addresstypeindex_to_anyaddressindex_reader_opt.take();
@@ -1967,13 +1967,13 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
self.0.into_iter().try_for_each(|(prev_height, v)| {
let prev_price = height_to_price_close_vec
.as_ref()
.map(|v| **v.get(prev_height.unwrap_to_usize()).unwrap());
.map(|v| **v.get(prev_height.to_usize()).unwrap());
let prev_timestamp = *height_to_timestamp_fixed_vec
.get(prev_height.unwrap_to_usize())
.get(prev_height.to_usize())
.unwrap();
let blocks_old = height.unwrap_to_usize() - prev_height.unwrap_to_usize();
let blocks_old = height.to_usize() - prev_height.to_usize();
let days_old = timestamp.difference_in_days_between_float(prev_timestamp);

View File

@@ -1514,13 +1514,13 @@ impl Vecs {
let chain_state_len = chain_state.len();
height_to_sent.into_iter().for_each(|(height, sent)| {
chain_state[height.unwrap_to_usize()].supply -= &sent.spendable_supply;
chain_state[height.to_usize()].supply -= &sent.spendable_supply;
let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
let block_state = chain_state.get(height.to_usize()).unwrap();
let prev_price = block_state.price;
let blocks_old = chain_state_len - 1 - height.unwrap_to_usize();
let blocks_old = chain_state_len - 1 - height.to_usize();
let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
let days_old_float =

View File

@@ -42,7 +42,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let price = *closes.into_owned();
let i_usize = i.unwrap_to_usize();
let i_usize = i.to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
@@ -92,7 +92,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let price = *closes.into_owned();
let i_usize = i.unwrap_to_usize();
let i_usize = i.to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
@@ -157,8 +157,8 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
if i > first_price_date {
avg_price = DCA_AMOUNT
* len
.min(i.unwrap_to_usize() + 1)
.min(i.checked_sub(first_price_date).unwrap().unwrap_to_usize() + 1)
.min(i.to_usize() + 1)
.min(i.checked_sub(first_price_date).unwrap().to_usize() + 1)
/ Bitcoin::from(stack);
}
self.forced_push_at(i, avg_price, exit)
@@ -182,14 +182,13 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
let index = max_from.min(DateIndex::from(self.len()));
let from_usize = from.unwrap_to_usize();
let from_usize = from.to_usize();
stacks.iter_at(index).try_for_each(|(i, stack)| {
let stack = stack.into_owned();
let mut avg_price = Dollars::from(f64::NAN);
if i >= from {
avg_price =
DCA_AMOUNT * (i.unwrap_to_usize() + 1 - from_usize) / Bitcoin::from(stack);
avg_price = DCA_AMOUNT * (i.to_usize() + 1 - from_usize) / Bitcoin::from(stack);
}
self.forced_push_at(i, avg_price, exit)
})?;