Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-27 16:19:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

snapshot

Browse Source
This commit is contained in:
nym21
2025-11-14 12:09:58 +01:00
parent 1d2c927d94
commit e8f77ab2e5
46 changed files with 1400 additions and 1394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/brk_computer/src/blks.rs
View File

@@ -48,6 +48,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}

194
crates/brk_computer/src/chain.rs
View File

@@ -10,8 +10,8 @@ use brk_types::{
TxVersion, Version, WeekIndex, Weight, YearIndex,
};
use vecdb::{
Database, EagerVec, Exit, IterableCloneableVec, IterableVec, LazyVecFrom1, LazyVecFrom2,
LazyVecFrom3, PAGE_SIZE, TypedVecIterator, VecIndex,
AnyVec, Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, IterableVec,
LazyVecFrom1, LazyVecFrom2, PAGE_SIZE, TypedVecIterator, VecIndex, unlikely,
};
use crate::grouped::{
@@ -71,13 +71,11 @@ pub struct Vecs {
pub indexes_to_fee: ComputedValueVecsFromTxindex,
pub indexes_to_fee_rate: ComputedVecsFromTxindex<FeeRate>,
/// Value == 0 when Coinbase
pub txindex_to_input_value:
LazyVecFrom3<TxIndex, Sats, TxIndex, TxInIndex, TxIndex, StoredU64, TxInIndex, Sats>,
pub txindex_to_input_value: EagerVec<TxIndex, Sats>,
pub indexes_to_sent: ComputedValueVecsFromHeight,
// pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU64>,
pub txindex_to_output_value:
LazyVecFrom3<TxIndex, Sats, TxIndex, TxOutIndex, TxIndex, StoredU64, TxOutIndex, Sats>,
pub txindex_to_output_value: EagerVec<TxIndex, Sats>,
// pub indexes_to_output_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredU64>,
@@ -151,7 +149,7 @@ impl Vecs {
price: Option<&price::Vecs>,
) -> Result<Self> {
let db = Database::open(&parent_path.join("chain"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
db.set_min_len(PAGE_SIZE * 50_000_000)?;
let version = parent_version + Version::ZERO;
@@ -202,82 +200,11 @@ impl Vecs {
},
);
let txindex_to_input_value = LazyVecFrom3::init(
"input_value",
version + Version::ZERO,
indexer.vecs.txindex_to_first_txinindex.boxed_clone(),
indexes.txindex_to_input_count.boxed_clone(),
txinindex_to_value.boxed_clone(),
|index: TxIndex,
txindex_to_first_txinindex_iter,
txindex_to_input_count_iter,
txinindex_to_value_iter| {
let txindex = index.to_usize();
txindex_to_first_txinindex_iter
.get_at(txindex)
.map(|first_index| {
let first_index = usize::from(first_index);
let count = *txindex_to_input_count_iter.get_at_unwrap(txindex);
let range = first_index..first_index + count as usize;
range.into_iter().fold(Sats::ZERO, |total, txinindex| {
total + txinindex_to_value_iter.get_at_unwrap(txinindex)
})
})
},
);
let txindex_to_input_value =
EagerVec::forced_import_compressed(&db, "input_value", version + Version::ZERO)?;
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// db,
// "input_value",
// true,
// version + Version::ZERO,
// format,
// computation,
// StorableVecGeneatorOptions::default()
// .add_average()
// .add_sum()
// .add_cumulative(),
// )?;
let txindex_to_output_value = LazyVecFrom3::init(
"output_value",
version + Version::ZERO,
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
indexes.txindex_to_output_count.boxed_clone(),
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index: TxIndex,
txindex_to_first_txoutindex_iter,
txindex_to_output_count_iter,
txoutindex_to_value_iter| {
let txindex = index.to_usize();
txindex_to_first_txoutindex_iter
.get_at(txindex)
.map(|first_index| {
let first_index = usize::from(first_index);
let count = *txindex_to_output_count_iter.get_at_unwrap(txindex);
let range = first_index..first_index + count as usize;
range.into_iter().fold(Sats::ZERO, |total, txoutindex| {
let v = txoutindex_to_value_iter.get_at_unwrap(txoutindex);
total + v
})
})
},
);
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// db,
// "output_value",
// true,
// version + Version::ZERO,
// format,
// computation,
// StorableVecGeneatorOptions::default()
// .add_average()
// .add_sum()
// .add_cumulative(),
// )?;
let txindex_to_output_value =
EagerVec::forced_import_compressed(&db, "output_value", version + Version::ZERO)?;
let txindex_to_fee =
EagerVec::forced_import_compressed(&db, "fee", version + Version::ZERO)?;
@@ -1084,8 +1011,6 @@ impl Vecs {
txindex_to_is_coinbase,
txinindex_to_value,
// indexes_to_input_value,
// indexes_to_output_value,
txindex_to_input_value,
txindex_to_output_value,
txindex_to_fee,
@@ -1102,6 +1027,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}
@@ -1365,7 +1292,9 @@ impl Vecs {
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
let mut txoutindex_to_value_iter = indexer.vecs.txoutindex_to_value.into_iter();
// Because random reads are needed, reading directly from the mmap is faster than using buffered iterators
let txoutindex_to_value = &indexer.vecs.txoutindex_to_value;
let txoutindex_to_value_reader = indexer.vecs.txoutindex_to_value.create_reader();
self.txinindex_to_value.compute_transform(
starting_indexes.txinindex,
&indexes.txinindex_to_txoutindex,
@@ -1373,64 +1302,56 @@ impl Vecs {
let value = if txoutindex == TxOutIndex::COINBASE {
Sats::MAX
} else {
txoutindex_to_value_iter.get_unwrap(txoutindex)
txoutindex_to_value
.unchecked_read(txoutindex, &txoutindex_to_value_reader)
.unwrap()
};
(txinindex, value)
},
exit,
)?;
// indexes.txinindex_to_txoutindex.boxed_clone(),
// indexer.vecs.txoutindex_to_value.boxed_clone(),
// |index: TxInIndex, txinindex_to_txoutindex_iter, txoutindex_to_value_iter| {
// txinindex_to_txoutindex_iter.get_unwrap(index.to_usize()).map(
// |(txinindex, txoutindex)| {
// let txoutindex = txoutindex;
// if txoutindex == TxOutIndex::COINBASE {
// Sats::MAX
// } else if let Some((_, value)) =
// txoutindex_to_value_iter.get_unwrap(txoutindex.to_usize())
// {
// value
// } else {
// dbg!(txinindex, txoutindex);
// panic!()
// }
// },
// )
// },
// Debug: verify the computed txinindex_to_value
dbg!("txinindex_to_value first 20:");
for i in 0..20.min(self.txinindex_to_value.len()) {
let val = self.txinindex_to_value.read_at_unwrap_once(i);
dbg!((TxInIndex::from(i), val));
}
// self.indexes_to_output_value.compute_all(
// indexer,
// indexes,
// starting_indexes,
// exit,
// |vec| {
// vec.compute_sum_from_indexes(
// starting_indexes.txindex,
// &indexer.vecs.txindex_to_first_txoutindex,
// self.indexes_to_output_count.txindex.as_ref().unwrap(),
// &indexer.vecs.txoutindex_to_value,
// exit,
// )
// },
// )?;
// Debug: verify the computed txindex_to_input_count
dbg!("txindex_to_input_count first 20:");
for i in 0..20.min(indexes.txindex_to_input_count.len()) {
let val = indexes.txindex_to_input_count.read_at_unwrap_once(i);
dbg!((TxInIndex::from(i), val));
}
// self.indexes_to_input_value.compute_all(
// indexer,
// indexes,
// starting_indexes,
// exit,
// |vec| {
// vec.compute_sum_from_indexes(
// starting_indexes.txindex,
// &indexer.vecs.txindex_to_first_txinindex,
// self.indexes_to_input_count.txindex.as_ref().unwrap(),
// &self.txinindex_to_value,
// exit,
// )
// },
// )?;
self.txindex_to_input_value.compute_sum_from_indexes(
starting_indexes.txindex,
&indexer.vecs.txindex_to_first_txinindex,
&indexes.txindex_to_input_count,
&self.txinindex_to_value,
exit,
)?;
// Debug: verify the computed input values
for i in 0..10.min(self.txindex_to_input_value.len()) {
let val = self.txindex_to_input_value.read_at_unwrap_once(i);
dbg!((TxIndex::from(i), "input_value", val));
}
self.txindex_to_output_value.compute_sum_from_indexes(
starting_indexes.txindex,
&indexer.vecs.txindex_to_first_txoutindex,
&indexes.txindex_to_output_count,
&indexer.vecs.txoutindex_to_value,
exit,
)?;
// Debug: verify the computed output values
for i in 0..10.min(self.txindex_to_output_value.len()) {
let val = self.txindex_to_output_value.read_at_unwrap_once(i);
dbg!((TxIndex::from(i), "output_value", val));
}
self.txindex_to_fee.compute_transform2(
starting_indexes.txindex,
@@ -1439,9 +1360,10 @@ impl Vecs {
|(i, input, output, ..)| {
(
i,
if input.is_max() {
if unlikely(input.is_max()) {
Sats::ZERO
} else {
dbg!((i, input, output));
input.checked_sub(output).unwrap()
},
)

2
crates/brk_computer/src/cointime.rs
View File

@@ -284,6 +284,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}

2
crates/brk_computer/src/constants.rs
View File

@@ -167,6 +167,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}

38
crates/brk_computer/src/fetched.rs
View File

@@ -7,7 +7,7 @@ use brk_traversable::Traversable;
use brk_types::{DateIndex, Height, OHLCCents, Version};
use vecdb::{
AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, IterableVec, PAGE_SIZE, RawVec,
VecIndex, VecIterator,
TypedVecIterator, VecIndex,
};
use super::{Indexes, indexes};
@@ -49,6 +49,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}
@@ -75,24 +77,22 @@ impl Vecs {
let index = starting_indexes
.height
.min(Height::from(self.height_to_price_ohlc_in_cents.len()));
let mut prev_timestamp = index
.decremented()
.map(|prev_i| height_to_timestamp.iter().unwrap().get_unwrap(prev_i));
height_to_timestamp
.iter()?
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(i, v)| -> Result<()> {
self.height_to_price_ohlc_in_cents.forced_push_at(
i,
self.fetcher
.get_height(
i.into(),
v,
i.decremented().map(|prev_i| {
height_to_timestamp.into_iter().get_at_unwrap(prev_i)
}),
)
.get_height(i.into(), v, prev_timestamp)
.unwrap(),
exit,
)?;
prev_timestamp = Some(v);
Ok(())
})?;
self.height_to_price_ohlc_in_cents.safe_flush(exit)?;
@@ -100,24 +100,18 @@ impl Vecs {
let index = starting_indexes
.dateindex
.min(DateIndex::from(self.dateindex_to_price_ohlc_in_cents.len()));
let mut prev = None;
let mut prev = Some(index.decremented().map_or(OHLCCents::default(), |prev_i| {
self.dateindex_to_price_ohlc_in_cents
.iter()
.unwrap()
.get_unwrap(prev_i)
}));
indexes
.dateindex_to_date
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(i, d)| -> Result<()> {
if prev.is_none() {
let i = i.to_usize();
prev.replace(if i > 0 {
self.dateindex_to_price_ohlc_in_cents
.into_iter()
.get_at_unwrap(i - 1)
} else {
OHLCCents::default()
});
}
let ohlc = if i.to_usize() + 100 >= self.dateindex_to_price_ohlc_in_cents.len()
&& let Ok(mut ohlc) = self.fetcher.get_date(d)
{

17
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -222,8 +222,8 @@ where
});
source
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(i, v)| -> Result<()> {
cumulative += v;
cumulative_vec.forced_push_at(i, cumulative, exit)?;
@@ -252,7 +252,6 @@ where
let index = self.starting_index(max_from);
let mut count_indexes_iter = count_indexes.iter();
let mut source_iter = source.iter();
let cumulative_vec = self.cumulative.as_mut();
@@ -263,12 +262,14 @@ where
})
});
let mut count_indexes_iter = count_indexes.iter().skip(index.to_usize());
first_indexes
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(index, first_index)| -> Result<()> {
let count_index = count_indexes_iter.get_at_unwrap(index);
let count_index = count_indexes_iter.next().unwrap();
if let Some(first) = self.first.as_mut() {
let f = source_iter
@@ -420,8 +421,6 @@ where
let index = self.starting_index(max_from);
let mut count_indexes_iter = count_indexes.iter();
let mut source_first_iter = source.first.as_ref().map(|f| f.iter());
let mut source_last_iter = source.last.as_ref().map(|f| f.iter());
let mut source_max_iter = source.max.as_ref().map(|f| f.iter());
@@ -435,12 +434,14 @@ where
})
});
let mut count_indexes_iter = count_indexes.iter().skip(index.to_usize());
first_indexes
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(index, first_index, ..)| -> Result<()> {
let count_index = count_indexes_iter.get_at_unwrap(index);
let count_index = count_indexes_iter.next().unwrap();
if let Some(first) = self.first.as_mut() {
let v = source_first_iter.as_mut().unwrap().get_unwrap(first_index);

26
crates/brk_computer/src/grouped/builder_lazy.rs
View File

@@ -65,7 +65,7 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
source.get_at(S1I::min_from(i))
@@ -90,10 +90,10 @@ where
),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
source.get_at(S1I::max_from(i, source.len()))
source.get_at(S1I::max_from(i, source.vec_len()))
},
))
}),
@@ -107,10 +107,10 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
S1I::inclusive_range_from(i, source.len())
S1I::inclusive_range_from(i, source.vec_len())
.flat_map(|i| source.get_at(i))
.min()
},
@@ -126,10 +126,10 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
S1I::inclusive_range_from(i, source.len())
S1I::inclusive_range_from(i, source.vec_len())
.flat_map(|i| source.get_at(i))
.max()
},
@@ -145,10 +145,10 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
let vec = S1I::inclusive_range_from(i, source.len())
let vec = S1I::inclusive_range_from(i, source.vec_len())
.flat_map(|i| source.get_at(i))
.collect::<Vec<_>>();
if vec.is_empty() {
@@ -175,10 +175,10 @@ where
.map_or_else(|| source.as_ref().unwrap().clone(), |v| v.clone()),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
let vec = S1I::inclusive_range_from(i, source.len())
let vec = S1I::inclusive_range_from(i, source.vec_len())
.flat_map(|i| source.get_at(i))
.collect::<Vec<_>>();
if vec.is_empty() {
@@ -197,10 +197,10 @@ where
source_extra.cumulative.as_ref().unwrap().boxed_clone(),
len_source.clone(),
|i: I, source, len_source| {
if i.to_usize() >= len_source.len() {
if i.to_usize() >= len_source.vec_len() {
return None;
}
source.get_at(S1I::max_from(i, source.len()))
source.get_at(S1I::max_from(i, source.vec_len()))
},
))
}),

1
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -110,6 +110,7 @@ where
.merge_branches()
.unwrap()
}
fn iter_any_writable(&self) -> impl Iterator<Item = &dyn AnyWritableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyWritableVec>> =
Box::new(self.height.iter_any_writable());

45
crates/brk_computer/src/grouped/ratio_from_dateindex.rs
View File

@@ -397,8 +397,8 @@ impl ComputedRatioVecsFromDateIndex {
.as_ref()
.unwrap()
.iter()
.skip(starting_dateindex.to_usize())
.enumerate()
.skip(starting_dateindex.to_usize())
.try_for_each(|(index, ratio)| -> Result<()> {
if index < min_ratio_date_usize {
self.ratio_pct5
@@ -611,28 +611,25 @@ impl ComputedRatioVecsFromDateIndex {
}
fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec<DateIndex, StoredF32>> {
[
self.ratio_pct1
.as_mut()
.map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]),
self.ratio_pct2
.as_mut()
.map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]),
self.ratio_pct5
.as_mut()
.map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]),
self.ratio_pct95
.as_mut()
.map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]),
self.ratio_pct98
.as_mut()
.map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]),
self.ratio_pct99
.as_mut()
.map_or(vec![], |v| vec![v.dateindex.as_mut().unwrap()]),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
let mut vecs = Vec::with_capacity(6);
if let Some(v) = self.ratio_pct1.as_mut() {
vecs.push(v.dateindex.as_mut().unwrap());
}
if let Some(v) = self.ratio_pct2.as_mut() {
vecs.push(v.dateindex.as_mut().unwrap());
}
if let Some(v) = self.ratio_pct5.as_mut() {
vecs.push(v.dateindex.as_mut().unwrap());
}
if let Some(v) = self.ratio_pct95.as_mut() {
vecs.push(v.dateindex.as_mut().unwrap());
}
if let Some(v) = self.ratio_pct98.as_mut() {
vecs.push(v.dateindex.as_mut().unwrap());
}
if let Some(v) = self.ratio_pct99.as_mut() {
vecs.push(v.dateindex.as_mut().unwrap());
}
vecs
}
}

9
crates/brk_computer/src/grouped/sd_from_dateindex.rs
View File

@@ -483,8 +483,6 @@ impl ComputedStandardDeviationVecsFromDateIndex {
sorted.sort_unstable();
let mut sma_iter = sma.iter();
let mut p0_5sd = self.p0_5sd.as_mut().map(|c| c.dateindex.as_mut().unwrap());
let mut p1sd = self.p1sd.as_mut().map(|c| c.dateindex.as_mut().unwrap());
let mut p1_5sd = self.p1_5sd.as_mut().map(|c| c.dateindex.as_mut().unwrap());
@@ -499,11 +497,12 @@ impl ComputedStandardDeviationVecsFromDateIndex {
let mut m3sd = self.m3sd.as_mut().map(|c| c.dateindex.as_mut().unwrap());
let min_date_usize = min_date.to_usize();
let mut sma_iter = sma.iter().skip(starting_dateindex.to_usize());
source
.iter()
.skip(starting_dateindex.to_usize())
.enumerate()
.skip(starting_dateindex.to_usize())
.try_for_each(|(index, ratio)| -> Result<()> {
if index < min_date_usize {
self.sd.dateindex.as_mut().unwrap().forced_push_at(
@@ -548,11 +547,13 @@ impl ComputedStandardDeviationVecsFromDateIndex {
if let Some(v) = m3sd.as_mut() {
v.forced_push_at(index, StoredF32::NAN, exit)?
}
// Advance iterator to stay in sync
sma_iter.next();
} else {
let pos = sorted.binary_search(&ratio).unwrap_or_else(|pos| pos);
sorted.insert(pos, ratio);
let avg = sma_iter.get_at_unwrap(index);
let avg = sma_iter.next().unwrap();
let population =
index.checked_sub(min_date_usize).unwrap().to_usize() as f32 + 1.0;

329
crates/brk_computer/src/indexes.rs
View File

@@ -13,8 +13,8 @@ use brk_types::{
YearIndex,
};
use vecdb::{
Database, EagerVec, Exit, IterableCloneableVec, LazyVecFrom1, LazyVecFrom2, PAGE_SIZE,
TypedVecIterator, VecIndex,
Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, LazyVecFrom1, PAGE_SIZE,
TypedVecIterator, unlikely,
};
const VERSION: Version = Version::ZERO;
@@ -79,14 +79,11 @@ pub struct Vecs {
pub semesterindex_to_first_monthindex: EagerVec<SemesterIndex, MonthIndex>,
pub semesterindex_to_monthindex_count: EagerVec<SemesterIndex, StoredU64>,
pub semesterindex_to_semesterindex: EagerVec<SemesterIndex, SemesterIndex>,
pub txindex_to_input_count:
LazyVecFrom2<TxIndex, StoredU64, TxIndex, TxInIndex, TxInIndex, TxOutIndex>,
pub txindex_to_output_count:
LazyVecFrom2<TxIndex, StoredU64, TxIndex, TxOutIndex, TxOutIndex, Sats>,
pub txindex_to_input_count: EagerVec<TxIndex, StoredU64>,
pub txindex_to_output_count: EagerVec<TxIndex, StoredU64>,
pub txindex_to_txindex: LazyVecFrom1<TxIndex, TxIndex, TxIndex, Txid>,
pub txinindex_to_txinindex: LazyVecFrom1<TxInIndex, TxInIndex, TxInIndex, OutPoint>,
pub txinindex_to_txoutindex:
LazyVecFrom2<TxInIndex, TxOutIndex, TxInIndex, OutPoint, TxIndex, TxOutIndex>,
pub txinindex_to_txoutindex: EagerVec<TxInIndex, TxOutIndex>,
pub txoutindex_to_txoutindex: LazyVecFrom1<TxOutIndex, TxOutIndex, TxOutIndex, Sats>,
pub unknownoutputindex_to_unknownoutputindex:
LazyVecFrom1<UnknownOutputIndex, UnknownOutputIndex, UnknownOutputIndex, TxIndex>,
@@ -110,179 +107,112 @@ impl Vecs {
let version = parent_version + VERSION;
let txinindex_to_txoutindex = LazyVecFrom2::init(
"txoutindex",
version,
indexer.vecs.txinindex_to_outpoint.boxed_clone(),
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
|index: TxInIndex, txinindex_to_outpoint_iter, txindex_to_first_txoutindex_iter| {
txinindex_to_outpoint_iter
.get_at(index.to_usize())
.map(|outpoint| {
if outpoint.is_coinbase() {
return TxOutIndex::COINBASE;
}
txindex_to_first_txoutindex_iter
.get_at_unwrap(outpoint.txindex().to_usize())
+ outpoint.vout()
})
},
);
let txoutindex_to_txoutindex = LazyVecFrom1::init(
"txoutindex",
version + Version::ZERO,
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index, _| Some(index),
);
let txinindex_to_txinindex = LazyVecFrom1::init(
"txinindex",
version + Version::ZERO,
indexer.vecs.txinindex_to_outpoint.boxed_clone(),
|index, _| Some(index),
);
let txindex_to_txindex = LazyVecFrom1::init(
"txindex",
version + Version::ZERO,
indexer.vecs.txindex_to_txid.boxed_clone(),
|index, _| Some(index),
);
let txindex_to_input_count = LazyVecFrom2::init(
"input_count",
version + Version::ZERO,
indexer.vecs.txindex_to_first_txinindex.boxed_clone(),
txinindex_to_txoutindex.boxed_clone(),
|index: TxIndex, txindex_to_first_txinindex_iter, txinindex_to_txoutindex_iter| {
let txindex = index.to_usize();
txindex_to_first_txinindex_iter
.get_at(txindex)
.map(|start| {
let start = usize::from(start);
let end = txindex_to_first_txinindex_iter
.get_at(txindex + 1)
.map(usize::from)
.unwrap_or_else(|| txinindex_to_txoutindex_iter.len());
StoredU64::from((start..end).count())
})
},
);
let txindex_to_output_count = LazyVecFrom2::init(
"output_count",
version + Version::ZERO,
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index: TxIndex, txindex_to_first_txoutindex_iter, txoutindex_to_value_iter| {
let txindex = index.to_usize();
txindex_to_first_txoutindex_iter
.get_at(txindex)
.map(|start| {
let start = usize::from(start);
let end = txindex_to_first_txoutindex_iter
.get_at(txindex + 1)
.map(usize::from)
.unwrap_or_else(|| txoutindex_to_value_iter.len());
StoredU64::from((start..end).count())
})
},
);
let p2pk33addressindex_to_p2pk33addressindex = LazyVecFrom1::init(
"p2pk33addressindex",
version + Version::ZERO,
indexer.vecs.p2pk33addressindex_to_p2pk33bytes.boxed_clone(),
|index, _| Some(index),
);
let p2pk65addressindex_to_p2pk65addressindex = LazyVecFrom1::init(
"p2pk65addressindex",
version + Version::ZERO,
indexer.vecs.p2pk65addressindex_to_p2pk65bytes.boxed_clone(),
|index, _| Some(index),
);
let p2pkhaddressindex_to_p2pkhaddressindex = LazyVecFrom1::init(
"p2pkhaddressindex",
version + Version::ZERO,
indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.boxed_clone(),
|index, _| Some(index),
);
let p2shaddressindex_to_p2shaddressindex = LazyVecFrom1::init(
"p2shaddressindex",
version + Version::ZERO,
indexer.vecs.p2shaddressindex_to_p2shbytes.boxed_clone(),
|index, _| Some(index),
);
let p2traddressindex_to_p2traddressindex = LazyVecFrom1::init(
"p2traddressindex",
version + Version::ZERO,
indexer.vecs.p2traddressindex_to_p2trbytes.boxed_clone(),
|index, _| Some(index),
);
let p2wpkhaddressindex_to_p2wpkhaddressindex = LazyVecFrom1::init(
"p2wpkhaddressindex",
version + Version::ZERO,
indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes.boxed_clone(),
|index, _| Some(index),
);
let p2wshaddressindex_to_p2wshaddressindex = LazyVecFrom1::init(
"p2wshaddressindex",
version + Version::ZERO,
indexer.vecs.p2wshaddressindex_to_p2wshbytes.boxed_clone(),
|index, _| Some(index),
);
let p2aaddressindex_to_p2aaddressindex = LazyVecFrom1::init(
"p2aaddressindex",
version + Version::ZERO,
indexer.vecs.p2aaddressindex_to_p2abytes.boxed_clone(),
|index, _| Some(index),
);
let p2msoutputindex_to_p2msoutputindex = LazyVecFrom1::init(
"p2msoutputindex",
version + Version::ZERO,
indexer.vecs.p2msoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let emptyoutputindex_to_emptyoutputindex = LazyVecFrom1::init(
"emptyoutputindex",
version + Version::ZERO,
indexer.vecs.emptyoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let unknownoutputindex_to_unknownoutputindex = LazyVecFrom1::init(
"unknownoutputindex",
version + Version::ZERO,
indexer.vecs.unknownoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let opreturnindex_to_opreturnindex = LazyVecFrom1::init(
"opreturnindex",
version + Version::ZERO,
indexer.vecs.opreturnindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let this = Self {
txinindex_to_txoutindex,
emptyoutputindex_to_emptyoutputindex,
txinindex_to_txinindex,
opreturnindex_to_opreturnindex,
txoutindex_to_txoutindex,
p2aaddressindex_to_p2aaddressindex,
p2msoutputindex_to_p2msoutputindex,
p2pk33addressindex_to_p2pk33addressindex,
p2pk65addressindex_to_p2pk65addressindex,
p2pkhaddressindex_to_p2pkhaddressindex,
p2shaddressindex_to_p2shaddressindex,
p2traddressindex_to_p2traddressindex,
p2wpkhaddressindex_to_p2wpkhaddressindex,
p2wshaddressindex_to_p2wshaddressindex,
txindex_to_input_count,
txindex_to_output_count,
txindex_to_txindex,
unknownoutputindex_to_unknownoutputindex,
txinindex_to_txoutindex: EagerVec::forced_import_compressed(
&db,
"txoutindex",
version,
)?,
txoutindex_to_txoutindex: LazyVecFrom1::init(
"txoutindex",
version + Version::ZERO,
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index, _| Some(index),
),
txinindex_to_txinindex: LazyVecFrom1::init(
"txinindex",
version + Version::ZERO,
indexer.vecs.txinindex_to_outpoint.boxed_clone(),
|index, _| Some(index),
),
p2pk33addressindex_to_p2pk33addressindex: LazyVecFrom1::init(
"p2pk33addressindex",
version + Version::ZERO,
indexer.vecs.p2pk33addressindex_to_p2pk33bytes.boxed_clone(),
|index, _| Some(index),
),
p2pk65addressindex_to_p2pk65addressindex: LazyVecFrom1::init(
"p2pk65addressindex",
version + Version::ZERO,
indexer.vecs.p2pk65addressindex_to_p2pk65bytes.boxed_clone(),
|index, _| Some(index),
),
p2pkhaddressindex_to_p2pkhaddressindex: LazyVecFrom1::init(
"p2pkhaddressindex",
version + Version::ZERO,
indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.boxed_clone(),
|index, _| Some(index),
),
p2shaddressindex_to_p2shaddressindex: LazyVecFrom1::init(
"p2shaddressindex",
version + Version::ZERO,
indexer.vecs.p2shaddressindex_to_p2shbytes.boxed_clone(),
|index, _| Some(index),
),
p2traddressindex_to_p2traddressindex: LazyVecFrom1::init(
"p2traddressindex",
version + Version::ZERO,
indexer.vecs.p2traddressindex_to_p2trbytes.boxed_clone(),
|index, _| Some(index),
),
p2wpkhaddressindex_to_p2wpkhaddressindex: LazyVecFrom1::init(
"p2wpkhaddressindex",
version + Version::ZERO,
indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes.boxed_clone(),
|index, _| Some(index),
),
p2wshaddressindex_to_p2wshaddressindex: LazyVecFrom1::init(
"p2wshaddressindex",
version + Version::ZERO,
indexer.vecs.p2wshaddressindex_to_p2wshbytes.boxed_clone(),
|index, _| Some(index),
),
p2aaddressindex_to_p2aaddressindex: LazyVecFrom1::init(
"p2aaddressindex",
version + Version::ZERO,
indexer.vecs.p2aaddressindex_to_p2abytes.boxed_clone(),
|index, _| Some(index),
),
p2msoutputindex_to_p2msoutputindex: LazyVecFrom1::init(
"p2msoutputindex",
version + Version::ZERO,
indexer.vecs.p2msoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
),
emptyoutputindex_to_emptyoutputindex: LazyVecFrom1::init(
"emptyoutputindex",
version + Version::ZERO,
indexer.vecs.emptyoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
),
unknownoutputindex_to_unknownoutputindex: LazyVecFrom1::init(
"unknownoutputindex",
version + Version::ZERO,
indexer.vecs.unknownoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
),
opreturnindex_to_opreturnindex: LazyVecFrom1::init(
"opreturnindex",
version + Version::ZERO,
indexer.vecs.opreturnindex_to_txindex.boxed_clone(),
|index, _| Some(index),
),
txindex_to_txindex: LazyVecFrom1::init(
"txindex",
version + Version::ZERO,
indexer.vecs.txindex_to_txid.boxed_clone(),
|index, _| Some(index),
),
txindex_to_input_count: EagerVec::forced_import_compressed(
&db,
"input_count",
version + Version::ZERO,
)?,
txindex_to_output_count: EagerVec::forced_import_compressed(
&db,
"output_count",
version + Version::ZERO,
)?,
dateindex_to_date: EagerVec::forced_import_compressed(
&db,
"date",
@@ -497,6 +427,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}
@@ -517,10 +449,46 @@ impl Vecs {
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> Result<Indexes> {
// ---
// TxInIndex
// ---
let txindex_to_first_txoutindex = &indexer.vecs.txindex_to_first_txoutindex;
let txindex_to_first_txoutindex_reader = txindex_to_first_txoutindex.create_reader();
self.txinindex_to_txoutindex.compute_transform(
starting_indexes.txinindex,
&indexer.vecs.txinindex_to_outpoint,
|(txinindex, outpoint, ..)| {
if unlikely(outpoint.is_coinbase()) {
(txinindex, TxOutIndex::COINBASE)
} else {
let txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &txindex_to_first_txoutindex_reader)
+ outpoint.vout();
(txinindex, txoutindex)
}
},
exit,
)?;
// ---
// TxIndex
// ---
self.txindex_to_input_count.compute_count_from_indexes(
starting_indexes.txindex,
&indexer.vecs.txindex_to_first_txinindex,
&self.txinindex_to_txoutindex,
exit,
)?;
self.txindex_to_output_count.compute_count_from_indexes(
starting_indexes.txindex,
&indexer.vecs.txindex_to_first_txoutindex,
&indexer.vecs.txoutindex_to_value,
exit,
)?;
self.height_to_txindex_count.compute_count_from_indexes(
starting_indexes.height,
&indexer.vecs.height_to_first_txindex,
@@ -928,6 +896,7 @@ impl Vecs {
}
}
#[derive(Debug)]
pub struct Indexes {
indexes: brk_indexer::Indexes,
pub dateindex: DateIndex,

54
crates/brk_computer/src/lib.rs
View File

@@ -174,35 +174,35 @@ impl Computer {
self.blks
.compute(indexer, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
let constants = scope.spawn(|| -> Result<()> {
info!("Computing constants...");
self.constants
.compute(&self.indexes, &starting_indexes, exit)?;
Ok(())
});
// std::thread::scope(|scope| -> Result<()> {
// let constants = scope.spawn(|| -> Result<()> {
info!("Computing constants...");
self.constants
.compute(&self.indexes, &starting_indexes, exit)?;
// Ok(())
// });
let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
self.chain.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
exit,
)?;
Ok(())
});
// let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
self.chain.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
exit,
)?;
// Ok(())
// });
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market.compute(price, &starting_indexes, exit)?;
}
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market.compute(price, &starting_indexes, exit)?;
}
constants.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;
// constants.join().unwrap()?;
// chain.join().unwrap()?;
// Ok(())
// })?;
self.pools.compute(
indexer,
@@ -213,6 +213,8 @@ impl Computer {
exit,
)?;
return Ok(());
info!("Computing stateful...");
self.stateful.compute(
indexer,

2
crates/brk_computer/src/market.rs
View File

@@ -1522,6 +1522,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}

2
crates/brk_computer/src/pools/mod.rs
View File

@@ -67,6 +67,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}

948
crates/brk_computer/src/price.rs
View File

File diff suppressed because it is too large Load Diff

2
crates/brk_computer/src/stateful/mod.rs
View File

@@ -487,6 +487,8 @@ impl Vecs {
.collect(),
)?;
this.db.compact()?;
Ok(this)
}

141
crates/brk_computer/src/traits.rs
View File

@@ -23,6 +23,7 @@ pub trait ComputeDCAStackViaLen {
exit: &Exit,
) -> Result<()>;
}
impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
fn compute_dca_stack_via_len(
&mut self,
@@ -35,41 +36,40 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
Version::ZERO + self.inner_version() + closes.version(),
)?;
let mut other_iter = closes.iter();
let mut prev = None;
let index = max_from.to_usize().min(self.len());
// Initialize prev before the loop to avoid checking on every iteration
let mut prev = if index == 0 {
Sats::ZERO
} else {
self.read_at_unwrap_once(index - 1)
};
let mut lookback = closes.create_lookback(index, len, 0);
closes
.iter()
.skip(index)
.enumerate()
.skip(index)
.try_for_each(|(i, closes)| {
let price = *closes;
let i_usize = i.to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.read_at_unwrap_once(i_usize - 1));
}
}
let mut stack = Sats::ZERO;
if price != Dollars::ZERO {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
stack = prev + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
if i_usize >= len {
let prev_price = *other_iter.get_at_unwrap(i_usize - len);
if prev_price != Dollars::ZERO {
stack = stack
.checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price)))
.unwrap();
}
let prev_price =
*lookback.get_and_push(i_usize, Close::new(price), Close::default());
if prev_price != Dollars::ZERO {
stack = stack
.checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price)))
.unwrap();
}
}
prev.replace(stack);
prev = stack;
self.forced_push_at(i, stack, exit)
})?;
@@ -90,32 +90,30 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
Version::ZERO + self.inner_version() + closes.version(),
)?;
let mut prev = None;
let from = from.to_usize();
let index = max_from.min(DateIndex::from(self.len()));
// Initialize prev before the loop to avoid checking on every iteration
let mut prev = if index.to_usize() == 0 {
Sats::ZERO
} else {
self.read_at_unwrap_once(index.to_usize() - 1)
};
closes
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(i, closes)| {
let price = *closes;
let i_usize = i.to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.read_at_unwrap_once(i_usize - 1));
}
}
let mut stack = Sats::ZERO;
if price != Dollars::ZERO && i >= from {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
stack = prev + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
}
prev.replace(stack);
prev = stack;
self.forced_push_at(i, stack, exit)
})?;
@@ -143,6 +141,7 @@ pub trait ComputeDCAAveragePriceViaLen {
exit: &Exit,
) -> Result<()>;
}
impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
fn compute_dca_avg_price_via_len(
&mut self,
@@ -163,8 +162,8 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
stacks
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(i, stack)| {
let mut avg_price = Dollars::from(f64::NAN);
if i > first_price_date {
@@ -199,8 +198,8 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
stacks
.iter()
.skip(index.to_usize())
.enumerate()
.skip(index.to_usize())
.try_for_each(|(i, stack)| {
let mut avg_price = Dollars::from(f64::NAN);
if i >= from {
@@ -223,6 +222,7 @@ pub trait ComputeFromSats<I> {
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromSats<I> for EagerVec<I, Bitcoin>
where
I: VecIndex,
@@ -233,21 +233,12 @@ where
sats: &impl IterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + sats.version(),
self.compute_transform(
max_from,
sats,
|(i, sats, _)| (i, Bitcoin::from(sats)),
exit,
)?;
let index = max_from.min(I::from(self.len()));
sats.iter()
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, sats)| {
let (i, v) = (i, Bitcoin::from(sats));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
@@ -261,6 +252,7 @@ pub trait ComputeFromBitcoin<I> {
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromBitcoin<I> for EagerVec<I, Dollars>
where
I: VecIndex,
@@ -272,24 +264,13 @@ where
price: &impl IterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + bitcoin.version(),
self.compute_transform2(
max_from,
bitcoin,
price,
|(i, bitcoin, price, _)| (i, *price * bitcoin),
exit,
)?;
let mut price_iter = price.iter();
let index = max_from.min(I::from(self.len()));
bitcoin
.iter()
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, bitcoin)| {
let dollars = price_iter.get_at_unwrap(i);
let (i, v) = (i, *dollars * bitcoin);
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)?;
Ok(())
}
}
@@ -303,6 +284,7 @@ pub trait ComputeDrawdown<I> {
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeDrawdown<I> for EagerVec<I, StoredF32>
where
I: VecIndex,
@@ -314,27 +296,20 @@ where
ath: &impl IterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + ath.version() + close.version(),
)?;
let index = max_from.min(I::from(self.len()));
let mut close_iter = close.iter();
ath.iter()
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, ath)| {
self.compute_transform2(
max_from,
ath,
close,
|(i, ath, close, _)| {
if ath == Dollars::ZERO {
self.forced_push_at(i, StoredF32::default(), exit)
(i, StoredF32::default())
} else {
let close = *close_iter.get_at_unwrap(i);
let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0);
self.forced_push_at(i, drawdown, exit)
let drawdown = StoredF32::from((*ath - **close) / *ath * -100.0);
(i, drawdown)
}
})?;
self.safe_flush(exit)?;
},
exit,
)?;
Ok(())
}
}