Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-05 16:18:13 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: lazy part 1

Browse Source
This commit is contained in:
nym21
2025-05-01 17:25:59 +02:00
parent 700352ec45
commit c77aecbfce
26 changed files with 466 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/lib.rs
+2 -1
View File
@@ -36,9 +36,10 @@ impl Computer {
}
}
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
pub fn import_vecs(&mut self, indexer: &Indexer) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(
&self.path.join("vecs/computed"),
indexer,
self.fetcher.is_some(),
self.compressed,
)?);
crates/brk_computer/src/storage/vecs/blocks.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, Version};
use brk_vec::{Compressed, VecIterator, Version};
use super::{
EagerVec, Indexes,
crates/brk_computer/src/storage/vecs/grouped/builder.rs
+3 -1
View File
@@ -2,7 +2,9 @@ use std::path::Path;
use brk_core::{CheckedSub, StoredUsize};
use brk_exit::Exit;
use brk_vec::{Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, Version};
use brk_vec::{
Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, VecIterator, Version,
};
use color_eyre::eyre::ContextCompat;
use crate::storage::{ComputedType, EagerVec};
crates/brk_computer/src/storage/vecs/indexes.rs
+1 -1
View File
@@ -8,7 +8,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, Version};
use brk_vec::{Compressed, VecIterator, Version};
use super::EagerVec;
crates/brk_computer/src/storage/vecs/marketprice.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{Compressed, Version};
use brk_vec::{Compressed, VecIterator, Version};
use super::{
EagerVec, Indexes,
crates/brk_computer/src/storage/vecs/mining.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use std::{fs, path::Path};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, Version};
use brk_vec::{Compressed, VecIterator, Version};
use super::{
Indexes,
crates/brk_computer/src/storage/vecs/mod.rs
+7 -2
View File
@@ -26,14 +26,19 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path, fetch: bool, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn import(
path: &Path,
indexer: &Indexer,
fetch: bool,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
blocks: blocks::Vecs::forced_import(path, compressed)?,
indexes: indexes::Vecs::forced_import(path, compressed)?,
mining: mining::Vecs::forced_import(path, compressed)?,
transactions: transactions::Vecs::forced_import(path, compressed, fetch)?,
transactions: transactions::Vecs::forced_import(path, indexer, compressed, fetch)?,
marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()),
})
}
crates/brk_computer/src/storage/vecs/transactions.rs
+64 -41
View File
@@ -6,10 +6,10 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, DynamicVec, StoredIndex, Version};
use brk_vec::{Compressed, StoredIndex, VecIterator, Version};
use super::{
EagerVec, Indexes,
EagerVec, Indexes, LazyVec,
grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
ComputedVecsFromTxindex, StorableVecGeneatorOptions,
@@ -47,7 +47,7 @@ pub struct Vecs {
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredUsize>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub inputindex_to_value: EagerVec<InputIndex, Sats>,
pub inputindex_to_value: LazyVec<InputIndex, Sats, OutputIndex, Sats, InputIndex, OutputIndex>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_is_coinbase: EagerVec<TxIndex, bool>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
@@ -58,6 +58,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
indexer: &Indexer,
compressed: Compressed,
compute_dollars: bool,
) -> color_eyre::Result<Self> {
@@ -109,11 +110,34 @@ impl Vecs {
.add_sum()
.add_total(),
)?,
inputindex_to_value: EagerVec::forced_import(
&path.join("inputindex_to_value"),
inputindex_to_value: LazyVec::init(
// &path.join("inputindex_to_value"),
Version::ZERO,
compressed,
)?,
indexer.vecs().outputindex_to_value.vec().clone(),
indexer.vecs().inputindex_to_outputindex.vec().clone(),
|index, outputindex_to_value_iter, inputindex_to_outputindex_iter| {
// outputindex_to_value_iter.get(i)
// inputindex_to_outputindex_iter.get
// let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
// self.inputindex_to_value.compute_transform(
// starting_indexes.inputindex,
// indexer.vecs().inputindex_to_outputindex.vec(),
// |(inputindex, outputindex, ..)| {
// let value = if outputindex == OutputIndex::COINBASE {
// Sats::ZERO
// } else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
// value.into_inner()
// } else {
// dbg!(inputindex, outputindex);
// panic!()
// };
// (inputindex, value)
// },
// exit,
// )?;
Some(Sats::ZERO)
},
),
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path,
"tx_v1",
@@ -517,24 +541,23 @@ impl Vecs {
exit,
)?;
let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
let inputs_len = indexer.vecs().inputindex_to_outputindex.vec().len();
self.inputindex_to_value.compute_transform(
starting_indexes.inputindex,
indexer.vecs().inputindex_to_outputindex.vec(),
|(inputindex, outputindex, ..)| {
let value = if outputindex == OutputIndex::COINBASE {
Sats::ZERO
} else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
value.into_inner()
} else {
dbg!(inputindex, outputindex, inputs_len);
panic!()
};
(inputindex, value)
},
exit,
)?;
// let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
// self.inputindex_to_value.compute_transform(
// starting_indexes.inputindex,
// indexer.vecs().inputindex_to_outputindex.vec(),
// |(inputindex, outputindex, ..)| {
// let value = if outputindex == OutputIndex::COINBASE {
// Sats::ZERO
// } else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
// value.into_inner()
// } else {
// dbg!(inputindex, outputindex);
// panic!()
// };
// (inputindex, value)
// },
// exit,
// )?;
self.indexes_to_output_value.compute_all(
indexer,
@@ -552,21 +575,21 @@ impl Vecs {
},
)?;
self.indexes_to_input_value.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, indexer, _, starting_indexes, exit| {
vec.compute_sum_from_indexes(
starting_indexes.txindex,
indexer.vecs().txindex_to_first_inputindex.vec(),
self.indexes_to_input_count.txindex.as_ref().unwrap().vec(),
self.inputindex_to_value.vec(),
exit,
)
},
)?;
// self.indexes_to_input_value.compute_all(
// indexer,
// indexes,
// starting_indexes,
// exit,
// |vec, indexer, _, starting_indexes, exit| {
// vec.compute_sum_from_indexes(
// starting_indexes.txindex,
// indexer.vecs().txindex_to_first_inputindex.vec(),
// self.indexes_to_input_count.txindex.as_ref().unwrap().vec(),
// self.inputindex_to_value.vec(),
// exit,
// )
// },
// )?;
self.indexes_to_fee.compute_all(
indexer,
@@ -868,7 +891,7 @@ impl Vecs {
[
vec![
self.txindex_to_is_coinbase.any_vec(),
self.inputindex_to_value.any_vec(),
// self.inputindex_to_value.any_vec(),
self.txindex_to_weight.any_vec(),
self.txindex_to_vsize.any_vec(),
],
crates/brk_computer/src/storage/vecs/vec/eager.rs
+2 -5
View File
@@ -10,7 +10,7 @@ use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, T
use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, Version,
StoredVecIterator, Value, VecIterator, Version,
};
use log::info;
@@ -211,10 +211,7 @@ where
)?;
let index = max_from.min(
self.inner
.iter()
.last()
.map_or_else(T::default, |(_, v)| v.into_inner()),
VecIterator::last(self.inner.iter()).map_or_else(T::default, |(_, v)| v.into_inner()),
);
let mut prev_i = None;
other.iter_at(index).try_for_each(|(v, i)| -> Result<()> {
crates/brk_computer/src/storage/vecs/vec/lazy.rs
+178 -17
View File
@@ -1,43 +1,204 @@
use std::fmt::Debug;
use std::marker::PhantomData;
use brk_vec::{DynamicVec, GenericVec, StoredIndex, StoredType, StoredVec, Version};
use brk_vec::{StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version};
#[derive(Debug)]
pub struct LazyVec<I, T>
// LazyVec owns SourceVecs
//
// Functions:
// init()
// version()
// iter()
// len() ?
//
// When .iter() called convert SourcesVecs into iterators
// iter owns source iters
// call compute function to convert index and source iters to T
#[derive(Clone)]
pub struct LazyVec<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
inner: StoredVec<I, T>,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
compute: for<'a> fn(
usize,
&mut (dyn Iterator<Item = (S1I, Value<'a, S1T>)>),
&mut (dyn Iterator<Item = (S2I, Value<'a, S2T>)>),
) -> Option<T>,
phantom: PhantomData<I>,
}
impl<I, T> LazyVec<I, T>
impl<I, T, S1I, S1T, S2I, S2T> LazyVec<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
const SIZE_OF: usize = size_of::<T>();
pub fn init(
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
compute: for<'a> fn(
usize,
&mut (dyn Iterator<Item = (S1I, Value<'a, S1T>)>),
&mut (dyn Iterator<Item = (S2I, Value<'a, S2T>)>),
) -> Option<T>,
) -> Self {
Self {
version,
source1,
source2,
compute,
phantom: PhantomData,
}
}
fn version(&self) -> Version {
self.inner.version()
self.version
}
pub fn len(&self) -> usize {
self.inner.len()
}
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
fn len(&self) -> usize {
todo!()
}
}
impl<I, T> Clone for LazyVec<I, T>
pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
lazy: &'a LazyVec<I, T, S1I, S1T, S2I, S2T>,
source1: StoredVecIterator<'a, S1I, S1T>,
source2: StoredVecIterator<'a, S2I, S2T>,
index: usize,
}
impl<I, T, S1I, S1T, S2I, S2T> LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
#[inline]
pub fn set(&mut self, i: I) -> &mut Self {
self.index = i.unwrap_to_usize();
self
}
#[inline]
pub fn set_(&mut self, i: usize) {
self.index = i;
}
#[inline]
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.set(i).next().map(|(_, v)| v)
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
self.set_(i);
self.next().map(|(_, v)| v)
}
}
// impl<'a, I, T, S1I, S1T, S2I, S2T> VecIterator<'a> for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>
// where
// I: StoredIndex,
// T: StoredType,
// S1I: StoredIndex,
// S1T: StoredType,
// S2I: StoredIndex,
// S2T: StoredType,
// {
// type I = I;
// type T = T;
// #[inline]
// fn mut_index(&mut self) -> &mut usize {
// &mut self.index
// }
// #[inline]
// fn len(&self) -> usize {
// todo!();
// // self.vec.len()
// }
// }
impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(
self.index,
&mut self.lazy.source1.iter(),
&mut self.lazy.source2.iter(),
)
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
self.index += 1;
}
opt
}
// fn last(mut self) -> Option<Self::Item>
// where
// Self: Sized,
// {
// let len = self.vec.len();
// if len == 0 {
// return None;
// }
// let i = len - 1;
// self.get_(i)
// .map(|v| (I::from(i), Value::Owned(v.into_inner())))
// }
}
impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVec<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
lazy: self,
source1: self.source1.iter(),
source2: self.source2.iter(),
index: 0,
}
}
}