Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 07:09:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: big vec refactor + lazy

Browse Source
This commit is contained in:
nym21
2025-05-05 12:47:52 +02:00
parent 320c708e10
commit 9ba3c2b7c5
51 changed files with 1608 additions and 1097 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crates/brk_computer/src/lib.rs
View File

@@ -12,9 +12,8 @@ pub use brk_parser::rpc;
mod storage;
use brk_vec::Compressed;
use brk_vec::{AnyCollectableVec, Compressed, Computation};
use log::info;
pub use storage::Computation;
use storage::{Stores, Vecs};
#[derive(Clone)]
@@ -82,13 +81,14 @@ impl Computer {
Ok(())
}
pub fn vecs(&self) -> &Vecs {
self.vecs.as_ref().unwrap()
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
// pub fn vecs(&self) -> &Vecs {
self.vecs.as_ref().unwrap().vecs()
}
pub fn mut_vecs(&mut self) -> &mut Vecs {
self.vecs.as_mut().unwrap()
}
// pub fn mut_vecs(&mut self) -> &mut Vecs {
// self.vecs.as_mut().unwrap()
// }
pub fn stores(&self) -> &Stores {
self.stores.as_ref().unwrap()

24
crates/brk_computer/src/storage/vecs/blocks.rs
View File

@@ -7,10 +7,10 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, VecIterator, Version};
use brk_vec::{AnyIterableVec, Compressed, Computation, EagerVec, VecIterator, Version};
use super::{
EagerVec, Indexes,
Indexes,
grouped::{ComputedVecsFromDateindex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
indexes,
};
@@ -30,7 +30,11 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -223,7 +227,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
vec![
self.height_to_interval.any_vec(),
@@ -231,12 +235,12 @@ impl Vecs {
self.difficultyepoch_to_timestamp.any_vec(),
self.halvingepoch_to_timestamp.any_vec(),
],
self.timeindexes_to_timestamp.any_vecs(),
self.indexes_to_block_count.any_vecs(),
self.indexes_to_block_interval.any_vecs(),
self.indexes_to_block_size.any_vecs(),
self.indexes_to_block_vbytes.any_vecs(),
self.indexes_to_block_weight.any_vecs(),
self.timeindexes_to_timestamp.vecs(),
self.indexes_to_block_count.vecs(),
self.indexes_to_block_interval.vecs(),
self.indexes_to_block_size.vecs(),
self.indexes_to_block_vbytes.vecs(),
self.indexes_to_block_weight.vecs(),
]
.concat()
}

32
crates/brk_computer/src/storage/vecs/grouped/builder.rs
View File

@@ -3,11 +3,12 @@ use std::path::Path;
use brk_core::{CheckedSub, StoredUsize};
use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, VecIterator, Version,
AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredIndex, StoredType, VecIterator,
Version,
};
use color_eyre::eyre::ContextCompat;
use crate::storage::{ComputedType, EagerVec};
use super::ComputedType;
#[derive(Clone, Debug)]
pub struct ComputedVecBuilder<I, T>
@@ -137,7 +138,12 @@ where
Ok(s)
}
pub fn extend(&mut self, max_from: I, source: &StoredVec<I, T>, exit: &Exit) -> Result<()> {
pub fn extend(
&mut self,
max_from: I,
source: &impl AnyIterableVec<I, T>,
exit: &Exit,
) -> Result<()> {
if self.total.is_none() {
return Ok(());
};
@@ -162,9 +168,9 @@ where
pub fn compute<I2>(
&mut self,
max_from: I,
source: &StoredVec<I2, T>,
first_indexes: &StoredVec<I, I2>,
count_indexes: &StoredVec<I, StoredUsize>,
source: &impl AnyIterableVec<I2, T>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
exit: &Exit,
) -> Result<()>
where
@@ -244,12 +250,12 @@ where
dbg!(
&values,
max.path(),
first_indexes.path(),
first_indexes.name(),
first_index,
count_indexes.path(),
count_indexes.name(),
count_index,
source.len(),
source.path()
source.name()
);
})
.unwrap()
@@ -319,8 +325,8 @@ where
&mut self,
max_from: I,
source: &ComputedVecBuilder<I2, T>,
first_indexes: &StoredVec<I, I2>,
count_indexes: &StoredVec<I, StoredUsize>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
exit: &Exit,
) -> Result<()>
where
@@ -482,7 +488,7 @@ where
fn starting_index(&self, max_from: I) -> I {
max_from.min(I::from(
self.any_vecs().into_iter().map(|v| v.len()).min().unwrap(),
self.vecs().into_iter().map(|v| v.len()).min().unwrap(),
))
}
@@ -523,7 +529,7 @@ where
self.total.as_mut().unwrap()
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
let mut v: Vec<&dyn brk_vec::AnyVec> = vec![];
if let Some(first) = self.first.as_ref() {

46
crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs
View File

@@ -3,11 +3,11 @@ use std::path::Path;
use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromDateindex<T>
@@ -95,60 +95,60 @@ where
)?;
self.dateindex_extra
.extend(starting_indexes.dateindex, self.dateindex.vec(), exit)?;
.extend(starting_indexes.dateindex, self.dateindex.iter_vec(), exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
self.dateindex.vec(),
indexes.weekindex_to_first_dateindex.vec(),
indexes.weekindex_to_dateindex_count.vec(),
self.dateindex.iter_vec(),
indexes.weekindex_to_first_dateindex.iter_vec(),
indexes.weekindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
self.dateindex.vec(),
indexes.monthindex_to_first_dateindex.vec(),
indexes.monthindex_to_dateindex_count.vec(),
self.dateindex.iter_vec(),
indexes.monthindex_to_first_dateindex.iter_vec(),
indexes.monthindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
&self.monthindex,
indexes.quarterindex_to_first_monthindex.vec(),
indexes.quarterindex_to_monthindex_count.vec(),
indexes.quarterindex_to_first_monthindex.iter_vec(),
indexes.quarterindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.yearindex.from_aligned(
starting_indexes.yearindex,
&self.monthindex,
indexes.yearindex_to_first_monthindex.vec(),
indexes.yearindex_to_monthindex_count.vec(),
indexes.yearindex_to_first_monthindex.iter_vec(),
indexes.yearindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.decadeindex.from_aligned(
starting_indexes.decadeindex,
&self.yearindex,
indexes.decadeindex_to_first_yearindex.vec(),
indexes.decadeindex_to_yearindex_count.vec(),
indexes.decadeindex_to_first_yearindex.iter_vec(),
indexes.decadeindex_to_yearindex_count.iter_vec(),
exit,
)?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
vec![self.dateindex.any_vec()],
self.dateindex_extra.any_vecs(),
self.weekindex.any_vecs(),
self.monthindex.any_vecs(),
self.quarterindex.any_vecs(),
self.yearindex.any_vecs(),
self.decadeindex.any_vecs(),
self.dateindex_extra.vecs(),
self.weekindex.vecs(),
self.monthindex.vecs(),
self.quarterindex.vecs(),
self.yearindex.vecs(),
self.decadeindex.vecs(),
]
.concat()
}

58
crates/brk_computer/src/storage/vecs/grouped/from_height.rs
View File

@@ -5,11 +5,11 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromHeight<T>
@@ -114,9 +114,9 @@ where
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&StoredVec<Height, T>>,
height: Option<&impl AnyIterableVec<Height, T>>,
) -> color_eyre::Result<()> {
let height = height.unwrap_or_else(|| self.height.as_ref().unwrap().vec());
let height = height.unwrap_or_else(|| self.height.as_ref().unwrap().iter_vec());
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -124,74 +124,74 @@ where
self.dateindex.compute(
starting_indexes.dateindex,
height,
indexes.dateindex_to_first_height.vec(),
indexes.dateindex_to_height_count.vec(),
indexes.dateindex_to_first_height.iter_vec(),
indexes.dateindex_to_height_count.iter_vec(),
exit,
)?;
self.weekindex.from_aligned(
starting_indexes.weekindex,
&self.dateindex,
indexes.weekindex_to_first_dateindex.vec(),
indexes.weekindex_to_dateindex_count.vec(),
indexes.weekindex_to_first_dateindex.iter_vec(),
indexes.weekindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.monthindex.from_aligned(
starting_indexes.monthindex,
&self.dateindex,
indexes.monthindex_to_first_dateindex.vec(),
indexes.monthindex_to_dateindex_count.vec(),
indexes.monthindex_to_first_dateindex.iter_vec(),
indexes.monthindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
&self.monthindex,
indexes.quarterindex_to_first_monthindex.vec(),
indexes.quarterindex_to_monthindex_count.vec(),
indexes.quarterindex_to_first_monthindex.iter_vec(),
indexes.quarterindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.yearindex.from_aligned(
starting_indexes.yearindex,
&self.monthindex,
indexes.yearindex_to_first_monthindex.vec(),
indexes.yearindex_to_monthindex_count.vec(),
indexes.yearindex_to_first_monthindex.iter_vec(),
indexes.yearindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.decadeindex.from_aligned(
starting_indexes.decadeindex,
&self.yearindex,
indexes.decadeindex_to_first_yearindex.vec(),
indexes.decadeindex_to_yearindex_count.vec(),
indexes.decadeindex_to_first_yearindex.iter_vec(),
indexes.decadeindex_to_yearindex_count.iter_vec(),
exit,
)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
height,
indexes.difficultyepoch_to_first_height.vec(),
indexes.difficultyepoch_to_height_count.vec(),
indexes.difficultyepoch_to_first_height.iter_vec(),
indexes.difficultyepoch_to_height_count.iter_vec(),
exit,
)?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height_extra.any_vecs(),
self.dateindex.any_vecs(),
self.weekindex.any_vecs(),
self.difficultyepoch.any_vecs(),
self.monthindex.any_vecs(),
self.quarterindex.any_vecs(),
self.yearindex.any_vecs(),
// self.halvingepoch.any_vecs(),
self.decadeindex.any_vecs(),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
self.difficultyepoch.vecs(),
self.monthindex.vecs(),
self.quarterindex.vecs(),
self.yearindex.vecs(),
// self.halvingepoch.vecs(),
self.decadeindex.vecs(),
]
.concat()
}

22
crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs
View File

@@ -3,11 +3,11 @@ use std::path::Path;
use brk_core::{DifficultyEpoch, Height};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromHeightStrict<T>
@@ -73,25 +73,25 @@ where
compute(&mut self.height, indexer, indexes, starting_indexes, exit)?;
self.height_extra
.extend(starting_indexes.height, self.height.vec(), exit)?;
.extend(starting_indexes.height, self.height.iter_vec(), exit)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
self.height.vec(),
indexes.difficultyepoch_to_first_height.vec(),
indexes.difficultyepoch_to_height_count.vec(),
self.height.iter_vec(),
indexes.difficultyepoch_to_first_height.iter_vec(),
indexes.difficultyepoch_to_height_count.iter_vec(),
exit,
)?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
vec![self.height.any_vec()],
self.height_extra.any_vecs(),
self.difficultyepoch.any_vecs(),
// self.halvingepoch.any_vecs(),
self.height_extra.vecs(),
self.difficultyepoch.vecs(),
// self.halvingepoch.vecs(),
]
.concat()
}

28
crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs
View File

@@ -6,11 +6,11 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromTxindex<T>
@@ -119,7 +119,7 @@ where
exit: &Exit,
txindex: Option<&StoredVec<TxIndex, T>>,
) -> color_eyre::Result<()> {
let txindex = txindex.unwrap_or_else(|| self.txindex.as_ref().unwrap().vec());
let txindex = txindex.unwrap_or_else(|| self.txindex.as_ref().unwrap().iter_vec());
self.height.compute(
starting_indexes.height,
@@ -188,18 +188,18 @@ where
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height.any_vecs(),
self.dateindex.any_vecs(),
self.weekindex.any_vecs(),
self.difficultyepoch.any_vecs(),
self.monthindex.any_vecs(),
self.quarterindex.any_vecs(),
self.yearindex.any_vecs(),
// self.halvingepoch.any_vecs(),
self.decadeindex.any_vecs(),
self.height.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
self.difficultyepoch.vecs(),
self.monthindex.vecs(),
self.quarterindex.vecs(),
self.yearindex.vecs(),
// self.halvingepoch.vecs(),
self.decadeindex.vecs(),
]
.concat()
}

2
crates/brk_computer/src/storage/vecs/grouped/mod.rs
View File

@@ -3,6 +3,7 @@ mod from_dateindex;
mod from_height;
mod from_height_strict;
mod from_txindex;
mod r#type;
mod value_from_height;
mod value_from_txindex;
@@ -11,5 +12,6 @@ pub use from_dateindex::*;
pub use from_height::*;
pub use from_height_strict::*;
pub use from_txindex::*;
use r#type::*;
pub use value_from_height::*;
pub use value_from_txindex::*;

0
crates/brk_computer/src/storage/vecs/vec/_type.rs → crates/brk_computer/src/storage/vecs/grouped/type.rs
View File

18
crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs
View File

@@ -3,10 +3,10 @@ use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Sats};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{
EagerVec, marketprice,
marketprice,
vecs::{Indexes, indexes},
};
@@ -110,7 +110,7 @@ impl ComputedValueVecsFromHeight {
.compute_rest(indexes, starting_indexes, exit, None)?;
}
let height = height.unwrap_or_else(|| self.sats.height.as_ref().unwrap().vec());
let height = height.unwrap_or_else(|| self.sats.height.as_ref().unwrap().iter_vec());
self.bitcoin.compute_all(
indexer,
@@ -122,13 +122,13 @@ impl ComputedValueVecsFromHeight {
},
)?;
let txindex = self.bitcoin.height.as_ref().unwrap().vec();
let txindex = self.bitcoin.height.as_ref().unwrap().iter_vec();
let price = marketprices
.as_ref()
.unwrap()
.chainindexes_to_close
.height
.vec();
.iter_vec();
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_all(
@@ -145,11 +145,11 @@ impl ComputedValueVecsFromHeight {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.sats.any_vecs(),
self.bitcoin.any_vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()),
self.sats.vecs(),
self.bitcoin.vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()
}

12
crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs
View File

@@ -3,10 +3,10 @@ use std::path::Path;
use brk_core::{Bitcoin, Dollars, Sats, TxIndex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{
EagerVec, marketprice,
marketprice,
vecs::{Indexes, indexes},
};
@@ -147,11 +147,11 @@ impl ComputedValueVecsFromTxindex {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.sats.any_vecs(),
self.bitcoin.any_vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()),
self.sats.vecs(),
self.bitcoin.vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()
}

289
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -8,13 +8,14 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, VecIterator, Version};
use super::EagerVec;
use brk_vec::{
AnyCollectableVec, CloneableAnyIterableVec, Compressed, Computation, ComputedVec,
ComputedVecFrom1, EagerVec, VecIterator, Version,
};
#[derive(Clone)]
pub struct Vecs {
pub dateindex_to_date: EagerVec<DateIndex, Date>,
pub dateindex_to_date: ComputedVecFrom1<DateIndex, Date, DateIndex, DateIndex>,
pub dateindex_to_dateindex: EagerVec<DateIndex, DateIndex>,
pub dateindex_to_first_height: EagerVec<DateIndex, Height>,
pub dateindex_to_height_count: EagerVec<DateIndex, StoredUsize>,
@@ -70,20 +71,36 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let dateindex_to_dateindex = EagerVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::ZERO,
compressed,
)?;
let dateindex_to_date = ComputedVec::forced_import_or_init_from_1(
computation,
path,
"dateindex_to_date",
Version::ZERO,
compressed,
dateindex_to_dateindex.boxed_clone(),
|index, dateindex_to_dateindex_iter| {
dateindex_to_dateindex_iter
.next_at(index)
.map(|(dateindex, _)| Date::from(dateindex))
},
)?;
Ok(Self {
dateindex_to_date: EagerVec::forced_import(
&path.join("dateindex_to_date"),
Version::ZERO,
compressed,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::ZERO,
compressed,
)?,
dateindex_to_date,
dateindex_to_dateindex,
dateindex_to_first_height: EagerVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::ZERO,
@@ -356,84 +373,84 @@ impl Vecs {
self.outputindex_to_outputindex.compute_range(
starting_indexes.outputindex,
indexer_vecs.outputindex_to_value.vec(),
&indexer_vecs.outputindex_to_value,
|i| (i, i),
exit,
)?;
self.p2pk33index_to_p2pk33index.compute_range(
starting_indexes.p2pk33index,
indexer_vecs.p2pk33index_to_p2pk33bytes.vec(),
&indexer_vecs.p2pk33index_to_p2pk33bytes,
|i| (i, i),
exit,
)?;
self.p2pk65index_to_p2pk65index.compute_range(
starting_indexes.p2pk65index,
indexer_vecs.p2pk65index_to_p2pk65bytes.vec(),
&indexer_vecs.p2pk65index_to_p2pk65bytes,
|i| (i, i),
exit,
)?;
self.p2pkhindex_to_p2pkhindex.compute_range(
starting_indexes.p2pkhindex,
indexer_vecs.p2pkhindex_to_p2pkhbytes.vec(),
&indexer_vecs.p2pkhindex_to_p2pkhbytes,
|i| (i, i),
exit,
)?;
self.p2shindex_to_p2shindex.compute_range(
starting_indexes.p2shindex,
indexer_vecs.p2shindex_to_p2shbytes.vec(),
&indexer_vecs.p2shindex_to_p2shbytes,
|i| (i, i),
exit,
)?;
self.p2trindex_to_p2trindex.compute_range(
starting_indexes.p2trindex,
indexer_vecs.p2trindex_to_p2trbytes.vec(),
&indexer_vecs.p2trindex_to_p2trbytes,
|i| (i, i),
exit,
)?;
self.p2wpkhindex_to_p2wpkhindex.compute_range(
starting_indexes.p2wpkhindex,
indexer_vecs.p2wpkhindex_to_p2wpkhbytes.vec(),
&indexer_vecs.p2wpkhindex_to_p2wpkhbytes,
|i| (i, i),
exit,
)?;
self.p2wshindex_to_p2wshindex.compute_range(
starting_indexes.p2wshindex,
indexer_vecs.p2wshindex_to_p2wshbytes.vec(),
&indexer_vecs.p2wshindex_to_p2wshbytes,
|i| (i, i),
exit,
)?;
self.emptyoutputindex_to_emptyoutputindex.compute_range(
starting_indexes.emptyoutputindex,
indexer_vecs.emptyoutputindex_to_txindex.vec(),
&indexer_vecs.emptyoutputindex_to_txindex,
|i| (i, i),
exit,
)?;
self.p2msindex_to_p2msindex.compute_range(
starting_indexes.p2msindex,
indexer_vecs.p2msindex_to_txindex.vec(),
&indexer_vecs.p2msindex_to_txindex,
|i| (i, i),
exit,
)?;
self.opreturnindex_to_opreturnindex.compute_range(
starting_indexes.opreturnindex,
indexer_vecs.opreturnindex_to_txindex.vec(),
&indexer_vecs.opreturnindex_to_txindex,
|i| (i, i),
exit,
)?;
self.p2aindex_to_p2aindex.compute_range(
starting_indexes.p2aindex,
indexer_vecs.p2aindex_to_p2abytes.vec(),
&indexer_vecs.p2aindex_to_p2abytes,
|i| (i, i),
exit,
)?;
@@ -441,7 +458,7 @@ impl Vecs {
self.unknownoutputindex_to_unknownoutputindex
.compute_range(
starting_indexes.unknownoutputindex,
indexer_vecs.unknownoutputindex_to_txindex.vec(),
&indexer_vecs.unknownoutputindex_to_txindex,
|i| (i, i),
exit,
)?;
@@ -452,7 +469,7 @@ impl Vecs {
self.inputindex_to_inputindex.compute_range(
starting_indexes.inputindex,
indexer_vecs.inputindex_to_outputindex.vec(),
&indexer_vecs.inputindex_to_outputindex,
|i| (i, i),
exit,
)?;
@@ -463,22 +480,22 @@ impl Vecs {
self.txindex_to_txindex.compute_range(
starting_indexes.txindex,
indexer_vecs.txindex_to_txid.vec(),
&indexer_vecs.txindex_to_txid,
|i| (i, i),
exit,
)?;
self.height_to_txindex_count.compute_count_from_indexes(
starting_indexes.height,
indexer_vecs.height_to_first_txindex.vec(),
indexer_vecs.txindex_to_txid.vec(),
&indexer_vecs.height_to_first_txindex,
&indexer_vecs.txindex_to_txid,
exit,
)?;
self.txindex_to_height.compute_inverse_less_to_more(
starting_indexes.height,
indexer_vecs.height_to_first_txindex.vec(),
self.height_to_txindex_count.vec(),
&indexer_vecs.height_to_first_txindex,
&self.height_to_txindex_count,
exit,
)?;
@@ -488,14 +505,14 @@ impl Vecs {
self.height_to_height.compute_range(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
&indexer_vecs.height_to_timestamp,
|h| (h, h),
exit,
)?;
self.height_to_date.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
&indexer_vecs.height_to_timestamp,
|(h, t, ..)| (h, Date::from(t)),
exit,
)?;
@@ -503,7 +520,7 @@ impl Vecs {
let mut prev_timestamp_fixed = None;
self.height_to_timestamp_fixed.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
&indexer_vecs.height_to_timestamp,
|(h, timestamp, height_to_timestamp_fixed_iter)| {
if prev_timestamp_fixed.is_none() {
if let Some(prev_h) = h.decremented() {
@@ -524,7 +541,7 @@ impl Vecs {
self.height_to_date_fixed.compute_transform(
starting_indexes.height,
self.height_to_timestamp_fixed.vec(),
&self.height_to_timestamp_fixed,
|(h, t, ..)| (h, Date::from(t)),
exit,
)?;
@@ -543,7 +560,7 @@ impl Vecs {
self.height_to_dateindex.compute_transform(
starting_indexes.height,
self.height_to_date_fixed.vec(),
&self.height_to_date_fixed,
|(h, d, ..)| (h, DateIndex::try_from(d).unwrap()),
exit,
)?;
@@ -561,28 +578,24 @@ impl Vecs {
self.dateindex_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
self.height_to_dateindex.vec(),
&self.height_to_dateindex,
exit,
)?;
self.dateindex_to_dateindex.compute_range(
starting_dateindex,
self.dateindex_to_first_height.vec(),
&self.dateindex_to_first_height,
|di| (di, di),
exit,
)?;
self.dateindex_to_date.compute_range(
starting_dateindex,
self.dateindex_to_dateindex.vec(),
|di| (di, Date::from(di)),
exit,
)?;
self.dateindex_to_date
.compute_if_necessary(starting_dateindex, exit)?;
self.dateindex_to_height_count.compute_count_from_indexes(
starting_dateindex,
self.dateindex_to_first_height.vec(),
indexer_vecs.height_to_weight.vec(),
&self.dateindex_to_first_height,
&indexer_vecs.height_to_weight,
exit,
)?;
@@ -598,21 +611,17 @@ impl Vecs {
self.dateindex_to_weekindex.compute_range(
starting_dateindex,
self.dateindex_to_dateindex.vec(),
&self.dateindex_to_dateindex,
|di| (di, WeekIndex::from(di)),
exit,
)?;
self.weekindex_to_first_dateindex
.compute_inverse_more_to_less(
starting_dateindex,
self.dateindex_to_weekindex.vec(),
exit,
)?;
.compute_inverse_more_to_less(starting_dateindex, &self.dateindex_to_weekindex, exit)?;
self.weekindex_to_weekindex.compute_range(
starting_weekindex,
self.weekindex_to_first_dateindex.vec(),
&self.weekindex_to_first_dateindex,
|wi| (wi, wi),
exit,
)?;
@@ -620,8 +629,8 @@ impl Vecs {
self.weekindex_to_dateindex_count
.compute_count_from_indexes(
starting_weekindex,
self.weekindex_to_first_dateindex.vec(),
self.dateindex_to_date.vec(),
&self.weekindex_to_first_dateindex,
&self.dateindex_to_date,
exit,
)?;
@@ -637,7 +646,7 @@ impl Vecs {
self.height_to_difficultyepoch.compute_range(
starting_indexes.height,
self.height_to_height.vec(),
&self.height_to_height,
|h| (h, DifficultyEpoch::from(h)),
exit,
)?;
@@ -645,13 +654,13 @@ impl Vecs {
self.difficultyepoch_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
self.height_to_difficultyepoch.vec(),
&self.height_to_difficultyepoch,
exit,
)?;
self.difficultyepoch_to_difficultyepoch.compute_range(
starting_difficultyepoch,
self.difficultyepoch_to_first_height.vec(),
&self.difficultyepoch_to_first_height,
|i| (i, i),
exit,
)?;
@@ -659,8 +668,8 @@ impl Vecs {
self.difficultyepoch_to_height_count
.compute_count_from_indexes(
starting_difficultyepoch,
self.difficultyepoch_to_first_height.vec(),
self.height_to_date.vec(),
&self.difficultyepoch_to_first_height,
&self.height_to_date,
exit,
)?;
@@ -676,7 +685,7 @@ impl Vecs {
self.dateindex_to_monthindex.compute_range(
starting_dateindex,
self.dateindex_to_dateindex.vec(),
&self.dateindex_to_dateindex,
|di| (di, MonthIndex::from(di)),
exit,
)?;
@@ -684,13 +693,13 @@ impl Vecs {
self.monthindex_to_first_dateindex
.compute_inverse_more_to_less(
starting_dateindex,
self.dateindex_to_monthindex.vec(),
&self.dateindex_to_monthindex,
exit,
)?;
self.monthindex_to_monthindex.compute_range(
starting_monthindex,
self.monthindex_to_first_dateindex.vec(),
&self.monthindex_to_first_dateindex,
|mi| (mi, mi),
exit,
)?;
@@ -698,8 +707,8 @@ impl Vecs {
self.monthindex_to_dateindex_count
.compute_count_from_indexes(
starting_monthindex,
self.monthindex_to_first_dateindex.vec(),
self.dateindex_to_date.vec(),
&self.monthindex_to_first_dateindex,
&self.dateindex_to_date,
exit,
)?;
@@ -715,7 +724,7 @@ impl Vecs {
self.monthindex_to_quarterindex.compute_range(
starting_monthindex,
self.monthindex_to_monthindex.vec(),
&self.monthindex_to_monthindex,
|mi| (mi, QuarterIndex::from(mi)),
exit,
)?;
@@ -723,7 +732,7 @@ impl Vecs {
self.quarterindex_to_first_monthindex
.compute_inverse_more_to_less(
starting_monthindex,
self.monthindex_to_quarterindex.vec(),
&self.monthindex_to_quarterindex,
exit,
)?;
@@ -731,7 +740,7 @@ impl Vecs {
self.quarterindex_to_quarterindex.compute_range(
starting_quarterindex,
self.quarterindex_to_first_monthindex.vec(),
&self.quarterindex_to_first_monthindex,
|i| (i, i),
exit,
)?;
@@ -739,8 +748,8 @@ impl Vecs {
self.quarterindex_to_monthindex_count
.compute_count_from_indexes(
starting_quarterindex,
self.quarterindex_to_first_monthindex.vec(),
self.monthindex_to_monthindex.vec(),
&self.quarterindex_to_first_monthindex,
&self.monthindex_to_monthindex,
exit,
)?;
@@ -756,7 +765,7 @@ impl Vecs {
self.monthindex_to_yearindex.compute_range(
starting_monthindex,
self.monthindex_to_monthindex.vec(),
&self.monthindex_to_monthindex,
|i| (i, YearIndex::from(i)),
exit,
)?;
@@ -764,13 +773,13 @@ impl Vecs {
self.yearindex_to_first_monthindex
.compute_inverse_more_to_less(
starting_monthindex,
self.monthindex_to_yearindex.vec(),
&self.monthindex_to_yearindex,
exit,
)?;
self.yearindex_to_yearindex.compute_range(
starting_yearindex,
self.yearindex_to_first_monthindex.vec(),
&self.yearindex_to_first_monthindex,
|i| (i, i),
exit,
)?;
@@ -778,8 +787,8 @@ impl Vecs {
self.yearindex_to_monthindex_count
.compute_count_from_indexes(
starting_yearindex,
self.yearindex_to_first_monthindex.vec(),
self.monthindex_to_monthindex.vec(),
&self.yearindex_to_first_monthindex,
&self.monthindex_to_monthindex,
exit,
)?;
// ---
@@ -794,7 +803,7 @@ impl Vecs {
self.height_to_halvingepoch.compute_range(
starting_indexes.height,
self.height_to_height.vec(),
&self.height_to_height,
|h| (h, HalvingEpoch::from(h)),
exit,
)?;
@@ -802,13 +811,13 @@ impl Vecs {
self.halvingepoch_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
self.height_to_halvingepoch.vec(),
&self.height_to_halvingepoch,
exit,
)?;
self.halvingepoch_to_halvingepoch.compute_range(
starting_halvingepoch,
self.halvingepoch_to_first_height.vec(),
&self.halvingepoch_to_first_height,
|i| (i, i),
exit,
)?;
@@ -825,7 +834,7 @@ impl Vecs {
self.yearindex_to_decadeindex.compute_range(
starting_yearindex,
self.yearindex_to_yearindex.vec(),
&self.yearindex_to_yearindex,
|i| (i, DecadeIndex::from(i)),
exit,
)?;
@@ -833,13 +842,13 @@ impl Vecs {
self.decadeindex_to_first_yearindex
.compute_inverse_more_to_less(
starting_yearindex,
self.yearindex_to_decadeindex.vec(),
&self.yearindex_to_decadeindex,
exit,
)?;
self.decadeindex_to_decadeindex.compute_range(
starting_decadeindex,
self.decadeindex_to_first_yearindex.vec(),
&self.decadeindex_to_first_yearindex,
|i| (i, i),
exit,
)?;
@@ -847,8 +856,8 @@ impl Vecs {
self.decadeindex_to_yearindex_count
.compute_count_from_indexes(
starting_decadeindex,
self.decadeindex_to_first_yearindex.vec(),
self.yearindex_to_yearindex.vec(),
&self.decadeindex_to_first_yearindex,
&self.yearindex_to_yearindex,
exit,
)?;
@@ -865,61 +874,61 @@ impl Vecs {
})
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
vec![
self.dateindex_to_date.any_vec(),
self.dateindex_to_dateindex.any_vec(),
self.dateindex_to_first_height.any_vec(),
self.dateindex_to_height_count.any_vec(),
self.dateindex_to_monthindex.any_vec(),
self.dateindex_to_weekindex.any_vec(),
self.decadeindex_to_decadeindex.any_vec(),
self.decadeindex_to_first_yearindex.any_vec(),
self.decadeindex_to_yearindex_count.any_vec(),
self.difficultyepoch_to_difficultyepoch.any_vec(),
self.difficultyepoch_to_first_height.any_vec(),
self.difficultyepoch_to_height_count.any_vec(),
self.emptyoutputindex_to_emptyoutputindex.any_vec(),
self.halvingepoch_to_first_height.any_vec(),
self.halvingepoch_to_halvingepoch.any_vec(),
self.height_to_date.any_vec(),
self.height_to_date_fixed.any_vec(),
self.height_to_dateindex.any_vec(),
self.height_to_difficultyepoch.any_vec(),
self.height_to_halvingepoch.any_vec(),
self.height_to_height.any_vec(),
self.height_to_timestamp_fixed.any_vec(),
self.height_to_txindex_count.any_vec(),
self.inputindex_to_inputindex.any_vec(),
self.monthindex_to_dateindex_count.any_vec(),
self.monthindex_to_first_dateindex.any_vec(),
self.monthindex_to_monthindex.any_vec(),
self.monthindex_to_quarterindex.any_vec(),
self.monthindex_to_yearindex.any_vec(),
self.opreturnindex_to_opreturnindex.any_vec(),
self.outputindex_to_outputindex.any_vec(),
self.p2aindex_to_p2aindex.any_vec(),
self.p2msindex_to_p2msindex.any_vec(),
self.p2pk33index_to_p2pk33index.any_vec(),
self.p2pk65index_to_p2pk65index.any_vec(),
self.p2pkhindex_to_p2pkhindex.any_vec(),
self.p2shindex_to_p2shindex.any_vec(),
self.p2trindex_to_p2trindex.any_vec(),
self.p2wpkhindex_to_p2wpkhindex.any_vec(),
self.p2wshindex_to_p2wshindex.any_vec(),
self.quarterindex_to_first_monthindex.any_vec(),
self.quarterindex_to_monthindex_count.any_vec(),
self.quarterindex_to_quarterindex.any_vec(),
self.txindex_to_height.any_vec(),
self.txindex_to_txindex.any_vec(),
self.unknownoutputindex_to_unknownoutputindex.any_vec(),
self.weekindex_to_dateindex_count.any_vec(),
self.weekindex_to_first_dateindex.any_vec(),
self.weekindex_to_weekindex.any_vec(),
self.yearindex_to_decadeindex.any_vec(),
self.yearindex_to_first_monthindex.any_vec(),
self.yearindex_to_monthindex_count.any_vec(),
self.yearindex_to_yearindex.any_vec(),
&self.dateindex_to_date,
&self.dateindex_to_dateindex,
&self.dateindex_to_first_height,
&self.dateindex_to_height_count,
&self.dateindex_to_monthindex,
&self.dateindex_to_weekindex,
&self.decadeindex_to_decadeindex,
&self.decadeindex_to_first_yearindex,
&self.decadeindex_to_yearindex_count,
&self.difficultyepoch_to_difficultyepoch,
&self.difficultyepoch_to_first_height,
&self.difficultyepoch_to_height_count,
&self.emptyoutputindex_to_emptyoutputindex,
&self.halvingepoch_to_first_height,
&self.halvingepoch_to_halvingepoch,
&self.height_to_date,
&self.height_to_date_fixed,
&self.height_to_dateindex,
&self.height_to_difficultyepoch,
&self.height_to_halvingepoch,
&self.height_to_height,
&self.height_to_timestamp_fixed,
&self.height_to_txindex_count,
&self.inputindex_to_inputindex,
&self.monthindex_to_dateindex_count,
&self.monthindex_to_first_dateindex,
&self.monthindex_to_monthindex,
&self.monthindex_to_quarterindex,
&self.monthindex_to_yearindex,
&self.opreturnindex_to_opreturnindex,
&self.outputindex_to_outputindex,
&self.p2aindex_to_p2aindex,
&self.p2msindex_to_p2msindex,
&self.p2pk33index_to_p2pk33index,
&self.p2pk65index_to_p2pk65index,
&self.p2pkhindex_to_p2pkhindex,
&self.p2shindex_to_p2shindex,
&self.p2trindex_to_p2trindex,
&self.p2wpkhindex_to_p2wpkhindex,
&self.p2wshindex_to_p2wshindex,
&self.quarterindex_to_first_monthindex,
&self.quarterindex_to_monthindex_count,
&self.quarterindex_to_quarterindex,
&self.txindex_to_height,
&self.txindex_to_txindex,
&self.unknownoutputindex_to_unknownoutputindex,
&self.weekindex_to_dateindex_count,
&self.weekindex_to_first_dateindex,
&self.weekindex_to_weekindex,
&self.yearindex_to_decadeindex,
&self.yearindex_to_first_monthindex,
&self.yearindex_to_monthindex_count,
&self.yearindex_to_yearindex,
]
}
}

137
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -7,10 +7,10 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{Compressed, VecIterator, Version};
use brk_vec::{AnyIterableVec, Compressed, Computation, EagerVec, VecIterator, Version};
use super::{
EagerVec, Indexes,
Indexes,
grouped::{
ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions,
},
@@ -69,7 +69,11 @@ const VERSION: Version = Version::ZERO;
const VERSION_IN_SATS: Version = Version::ONE;
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let mut fetched_path = path.to_owned();
@@ -336,7 +340,7 @@ impl Vecs {
let mut height_to_timestamp_iter = indexer_vecs.height_to_timestamp.iter();
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
indexer_vecs.height_to_timestamp.iter_vec(),
|(h, t, ..)| {
let ohlc = fetcher
.get_height(
@@ -353,42 +357,42 @@ impl Vecs {
self.height_to_open_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.height_to_high_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.height_to_low_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.height_to_close_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.height_to_ohlc.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
indexes.dateindex_to_date.vec(),
indexes.dateindex_to_date.iter_vec(),
|(di, d, ..)| {
let ohlc = fetcher.get_date(d).unwrap();
(di, ohlc)
@@ -398,35 +402,35 @@ impl Vecs {
self.dateindex_to_open_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.dateindex_to_high_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.dateindex_to_low_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.dateindex_to_close_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.dateindex_to_ohlc.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
@@ -439,7 +443,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)
@@ -454,7 +458,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)
@@ -469,7 +473,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)
@@ -484,7 +488,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)
@@ -499,7 +503,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)
@@ -514,7 +518,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)
@@ -529,7 +533,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)
@@ -544,7 +548,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)
@@ -556,7 +560,7 @@ impl Vecs {
let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter();
self.weekindex_to_ohlc.compute_transform(
starting_indexes.weekindex,
self.timeindexes_to_close.weekindex.unwrap_last().vec(),
self.timeindexes_to_close.weekindex.unwrap_last().iter_vec(),
|(i, close, ..)| {
(
i,
@@ -588,7 +592,7 @@ impl Vecs {
self.chainindexes_to_close
.difficultyepoch
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -608,7 +612,10 @@ impl Vecs {
let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter();
self.monthindex_to_ohlc.compute_transform(
starting_indexes.monthindex,
self.timeindexes_to_close.monthindex.unwrap_last().vec(),
self.timeindexes_to_close
.monthindex
.unwrap_last()
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -629,7 +636,10 @@ impl Vecs {
let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter();
self.quarterindex_to_ohlc.compute_transform(
starting_indexes.quarterindex,
self.timeindexes_to_close.quarterindex.unwrap_last().vec(),
self.timeindexes_to_close
.quarterindex
.unwrap_last()
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -649,7 +659,7 @@ impl Vecs {
let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter();
self.yearindex_to_ohlc.compute_transform(
starting_indexes.yearindex,
self.timeindexes_to_close.yearindex.unwrap_last().vec(),
self.timeindexes_to_close.yearindex.unwrap_last().iter_vec(),
|(i, close, ..)| {
(
i,
@@ -672,7 +682,10 @@ impl Vecs {
let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter();
self.decadeindex_to_ohlc.compute_transform(
starting_indexes.decadeindex,
self.timeindexes_to_close.decadeindex.unwrap_last().vec(),
self.timeindexes_to_close
.decadeindex
.unwrap_last()
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -695,7 +708,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_open.height.vec(),
self.chainindexes_to_open.height.iter_vec(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
@@ -710,7 +723,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_low.height.vec(),
self.chainindexes_to_low.height.iter_vec(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
@@ -725,7 +738,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_high.height.vec(),
self.chainindexes_to_high.height.iter_vec(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
@@ -740,7 +753,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_close.height.vec(),
self.chainindexes_to_close.height.iter_vec(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -755,7 +768,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_open.dateindex.vec(),
self.timeindexes_to_open.dateindex.iter_vec(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
@@ -770,7 +783,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_low.dateindex.vec(),
self.timeindexes_to_low.dateindex.iter_vec(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
@@ -785,7 +798,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_high.dateindex.vec(),
self.timeindexes_to_high.dateindex.iter_vec(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
@@ -800,7 +813,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_close.dateindex.vec(),
self.timeindexes_to_close.dateindex.iter_vec(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -812,7 +825,7 @@ impl Vecs {
let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter();
self.height_to_ohlc_in_sats.compute_transform(
starting_indexes.height,
self.chainindexes_to_close_in_sats.height.vec(),
self.chainindexes_to_close_in_sats.height.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -832,7 +845,7 @@ impl Vecs {
let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
self.dateindex_to_ohlc_in_sats.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_close_in_sats.dateindex.vec(),
self.timeindexes_to_close_in_sats.dateindex.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -867,7 +880,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.weekindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -902,7 +915,7 @@ impl Vecs {
self.chainindexes_to_close_in_sats
.difficultyepoch
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -937,7 +950,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.monthindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -972,7 +985,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.quarterindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -1007,7 +1020,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.yearindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -1045,7 +1058,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.decadeindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -1063,7 +1076,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
vec![
vec![
self.dateindex_to_close_in_cents.any_vec(),
@@ -1095,22 +1108,22 @@ impl Vecs {
// self.halvingepoch_to_ohlc_in_sats.any_vec(),
self.decadeindex_to_ohlc_in_sats.any_vec(),
],
self.timeindexes_to_close.any_vecs(),
self.timeindexes_to_high.any_vecs(),
self.timeindexes_to_low.any_vecs(),
self.timeindexes_to_open.any_vecs(),
self.chainindexes_to_close.any_vecs(),
self.chainindexes_to_high.any_vecs(),
self.chainindexes_to_low.any_vecs(),
self.chainindexes_to_open.any_vecs(),
self.timeindexes_to_close_in_sats.any_vecs(),
self.timeindexes_to_high_in_sats.any_vecs(),
self.timeindexes_to_low_in_sats.any_vecs(),
self.timeindexes_to_open_in_sats.any_vecs(),
self.chainindexes_to_close_in_sats.any_vecs(),
self.chainindexes_to_high_in_sats.any_vecs(),
self.chainindexes_to_low_in_sats.any_vecs(),
self.chainindexes_to_open_in_sats.any_vecs(),
self.timeindexes_to_close.vecs(),
self.timeindexes_to_high.vecs(),
self.timeindexes_to_low.vecs(),
self.timeindexes_to_open.vecs(),
self.chainindexes_to_close.vecs(),
self.chainindexes_to_high.vecs(),
self.chainindexes_to_low.vecs(),
self.chainindexes_to_open.vecs(),
self.timeindexes_to_close_in_sats.vecs(),
self.timeindexes_to_high_in_sats.vecs(),
self.timeindexes_to_low_in_sats.vecs(),
self.timeindexes_to_open_in_sats.vecs(),
self.chainindexes_to_close_in_sats.vecs(),
self.chainindexes_to_high_in_sats.vecs(),
self.chainindexes_to_low_in_sats.vecs(),
self.chainindexes_to_open_in_sats.vecs(),
]
.concat()
}

16
crates/brk_computer/src/storage/vecs/mining.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, path::Path};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, VecIterator, Version};
use brk_vec::{Compressed, Computation, VecIterator, Version};
use super::{
Indexes,
@@ -19,7 +19,11 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -113,11 +117,11 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
self.indexes_to_difficulty.any_vecs(),
self.indexes_to_difficultyepoch.any_vecs(),
self.indexes_to_halvingepoch.any_vecs(),
self.indexes_to_difficulty.vecs(),
self.indexes_to_difficultyepoch.vecs(),
self.indexes_to_halvingepoch.vecs(),
]
.concat()
}

97
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -3,26 +3,24 @@ use std::{fs, path::Path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed};
use brk_vec::{AnyCollectableVec, AnyVec, CollectableVec, Compressed, Computation};
pub mod blocks;
pub mod grouped;
// pub mod blocks;
// pub mod grouped;
pub mod indexes;
pub mod marketprice;
pub mod mining;
pub mod transactions;
pub mod vec;
// pub mod marketprice;
// pub mod mining;
// pub mod transactions;
pub use indexes::Indexes;
pub use vec::*;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub transactions: transactions::Vecs,
pub marketprice: Option<marketprice::Vecs>,
// pub blocks: blocks::Vecs,
// pub mining: mining::Vecs,
// pub transactions: transactions::Vecs,
// pub marketprice: Option<marketprice::Vecs>,
}
impl Vecs {
@@ -36,17 +34,18 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
blocks: blocks::Vecs::forced_import(path, compressed)?,
indexes: indexes::Vecs::forced_import(path, compressed)?,
mining: mining::Vecs::forced_import(path, compressed)?,
transactions: transactions::Vecs::forced_import(
path,
indexer,
computation,
compressed,
fetch,
)?,
marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()),
// blocks: blocks::Vecs::forced_import(path, computation, compressed)?,
indexes: indexes::Vecs::forced_import(path, computation, compressed)?,
// mining: mining::Vecs::forced_import(path, computation, compressed)?,
// transactions: transactions::Vecs::forced_import(
// path,
// indexer,
// computation,
// compressed,
// fetch,
// )?,
// marketprice: fetch
// .then(|| marketprice::Vecs::forced_import(path, computation, compressed).unwrap()),
})
}
@@ -59,40 +58,40 @@ impl Vecs {
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
// self.blocks
// .compute(indexer, &self.indexes, &starting_indexes, exit)?;
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
// self.mining
// .compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(marketprice) = self.marketprice.as_mut() {
marketprice.compute(
indexer,
&self.indexes,
&starting_indexes,
fetcher.unwrap(),
exit,
)?;
}
// if let Some(marketprice) = self.marketprice.as_mut() {
// marketprice.compute(
// indexer,
// &self.indexes,
// &starting_indexes,
// fetcher.unwrap(),
// exit,
// )?;
// }
self.transactions.compute(
indexer,
&self.indexes,
&starting_indexes,
self.marketprice.as_ref(),
exit,
)?;
// self.transactions.compute(
// indexer,
// &self.indexes,
// &starting_indexes,
// self.marketprice.as_ref(),
// exit,
// )?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.indexes.any_vecs(),
self.blocks.any_vecs(),
self.mining.any_vecs(),
self.transactions.any_vecs(),
self.marketprice.as_ref().map_or(vec![], |v| v.any_vecs()),
self.indexes.vecs(),
// self.blocks.vecs(),
// self.mining.vecs(),
// self.transactions.vecs(),
// self.marketprice.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()
}

81
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -6,10 +6,13 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{AnyVec, Compressed, StoredIndex, VecIterator, Version};
use brk_vec::{
AnyIterableVec, AnyVec, Compressed, Computation, ComputedVec, ComputedVecFrom2, EagerVec,
StoredIndex, VecIterator, Version,
};
use super::{
Computation, ComputedVec, ComputedVecFrom2, EagerVec, Indexes,
Indexes,
grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
ComputedVecsFromTxindex, StorableVecGeneatorOptions,
@@ -49,6 +52,7 @@ pub struct Vecs {
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub inputindex_to_value:
ComputedVecFrom2<InputIndex, Sats, OutputIndex, Sats, InputIndex, OutputIndex>,
// Bitcoin and dollar version too
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_is_coinbase: EagerVec<TxIndex, bool>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
@@ -541,23 +545,8 @@ impl Vecs {
exit,
)?;
// let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
self.inputindex_to_value.compute_if_necessary(
starting_indexes.inputindex,
// indexer.vecs().inputindex_to_outputindex.vec(),
// |(inputindex, outputindex, ..)| {
// let value = if outputindex == OutputIndex::COINBASE {
// Sats::ZERO
// } else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
// value.into_inner()
// } else {
// dbg!(inputindex, outputindex);
// panic!()
// };
// (inputindex, value)
// },
exit,
)?;
self.inputindex_to_value
.compute_if_necessary(starting_indexes.inputindex, exit)?;
self.indexes_to_output_value.compute_all(
indexer,
@@ -898,7 +887,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
vec![
self.inputindex_to_value.any_vec(),
@@ -906,32 +895,32 @@ impl Vecs {
self.txindex_to_vsize.any_vec(),
self.txindex_to_weight.any_vec(),
],
self.indexes_to_tx_count.any_vecs(),
self.indexes_to_coinbase.any_vecs(),
self.indexes_to_fee.any_vecs(),
self.indexes_to_feerate.any_vecs(),
self.indexes_to_input_value.any_vecs(),
self.indexes_to_output_value.any_vecs(),
self.indexes_to_subsidy.any_vecs(),
self.indexes_to_tx_v1.any_vecs(),
self.indexes_to_tx_v2.any_vecs(),
self.indexes_to_tx_v3.any_vecs(),
self.indexes_to_tx_vsize.any_vecs(),
self.indexes_to_tx_weight.any_vecs(),
self.indexes_to_input_count.any_vecs(),
self.indexes_to_output_count.any_vecs(),
self.indexes_to_p2a_count.any_vecs(),
self.indexes_to_p2ms_count.any_vecs(),
self.indexes_to_p2pk33_count.any_vecs(),
self.indexes_to_p2pk65_count.any_vecs(),
self.indexes_to_p2pkh_count.any_vecs(),
self.indexes_to_p2sh_count.any_vecs(),
self.indexes_to_p2tr_count.any_vecs(),
self.indexes_to_p2wpkh_count.any_vecs(),
self.indexes_to_p2wsh_count.any_vecs(),
self.indexes_to_opreturn_count.any_vecs(),
self.indexes_to_unknownoutput_count.any_vecs(),
self.indexes_to_emptyoutput_count.any_vecs(),
self.indexes_to_tx_count.vecs(),
self.indexes_to_coinbase.vecs(),
self.indexes_to_fee.vecs(),
self.indexes_to_feerate.vecs(),
self.indexes_to_input_value.vecs(),
self.indexes_to_output_value.vecs(),
self.indexes_to_subsidy.vecs(),
self.indexes_to_tx_v1.vecs(),
self.indexes_to_tx_v2.vecs(),
self.indexes_to_tx_v3.vecs(),
self.indexes_to_tx_vsize.vecs(),
self.indexes_to_tx_weight.vecs(),
self.indexes_to_input_count.vecs(),
self.indexes_to_output_count.vecs(),
self.indexes_to_p2a_count.vecs(),
self.indexes_to_p2ms_count.vecs(),
self.indexes_to_p2pk33_count.vecs(),
self.indexes_to_p2pk65_count.vecs(),
self.indexes_to_p2pkh_count.vecs(),
self.indexes_to_p2sh_count.vecs(),
self.indexes_to_p2tr_count.vecs(),
self.indexes_to_p2wpkh_count.vecs(),
self.indexes_to_p2wsh_count.vecs(),
self.indexes_to_opreturn_count.vecs(),
self.indexes_to_unknownoutput_count.vecs(),
self.indexes_to_emptyoutput_count.vecs(),
]
.concat()
}

536
crates/brk_computer/src/storage/vecs/vec/eager.rs
View File

@@ -1,536 +0,0 @@
use core::error;
use std::{
cmp::Ordering,
fmt::Debug,
ops::Add,
path::{Path, PathBuf},
time::Duration,
};
use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, TxIndex};
use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, VecIterator, Version,
};
use log::info;
const ONE_KIB: usize = 1024;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
const MAX_CACHE_SIZE: usize = 210 * ONE_MIB;
#[derive(Debug, Clone)]
pub struct EagerVec<I, T> {
computed_version: Option<Version>,
inner: StoredVec<I, T>,
}
impl<I, T> EagerVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF: usize = size_of::<T>();
pub fn forced_import(
path: &Path,
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
let inner = StoredVec::forced_import(path, version, compressed)?;
Ok(Self {
computed_version: None,
inner,
})
}
fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> {
if exit.triggered() {
return Ok(());
}
exit.block();
self.inner.truncate_if_needed(index)?;
exit.release();
Ok(())
}
#[inline]
pub fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
match self.len().cmp(&index.to_usize()?) {
Ordering::Less => {
return Err(Error::IndexTooHigh);
}
ord => {
if ord == Ordering::Greater {
self.safe_truncate_if_needed(index, exit)?;
}
self.inner.push(value);
}
}
if self.inner.pushed_len() * Self::SIZE_OF >= MAX_CACHE_SIZE {
self.safe_flush(exit)
} else {
Ok(())
}
}
pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
if exit.triggered() {
return Ok(());
}
exit.block();
self.inner.flush()?;
exit.release();
Ok(())
}
pub fn version(&self) -> Version {
self.computed_version.unwrap()
}
pub fn len(&self) -> usize {
self.inner.len()
}
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
pub fn name(&self) -> String {
self.inner.name()
}
pub fn modified_time(&self) -> Result<Duration> {
self.inner.modified_time()
}
pub fn vec(&self) -> &StoredVec<I, T> {
&self.inner
}
pub fn mut_vec(&mut self) -> &StoredVec<I, T> {
&mut self.inner
}
pub fn any_vec(&self) -> &dyn brk_vec::AnyVec {
&self.inner
}
pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyVec {
&mut self.inner
}
pub fn path(&self) -> &Path {
self.inner.path()
}
#[inline]
fn path_computed_version(&self) -> PathBuf {
self.inner.path().join("computed_version")
}
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
let path = self.path_computed_version();
if version.validate(path.as_ref()).is_err() {
self.inner.reset()?;
}
version.write(path.as_ref())?;
if self.is_empty() {
info!("Computing {}...", self.name())
}
Ok(())
}
pub fn iter(&self) -> StoredVecIterator<I, T> {
self.into_iter()
}
pub fn compute_to<F>(
&mut self,
max_from: I,
to: usize,
version: Version,
mut t: F,
exit: &Exit,
) -> Result<()>
where
F: FnMut(I) -> (I, T),
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + version,
)?;
let index = max_from.min(I::from(self.len()));
(index.to_usize()?..to).try_for_each(|i| {
let (i, v) = t(I::from(i));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
pub fn compute_range<A, F>(
&mut self,
max_from: I,
other: &StoredVec<I, A>,
t: F,
exit: &Exit,
) -> Result<()>
where
A: StoredType,
F: FnMut(I) -> (I, T),
{
self.compute_to(max_from, other.len(), other.version(), t, exit)
}
pub fn compute_transform<A, B, F>(
&mut self,
max_from: A,
other: &StoredVec<A, B>,
mut t: F,
exit: &Exit,
) -> Result<()>
where
A: StoredIndex,
B: StoredType,
F: FnMut((A, B, &Self)) -> (I, T),
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + other.version(),
)?;
let index = max_from.min(A::from(self.len()));
other.iter_at(index).try_for_each(|(a, b)| {
let (i, v) = t((a, b.into_inner(), self));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
pub fn compute_inverse_more_to_less(
&mut self,
max_from: T,
other: &StoredVec<T, I>,
exit: &Exit,
) -> Result<()>
where
I: StoredType + StoredIndex,
T: StoredIndex,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + other.version(),
)?;
let index = max_from.min(
VecIterator::last(self.inner.iter()).map_or_else(T::default, |(_, v)| v.into_inner()),
);
let mut prev_i = None;
other.iter_at(index).try_for_each(|(v, i)| -> Result<()> {
let i = i.into_inner();
if prev_i.is_some_and(|prev_i| prev_i == i) {
return Ok(());
}
if self.iter().get_inner(i).is_none_or(|old_v| old_v > v) {
self.forced_push_at(i, v, exit)?;
}
prev_i.replace(i);
Ok(())
})?;
self.safe_flush(exit)
}
pub fn compute_inverse_less_to_more(
&mut self,
max_from: T,
first_indexes: &StoredVec<T, I>,
indexes_count: &StoredVec<T, StoredUsize>,
exit: &Exit,
) -> Result<()>
where
I: StoredType,
T: StoredIndex,
{
self.validate_computed_version_or_reset_file(
Version::ZERO
+ self.inner.version()
+ first_indexes.version()
+ indexes_count.version(),
)?;
let mut indexes_count_iter = indexes_count.iter();
let index = max_from.min(T::from(self.len()));
first_indexes
.iter_at(index)
.try_for_each(|(value, first_index)| {
let first_index = (first_index).to_usize()?;
let count = *indexes_count_iter.unwrap_get_inner(value);
(first_index..first_index + count)
.try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
})?;
self.safe_flush(exit)
}
pub fn compute_count_from_indexes<T2, T3>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
other_to_else: &StoredVec<T2, T3>,
exit: &Exit,
) -> Result<()>
where
T: From<T2>,
T2: StoredType
+ StoredIndex
+ Copy
+ Add<usize, Output = T2>
+ CheckedSub<T2>
+ TryInto<T>
+ Default,
<T2 as TryInto<T>>::Error: error::Error + 'static,
T3: StoredType,
{
let opt: Option<Box<dyn FnMut(T2) -> bool>> = None;
self.compute_filtered_count_from_indexes_(max_from, first_indexes, other_to_else, opt, exit)
}
pub fn compute_filtered_count_from_indexes<T2, T3, F>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
other_to_else: &StoredVec<T2, T3>,
filter: F,
exit: &Exit,
) -> Result<()>
where
T: From<T2>,
T2: StoredType
+ StoredIndex
+ Copy
+ Add<usize, Output = T2>
+ CheckedSub<T2>
+ TryInto<T>
+ Default,
<T2 as TryInto<T>>::Error: error::Error + 'static,
T3: StoredType,
F: FnMut(T2) -> bool,
{
self.compute_filtered_count_from_indexes_(
max_from,
first_indexes,
other_to_else,
Some(Box::new(filter)),
exit,
)
}
fn compute_filtered_count_from_indexes_<T2, T3>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
other_to_else: &StoredVec<T2, T3>,
mut filter: Option<Box<dyn FnMut(T2) -> bool + '_>>,
exit: &Exit,
) -> Result<()>
where
T: From<T2>,
T2: StoredType
+ StoredIndex
+ Copy
+ Add<usize, Output = T2>
+ CheckedSub<T2>
+ TryInto<T>
+ Default,
T3: StoredType,
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + first_indexes.version(), // + last_indexes.version(),
)?;
let mut other_iter = first_indexes.iter();
let index = max_from.min(I::from(self.len()));
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| {
let end = other_iter
.get_inner(i + 1)
.map(|v| v.unwrap_to_usize())
.unwrap_or_else(|| other_to_else.len());
let range = first_index.unwrap_to_usize()..end;
let count = if let Some(filter) = filter.as_mut() {
range.into_iter().filter(|i| filter(T2::from(*i))).count()
} else {
range.count()
};
self.forced_push_at(i, T::from(T2::from(count)), exit)
})?;
self.safe_flush(exit)
}
pub fn compute_is_first_ordered<A>(
&mut self,
max_from: I,
self_to_other: &StoredVec<I, A>,
other_to_self: &StoredVec<A, I>,
exit: &Exit,
) -> Result<()>
where
I: StoredType,
T: From<bool>,
A: StoredIndex + StoredType,
{
self.validate_computed_version_or_reset_file(
Version::ZERO
+ self.inner.version()
+ self_to_other.version()
+ other_to_self.version(),
)?;
let mut other_to_self_iter = other_to_self.iter();
let index = max_from.min(I::from(self.len()));
self_to_other.iter_at(index).try_for_each(|(i, other)| {
self.forced_push_at(
i,
T::from(other_to_self_iter.unwrap_get_inner(other.into_inner()) == i),
exit,
)
})?;
self.safe_flush(exit)
}
pub fn compute_sum_from_indexes<T2>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
indexes_count: &StoredVec<I, StoredUsize>,
source: &StoredVec<T2, T>,
exit: &Exit,
) -> Result<()>
where
T: From<usize> + Add<T, Output = T>,
T2: StoredIndex + StoredType,
{
self.validate_computed_version_or_reset_file(
Version::ZERO
+ self.inner.version()
+ first_indexes.version()
+ indexes_count.version(),
)?;
let mut indexes_count_iter = indexes_count.iter();
let mut source_iter = source.iter();
let index = max_from.min(I::from(self.len()));
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| {
let count = *indexes_count_iter.unwrap_get_inner(i);
let first_index = first_index.unwrap_to_usize();
let range = first_index..first_index + count;
let mut sum = T::from(0_usize);
range.into_iter().for_each(|i| {
sum = sum.clone() + source_iter.unwrap_get_inner(T2::from(i));
});
self.forced_push_at(i, sum, exit)
})?;
self.safe_flush(exit)
}
}
impl<I> EagerVec<I, Bitcoin>
where
I: StoredIndex,
{
pub fn compute_from_sats(
&mut self,
max_from: I,
sats: &StoredVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + sats.version(),
)?;
let index = max_from.min(I::from(self.len()));
sats.iter_at(index).try_for_each(|(i, sats)| {
let (i, v) = (i, Bitcoin::from(sats.into_inner()));
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
}
impl EagerVec<Height, Dollars> {
pub fn compute_from_bitcoin(
&mut self,
max_from: Height,
bitcoin: &StoredVec<Height, Bitcoin>,
price: &StoredVec<Height, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + bitcoin.version(),
)?;
let mut price_iter = price.iter();
let index = max_from.min(Height::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
let dollars = price_iter.unwrap_get_inner(i);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
}
impl EagerVec<TxIndex, Dollars> {
pub fn compute_from_bitcoin(
&mut self,
max_from: TxIndex,
bitcoin: &StoredVec<TxIndex, Bitcoin>,
i_to_height: &StoredVec<TxIndex, Height>,
price: &StoredVec<Height, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + bitcoin.version(),
)?;
let mut i_to_height_iter = i_to_height.iter();
let mut price_iter = price.iter();
let index = max_from.min(TxIndex::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| {
let height = i_to_height_iter.unwrap_get_inner(i);
let dollars = price_iter.unwrap_get_inner(height);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
self.safe_flush(exit)
}
}
impl<'a, I, T> IntoIterator for &'a EagerVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = StoredVecIterator<'a, I, T>;
fn into_iter(self) -> Self::IntoIter {
self.inner.into_iter()
}
}

139
crates/brk_computer/src/storage/vecs/vec/lazy1.rs
View File

@@ -1,139 +0,0 @@
use std::marker::PhantomData;
use brk_vec::{
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
};
pub type ComputeFrom1<T, S1I, S1T> =
for<'a> fn(usize, &mut dyn BaseVecIterator<Item = (S1I, Value<'a, S1T>)>) -> Option<T>;
#[derive(Clone)]
pub struct LazyVecFrom1<I, T, S1I, S1T> {
name: String,
version: Version,
source: StoredVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
phantom: PhantomData<I>,
}
impl<I, T, S1I, S1T> LazyVecFrom1<I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
{
pub fn init(
name: &str,
version: Version,
source: StoredVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
) -> Self {
Self {
name: name.to_owned(),
version,
source,
compute,
phantom: PhantomData,
}
}
fn version(&self) -> Version {
self.version
}
}
pub struct LazyVecIterator<'a, I, T, S1I, S1T> {
lazy: &'a LazyVecFrom1<I, T, S1I, S1T>,
source: StoredVecIterator<'a, S1I, S1T>,
index: usize,
}
impl<'a, I, T, S1I, S1T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(self.index, &mut self.lazy.source.iter())
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
self.index += 1;
}
opt
}
}
impl<I, T, S1I, S1T> BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
{
#[inline]
fn mut_index(&mut self) -> &mut usize {
&mut self.index
}
#[inline]
fn len(&self) -> usize {
todo!();
}
}
impl<'a, I, T, S1I, S1T> IntoIterator for &'a LazyVecFrom1<I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
lazy: self,
source: self.source.iter(),
index: 0,
}
}
}
impl<I, T, S1I, S1T> AnyVec for LazyVecFrom1<I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
{
fn name(&self) -> String {
self.name.clone()
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
self.source.len()
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
self.source.modified_time()
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
}
}

164
crates/brk_computer/src/storage/vecs/vec/lazy2.rs
View File

@@ -1,164 +0,0 @@
use std::marker::PhantomData;
use brk_vec::{
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
};
pub type ComputeFrom2<T, S1I, S1T, S2I, S2T> = for<'a> fn(
usize,
&mut dyn BaseVecIterator<Item = (S1I, Value<'a, S1T>)>,
&mut dyn BaseVecIterator<Item = (S2I, Value<'a, S2T>)>,
) -> Option<T>;
#[derive(Clone)]
pub struct LazyVecFrom2<I, T, S1I, S1T, S2I, S2T> {
name: String,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
phantom: PhantomData<I>,
}
impl<I, T, S1I, S1T, S2I, S2T> LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
pub fn init(
name: &str,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
) -> Self {
Self {
name: name.to_string(),
version,
source1,
source2,
compute,
phantom: PhantomData,
}
}
fn version(&self) -> Version {
self.version
}
}
pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> {
lazy: &'a LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>,
source1: StoredVecIterator<'a, S1I, S1T>,
source2: StoredVecIterator<'a, S2I, S2T>,
index: usize,
}
impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(
self.index,
&mut self.lazy.source1.iter(),
&mut self.lazy.source2.iter(),
)
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
self.index += 1;
}
opt
}
}
impl<I, T, S1I, S1T, S2I, S2T> BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
#[inline]
fn mut_index(&mut self) -> &mut usize {
&mut self.index
}
#[inline]
fn len(&self) -> usize {
todo!();
}
}
impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
lazy: self,
source1: self.source1.iter(),
source2: self.source2.iter(),
index: 0,
}
}
}
impl<I, T, S1I, S1T, S2I, S2T> AnyVec for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
fn name(&self) -> String {
self.name.clone()
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
self.source1.len().min(self.source2.len())
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
Ok(self
.source1
.modified_time()?
.min(self.source2.modified_time()?))
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
}
}

184
crates/brk_computer/src/storage/vecs/vec/lazy3.rs
View File

@@ -1,184 +0,0 @@
use std::marker::PhantomData;
use brk_vec::{
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
};
pub type ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(
usize,
&mut dyn BaseVecIterator<Item = (S1I, Value<'a, S1T>)>,
&mut dyn BaseVecIterator<Item = (S2I, Value<'a, S2T>)>,
&mut dyn BaseVecIterator<Item = (S3I, Value<'a, S3T>)>,
) -> Option<T>;
#[derive(Clone)]
pub struct LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
name: String,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source3: StoredVec<S3I, S3T>,
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
phantom: PhantomData<I>,
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
pub fn init(
name: &str,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source3: StoredVec<S3I, S3T>,
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
) -> Self {
Self {
name: name.to_string(),
version,
source1,
source2,
source3,
compute,
phantom: PhantomData,
}
}
fn version(&self) -> Version {
self.version
}
}
pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
lazy: &'a LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>,
source1: StoredVecIterator<'a, S1I, S1T>,
source2: StoredVecIterator<'a, S2I, S2T>,
source3: StoredVecIterator<'a, S3I, S3T>,
index: usize,
}
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator
for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(
self.index,
&mut self.lazy.source1.iter(),
&mut self.lazy.source2.iter(),
&mut self.lazy.source3.iter(),
)
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
self.index += 1;
}
opt
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> BaseVecIterator
for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
#[inline]
fn mut_index(&mut self) -> &mut usize {
&mut self.index
}
#[inline]
fn len(&self) -> usize {
todo!();
}
}
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator
for &'a LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType + 'a,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
lazy: self,
source1: self.source1.iter(),
source2: self.source2.iter(),
source3: self.source3.iter(),
index: 0,
}
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyVec for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn name(&self) -> String {
self.name.clone()
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
self.source1.len().min(self.source2.len())
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
Ok(self
.source1
.modified_time()?
.min(self.source2.modified_time()?))
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
}
}

251
crates/brk_computer/src/storage/vecs/vec/mod.rs
View File

@@ -1,251 +0,0 @@
use std::{path::Path, time::Duration};
use brk_exit::Exit;
use clap_derive::ValueEnum;
use serde::{Deserialize, Serialize};
mod _type;
mod eager;
mod lazy1;
mod lazy2;
mod lazy3;
pub use _type::*;
use brk_core::StoredPhantom;
use brk_vec::{
AnyVec, Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
};
pub use eager::*;
pub use lazy1::*;
pub use lazy2::*;
pub use lazy3::*;
#[derive(
Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum,
)]
pub enum Computation {
Eager,
#[default]
Lazy,
}
impl Computation {
pub fn eager(&self) -> bool {
*self == Self::Eager
}
pub fn lazy(&self) -> bool {
*self == Self::Lazy
}
}
#[derive(Clone)]
pub enum Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T> {
From1(StoredVec<S1I, S1T>, ComputeFrom1<T, S1I, S1T>),
From2(
(StoredVec<S1I, S1T>, StoredVec<S2I, S2T>),
ComputeFrom2<T, S1I, S1T, S2I, S2T>,
),
From3(
(
StoredVec<S1I, S1T>,
StoredVec<S2I, S2T>,
StoredVec<S3I, S3T>,
),
ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
),
}
pub type ComputedVecFrom1<I, T, S1I, S1T> =
ComputedVec<I, T, S1I, S1T, StoredPhantom, StoredPhantom, StoredPhantom, StoredPhantom>;
pub type ComputedVecFrom2<I, T, S1I, S1T, S2I, S2T> =
ComputedVec<I, T, S1I, S1T, S2I, S2T, StoredPhantom, StoredPhantom>;
pub type ComputedVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> =
ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
#[derive(Clone)]
pub enum ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
Eager {
vec: EagerVec<I, T>,
deps: Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T>,
},
LazyFrom1(LazyVecFrom1<I, T, S1I, S1T>),
LazyFrom2(LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>),
LazyFrom3(LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>),
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
pub fn forced_import_or_init_from_1(
mode: Computation,
path: &Path,
name: &str,
version: Version,
compressed: Compressed,
source: StoredVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
) -> brk_vec::Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, version, compressed)?,
deps: Dependencies::From1(source, compute),
},
Computation::Lazy => {
Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute))
}
})
}
#[allow(clippy::too_many_arguments)]
pub fn forced_import_or_init_from_2(
path: &Path,
name: &str,
mode: Computation,
version: Version,
compressed: Compressed,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
) -> brk_vec::Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, version, compressed)?,
deps: Dependencies::From2((source1, source2), compute),
},
Computation::Lazy => {
Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute))
}
})
}
#[allow(clippy::too_many_arguments)]
pub fn forced_import_or_init_from_3(
mode: Computation,
path: &Path,
name: &str,
version: Version,
compressed: Compressed,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source3: StoredVec<S3I, S3T>,
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
) -> brk_vec::Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, version, compressed)?,
deps: Dependencies::From3((source1, source2, source3), compute),
},
Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init(
name, version, source1, source2, source3, compute,
)),
})
}
pub fn compute_if_necessary(&mut self, max_from: I, exit: &Exit) -> Result<()> {
let (vec, dependencies) = if let ComputedVec::Eager {
vec,
deps: dependencies,
} = self
{
(vec, dependencies)
} else {
return Ok(());
};
match dependencies {
Dependencies::From1(source, compute) => {
let version = source.version();
let mut iter = source.iter();
let t = |i: I| {
compute(i.unwrap_to_usize(), &mut iter)
.map(|v| (i, v))
.unwrap()
};
vec.compute_to(max_from, 1, version, t, exit)
}
Dependencies::From2((source1, source2), compute) => {
let version = source1.version() + source2.version();
let mut iter1 = source1.iter();
let mut iter2 = source2.iter();
let t = |i: I| {
compute(i.unwrap_to_usize(), &mut iter1, &mut iter2)
.map(|v| (i, v))
.unwrap()
};
vec.compute_to(max_from, 1, version, t, exit)
}
Dependencies::From3((source1, source2, source3), compute) => {
let version = source1.version() + source2.version() + source3.version();
let mut iter1 = source1.iter();
let mut iter2 = source2.iter();
let mut iter3 = source3.iter();
let t = |i: I| {
compute(i.unwrap_to_usize(), &mut iter1, &mut iter2, &mut iter3)
.map(|v| (i, v))
.unwrap()
};
vec.compute_to(max_from, 1, version, t, exit)
}
}
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyVec for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn name(&self) -> String {
match self {
ComputedVec::Eager { vec, .. } => vec.name(),
ComputedVec::LazyFrom1(v) => v.name(),
ComputedVec::LazyFrom2(v) => v.name(),
ComputedVec::LazyFrom3(v) => v.name(),
}
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
match self {
ComputedVec::Eager { vec, .. } => vec.len(),
ComputedVec::LazyFrom1(v) => v.len(),
ComputedVec::LazyFrom2(v) => v.len(),
ComputedVec::LazyFrom3(v) => v.len(),
}
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
todo!()
}
fn modified_time(&self) -> Result<Duration> {
match self {
ComputedVec::Eager { vec, .. } => vec.modified_time(),
ComputedVec::LazyFrom1(v) => v.modified_time(),
ComputedVec::LazyFrom2(v) => v.modified_time(),
ComputedVec::LazyFrom3(v) => v.modified_time(),
}
}
}