Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: big vec refactor + lazy

Browse Source
This commit is contained in:
nym21
2025-05-05 12:47:52 +02:00
parent 320c708e10
commit 9ba3c2b7c5
51 changed files with 1608 additions and 1097 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -33,6 +33,5 @@ paths.d.ts
# Outputs
_outputs
# Logs
.log

7
Cargo.lock generated
View File

@@ -409,6 +409,7 @@ dependencies = [
"brk_parser",
"brk_query",
"brk_server",
"brk_vec",
"clap",
"clap_derive",
"color-eyre",
@@ -553,6 +554,7 @@ dependencies = [
"brk_logger",
"brk_parser",
"brk_query",
"brk_vec",
"clap",
"clap_derive",
"color-eyre",
@@ -572,6 +574,11 @@ name = "brk_vec"
version = "0.0.31"
dependencies = [
"arc-swap",
"brk_core",
"brk_exit",
"clap",
"clap_derive",
"log",
"memmap2",
"rayon",
"serde",

1
crates/brk_cli/Cargo.toml
View File

@@ -16,6 +16,7 @@ brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_query = { workspace = true }
brk_server = { workspace = true }
brk_vec = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }

3
crates/brk_cli/src/run.rs
View File

@@ -5,13 +5,14 @@ use std::{
time::Duration,
};
use brk_computer::{Computation, Computer};
use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::rpc::{self, Auth, Client, RpcApi};
use brk_server::{Server, Website, tokio};
use brk_vec::Computation;
use clap_derive::{Parser, ValueEnum};
use color_eyre::eyre::eyre;
use log::info;

3
crates/brk_computer/examples/main.rs
View File

@@ -1,11 +1,12 @@
use std::path::Path;
use brk_computer::{Computation, Computer};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{Parser, rpc};
use brk_vec::Computation;
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

14
crates/brk_computer/src/lib.rs
View File

@@ -12,9 +12,8 @@ pub use brk_parser::rpc;
mod storage;
use brk_vec::Compressed;
use brk_vec::{AnyCollectableVec, Compressed, Computation};
use log::info;
pub use storage::Computation;
use storage::{Stores, Vecs};
#[derive(Clone)]
@@ -82,13 +81,14 @@ impl Computer {
Ok(())
}
pub fn vecs(&self) -> &Vecs {
self.vecs.as_ref().unwrap()
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
// pub fn vecs(&self) -> &Vecs {
self.vecs.as_ref().unwrap().vecs()
}
pub fn mut_vecs(&mut self) -> &mut Vecs {
self.vecs.as_mut().unwrap()
}
// pub fn mut_vecs(&mut self) -> &mut Vecs {
// self.vecs.as_mut().unwrap()
// }
pub fn stores(&self) -> &Stores {
self.stores.as_ref().unwrap()

24
crates/brk_computer/src/storage/vecs/blocks.rs
View File

@@ -7,10 +7,10 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, VecIterator, Version};
use brk_vec::{AnyIterableVec, Compressed, Computation, EagerVec, VecIterator, Version};
use super::{
EagerVec, Indexes,
Indexes,
grouped::{ComputedVecsFromDateindex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
indexes,
};
@@ -30,7 +30,11 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -223,7 +227,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
vec![
self.height_to_interval.any_vec(),
@@ -231,12 +235,12 @@ impl Vecs {
self.difficultyepoch_to_timestamp.any_vec(),
self.halvingepoch_to_timestamp.any_vec(),
],
self.timeindexes_to_timestamp.any_vecs(),
self.indexes_to_block_count.any_vecs(),
self.indexes_to_block_interval.any_vecs(),
self.indexes_to_block_size.any_vecs(),
self.indexes_to_block_vbytes.any_vecs(),
self.indexes_to_block_weight.any_vecs(),
self.timeindexes_to_timestamp.vecs(),
self.indexes_to_block_count.vecs(),
self.indexes_to_block_interval.vecs(),
self.indexes_to_block_size.vecs(),
self.indexes_to_block_vbytes.vecs(),
self.indexes_to_block_weight.vecs(),
]
.concat()
}

32
crates/brk_computer/src/storage/vecs/grouped/builder.rs
View File

@@ -3,11 +3,12 @@ use std::path::Path;
use brk_core::{CheckedSub, StoredUsize};
use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, VecIterator, Version,
AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredIndex, StoredType, VecIterator,
Version,
};
use color_eyre::eyre::ContextCompat;
use crate::storage::{ComputedType, EagerVec};
use super::ComputedType;
#[derive(Clone, Debug)]
pub struct ComputedVecBuilder<I, T>
@@ -137,7 +138,12 @@ where
Ok(s)
}
pub fn extend(&mut self, max_from: I, source: &StoredVec<I, T>, exit: &Exit) -> Result<()> {
pub fn extend(
&mut self,
max_from: I,
source: &impl AnyIterableVec<I, T>,
exit: &Exit,
) -> Result<()> {
if self.total.is_none() {
return Ok(());
};
@@ -162,9 +168,9 @@ where
pub fn compute<I2>(
&mut self,
max_from: I,
source: &StoredVec<I2, T>,
first_indexes: &StoredVec<I, I2>,
count_indexes: &StoredVec<I, StoredUsize>,
source: &impl AnyIterableVec<I2, T>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
exit: &Exit,
) -> Result<()>
where
@@ -244,12 +250,12 @@ where
dbg!(
&values,
max.path(),
first_indexes.path(),
first_indexes.name(),
first_index,
count_indexes.path(),
count_indexes.name(),
count_index,
source.len(),
source.path()
source.name()
);
})
.unwrap()
@@ -319,8 +325,8 @@ where
&mut self,
max_from: I,
source: &ComputedVecBuilder<I2, T>,
first_indexes: &StoredVec<I, I2>,
count_indexes: &StoredVec<I, StoredUsize>,
first_indexes: &impl AnyIterableVec<I, I2>,
count_indexes: &impl AnyIterableVec<I, StoredUsize>,
exit: &Exit,
) -> Result<()>
where
@@ -482,7 +488,7 @@ where
fn starting_index(&self, max_from: I) -> I {
max_from.min(I::from(
self.any_vecs().into_iter().map(|v| v.len()).min().unwrap(),
self.vecs().into_iter().map(|v| v.len()).min().unwrap(),
))
}
@@ -523,7 +529,7 @@ where
self.total.as_mut().unwrap()
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
let mut v: Vec<&dyn brk_vec::AnyVec> = vec![];
if let Some(first) = self.first.as_ref() {

46
crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs
View File

@@ -3,11 +3,11 @@ use std::path::Path;
use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromDateindex<T>
@@ -95,60 +95,60 @@ where
)?;
self.dateindex_extra
.extend(starting_indexes.dateindex, self.dateindex.vec(), exit)?;
.extend(starting_indexes.dateindex, self.dateindex.iter_vec(), exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
self.dateindex.vec(),
indexes.weekindex_to_first_dateindex.vec(),
indexes.weekindex_to_dateindex_count.vec(),
self.dateindex.iter_vec(),
indexes.weekindex_to_first_dateindex.iter_vec(),
indexes.weekindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
self.dateindex.vec(),
indexes.monthindex_to_first_dateindex.vec(),
indexes.monthindex_to_dateindex_count.vec(),
self.dateindex.iter_vec(),
indexes.monthindex_to_first_dateindex.iter_vec(),
indexes.monthindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
&self.monthindex,
indexes.quarterindex_to_first_monthindex.vec(),
indexes.quarterindex_to_monthindex_count.vec(),
indexes.quarterindex_to_first_monthindex.iter_vec(),
indexes.quarterindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.yearindex.from_aligned(
starting_indexes.yearindex,
&self.monthindex,
indexes.yearindex_to_first_monthindex.vec(),
indexes.yearindex_to_monthindex_count.vec(),
indexes.yearindex_to_first_monthindex.iter_vec(),
indexes.yearindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.decadeindex.from_aligned(
starting_indexes.decadeindex,
&self.yearindex,
indexes.decadeindex_to_first_yearindex.vec(),
indexes.decadeindex_to_yearindex_count.vec(),
indexes.decadeindex_to_first_yearindex.iter_vec(),
indexes.decadeindex_to_yearindex_count.iter_vec(),
exit,
)?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
vec![self.dateindex.any_vec()],
self.dateindex_extra.any_vecs(),
self.weekindex.any_vecs(),
self.monthindex.any_vecs(),
self.quarterindex.any_vecs(),
self.yearindex.any_vecs(),
self.decadeindex.any_vecs(),
self.dateindex_extra.vecs(),
self.weekindex.vecs(),
self.monthindex.vecs(),
self.quarterindex.vecs(),
self.yearindex.vecs(),
self.decadeindex.vecs(),
]
.concat()
}

58
crates/brk_computer/src/storage/vecs/grouped/from_height.rs
View File

@@ -5,11 +5,11 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromHeight<T>
@@ -114,9 +114,9 @@ where
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&StoredVec<Height, T>>,
height: Option<&impl AnyIterableVec<Height, T>>,
) -> color_eyre::Result<()> {
let height = height.unwrap_or_else(|| self.height.as_ref().unwrap().vec());
let height = height.unwrap_or_else(|| self.height.as_ref().unwrap().iter_vec());
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -124,74 +124,74 @@ where
self.dateindex.compute(
starting_indexes.dateindex,
height,
indexes.dateindex_to_first_height.vec(),
indexes.dateindex_to_height_count.vec(),
indexes.dateindex_to_first_height.iter_vec(),
indexes.dateindex_to_height_count.iter_vec(),
exit,
)?;
self.weekindex.from_aligned(
starting_indexes.weekindex,
&self.dateindex,
indexes.weekindex_to_first_dateindex.vec(),
indexes.weekindex_to_dateindex_count.vec(),
indexes.weekindex_to_first_dateindex.iter_vec(),
indexes.weekindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.monthindex.from_aligned(
starting_indexes.monthindex,
&self.dateindex,
indexes.monthindex_to_first_dateindex.vec(),
indexes.monthindex_to_dateindex_count.vec(),
indexes.monthindex_to_first_dateindex.iter_vec(),
indexes.monthindex_to_dateindex_count.iter_vec(),
exit,
)?;
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
&self.monthindex,
indexes.quarterindex_to_first_monthindex.vec(),
indexes.quarterindex_to_monthindex_count.vec(),
indexes.quarterindex_to_first_monthindex.iter_vec(),
indexes.quarterindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.yearindex.from_aligned(
starting_indexes.yearindex,
&self.monthindex,
indexes.yearindex_to_first_monthindex.vec(),
indexes.yearindex_to_monthindex_count.vec(),
indexes.yearindex_to_first_monthindex.iter_vec(),
indexes.yearindex_to_monthindex_count.iter_vec(),
exit,
)?;
self.decadeindex.from_aligned(
starting_indexes.decadeindex,
&self.yearindex,
indexes.decadeindex_to_first_yearindex.vec(),
indexes.decadeindex_to_yearindex_count.vec(),
indexes.decadeindex_to_first_yearindex.iter_vec(),
indexes.decadeindex_to_yearindex_count.iter_vec(),
exit,
)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
height,
indexes.difficultyepoch_to_first_height.vec(),
indexes.difficultyepoch_to_height_count.vec(),
indexes.difficultyepoch_to_first_height.iter_vec(),
indexes.difficultyepoch_to_height_count.iter_vec(),
exit,
)?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height_extra.any_vecs(),
self.dateindex.any_vecs(),
self.weekindex.any_vecs(),
self.difficultyepoch.any_vecs(),
self.monthindex.any_vecs(),
self.quarterindex.any_vecs(),
self.yearindex.any_vecs(),
// self.halvingepoch.any_vecs(),
self.decadeindex.any_vecs(),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
self.difficultyepoch.vecs(),
self.monthindex.vecs(),
self.quarterindex.vecs(),
self.yearindex.vecs(),
// self.halvingepoch.vecs(),
self.decadeindex.vecs(),
]
.concat()
}

22
crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs
View File

@@ -3,11 +3,11 @@ use std::path::Path;
use brk_core::{DifficultyEpoch, Height};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromHeightStrict<T>
@@ -73,25 +73,25 @@ where
compute(&mut self.height, indexer, indexes, starting_indexes, exit)?;
self.height_extra
.extend(starting_indexes.height, self.height.vec(), exit)?;
.extend(starting_indexes.height, self.height.iter_vec(), exit)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
self.height.vec(),
indexes.difficultyepoch_to_first_height.vec(),
indexes.difficultyepoch_to_height_count.vec(),
self.height.iter_vec(),
indexes.difficultyepoch_to_first_height.iter_vec(),
indexes.difficultyepoch_to_height_count.iter_vec(),
exit,
)?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
vec![self.height.any_vec()],
self.height_extra.any_vecs(),
self.difficultyepoch.any_vecs(),
// self.halvingepoch.any_vecs(),
self.height_extra.vecs(),
self.difficultyepoch.vecs(),
// self.halvingepoch.vecs(),
]
.concat()
}

28
crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs
View File

@@ -6,11 +6,11 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
use crate::storage::{Indexes, indexes};
use super::{ComputedVecBuilder, StorableVecGeneatorOptions};
use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
#[derive(Clone)]
pub struct ComputedVecsFromTxindex<T>
@@ -119,7 +119,7 @@ where
exit: &Exit,
txindex: Option<&StoredVec<TxIndex, T>>,
) -> color_eyre::Result<()> {
let txindex = txindex.unwrap_or_else(|| self.txindex.as_ref().unwrap().vec());
let txindex = txindex.unwrap_or_else(|| self.txindex.as_ref().unwrap().iter_vec());
self.height.compute(
starting_indexes.height,
@@ -188,18 +188,18 @@ where
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height.any_vecs(),
self.dateindex.any_vecs(),
self.weekindex.any_vecs(),
self.difficultyepoch.any_vecs(),
self.monthindex.any_vecs(),
self.quarterindex.any_vecs(),
self.yearindex.any_vecs(),
// self.halvingepoch.any_vecs(),
self.decadeindex.any_vecs(),
self.height.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
self.difficultyepoch.vecs(),
self.monthindex.vecs(),
self.quarterindex.vecs(),
self.yearindex.vecs(),
// self.halvingepoch.vecs(),
self.decadeindex.vecs(),
]
.concat()
}

2
crates/brk_computer/src/storage/vecs/grouped/mod.rs
View File

@@ -3,6 +3,7 @@ mod from_dateindex;
mod from_height;
mod from_height_strict;
mod from_txindex;
mod r#type;
mod value_from_height;
mod value_from_txindex;
@@ -11,5 +12,6 @@ pub use from_dateindex::*;
pub use from_height::*;
pub use from_height_strict::*;
pub use from_txindex::*;
use r#type::*;
pub use value_from_height::*;
pub use value_from_txindex::*;

0
crates/brk_computer/src/storage/vecs/vec/_type.rs → crates/brk_computer/src/storage/vecs/grouped/type.rs
View File

18
crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs
View File

@@ -3,10 +3,10 @@ use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Sats};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{
EagerVec, marketprice,
marketprice,
vecs::{Indexes, indexes},
};
@@ -110,7 +110,7 @@ impl ComputedValueVecsFromHeight {
.compute_rest(indexes, starting_indexes, exit, None)?;
}
let height = height.unwrap_or_else(|| self.sats.height.as_ref().unwrap().vec());
let height = height.unwrap_or_else(|| self.sats.height.as_ref().unwrap().iter_vec());
self.bitcoin.compute_all(
indexer,
@@ -122,13 +122,13 @@ impl ComputedValueVecsFromHeight {
},
)?;
let txindex = self.bitcoin.height.as_ref().unwrap().vec();
let txindex = self.bitcoin.height.as_ref().unwrap().iter_vec();
let price = marketprices
.as_ref()
.unwrap()
.chainindexes_to_close
.height
.vec();
.iter_vec();
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_all(
@@ -145,11 +145,11 @@ impl ComputedValueVecsFromHeight {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.sats.any_vecs(),
self.bitcoin.any_vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()),
self.sats.vecs(),
self.bitcoin.vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()
}

12
crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs
View File

@@ -3,10 +3,10 @@ use std::path::Path;
use brk_core::{Bitcoin, Dollars, Sats, TxIndex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyVec, Compressed, EagerVec, Result, StoredVec, Version};
use crate::storage::{
EagerVec, marketprice,
marketprice,
vecs::{Indexes, indexes},
};
@@ -147,11 +147,11 @@ impl ComputedValueVecsFromTxindex {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyVec> {
[
self.sats.any_vecs(),
self.bitcoin.any_vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()),
self.sats.vecs(),
self.bitcoin.vecs(),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()
}

289
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -8,13 +8,14 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, VecIterator, Version};
use super::EagerVec;
use brk_vec::{
AnyCollectableVec, CloneableAnyIterableVec, Compressed, Computation, ComputedVec,
ComputedVecFrom1, EagerVec, VecIterator, Version,
};
#[derive(Clone)]
pub struct Vecs {
pub dateindex_to_date: EagerVec<DateIndex, Date>,
pub dateindex_to_date: ComputedVecFrom1<DateIndex, Date, DateIndex, DateIndex>,
pub dateindex_to_dateindex: EagerVec<DateIndex, DateIndex>,
pub dateindex_to_first_height: EagerVec<DateIndex, Height>,
pub dateindex_to_height_count: EagerVec<DateIndex, StoredUsize>,
@@ -70,20 +71,36 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let dateindex_to_dateindex = EagerVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::ZERO,
compressed,
)?;
let dateindex_to_date = ComputedVec::forced_import_or_init_from_1(
computation,
path,
"dateindex_to_date",
Version::ZERO,
compressed,
dateindex_to_dateindex.boxed_clone(),
|index, dateindex_to_dateindex_iter| {
dateindex_to_dateindex_iter
.next_at(index)
.map(|(dateindex, _)| Date::from(dateindex))
},
)?;
Ok(Self {
dateindex_to_date: EagerVec::forced_import(
&path.join("dateindex_to_date"),
Version::ZERO,
compressed,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::ZERO,
compressed,
)?,
dateindex_to_date,
dateindex_to_dateindex,
dateindex_to_first_height: EagerVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::ZERO,
@@ -356,84 +373,84 @@ impl Vecs {
self.outputindex_to_outputindex.compute_range(
starting_indexes.outputindex,
indexer_vecs.outputindex_to_value.vec(),
&indexer_vecs.outputindex_to_value,
|i| (i, i),
exit,
)?;
self.p2pk33index_to_p2pk33index.compute_range(
starting_indexes.p2pk33index,
indexer_vecs.p2pk33index_to_p2pk33bytes.vec(),
&indexer_vecs.p2pk33index_to_p2pk33bytes,
|i| (i, i),
exit,
)?;
self.p2pk65index_to_p2pk65index.compute_range(
starting_indexes.p2pk65index,
indexer_vecs.p2pk65index_to_p2pk65bytes.vec(),
&indexer_vecs.p2pk65index_to_p2pk65bytes,
|i| (i, i),
exit,
)?;
self.p2pkhindex_to_p2pkhindex.compute_range(
starting_indexes.p2pkhindex,
indexer_vecs.p2pkhindex_to_p2pkhbytes.vec(),
&indexer_vecs.p2pkhindex_to_p2pkhbytes,
|i| (i, i),
exit,
)?;
self.p2shindex_to_p2shindex.compute_range(
starting_indexes.p2shindex,
indexer_vecs.p2shindex_to_p2shbytes.vec(),
&indexer_vecs.p2shindex_to_p2shbytes,
|i| (i, i),
exit,
)?;
self.p2trindex_to_p2trindex.compute_range(
starting_indexes.p2trindex,
indexer_vecs.p2trindex_to_p2trbytes.vec(),
&indexer_vecs.p2trindex_to_p2trbytes,
|i| (i, i),
exit,
)?;
self.p2wpkhindex_to_p2wpkhindex.compute_range(
starting_indexes.p2wpkhindex,
indexer_vecs.p2wpkhindex_to_p2wpkhbytes.vec(),
&indexer_vecs.p2wpkhindex_to_p2wpkhbytes,
|i| (i, i),
exit,
)?;
self.p2wshindex_to_p2wshindex.compute_range(
starting_indexes.p2wshindex,
indexer_vecs.p2wshindex_to_p2wshbytes.vec(),
&indexer_vecs.p2wshindex_to_p2wshbytes,
|i| (i, i),
exit,
)?;
self.emptyoutputindex_to_emptyoutputindex.compute_range(
starting_indexes.emptyoutputindex,
indexer_vecs.emptyoutputindex_to_txindex.vec(),
&indexer_vecs.emptyoutputindex_to_txindex,
|i| (i, i),
exit,
)?;
self.p2msindex_to_p2msindex.compute_range(
starting_indexes.p2msindex,
indexer_vecs.p2msindex_to_txindex.vec(),
&indexer_vecs.p2msindex_to_txindex,
|i| (i, i),
exit,
)?;
self.opreturnindex_to_opreturnindex.compute_range(
starting_indexes.opreturnindex,
indexer_vecs.opreturnindex_to_txindex.vec(),
&indexer_vecs.opreturnindex_to_txindex,
|i| (i, i),
exit,
)?;
self.p2aindex_to_p2aindex.compute_range(
starting_indexes.p2aindex,
indexer_vecs.p2aindex_to_p2abytes.vec(),
&indexer_vecs.p2aindex_to_p2abytes,
|i| (i, i),
exit,
)?;
@@ -441,7 +458,7 @@ impl Vecs {
self.unknownoutputindex_to_unknownoutputindex
.compute_range(
starting_indexes.unknownoutputindex,
indexer_vecs.unknownoutputindex_to_txindex.vec(),
&indexer_vecs.unknownoutputindex_to_txindex,
|i| (i, i),
exit,
)?;
@@ -452,7 +469,7 @@ impl Vecs {
self.inputindex_to_inputindex.compute_range(
starting_indexes.inputindex,
indexer_vecs.inputindex_to_outputindex.vec(),
&indexer_vecs.inputindex_to_outputindex,
|i| (i, i),
exit,
)?;
@@ -463,22 +480,22 @@ impl Vecs {
self.txindex_to_txindex.compute_range(
starting_indexes.txindex,
indexer_vecs.txindex_to_txid.vec(),
&indexer_vecs.txindex_to_txid,
|i| (i, i),
exit,
)?;
self.height_to_txindex_count.compute_count_from_indexes(
starting_indexes.height,
indexer_vecs.height_to_first_txindex.vec(),
indexer_vecs.txindex_to_txid.vec(),
&indexer_vecs.height_to_first_txindex,
&indexer_vecs.txindex_to_txid,
exit,
)?;
self.txindex_to_height.compute_inverse_less_to_more(
starting_indexes.height,
indexer_vecs.height_to_first_txindex.vec(),
self.height_to_txindex_count.vec(),
&indexer_vecs.height_to_first_txindex,
&self.height_to_txindex_count,
exit,
)?;
@@ -488,14 +505,14 @@ impl Vecs {
self.height_to_height.compute_range(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
&indexer_vecs.height_to_timestamp,
|h| (h, h),
exit,
)?;
self.height_to_date.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
&indexer_vecs.height_to_timestamp,
|(h, t, ..)| (h, Date::from(t)),
exit,
)?;
@@ -503,7 +520,7 @@ impl Vecs {
let mut prev_timestamp_fixed = None;
self.height_to_timestamp_fixed.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
&indexer_vecs.height_to_timestamp,
|(h, timestamp, height_to_timestamp_fixed_iter)| {
if prev_timestamp_fixed.is_none() {
if let Some(prev_h) = h.decremented() {
@@ -524,7 +541,7 @@ impl Vecs {
self.height_to_date_fixed.compute_transform(
starting_indexes.height,
self.height_to_timestamp_fixed.vec(),
&self.height_to_timestamp_fixed,
|(h, t, ..)| (h, Date::from(t)),
exit,
)?;
@@ -543,7 +560,7 @@ impl Vecs {
self.height_to_dateindex.compute_transform(
starting_indexes.height,
self.height_to_date_fixed.vec(),
&self.height_to_date_fixed,
|(h, d, ..)| (h, DateIndex::try_from(d).unwrap()),
exit,
)?;
@@ -561,28 +578,24 @@ impl Vecs {
self.dateindex_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
self.height_to_dateindex.vec(),
&self.height_to_dateindex,
exit,
)?;
self.dateindex_to_dateindex.compute_range(
starting_dateindex,
self.dateindex_to_first_height.vec(),
&self.dateindex_to_first_height,
|di| (di, di),
exit,
)?;
self.dateindex_to_date.compute_range(
starting_dateindex,
self.dateindex_to_dateindex.vec(),
|di| (di, Date::from(di)),
exit,
)?;
self.dateindex_to_date
.compute_if_necessary(starting_dateindex, exit)?;
self.dateindex_to_height_count.compute_count_from_indexes(
starting_dateindex,
self.dateindex_to_first_height.vec(),
indexer_vecs.height_to_weight.vec(),
&self.dateindex_to_first_height,
&indexer_vecs.height_to_weight,
exit,
)?;
@@ -598,21 +611,17 @@ impl Vecs {
self.dateindex_to_weekindex.compute_range(
starting_dateindex,
self.dateindex_to_dateindex.vec(),
&self.dateindex_to_dateindex,
|di| (di, WeekIndex::from(di)),
exit,
)?;
self.weekindex_to_first_dateindex
.compute_inverse_more_to_less(
starting_dateindex,
self.dateindex_to_weekindex.vec(),
exit,
)?;
.compute_inverse_more_to_less(starting_dateindex, &self.dateindex_to_weekindex, exit)?;
self.weekindex_to_weekindex.compute_range(
starting_weekindex,
self.weekindex_to_first_dateindex.vec(),
&self.weekindex_to_first_dateindex,
|wi| (wi, wi),
exit,
)?;
@@ -620,8 +629,8 @@ impl Vecs {
self.weekindex_to_dateindex_count
.compute_count_from_indexes(
starting_weekindex,
self.weekindex_to_first_dateindex.vec(),
self.dateindex_to_date.vec(),
&self.weekindex_to_first_dateindex,
&self.dateindex_to_date,
exit,
)?;
@@ -637,7 +646,7 @@ impl Vecs {
self.height_to_difficultyepoch.compute_range(
starting_indexes.height,
self.height_to_height.vec(),
&self.height_to_height,
|h| (h, DifficultyEpoch::from(h)),
exit,
)?;
@@ -645,13 +654,13 @@ impl Vecs {
self.difficultyepoch_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
self.height_to_difficultyepoch.vec(),
&self.height_to_difficultyepoch,
exit,
)?;
self.difficultyepoch_to_difficultyepoch.compute_range(
starting_difficultyepoch,
self.difficultyepoch_to_first_height.vec(),
&self.difficultyepoch_to_first_height,
|i| (i, i),
exit,
)?;
@@ -659,8 +668,8 @@ impl Vecs {
self.difficultyepoch_to_height_count
.compute_count_from_indexes(
starting_difficultyepoch,
self.difficultyepoch_to_first_height.vec(),
self.height_to_date.vec(),
&self.difficultyepoch_to_first_height,
&self.height_to_date,
exit,
)?;
@@ -676,7 +685,7 @@ impl Vecs {
self.dateindex_to_monthindex.compute_range(
starting_dateindex,
self.dateindex_to_dateindex.vec(),
&self.dateindex_to_dateindex,
|di| (di, MonthIndex::from(di)),
exit,
)?;
@@ -684,13 +693,13 @@ impl Vecs {
self.monthindex_to_first_dateindex
.compute_inverse_more_to_less(
starting_dateindex,
self.dateindex_to_monthindex.vec(),
&self.dateindex_to_monthindex,
exit,
)?;
self.monthindex_to_monthindex.compute_range(
starting_monthindex,
self.monthindex_to_first_dateindex.vec(),
&self.monthindex_to_first_dateindex,
|mi| (mi, mi),
exit,
)?;
@@ -698,8 +707,8 @@ impl Vecs {
self.monthindex_to_dateindex_count
.compute_count_from_indexes(
starting_monthindex,
self.monthindex_to_first_dateindex.vec(),
self.dateindex_to_date.vec(),
&self.monthindex_to_first_dateindex,
&self.dateindex_to_date,
exit,
)?;
@@ -715,7 +724,7 @@ impl Vecs {
self.monthindex_to_quarterindex.compute_range(
starting_monthindex,
self.monthindex_to_monthindex.vec(),
&self.monthindex_to_monthindex,
|mi| (mi, QuarterIndex::from(mi)),
exit,
)?;
@@ -723,7 +732,7 @@ impl Vecs {
self.quarterindex_to_first_monthindex
.compute_inverse_more_to_less(
starting_monthindex,
self.monthindex_to_quarterindex.vec(),
&self.monthindex_to_quarterindex,
exit,
)?;
@@ -731,7 +740,7 @@ impl Vecs {
self.quarterindex_to_quarterindex.compute_range(
starting_quarterindex,
self.quarterindex_to_first_monthindex.vec(),
&self.quarterindex_to_first_monthindex,
|i| (i, i),
exit,
)?;
@@ -739,8 +748,8 @@ impl Vecs {
self.quarterindex_to_monthindex_count
.compute_count_from_indexes(
starting_quarterindex,
self.quarterindex_to_first_monthindex.vec(),
self.monthindex_to_monthindex.vec(),
&self.quarterindex_to_first_monthindex,
&self.monthindex_to_monthindex,
exit,
)?;
@@ -756,7 +765,7 @@ impl Vecs {
self.monthindex_to_yearindex.compute_range(
starting_monthindex,
self.monthindex_to_monthindex.vec(),
&self.monthindex_to_monthindex,
|i| (i, YearIndex::from(i)),
exit,
)?;
@@ -764,13 +773,13 @@ impl Vecs {
self.yearindex_to_first_monthindex
.compute_inverse_more_to_less(
starting_monthindex,
self.monthindex_to_yearindex.vec(),
&self.monthindex_to_yearindex,
exit,
)?;
self.yearindex_to_yearindex.compute_range(
starting_yearindex,
self.yearindex_to_first_monthindex.vec(),
&self.yearindex_to_first_monthindex,
|i| (i, i),
exit,
)?;
@@ -778,8 +787,8 @@ impl Vecs {
self.yearindex_to_monthindex_count
.compute_count_from_indexes(
starting_yearindex,
self.yearindex_to_first_monthindex.vec(),
self.monthindex_to_monthindex.vec(),
&self.yearindex_to_first_monthindex,
&self.monthindex_to_monthindex,
exit,
)?;
// ---
@@ -794,7 +803,7 @@ impl Vecs {
self.height_to_halvingepoch.compute_range(
starting_indexes.height,
self.height_to_height.vec(),
&self.height_to_height,
|h| (h, HalvingEpoch::from(h)),
exit,
)?;
@@ -802,13 +811,13 @@ impl Vecs {
self.halvingepoch_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
self.height_to_halvingepoch.vec(),
&self.height_to_halvingepoch,
exit,
)?;
self.halvingepoch_to_halvingepoch.compute_range(
starting_halvingepoch,
self.halvingepoch_to_first_height.vec(),
&self.halvingepoch_to_first_height,
|i| (i, i),
exit,
)?;
@@ -825,7 +834,7 @@ impl Vecs {
self.yearindex_to_decadeindex.compute_range(
starting_yearindex,
self.yearindex_to_yearindex.vec(),
&self.yearindex_to_yearindex,
|i| (i, DecadeIndex::from(i)),
exit,
)?;
@@ -833,13 +842,13 @@ impl Vecs {
self.decadeindex_to_first_yearindex
.compute_inverse_more_to_less(
starting_yearindex,
self.yearindex_to_decadeindex.vec(),
&self.yearindex_to_decadeindex,
exit,
)?;
self.decadeindex_to_decadeindex.compute_range(
starting_decadeindex,
self.decadeindex_to_first_yearindex.vec(),
&self.decadeindex_to_first_yearindex,
|i| (i, i),
exit,
)?;
@@ -847,8 +856,8 @@ impl Vecs {
self.decadeindex_to_yearindex_count
.compute_count_from_indexes(
starting_decadeindex,
self.decadeindex_to_first_yearindex.vec(),
self.yearindex_to_yearindex.vec(),
&self.decadeindex_to_first_yearindex,
&self.yearindex_to_yearindex,
exit,
)?;
@@ -865,61 +874,61 @@ impl Vecs {
})
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
vec![
self.dateindex_to_date.any_vec(),
self.dateindex_to_dateindex.any_vec(),
self.dateindex_to_first_height.any_vec(),
self.dateindex_to_height_count.any_vec(),
self.dateindex_to_monthindex.any_vec(),
self.dateindex_to_weekindex.any_vec(),
self.decadeindex_to_decadeindex.any_vec(),
self.decadeindex_to_first_yearindex.any_vec(),
self.decadeindex_to_yearindex_count.any_vec(),
self.difficultyepoch_to_difficultyepoch.any_vec(),
self.difficultyepoch_to_first_height.any_vec(),
self.difficultyepoch_to_height_count.any_vec(),
self.emptyoutputindex_to_emptyoutputindex.any_vec(),
self.halvingepoch_to_first_height.any_vec(),
self.halvingepoch_to_halvingepoch.any_vec(),
self.height_to_date.any_vec(),
self.height_to_date_fixed.any_vec(),
self.height_to_dateindex.any_vec(),
self.height_to_difficultyepoch.any_vec(),
self.height_to_halvingepoch.any_vec(),
self.height_to_height.any_vec(),
self.height_to_timestamp_fixed.any_vec(),
self.height_to_txindex_count.any_vec(),
self.inputindex_to_inputindex.any_vec(),
self.monthindex_to_dateindex_count.any_vec(),
self.monthindex_to_first_dateindex.any_vec(),
self.monthindex_to_monthindex.any_vec(),
self.monthindex_to_quarterindex.any_vec(),
self.monthindex_to_yearindex.any_vec(),
self.opreturnindex_to_opreturnindex.any_vec(),
self.outputindex_to_outputindex.any_vec(),
self.p2aindex_to_p2aindex.any_vec(),
self.p2msindex_to_p2msindex.any_vec(),
self.p2pk33index_to_p2pk33index.any_vec(),
self.p2pk65index_to_p2pk65index.any_vec(),
self.p2pkhindex_to_p2pkhindex.any_vec(),
self.p2shindex_to_p2shindex.any_vec(),
self.p2trindex_to_p2trindex.any_vec(),
self.p2wpkhindex_to_p2wpkhindex.any_vec(),
self.p2wshindex_to_p2wshindex.any_vec(),
self.quarterindex_to_first_monthindex.any_vec(),
self.quarterindex_to_monthindex_count.any_vec(),
self.quarterindex_to_quarterindex.any_vec(),
self.txindex_to_height.any_vec(),
self.txindex_to_txindex.any_vec(),
self.unknownoutputindex_to_unknownoutputindex.any_vec(),
self.weekindex_to_dateindex_count.any_vec(),
self.weekindex_to_first_dateindex.any_vec(),
self.weekindex_to_weekindex.any_vec(),
self.yearindex_to_decadeindex.any_vec(),
self.yearindex_to_first_monthindex.any_vec(),
self.yearindex_to_monthindex_count.any_vec(),
self.yearindex_to_yearindex.any_vec(),
&self.dateindex_to_date,
&self.dateindex_to_dateindex,
&self.dateindex_to_first_height,
&self.dateindex_to_height_count,
&self.dateindex_to_monthindex,
&self.dateindex_to_weekindex,
&self.decadeindex_to_decadeindex,
&self.decadeindex_to_first_yearindex,
&self.decadeindex_to_yearindex_count,
&self.difficultyepoch_to_difficultyepoch,
&self.difficultyepoch_to_first_height,
&self.difficultyepoch_to_height_count,
&self.emptyoutputindex_to_emptyoutputindex,
&self.halvingepoch_to_first_height,
&self.halvingepoch_to_halvingepoch,
&self.height_to_date,
&self.height_to_date_fixed,
&self.height_to_dateindex,
&self.height_to_difficultyepoch,
&self.height_to_halvingepoch,
&self.height_to_height,
&self.height_to_timestamp_fixed,
&self.height_to_txindex_count,
&self.inputindex_to_inputindex,
&self.monthindex_to_dateindex_count,
&self.monthindex_to_first_dateindex,
&self.monthindex_to_monthindex,
&self.monthindex_to_quarterindex,
&self.monthindex_to_yearindex,
&self.opreturnindex_to_opreturnindex,
&self.outputindex_to_outputindex,
&self.p2aindex_to_p2aindex,
&self.p2msindex_to_p2msindex,
&self.p2pk33index_to_p2pk33index,
&self.p2pk65index_to_p2pk65index,
&self.p2pkhindex_to_p2pkhindex,
&self.p2shindex_to_p2shindex,
&self.p2trindex_to_p2trindex,
&self.p2wpkhindex_to_p2wpkhindex,
&self.p2wshindex_to_p2wshindex,
&self.quarterindex_to_first_monthindex,
&self.quarterindex_to_monthindex_count,
&self.quarterindex_to_quarterindex,
&self.txindex_to_height,
&self.txindex_to_txindex,
&self.unknownoutputindex_to_unknownoutputindex,
&self.weekindex_to_dateindex_count,
&self.weekindex_to_first_dateindex,
&self.weekindex_to_weekindex,
&self.yearindex_to_decadeindex,
&self.yearindex_to_first_monthindex,
&self.yearindex_to_monthindex_count,
&self.yearindex_to_yearindex,
]
}
}

137
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -7,10 +7,10 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{Compressed, VecIterator, Version};
use brk_vec::{AnyIterableVec, Compressed, Computation, EagerVec, VecIterator, Version};
use super::{
EagerVec, Indexes,
Indexes,
grouped::{
ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions,
},
@@ -69,7 +69,11 @@ const VERSION: Version = Version::ZERO;
const VERSION_IN_SATS: Version = Version::ONE;
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let mut fetched_path = path.to_owned();
@@ -336,7 +340,7 @@ impl Vecs {
let mut height_to_timestamp_iter = indexer_vecs.height_to_timestamp.iter();
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.vec(),
indexer_vecs.height_to_timestamp.iter_vec(),
|(h, t, ..)| {
let ohlc = fetcher
.get_height(
@@ -353,42 +357,42 @@ impl Vecs {
self.height_to_open_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.height_to_high_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.height_to_low_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.height_to_close_in_cents.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.height_to_ohlc.compute_transform(
starting_indexes.height,
self.height_to_ohlc_in_cents.vec(),
self.height_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
indexes.dateindex_to_date.vec(),
indexes.dateindex_to_date.iter_vec(),
|(di, d, ..)| {
let ohlc = fetcher.get_date(d).unwrap();
(di, ohlc)
@@ -398,35 +402,35 @@ impl Vecs {
self.dateindex_to_open_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.dateindex_to_high_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.dateindex_to_low_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.dateindex_to_close_in_cents.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.dateindex_to_ohlc.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc_in_cents.vec(),
self.dateindex_to_ohlc_in_cents.iter_vec(),
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
@@ -439,7 +443,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)
@@ -454,7 +458,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)
@@ -469,7 +473,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)
@@ -484,7 +488,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.dateindex_to_ohlc.vec(),
self.dateindex_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)
@@ -499,7 +503,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)
@@ -514,7 +518,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)
@@ -529,7 +533,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)
@@ -544,7 +548,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.height_to_ohlc.vec(),
self.height_to_ohlc.iter_vec(),
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)
@@ -556,7 +560,7 @@ impl Vecs {
let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter();
self.weekindex_to_ohlc.compute_transform(
starting_indexes.weekindex,
self.timeindexes_to_close.weekindex.unwrap_last().vec(),
self.timeindexes_to_close.weekindex.unwrap_last().iter_vec(),
|(i, close, ..)| {
(
i,
@@ -588,7 +592,7 @@ impl Vecs {
self.chainindexes_to_close
.difficultyepoch
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -608,7 +612,10 @@ impl Vecs {
let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter();
self.monthindex_to_ohlc.compute_transform(
starting_indexes.monthindex,
self.timeindexes_to_close.monthindex.unwrap_last().vec(),
self.timeindexes_to_close
.monthindex
.unwrap_last()
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -629,7 +636,10 @@ impl Vecs {
let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter();
self.quarterindex_to_ohlc.compute_transform(
starting_indexes.quarterindex,
self.timeindexes_to_close.quarterindex.unwrap_last().vec(),
self.timeindexes_to_close
.quarterindex
.unwrap_last()
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -649,7 +659,7 @@ impl Vecs {
let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter();
self.yearindex_to_ohlc.compute_transform(
starting_indexes.yearindex,
self.timeindexes_to_close.yearindex.unwrap_last().vec(),
self.timeindexes_to_close.yearindex.unwrap_last().iter_vec(),
|(i, close, ..)| {
(
i,
@@ -672,7 +682,10 @@ impl Vecs {
let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter();
self.decadeindex_to_ohlc.compute_transform(
starting_indexes.decadeindex,
self.timeindexes_to_close.decadeindex.unwrap_last().vec(),
self.timeindexes_to_close
.decadeindex
.unwrap_last()
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -695,7 +708,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_open.height.vec(),
self.chainindexes_to_open.height.iter_vec(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
@@ -710,7 +723,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_low.height.vec(),
self.chainindexes_to_low.height.iter_vec(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
@@ -725,7 +738,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_high.height.vec(),
self.chainindexes_to_high.height.iter_vec(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
@@ -740,7 +753,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
self.chainindexes_to_close.height.vec(),
self.chainindexes_to_close.height.iter_vec(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -755,7 +768,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_open.dateindex.vec(),
self.timeindexes_to_open.dateindex.iter_vec(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
@@ -770,7 +783,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_low.dateindex.vec(),
self.timeindexes_to_low.dateindex.iter_vec(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
@@ -785,7 +798,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_high.dateindex.vec(),
self.timeindexes_to_high.dateindex.iter_vec(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
@@ -800,7 +813,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_close.dateindex.vec(),
self.timeindexes_to_close.dateindex.iter_vec(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -812,7 +825,7 @@ impl Vecs {
let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter();
self.height_to_ohlc_in_sats.compute_transform(
starting_indexes.height,
self.chainindexes_to_close_in_sats.height.vec(),
self.chainindexes_to_close_in_sats.height.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -832,7 +845,7 @@ impl Vecs {
let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
self.dateindex_to_ohlc_in_sats.compute_transform(
starting_indexes.dateindex,
self.timeindexes_to_close_in_sats.dateindex.vec(),
self.timeindexes_to_close_in_sats.dateindex.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -867,7 +880,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.weekindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -902,7 +915,7 @@ impl Vecs {
self.chainindexes_to_close_in_sats
.difficultyepoch
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -937,7 +950,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.monthindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -972,7 +985,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.quarterindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -1007,7 +1020,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.yearindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -1045,7 +1058,7 @@ impl Vecs {
self.timeindexes_to_close_in_sats
.decadeindex
.unwrap_last()
.vec(),
.iter_vec(),
|(i, close, ..)| {
(
i,
@@ -1063,7 +1076,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
vec![
vec![
self.dateindex_to_close_in_cents.any_vec(),
@@ -1095,22 +1108,22 @@ impl Vecs {
// self.halvingepoch_to_ohlc_in_sats.any_vec(),
self.decadeindex_to_ohlc_in_sats.any_vec(),
],
self.timeindexes_to_close.any_vecs(),
self.timeindexes_to_high.any_vecs(),
self.timeindexes_to_low.any_vecs(),
self.timeindexes_to_open.any_vecs(),
self.chainindexes_to_close.any_vecs(),
self.chainindexes_to_high.any_vecs(),
self.chainindexes_to_low.any_vecs(),
self.chainindexes_to_open.any_vecs(),
self.timeindexes_to_close_in_sats.any_vecs(),
self.timeindexes_to_high_in_sats.any_vecs(),
self.timeindexes_to_low_in_sats.any_vecs(),
self.timeindexes_to_open_in_sats.any_vecs(),
self.chainindexes_to_close_in_sats.any_vecs(),
self.chainindexes_to_high_in_sats.any_vecs(),
self.chainindexes_to_low_in_sats.any_vecs(),
self.chainindexes_to_open_in_sats.any_vecs(),
self.timeindexes_to_close.vecs(),
self.timeindexes_to_high.vecs(),
self.timeindexes_to_low.vecs(),
self.timeindexes_to_open.vecs(),
self.chainindexes_to_close.vecs(),
self.chainindexes_to_high.vecs(),
self.chainindexes_to_low.vecs(),
self.chainindexes_to_open.vecs(),
self.timeindexes_to_close_in_sats.vecs(),
self.timeindexes_to_high_in_sats.vecs(),
self.timeindexes_to_low_in_sats.vecs(),
self.timeindexes_to_open_in_sats.vecs(),
self.chainindexes_to_close_in_sats.vecs(),
self.chainindexes_to_high_in_sats.vecs(),
self.chainindexes_to_low_in_sats.vecs(),
self.chainindexes_to_open_in_sats.vecs(),
]
.concat()
}

16
crates/brk_computer/src/storage/vecs/mining.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, path::Path};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, VecIterator, Version};
use brk_vec::{Compressed, Computation, VecIterator, Version};
use super::{
Indexes,
@@ -19,7 +19,11 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
pub fn forced_import(
path: &Path,
computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -113,11 +117,11 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
self.indexes_to_difficulty.any_vecs(),
self.indexes_to_difficultyepoch.any_vecs(),
self.indexes_to_halvingepoch.any_vecs(),
self.indexes_to_difficulty.vecs(),
self.indexes_to_difficultyepoch.vecs(),
self.indexes_to_halvingepoch.vecs(),
]
.concat()
}

97
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -3,26 +3,24 @@ use std::{fs, path::Path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyVec, Compressed};
use brk_vec::{AnyCollectableVec, AnyVec, CollectableVec, Compressed, Computation};
pub mod blocks;
pub mod grouped;
// pub mod blocks;
// pub mod grouped;
pub mod indexes;
pub mod marketprice;
pub mod mining;
pub mod transactions;
pub mod vec;
// pub mod marketprice;
// pub mod mining;
// pub mod transactions;
pub use indexes::Indexes;
pub use vec::*;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub transactions: transactions::Vecs,
pub marketprice: Option<marketprice::Vecs>,
// pub blocks: blocks::Vecs,
// pub mining: mining::Vecs,
// pub transactions: transactions::Vecs,
// pub marketprice: Option<marketprice::Vecs>,
}
impl Vecs {
@@ -36,17 +34,18 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
blocks: blocks::Vecs::forced_import(path, compressed)?,
indexes: indexes::Vecs::forced_import(path, compressed)?,
mining: mining::Vecs::forced_import(path, compressed)?,
transactions: transactions::Vecs::forced_import(
path,
indexer,
computation,
compressed,
fetch,
)?,
marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()),
// blocks: blocks::Vecs::forced_import(path, computation, compressed)?,
indexes: indexes::Vecs::forced_import(path, computation, compressed)?,
// mining: mining::Vecs::forced_import(path, computation, compressed)?,
// transactions: transactions::Vecs::forced_import(
// path,
// indexer,
// computation,
// compressed,
// fetch,
// )?,
// marketprice: fetch
// .then(|| marketprice::Vecs::forced_import(path, computation, compressed).unwrap()),
})
}
@@ -59,40 +58,40 @@ impl Vecs {
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
// self.blocks
// .compute(indexer, &self.indexes, &starting_indexes, exit)?;
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
// self.mining
// .compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(marketprice) = self.marketprice.as_mut() {
marketprice.compute(
indexer,
&self.indexes,
&starting_indexes,
fetcher.unwrap(),
exit,
)?;
}
// if let Some(marketprice) = self.marketprice.as_mut() {
// marketprice.compute(
// indexer,
// &self.indexes,
// &starting_indexes,
// fetcher.unwrap(),
// exit,
// )?;
// }
self.transactions.compute(
indexer,
&self.indexes,
&starting_indexes,
self.marketprice.as_ref(),
exit,
)?;
// self.transactions.compute(
// indexer,
// &self.indexes,
// &starting_indexes,
// self.marketprice.as_ref(),
// exit,
// )?;
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.indexes.any_vecs(),
self.blocks.any_vecs(),
self.mining.any_vecs(),
self.transactions.any_vecs(),
self.marketprice.as_ref().map_or(vec![], |v| v.any_vecs()),
self.indexes.vecs(),
// self.blocks.vecs(),
// self.mining.vecs(),
// self.transactions.vecs(),
// self.marketprice.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()
}

81
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -6,10 +6,13 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{AnyVec, Compressed, StoredIndex, VecIterator, Version};
use brk_vec::{
AnyIterableVec, AnyVec, Compressed, Computation, ComputedVec, ComputedVecFrom2, EagerVec,
StoredIndex, VecIterator, Version,
};
use super::{
Computation, ComputedVec, ComputedVecFrom2, EagerVec, Indexes,
Indexes,
grouped::{
ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
ComputedVecsFromTxindex, StorableVecGeneatorOptions,
@@ -49,6 +52,7 @@ pub struct Vecs {
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub inputindex_to_value:
ComputedVecFrom2<InputIndex, Sats, OutputIndex, Sats, InputIndex, OutputIndex>,
// Bitcoin and dollar version too
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_is_coinbase: EagerVec<TxIndex, bool>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
@@ -541,23 +545,8 @@ impl Vecs {
exit,
)?;
// let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
self.inputindex_to_value.compute_if_necessary(
starting_indexes.inputindex,
// indexer.vecs().inputindex_to_outputindex.vec(),
// |(inputindex, outputindex, ..)| {
// let value = if outputindex == OutputIndex::COINBASE {
// Sats::ZERO
// } else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
// value.into_inner()
// } else {
// dbg!(inputindex, outputindex);
// panic!()
// };
// (inputindex, value)
// },
exit,
)?;
self.inputindex_to_value
.compute_if_necessary(starting_indexes.inputindex, exit)?;
self.indexes_to_output_value.compute_all(
indexer,
@@ -898,7 +887,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
vec![
self.inputindex_to_value.any_vec(),
@@ -906,32 +895,32 @@ impl Vecs {
self.txindex_to_vsize.any_vec(),
self.txindex_to_weight.any_vec(),
],
self.indexes_to_tx_count.any_vecs(),
self.indexes_to_coinbase.any_vecs(),
self.indexes_to_fee.any_vecs(),
self.indexes_to_feerate.any_vecs(),
self.indexes_to_input_value.any_vecs(),
self.indexes_to_output_value.any_vecs(),
self.indexes_to_subsidy.any_vecs(),
self.indexes_to_tx_v1.any_vecs(),
self.indexes_to_tx_v2.any_vecs(),
self.indexes_to_tx_v3.any_vecs(),
self.indexes_to_tx_vsize.any_vecs(),
self.indexes_to_tx_weight.any_vecs(),
self.indexes_to_input_count.any_vecs(),
self.indexes_to_output_count.any_vecs(),
self.indexes_to_p2a_count.any_vecs(),
self.indexes_to_p2ms_count.any_vecs(),
self.indexes_to_p2pk33_count.any_vecs(),
self.indexes_to_p2pk65_count.any_vecs(),
self.indexes_to_p2pkh_count.any_vecs(),
self.indexes_to_p2sh_count.any_vecs(),
self.indexes_to_p2tr_count.any_vecs(),
self.indexes_to_p2wpkh_count.any_vecs(),
self.indexes_to_p2wsh_count.any_vecs(),
self.indexes_to_opreturn_count.any_vecs(),
self.indexes_to_unknownoutput_count.any_vecs(),
self.indexes_to_emptyoutput_count.any_vecs(),
self.indexes_to_tx_count.vecs(),
self.indexes_to_coinbase.vecs(),
self.indexes_to_fee.vecs(),
self.indexes_to_feerate.vecs(),
self.indexes_to_input_value.vecs(),
self.indexes_to_output_value.vecs(),
self.indexes_to_subsidy.vecs(),
self.indexes_to_tx_v1.vecs(),
self.indexes_to_tx_v2.vecs(),
self.indexes_to_tx_v3.vecs(),
self.indexes_to_tx_vsize.vecs(),
self.indexes_to_tx_weight.vecs(),
self.indexes_to_input_count.vecs(),
self.indexes_to_output_count.vecs(),
self.indexes_to_p2a_count.vecs(),
self.indexes_to_p2ms_count.vecs(),
self.indexes_to_p2pk33_count.vecs(),
self.indexes_to_p2pk65_count.vecs(),
self.indexes_to_p2pkh_count.vecs(),
self.indexes_to_p2sh_count.vecs(),
self.indexes_to_p2tr_count.vecs(),
self.indexes_to_p2wpkh_count.vecs(),
self.indexes_to_p2wsh_count.vecs(),
self.indexes_to_opreturn_count.vecs(),
self.indexes_to_unknownoutput_count.vecs(),
self.indexes_to_emptyoutput_count.vecs(),
]
.concat()
}

4
crates/brk_indexer/src/indexes.rs
View File

@@ -5,10 +5,10 @@ use brk_core::{
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{StoredIndex, StoredType, VecIterator};
use brk_vec::{AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::{IndexedVec, Stores, Vecs};
use crate::{Stores, Vecs};
#[derive(Debug, Default, Clone)]
pub struct Indexes {

4
crates/brk_indexer/src/lib.rs
View File

@@ -18,7 +18,7 @@ pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_vec::{Compressed, DynamicVec, VecIterator};
use brk_vec::{AnyVec, Compressed, GenericStoredVec, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -648,7 +648,7 @@ impl Indexer {
let mut txindex_to_tx_and_txid: BTreeMap<TxIndex, (&Transaction, Txid)> = BTreeMap::default();
let mut txindex_to_txid_iter = vecs
.txindex_to_txid.iter();
.txindex_to_txid.into_iter();
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()

2
crates/brk_indexer/src/stores/mod.rs
View File

@@ -4,7 +4,7 @@ use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
TxidPrefix,
};
use brk_vec::{Value, VecIterator, Version};
use brk_vec::{AnyIterableVec, Value, Version};
use fjall::{PersistMode, TransactionalKeyspace};
use crate::Indexes;

102
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -7,15 +7,11 @@ use brk_core::{
P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Sats, StoredF64, StoredU32,
StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Weight,
};
use brk_vec::{AnyVec, Compressed, Result, Version};
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Compressed, IndexedVec, Result, Version};
use rayon::prelude::*;
use crate::Indexes;
mod base;
pub use base::*;
#[derive(Clone)]
pub struct Vecs {
pub emptyoutputindex_to_txindex: IndexedVec<EmptyOutputIndex, TxIndex>,
@@ -442,69 +438,69 @@ impl Vecs {
}
pub fn flush(&mut self, height: Height) -> Result<()> {
self.mut_any_vecs()
self.mut_vecs()
.into_par_iter()
.try_for_each(|vec| vec.flush(height))
}
pub fn starting_height(&mut self) -> Height {
self.mut_any_vecs()
self.mut_vecs()
.into_iter()
.map(|vec| vec.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
vec![
self.emptyoutputindex_to_txindex.any_vec(),
self.height_to_blockhash.any_vec(),
self.height_to_difficulty.any_vec(),
self.height_to_first_emptyoutputindex.any_vec(),
self.height_to_first_inputindex.any_vec(),
self.height_to_first_opreturnindex.any_vec(),
self.height_to_first_outputindex.any_vec(),
self.height_to_first_p2aindex.any_vec(),
self.height_to_first_p2msindex.any_vec(),
self.height_to_first_p2pk33index.any_vec(),
self.height_to_first_p2pk65index.any_vec(),
self.height_to_first_p2pkhindex.any_vec(),
self.height_to_first_p2shindex.any_vec(),
self.height_to_first_p2trindex.any_vec(),
self.height_to_first_p2wpkhindex.any_vec(),
self.height_to_first_p2wshindex.any_vec(),
self.height_to_first_txindex.any_vec(),
self.height_to_first_unknownoutputindex.any_vec(),
self.height_to_timestamp.any_vec(),
self.height_to_total_size.any_vec(),
self.height_to_weight.any_vec(),
self.inputindex_to_outputindex.any_vec(),
self.opreturnindex_to_txindex.any_vec(),
self.outputindex_to_outputtype.any_vec(),
self.outputindex_to_outputtypeindex.any_vec(),
self.outputindex_to_value.any_vec(),
self.p2aindex_to_p2abytes.any_vec(),
self.p2msindex_to_txindex.any_vec(),
self.p2pk33index_to_p2pk33bytes.any_vec(),
self.p2pk65index_to_p2pk65bytes.any_vec(),
self.p2pkhindex_to_p2pkhbytes.any_vec(),
self.p2shindex_to_p2shbytes.any_vec(),
self.p2trindex_to_p2trbytes.any_vec(),
self.p2wpkhindex_to_p2wpkhbytes.any_vec(),
self.p2wshindex_to_p2wshbytes.any_vec(),
self.txindex_to_base_size.any_vec(),
self.txindex_to_first_inputindex.any_vec(),
self.txindex_to_first_outputindex.any_vec(),
self.txindex_to_is_explicitly_rbf.any_vec(),
self.txindex_to_rawlocktime.any_vec(),
self.txindex_to_total_size.any_vec(),
self.txindex_to_txid.any_vec(),
self.txindex_to_txversion.any_vec(),
self.unknownoutputindex_to_txindex.any_vec(),
&self.emptyoutputindex_to_txindex,
&self.height_to_blockhash,
&self.height_to_difficulty,
&self.height_to_first_emptyoutputindex,
&self.height_to_first_inputindex,
&self.height_to_first_opreturnindex,
&self.height_to_first_outputindex,
&self.height_to_first_p2aindex,
&self.height_to_first_p2msindex,
&self.height_to_first_p2pk33index,
&self.height_to_first_p2pk65index,
&self.height_to_first_p2pkhindex,
&self.height_to_first_p2shindex,
&self.height_to_first_p2trindex,
&self.height_to_first_p2wpkhindex,
&self.height_to_first_p2wshindex,
&self.height_to_first_txindex,
&self.height_to_first_unknownoutputindex,
&self.height_to_timestamp,
&self.height_to_total_size,
&self.height_to_weight,
&self.inputindex_to_outputindex,
&self.opreturnindex_to_txindex,
&self.outputindex_to_outputtype,
&self.outputindex_to_outputtypeindex,
&self.outputindex_to_value,
&self.p2aindex_to_p2abytes,
&self.p2msindex_to_txindex,
&self.p2pk33index_to_p2pk33bytes,
&self.p2pk65index_to_p2pk65bytes,
&self.p2pkhindex_to_p2pkhbytes,
&self.p2shindex_to_p2shbytes,
&self.p2trindex_to_p2trbytes,
&self.p2wpkhindex_to_p2wpkhbytes,
&self.p2wshindex_to_p2wshbytes,
&self.txindex_to_base_size,
&self.txindex_to_first_inputindex,
&self.txindex_to_first_outputindex,
&self.txindex_to_is_explicitly_rbf,
&self.txindex_to_rawlocktime,
&self.txindex_to_total_size,
&self.txindex_to_txid,
&self.txindex_to_txversion,
&self.unknownoutputindex_to_txindex,
]
}
fn mut_any_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> {
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> {
vec![
&mut self.emptyoutputindex_to_txindex,
&mut self.height_to_blockhash,

3
crates/brk_query/examples/main.rs
View File

@@ -1,8 +1,9 @@
use std::path::Path;
use brk_computer::{Computation, Computer};
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_query::{Index, Query};
use brk_vec::Computation;
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

9
crates/brk_query/src/lib.rs
View File

@@ -5,7 +5,7 @@
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_vec::AnyVec;
use brk_vec::AnyCollectableVec;
use tabled::settings::Style;
mod format;
@@ -34,13 +34,12 @@ impl<'a> Query<'a> {
indexer
.vecs()
.any_vecs()
.vecs()
.into_iter()
.for_each(|vec| vec_trees.insert(vec));
computer
.vecs()
.any_vecs()
.into_iter()
.for_each(|vec| vec_trees.insert(vec));
@@ -51,7 +50,7 @@ impl<'a> Query<'a> {
}
}
pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyVec)> {
pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyCollectableVec)> {
let tuples = ids
.iter()
.flat_map(|s| {
@@ -86,7 +85,7 @@ impl<'a> Query<'a> {
pub fn format(
&self,
vecs: Vec<(String, &&dyn AnyVec)>,
vecs: Vec<(String, &&dyn AnyCollectableVec)>,
from: Option<i64>,
to: Option<i64>,
format: Option<Format>,

8
crates/brk_query/src/vec_trees.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_vec::AnyVec;
use brk_vec::AnyCollectableVec;
use derive_deref::{Deref, DerefMut};
use super::index::Index;
@@ -13,7 +13,7 @@ pub struct VecTrees<'a> {
impl<'a> VecTrees<'a> {
// Not the most performant or type safe but only built once so that's okay
pub fn insert(&mut self, vec: &'a dyn AnyVec) {
pub fn insert(&mut self, vec: &'a dyn AnyCollectableVec) {
let name = vec.name();
let split = name.split("_to_").collect::<Vec<_>>();
if split.len() != 2 {
@@ -88,7 +88,7 @@ impl<'a> VecTrees<'a> {
}
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyVec>);
pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyCollectableVec>);
#[derive(Default, Deref, DerefMut)]
pub struct IdToVec<'a>(BTreeMap<String, &'a dyn AnyVec>);
pub struct IdToVec<'a>(BTreeMap<String, &'a dyn AnyCollectableVec>);

3
crates/brk_server/Cargo.toml
View File

@@ -10,12 +10,13 @@ repository.workspace = true
axum = { workspace = true }
brk_computer = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_core = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_query = { workspace = true }
brk_vec = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }

3
crates/brk_server/examples/main.rs
View File

@@ -1,6 +1,6 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::{Computation, Computer};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
@@ -10,6 +10,7 @@ use brk_parser::{
rpc::{self, RpcApi},
};
use brk_server::{Server, Website};
use brk_vec::Computation;
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

5
crates/brk_vec/Cargo.toml
View File

@@ -10,6 +10,11 @@ repository.workspace = true
[dependencies]
arc-swap = "1.7.1"
brk_core = { workspace = true }
brk_exit = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
log = { workspace = true }
memmap2 = "0.9.5"
rayon = { workspace = true }
serde = { workspace = true }

16
crates/brk_vec/examples/main.rs
View File

@@ -1,6 +1,8 @@
use std::{fs, path::Path};
use brk_vec::{Compressed, DynamicVec, GenericVec, StoredVec, VecIterator, Version};
use brk_vec::{
AnyVec, CollectableVec, Compressed, GenericStoredVec, StoredVec, VecIterator, Version,
};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let _ = fs::remove_dir_all("./vec");
@@ -16,7 +18,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.push(v);
});
let mut iter = vec.iter();
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
@@ -27,7 +29,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StoredVec<usize, u32> =
StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
let mut iter = vec.iter();
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(0));
@@ -40,7 +42,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.push(21);
vec.push(22);
let mut iter = vec.iter();
let mut iter = vec.into_iter();
dbg!(iter.get(20));
dbg!(iter.get(21));
@@ -53,7 +55,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StoredVec<usize, u32> =
StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
let mut iter = vec.iter();
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
@@ -62,7 +64,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.truncate_if_needed(14)?;
let mut iter = vec.iter();
let mut iter = vec.into_iter();
iter.get(0);
iter.get(5);
@@ -71,7 +73,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
dbg!(vec.collect_signed_range(Some(-5), None)?);
vec.push(vec.len() as u32);
dbg!(VecIterator::last(vec.iter()));
dbg!(VecIterator::last(vec.into_iter()));
dbg!(vec.into_iter().collect::<Vec<_>>());
}

219
crates/brk_vec/src/lib.rs
View File

@@ -8,225 +8,8 @@ mod structs;
mod traits;
mod variants;
use std::{path::Path, time::Duration};
use arc_swap::ArcSwap;
pub use enums::*;
pub use memmap2::Mmap;
pub use structs::*;
pub use traits::*;
use variants::*;
#[derive(Debug, Clone)]
pub enum StoredVec<I, T> {
Raw(RawVec<I, T>),
Compressed(CompressedVec<I, T>),
}
impl<I, T> StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
if *compressed {
Ok(Self::Compressed(CompressedVec::forced_import(
path, version,
)?))
} else {
Ok(Self::Raw(RawVec::forced_import(path, version)?))
}
}
pub fn iter(&self) -> StoredVecIterator<'_, I, T> {
self.into_iter()
}
pub fn iter_at(&self, i: I) -> StoredVecIterator<'_, I, T> {
let mut iter = self.into_iter();
iter.set(i);
iter
}
}
impl<I, T> DynamicVec for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type I = I;
type T = T;
#[inline]
fn read_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
match self {
StoredVec::Raw(v) => v.read_(index, guard),
StoredVec::Compressed(v) => v.read_(index, guard),
}
}
#[inline]
fn mmap(&self) -> &ArcSwap<Mmap> {
match self {
StoredVec::Raw(v) => v.mmap(),
StoredVec::Compressed(v) => v.mmap(),
}
}
#[inline]
fn stored_len(&self) -> usize {
match self {
StoredVec::Raw(v) => v.stored_len(),
StoredVec::Compressed(v) => v.stored_len(),
}
}
#[inline]
fn pushed(&self) -> &[T] {
match self {
StoredVec::Raw(v) => v.pushed(),
StoredVec::Compressed(v) => v.pushed(),
}
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
match self {
StoredVec::Raw(v) => v.mut_pushed(),
StoredVec::Compressed(v) => v.mut_pushed(),
}
}
#[inline]
fn path(&self) -> &Path {
match self {
StoredVec::Raw(v) => v.path(),
StoredVec::Compressed(v) => v.path(),
}
}
}
impl<I, T> GenericVec<I, T> for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>> {
match self {
StoredVec::Raw(v) => v.collect_range(from, to),
StoredVec::Compressed(v) => v.collect_range(from, to),
}
}
fn flush(&mut self) -> Result<()> {
match self {
StoredVec::Raw(v) => v.flush(),
StoredVec::Compressed(v) => v.flush(),
}
}
fn truncate_if_needed(&mut self, index: I) -> Result<()> {
match self {
StoredVec::Raw(v) => v.truncate_if_needed(index),
StoredVec::Compressed(v) => v.truncate_if_needed(index),
}
}
#[inline]
fn version(&self) -> Version {
match self {
StoredVec::Raw(v) => v.version(),
StoredVec::Compressed(v) => v.version(),
}
}
}
impl<I, T> AnyVec for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn len(&self) -> usize {
DynamicVec::len(self)
}
#[inline]
fn index_type_to_string(&self) -> &str {
GenericVec::index_type_to_string(self)
}
#[inline]
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
GenericVec::collect_range_serde_json(self, from, to)
}
#[inline]
fn modified_time(&self) -> Result<Duration> {
GenericVec::modified_time(self)
}
fn name(&self) -> String {
GenericVec::name(self)
}
}
#[derive(Debug)]
pub enum StoredVecIterator<'a, I, T> {
Raw(RawVecIterator<'a, I, T>),
Compressed(CompressedVecIterator<'a, I, T>),
}
impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T>
where
I: StoredIndex,
T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Compressed(i) => i.next(),
Self::Raw(i) => i.next(),
}
}
}
impl<I, T> BaseVecIterator for StoredVecIterator<'_, I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn mut_index(&mut self) -> &mut usize {
match self {
Self::Compressed(iter) => iter.mut_index(),
Self::Raw(iter) => iter.mut_index(),
}
}
fn len(&self) -> usize {
match self {
Self::Compressed(i) => i.len(),
Self::Raw(i) => i.len(),
}
}
}
impl<'a, I, T> IntoIterator for &'a StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = StoredVecIterator<'a, I, T>;
fn into_iter(self) -> Self::IntoIter {
match self {
StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()),
StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()),
}
}
}
pub use variants::*;

69
crates/brk_vec/src/traits/any.rs
View File

@@ -1,24 +1,73 @@
use std::time::Duration;
use crate::Result;
use crate::{Result, Version};
use super::{BoxedVecIterator, StoredIndex, StoredType};
pub trait AnyVec: Send + Sync {
fn version(&self) -> Version;
fn name(&self) -> String;
fn index_type_to_string(&self) -> &str;
fn len(&self) -> usize;
fn is_empty(&self) -> bool {
self.len() == 0
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>>;
fn modified_time(&self) -> Result<Duration>;
fn any_vec(&self) -> &dyn AnyVec
fn index_type_to_string(&self) -> &str;
}
pub trait AnyIterableVec<I, T>: AnyVec {
#[allow(clippy::wrong_self_convention)]
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
Self: Sized,
I: StoredIndex,
T: StoredType + 'a;
fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
I: StoredIndex,
T: StoredType + 'a,
{
self
self.boxed_iter()
}
fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T>
where
I: StoredIndex,
T: StoredType + 'a,
{
let mut iter = self.boxed_iter();
iter.set(i);
iter
}
fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T>
where
I: StoredIndex,
T: StoredType + 'a,
{
let mut iter = self.boxed_iter();
iter.set_(i);
iter
}
}
pub trait CloneableAnyIterableVec<I, T>: AnyIterableVec<I, T> {
fn boxed_clone(&self) -> Box<dyn CloneableAnyIterableVec<I, T>>;
}
impl<I, T, U> CloneableAnyIterableVec<I, T> for U
where
U: 'static + AnyIterableVec<I, T> + Clone,
{
fn boxed_clone(&self) -> Box<dyn CloneableAnyIterableVec<I, T>> {
Box::new(self.clone())
}
}
impl<I, T> Clone for Box<dyn CloneableAnyIterableVec<I, T>> {
fn clone(&self) -> Self {
self.boxed_clone()
}
}
pub type BoxedAnyIterableVec<I, T> = Box<dyn CloneableAnyIterableVec<I, T>>;

71
crates/brk_vec/src/traits/collectable.rs Normal file
View File

@@ -0,0 +1,71 @@
use crate::{Error, Result};
use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType};
pub trait CollectableVec<I, T>: AnyVec + AnyIterableVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
let len = self.len();
let from = from.unwrap_or_default();
let to = to.map_or(len, |i| i.min(len));
if from >= len || from >= to {
return Ok(vec![]);
}
Ok(self
.iter_at_(from)
.take(to - from)
.map(|(_, v)| v.into_inner())
.collect::<Vec<_>>())
}
#[inline]
fn i64_to_usize(i: i64, len: usize) -> usize {
if i >= 0 {
i as usize
} else {
let v = len as i64 + i;
if v < 0 { 0 } else { v as usize }
}
}
#[doc(hidden)]
fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
let len = self.len();
let from = from.map(|i| Self::i64_to_usize(i, len));
let to = to.map(|i| Self::i64_to_usize(i, len));
self.collect_range(from, to)
}
#[inline]
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
self.collect_signed_range(from, to)?
.into_iter()
.map(|v| serde_json::to_value(v).map_err(Error::from))
.collect::<Result<Vec<_>>>()
}
}
impl<I, T, V> CollectableVec<I, T> for V
where
V: AnyVec + AnyIterableVec<I, T>,
I: StoredIndex,
T: StoredType,
{
}
pub trait AnyCollectableVec: AnyVec {
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>>;
}

66
crates/brk_vec/src/traits/dynamic.rs
View File

@@ -1,66 +0,0 @@
use std::path::Path;
use arc_swap::ArcSwap;
use memmap2::Mmap;
use crate::{Result, Value};
use super::{StoredIndex, StoredType};
pub trait DynamicVec: Send + Sync {
type I: StoredIndex;
type T: StoredType;
const SIZE_OF_T: usize = size_of::<Self::T>();
#[inline]
fn read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Self::T>> {
self.read_(index.to_usize()?, mmap)
}
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
#[inline]
fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
self.get_or_read_(index.to_usize()?, mmap)
}
#[inline]
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
let stored_len = mmap.len() / Self::SIZE_OF_T;
if index >= stored_len {
let pushed = self.pushed();
let j = index - stored_len;
if j >= pushed.len() {
return Ok(None);
}
Ok(pushed.get(j).map(Value::Ref))
} else {
Ok(self.read_(index, mmap)?.map(Value::Owned))
}
}
#[inline]
fn len(&self) -> usize {
self.stored_len() + self.pushed_len()
}
#[inline]
fn is_empty(&self) -> bool {
self.len() == 0
}
fn mmap(&self) -> &ArcSwap<Mmap>;
fn stored_len(&self) -> usize;
fn pushed(&self) -> &[Self::T];
#[inline]
fn pushed_len(&self) -> usize {
self.pushed().len()
}
fn mut_pushed(&mut self) -> &mut Vec<Self::T>;
#[inline]
fn push(&mut self, value: Self::T) {
self.mut_pushed().push(value)
}
fn path(&self) -> &Path;
}

108
crates/brk_vec/src/traits/generic.rs
View File

@@ -6,17 +6,70 @@ use std::{
time::{self, Duration},
};
use arc_swap::ArcSwap;
use memmap2::Mmap;
use crate::{Error, Result, Version};
use crate::{Result, Value};
use super::{DynamicVec, StoredIndex, StoredType};
use super::{StoredIndex, StoredType};
pub trait GenericVec<I, T>: DynamicVec<I = I, T = T>
pub trait GenericStoredVec<I, T>: Send + Sync
where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<T>();
#[inline]
fn read(&self, index: I, mmap: &Mmap) -> Result<Option<T>> {
self.read_(index.to_usize()?, mmap)
}
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>>;
#[inline]
fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
self.get_or_read_(index.to_usize()?, mmap)
}
#[inline]
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<T>>> {
let stored_len = mmap.len() / Self::SIZE_OF_T;
if index >= stored_len {
let pushed = self.pushed();
let j = index - stored_len;
if j >= pushed.len() {
return Ok(None);
}
Ok(pushed.get(j).map(Value::Ref))
} else {
Ok(self.read_(index, mmap)?.map(Value::Owned))
}
}
#[inline]
fn len_(&self) -> usize {
self.stored_len() + self.pushed_len()
}
fn mmap(&self) -> &ArcSwap<Mmap>;
fn stored_len(&self) -> usize;
fn pushed(&self) -> &[T];
#[inline]
fn pushed_len(&self) -> usize {
self.pushed().len()
}
fn mut_pushed(&mut self) -> &mut Vec<T>;
#[inline]
fn push(&mut self, value: T) {
self.mut_pushed().push(value)
}
fn path(&self) -> &Path;
// ---
fn open_file(&self) -> io::Result<File> {
Self::open_file_(&self.path_vec())
}
@@ -74,54 +127,17 @@ where
}
#[inline]
fn has(&self, index: Self::I) -> Result<bool> {
fn has(&self, index: I) -> Result<bool> {
Ok(self.has_(index.to_usize()?))
}
#[inline]
fn has_(&self, index: usize) -> bool {
index < self.len()
}
#[inline]
fn index_type_to_string(&self) -> &str {
Self::I::to_string()
index < self.len_()
}
fn flush(&mut self) -> Result<()>;
fn truncate_if_needed(&mut self, index: Self::I) -> Result<()>;
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>>;
#[inline]
fn i64_to_usize(i: i64, len: usize) -> usize {
if i >= 0 {
i as usize
} else {
let v = len as i64 + i;
if v < 0 { 0 } else { v as usize }
}
}
#[doc(hidden)]
fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<Self::T>> {
let len = self.len();
let from = from.map(|i| Self::i64_to_usize(i, len));
let to = to.map(|i| Self::i64_to_usize(i, len));
self.collect_range(from, to)
}
#[inline]
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
self.collect_signed_range(from, to)?
.into_iter()
.map(|v| serde_json::to_value(v).map_err(Error::from))
.collect::<Result<Vec<_>>>()
}
fn truncate_if_needed(&mut self, index: I) -> Result<()>;
#[inline]
fn path_vec(&self) -> PathBuf {
@@ -143,7 +159,7 @@ where
}
#[inline]
fn name(&self) -> String {
fn name_(&self) -> String {
self.path()
.file_name()
.unwrap()
@@ -152,13 +168,11 @@ where
.to_owned()
}
fn modified_time(&self) -> Result<Duration> {
fn modified_time_(&self) -> Result<Duration> {
Ok(self
.path_vec()
.metadata()?
.modified()?
.duration_since(time::UNIX_EPOCH)?)
}
fn version(&self) -> Version;
}

1
crates/brk_vec/src/traits/stored_index.rs → crates/brk_vec/src/traits/index.rs
View File

@@ -23,6 +23,7 @@ where
fn to_string<'a>() -> &'a str;
fn decremented(self) -> Option<Self>;
}
impl<I> StoredIndex for I
where
I: Debug

9
crates/brk_vec/src/traits/iterator.rs
View File

@@ -32,9 +32,9 @@ pub trait BaseVecIterator: Iterator {
}
}
pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)> + 'a {
pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)> {
type I: StoredIndex;
type T: StoredType;
type T: StoredType + 'a;
#[inline]
fn set(&mut self, i: Self::I) {
@@ -77,10 +77,13 @@ pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)>
impl<'a, I, T, Iter> VecIterator<'a> for Iter
where
Iter: BaseVecIterator<Item = (I, Value<'a, T>)> + 'a,
Iter: BaseVecIterator<Item = (I, Value<'a, T>)>,
I: StoredIndex,
T: StoredType + 'a,
{
type I = I;
type T = T;
}
pub type BoxedVecIterator<'a, I, T> =
Box<dyn VecIterator<'a, I = I, T = T, Item = (I, Value<'a, T>)> + 'a>;

12
crates/brk_vec/src/traits/mod.rs
View File

@@ -1,13 +1,13 @@
mod any;
mod dynamic;
mod collectable;
mod generic;
mod index;
mod iterator;
mod stored_index;
mod stored_type;
mod r#type;
pub use any::*;
pub use dynamic::*;
pub use collectable::*;
pub use generic::*;
pub use index::*;
pub use iterator::*;
pub use stored_index::*;
pub use stored_type::*;
pub use r#type::*;

1
crates/brk_vec/src/traits/stored_type.rs → crates/brk_vec/src/traits/type.rs
View File

@@ -17,6 +17,7 @@ where
+ Serialize,
{
}
impl<T> StoredType for T where
T: Sized
+ Debug

86
crates/brk_vec/src/variants/compressed.rs
View File

@@ -3,6 +3,7 @@ use std::{
mem,
path::Path,
sync::Arc,
time::Duration,
};
use arc_swap::{ArcSwap, Guard};
@@ -11,8 +12,9 @@ use rayon::prelude::*;
use zstd::DEFAULT_COMPRESSION_LEVEL;
use crate::{
BaseVecIterator, CompressedPageMetadata, CompressedPagesMetadata, DynamicVec, Error,
GenericVec, RawVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version,
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
CompressedPageMetadata, CompressedPagesMetadata, Error, GenericStoredVec, RawVec, Result,
StoredIndex, StoredType, UnsafeSlice, Value, Version,
};
const ONE_KIB: usize = 1024;
@@ -152,14 +154,11 @@ where
}
}
impl<I, T> DynamicVec for CompressedVec<I, T>
impl<I, T> GenericStoredVec<I, T> for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type I = I;
type T = T;
#[inline]
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let page_index = Self::index_to_page_index(index);
@@ -194,28 +193,6 @@ where
fn path(&self) -> &Path {
self.inner.path()
}
}
impl<I, T> GenericVec<I, T> for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
let stored_len = self.stored_len();
let from = from.unwrap_or_default();
let to = to.map_or(stored_len, |i| i.min(stored_len));
if from >= stored_len || from >= to {
return Ok(vec![]);
}
Ok(self
.iter_at_(from)
.take(to - from)
.map(|(_, v)| v.into_inner())
.collect::<Vec<_>>())
}
fn flush(&mut self) -> Result<()> {
let pushed_len = self.pushed_len();
@@ -352,11 +329,37 @@ where
Ok(())
}
}
impl<I, T> AnyVec for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn version(&self) -> Version {
self.inner.version()
}
#[inline]
fn name(&self) -> String {
self.name_()
}
#[inline]
fn len(&self) -> usize {
self.len_()
}
#[inline]
fn modified_time(&self) -> Result<Duration> {
self.modified_time_()
}
#[inline]
fn index_type_to_string(&self) -> &str {
I::to_string()
}
}
impl<I, T> Clone for CompressedVec<I, T> {
@@ -474,3 +477,30 @@ where
}
}
}
impl<I, T> AnyIterableVec<I, T> for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T> AnyCollectableVec for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

201
crates/brk_computer/src/storage/vecs/vec/mod.rs → crates/brk_vec/src/variants/computed.rs
View File

@@ -4,21 +4,17 @@ use brk_exit::Exit;
use clap_derive::ValueEnum;
use serde::{Deserialize, Serialize};
mod _type;
mod eager;
mod lazy1;
mod lazy2;
mod lazy3;
pub use _type::*;
use brk_core::StoredPhantom;
use brk_vec::{
AnyVec, Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Compressed, Result, StoredIndex, StoredType, Value, Version,
};
use super::{
ComputeFrom1, ComputeFrom2, ComputeFrom3, EagerVec, LazyVecFrom1, LazyVecFrom1Iterator,
LazyVecFrom2, LazyVecFrom2Iterator, LazyVecFrom3, LazyVecFrom3Iterator, StoredVecIterator,
};
pub use eager::*;
pub use lazy1::*;
pub use lazy2::*;
pub use lazy3::*;
#[derive(
Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum,
@@ -41,16 +37,16 @@ impl Computation {
#[derive(Clone)]
pub enum Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T> {
From1(StoredVec<S1I, S1T>, ComputeFrom1<T, S1I, S1T>),
From1(BoxedAnyIterableVec<S1I, S1T>, ComputeFrom1<T, S1I, S1T>),
From2(
(StoredVec<S1I, S1T>, StoredVec<S2I, S2T>),
(BoxedAnyIterableVec<S1I, S1T>, BoxedAnyIterableVec<S2I, S2T>),
ComputeFrom2<T, S1I, S1T, S2I, S2T>,
),
From3(
(
StoredVec<S1I, S1T>,
StoredVec<S2I, S2T>,
StoredVec<S3I, S3T>,
BoxedAnyIterableVec<S1I, S1T>,
BoxedAnyIterableVec<S2I, S2T>,
BoxedAnyIterableVec<S3I, S3T>,
),
ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
),
@@ -91,9 +87,9 @@ where
name: &str,
version: Version,
compressed: Compressed,
source: StoredVec<S1I, S1T>,
source: BoxedAnyIterableVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
) -> brk_vec::Result<Self> {
) -> Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, version, compressed)?,
@@ -112,10 +108,10 @@ where
mode: Computation,
version: Version,
compressed: Compressed,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
) -> brk_vec::Result<Self> {
) -> Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, version, compressed)?,
@@ -134,11 +130,11 @@ where
name: &str,
version: Version,
compressed: Compressed,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source3: StoredVec<S3I, S3T>,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
source3: BoxedAnyIterableVec<S3I, S3T>,
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
) -> brk_vec::Result<Self> {
) -> Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, version, compressed)?,
@@ -166,7 +162,7 @@ where
let version = source.version();
let mut iter = source.iter();
let t = |i: I| {
compute(i.unwrap_to_usize(), &mut iter)
compute(i.unwrap_to_usize(), &mut *iter)
.map(|v| (i, v))
.unwrap()
};
@@ -177,7 +173,7 @@ where
let mut iter1 = source1.iter();
let mut iter2 = source2.iter();
let t = |i: I| {
compute(i.unwrap_to_usize(), &mut iter1, &mut iter2)
compute(i.unwrap_to_usize(), &mut *iter1, &mut *iter2)
.map(|v| (i, v))
.unwrap()
};
@@ -189,7 +185,7 @@ where
let mut iter2 = source2.iter();
let mut iter3 = source3.iter();
let t = |i: I| {
compute(i.unwrap_to_usize(), &mut iter1, &mut iter2, &mut iter3)
compute(i.unwrap_to_usize(), &mut *iter1, &mut *iter2, &mut *iter3)
.map(|v| (i, v))
.unwrap()
};
@@ -210,6 +206,15 @@ where
S3I: StoredIndex,
S3T: StoredType,
{
fn version(&self) -> Version {
match self {
ComputedVec::Eager { vec, .. } => vec.version(),
ComputedVec::LazyFrom1(v) => v.version(),
ComputedVec::LazyFrom2(v) => v.version(),
ComputedVec::LazyFrom3(v) => v.version(),
}
}
fn name(&self) -> String {
match self {
ComputedVec::Eager { vec, .. } => vec.name(),
@@ -232,14 +237,6 @@ where
}
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
todo!()
}
fn modified_time(&self) -> Result<Duration> {
match self {
ComputedVec::Eager { vec, .. } => vec.modified_time(),
@@ -249,3 +246,131 @@ where
}
}
}
pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
Eager(StoredVecIterator<'a, I, T>),
LazyFrom1(LazyVecFrom1Iterator<'a, I, T, S1I, S1T>),
LazyFrom2(LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>),
LazyFrom3(LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>),
}
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator
for ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Eager(i) => i.next(),
Self::LazyFrom1(i) => i.next(),
Self::LazyFrom2(i) => i.next(),
Self::LazyFrom3(i) => i.next(),
}
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> BaseVecIterator
for ComputedVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
#[inline]
fn mut_index(&mut self) -> &mut usize {
match self {
Self::Eager(i) => i.mut_index(),
Self::LazyFrom1(i) => i.mut_index(),
Self::LazyFrom2(i) => i.mut_index(),
Self::LazyFrom3(i) => i.mut_index(),
}
}
fn len(&self) -> usize {
match self {
Self::Eager(i) => i.len(),
Self::LazyFrom1(i) => i.len(),
Self::LazyFrom2(i) => i.len(),
Self::LazyFrom3(i) => i.len(),
}
}
}
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator
for &'a ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
fn into_iter(self) -> Self::IntoIter {
match self {
ComputedVec::Eager { vec, .. } => ComputedVecIterator::Eager(vec.into_iter()),
ComputedVec::LazyFrom1(v) => ComputedVecIterator::LazyFrom1(v.into_iter()),
ComputedVec::LazyFrom2(v) => ComputedVecIterator::LazyFrom2(v.into_iter()),
ComputedVec::LazyFrom3(v) => ComputedVecIterator::LazyFrom3(v.into_iter()),
}
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>
for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyCollectableVec
for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

148
crates/brk_computer/src/storage/vecs/vec/eager.rs → crates/brk_vec/src/variants/eager.rs
View File

@@ -9,12 +9,14 @@ use std::{
use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, TxIndex};
use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, VecIterator, Version,
};
use log::info;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error,
GenericStoredVec, Result, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value,
VecIterator, Version,
};
const ONE_KIB: usize = 1024;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
const MAX_CACHE_SIZE: usize = 210 * ONE_MIB;
@@ -32,11 +34,7 @@ where
{
const SIZE_OF: usize = size_of::<T>();
pub fn forced_import(
path: &Path,
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
let inner = StoredVec::forced_import(path, version, compressed)?;
Ok(Self {
@@ -105,21 +103,21 @@ where
self.inner.modified_time()
}
pub fn vec(&self) -> &StoredVec<I, T> {
&self.inner
}
// pub fn vec(&self) -> &StoredVec<I, T> {
// &self.inner
// }
pub fn mut_vec(&mut self) -> &StoredVec<I, T> {
&mut self.inner
}
// pub fn mut_vec(&mut self) -> &StoredVec<I, T> {
// &mut self.inner
// }
pub fn any_vec(&self) -> &dyn brk_vec::AnyVec {
&self.inner
}
// pub fn any_vec(&self) -> &dyn AnyVec {
// &self.inner
// }
pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyVec {
&mut self.inner
}
// pub fn mut_any_vec(&mut self) -> &mut dyn AnyVec {
// &mut self.inner
// }
pub fn path(&self) -> &Path {
self.inner.path()
@@ -175,7 +173,7 @@ where
pub fn compute_range<A, F>(
&mut self,
max_from: I,
other: &StoredVec<I, A>,
other: &impl AnyIterableVec<I, A>,
t: F,
exit: &Exit,
) -> Result<()>
@@ -189,7 +187,7 @@ where
pub fn compute_transform<A, B, F>(
&mut self,
max_from: A,
other: &StoredVec<A, B>,
other: &impl AnyIterableVec<A, B>,
mut t: F,
exit: &Exit,
) -> Result<()>
@@ -214,7 +212,7 @@ where
pub fn compute_inverse_more_to_less(
&mut self,
max_from: T,
other: &StoredVec<T, I>,
other: &impl AnyIterableVec<T, I>,
exit: &Exit,
) -> Result<()>
where
@@ -226,7 +224,8 @@ where
)?;
let index = max_from.min(
VecIterator::last(self.inner.iter()).map_or_else(T::default, |(_, v)| v.into_inner()),
VecIterator::last(self.inner.into_iter())
.map_or_else(T::default, |(_, v)| v.into_inner()),
);
let mut prev_i = None;
other.iter_at(index).try_for_each(|(v, i)| -> Result<()> {
@@ -247,8 +246,8 @@ where
pub fn compute_inverse_less_to_more(
&mut self,
max_from: T,
first_indexes: &StoredVec<T, I>,
indexes_count: &StoredVec<T, StoredUsize>,
first_indexes: &impl AnyIterableVec<T, I>,
indexes_count: &impl AnyIterableVec<T, StoredUsize>,
exit: &Exit,
) -> Result<()>
where
@@ -280,8 +279,8 @@ where
pub fn compute_count_from_indexes<T2, T3>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
other_to_else: &StoredVec<T2, T3>,
first_indexes: &impl AnyIterableVec<I, T2>,
other_to_else: &impl AnyIterableVec<T2, T3>,
exit: &Exit,
) -> Result<()>
where
@@ -303,8 +302,8 @@ where
pub fn compute_filtered_count_from_indexes<T2, T3, F>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
other_to_else: &StoredVec<T2, T3>,
first_indexes: &impl AnyIterableVec<I, T2>,
other_to_else: &impl AnyIterableVec<T2, T3>,
filter: F,
exit: &Exit,
) -> Result<()>
@@ -333,8 +332,8 @@ where
fn compute_filtered_count_from_indexes_<T2, T3>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
other_to_else: &StoredVec<T2, T3>,
first_indexes: &impl AnyIterableVec<I, T2>,
other_to_else: &impl AnyIterableVec<T2, T3>,
mut filter: Option<Box<dyn FnMut(T2) -> bool + '_>>,
exit: &Exit,
) -> Result<()>
@@ -351,7 +350,7 @@ where
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + first_indexes.version(), // + last_indexes.version(),
Version::ZERO + self.inner.version() + first_indexes.version(),
)?;
let mut other_iter = first_indexes.iter();
@@ -379,8 +378,8 @@ where
pub fn compute_is_first_ordered<A>(
&mut self,
max_from: I,
self_to_other: &StoredVec<I, A>,
other_to_self: &StoredVec<A, I>,
self_to_other: &impl AnyIterableVec<I, A>,
other_to_self: &impl AnyIterableVec<A, I>,
exit: &Exit,
) -> Result<()>
where
@@ -411,9 +410,9 @@ where
pub fn compute_sum_from_indexes<T2>(
&mut self,
max_from: I,
first_indexes: &StoredVec<I, T2>,
indexes_count: &StoredVec<I, StoredUsize>,
source: &StoredVec<T2, T>,
first_indexes: &impl AnyIterableVec<I, T2>,
indexes_count: &impl AnyIterableVec<I, StoredUsize>,
source: &impl AnyIterableVec<T2, T>,
exit: &Exit,
) -> Result<()>
where
@@ -454,7 +453,7 @@ where
pub fn compute_from_sats(
&mut self,
max_from: I,
sats: &StoredVec<I, Sats>,
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
@@ -475,8 +474,8 @@ impl EagerVec<Height, Dollars> {
pub fn compute_from_bitcoin(
&mut self,
max_from: Height,
bitcoin: &StoredVec<Height, Bitcoin>,
price: &StoredVec<Height, Close<Dollars>>,
bitcoin: &impl AnyIterableVec<Height, Bitcoin>,
price: &impl AnyIterableVec<Height, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
@@ -499,9 +498,9 @@ impl EagerVec<TxIndex, Dollars> {
pub fn compute_from_bitcoin(
&mut self,
max_from: TxIndex,
bitcoin: &StoredVec<TxIndex, Bitcoin>,
i_to_height: &StoredVec<TxIndex, Height>,
price: &StoredVec<Height, Close<Dollars>>,
bitcoin: &impl AnyIterableVec<TxIndex, Bitcoin>,
i_to_height: &impl AnyIterableVec<TxIndex, Height>,
price: &impl AnyIterableVec<Height, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
@@ -534,3 +533,62 @@ where
self.inner.into_iter()
}
}
impl<I, T> AnyVec for EagerVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn version(&self) -> Version {
self.inner.version()
}
#[inline]
fn name(&self) -> String {
self.inner.name()
}
#[inline]
fn len(&self) -> usize {
self.inner.len()
}
#[inline]
fn modified_time(&self) -> Result<Duration> {
self.inner.modified_time()
}
#[inline]
fn index_type_to_string(&self) -> &str {
I::to_string()
}
}
impl<I, T> AnyIterableVec<I, T> for EagerVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
I: StoredIndex,
T: StoredType + 'a,
{
Box::new(self.inner.into_iter())
}
}
impl<I, T> AnyCollectableVec for EagerVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

104
crates/brk_indexer/src/vecs/base.rs → crates/brk_vec/src/variants/indexed.rs
View File

@@ -2,21 +2,20 @@ use std::{
cmp::Ordering,
fmt::Debug,
path::{Path, PathBuf},
time::Duration,
};
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, VecIterator, Version,
use brk_core::Height;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error,
GenericStoredVec, Mmap, Result, StoredIndex, StoredType, StoredVec, Value, Version,
};
use super::Height;
use super::StoredVecIterator;
#[derive(Debug, Clone)]
pub struct IndexedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
pub struct IndexedVec<I, T> {
height: Option<Height>,
inner: StoredVec<I, T>,
}
@@ -26,11 +25,7 @@ where
I: StoredIndex,
T: StoredType,
{
pub fn forced_import(
path: &Path,
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
inner: StoredVec::forced_import(path, version, compressed)?,
@@ -44,7 +39,8 @@ where
#[inline]
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
match self.inner.len().cmp(&index.to_usize()?) {
let len = self.inner.len();
match len.cmp(&index.to_usize()?) {
Ordering::Greater => {
// dbg!(len, index, &self.pathbuf);
// panic!();
@@ -55,13 +51,13 @@ where
Ok(())
}
Ordering::Less => {
dbg!(index, value, self.inner.len(), self.path_height());
dbg!(index, value, len, self.path_height());
Err(Error::IndexTooHigh)
}
}
}
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result<()> {
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> Result<()> {
if self.height.is_none_or(|self_height| self_height != height) {
height.write(&self.path_height())?;
}
@@ -82,18 +78,6 @@ where
&mut self.inner
}
pub fn any_vec(&self) -> &dyn brk_vec::AnyVec {
&self.inner
}
pub fn len(&self) -> usize {
self.inner.len()
}
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
#[inline]
pub fn hasnt(&self, index: I) -> Result<bool> {
self.inner.has(index).map(|b| !b)
@@ -108,19 +92,40 @@ where
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
}
pub fn iter(&self) -> StoredVecIterator<'_, I, T> {
self.into_iter()
impl<I, T> AnyVec for IndexedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn version(&self) -> Version {
self.inner.version()
}
pub fn iter_at(&self, i: I) -> StoredVecIterator<'_, I, T> {
let mut iter = self.into_iter();
iter.set(i);
iter
#[inline]
fn name(&self) -> String {
self.inner.name()
}
#[inline]
fn len(&self) -> usize {
self.inner.len()
}
#[inline]
fn modified_time(&self) -> Result<Duration> {
self.inner.modified_time()
}
#[inline]
fn index_type_to_string(&self) -> &str {
I::to_string()
}
}
pub trait AnyIndexedVec: Send + Sync {
pub trait AnyIndexedVec: AnyVec {
fn height(&self) -> brk_core::Result<Height>;
fn flush(&mut self, height: Height) -> Result<()>;
}
@@ -151,3 +156,30 @@ where
self.inner.into_iter()
}
}
impl<I, T> AnyIterableVec<I, T> for IndexedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T> AnyCollectableVec for IndexedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

59
crates/brk_computer/src/storage/vecs/vec/lazy1.rs → crates/brk_vec/src/variants/lazy1.rs
View File

@@ -1,7 +1,8 @@
use std::marker::PhantomData;
use brk_vec::{
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
};
pub type ComputeFrom1<T, S1I, S1T> =
@@ -11,7 +12,7 @@ pub type ComputeFrom1<T, S1I, S1T> =
pub struct LazyVecFrom1<I, T, S1I, S1T> {
name: String,
version: Version,
source: StoredVec<S1I, S1T>,
source: BoxedAnyIterableVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
phantom: PhantomData<I>,
}
@@ -26,7 +27,7 @@ where
pub fn init(
name: &str,
version: Version,
source: StoredVec<S1I, S1T>,
source: BoxedAnyIterableVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
) -> Self {
Self {
@@ -43,13 +44,13 @@ where
}
}
pub struct LazyVecIterator<'a, I, T, S1I, S1T> {
pub struct LazyVecFrom1Iterator<'a, I, T, S1I, S1T> {
lazy: &'a LazyVecFrom1<I, T, S1I, S1T>,
source: StoredVecIterator<'a, S1I, S1T>,
source: BoxedVecIterator<'a, S1I, S1T>,
index: usize,
}
impl<'a, I, T, S1I, S1T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T>
impl<'a, I, T, S1I, S1T> Iterator for LazyVecFrom1Iterator<'a, I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType + 'a,
@@ -59,7 +60,7 @@ where
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(self.index, &mut self.lazy.source.iter())
let opt = (self.lazy.compute)(self.index, &mut *self.source)
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
self.index += 1;
@@ -68,7 +69,7 @@ where
}
}
impl<I, T, S1I, S1T> BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T>
impl<I, T, S1I, S1T> BaseVecIterator for LazyVecFrom1Iterator<'_, I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
@@ -82,7 +83,7 @@ where
#[inline]
fn len(&self) -> usize {
todo!();
self.source.len()
}
}
@@ -94,10 +95,10 @@ where
S1T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T>;
type IntoIter = LazyVecFrom1Iterator<'a, I, T, S1I, S1T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
LazyVecFrom1Iterator {
lazy: self,
source: self.source.iter(),
index: 0,
@@ -112,6 +113,10 @@ where
S1I: StoredIndex,
S1T: StoredType,
{
fn version(&self) -> Version {
self.version()
}
fn name(&self) -> String {
self.name.clone()
}
@@ -124,16 +129,38 @@ where
self.source.len()
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
fn modified_time(&self) -> Result<std::time::Duration> {
self.source.modified_time()
}
}
impl<I, T, S1I, S1T> AnyIterableVec<I, T> for LazyVecFrom1<I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T, S1I, S1T> AnyCollectableVec for LazyVecFrom1<I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

76
crates/brk_computer/src/storage/vecs/vec/lazy2.rs → crates/brk_vec/src/variants/lazy2.rs
View File

@@ -1,7 +1,8 @@
use std::marker::PhantomData;
use brk_vec::{
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
};
pub type ComputeFrom2<T, S1I, S1T, S2I, S2T> = for<'a> fn(
@@ -14,8 +15,8 @@ pub type ComputeFrom2<T, S1I, S1T, S2I, S2T> = for<'a> fn(
pub struct LazyVecFrom2<I, T, S1I, S1T, S2I, S2T> {
name: String,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
phantom: PhantomData<I>,
}
@@ -32,8 +33,8 @@ where
pub fn init(
name: &str,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
) -> Self {
Self {
@@ -51,14 +52,14 @@ where
}
}
pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> {
pub struct LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> {
lazy: &'a LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>,
source1: StoredVecIterator<'a, S1I, S1T>,
source2: StoredVecIterator<'a, S2I, S2T>,
source1: BoxedVecIterator<'a, S1I, S1T>,
source2: BoxedVecIterator<'a, S2I, S2T>,
index: usize,
}
impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>
impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType + 'a,
@@ -70,12 +71,8 @@ where
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(
self.index,
&mut self.lazy.source1.iter(),
&mut self.lazy.source2.iter(),
)
.map(|v| (I::from(self.index), Value::Owned(v)));
let opt = (self.lazy.compute)(self.index, &mut *self.source1, &mut *self.source2)
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
self.index += 1;
}
@@ -83,7 +80,8 @@ where
}
}
impl<I, T, S1I, S1T, S2I, S2T> BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T>
impl<I, T, S1I, S1T, S2I, S2T> BaseVecIterator
for LazyVecFrom2Iterator<'_, I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
@@ -99,7 +97,7 @@ where
#[inline]
fn len(&self) -> usize {
todo!();
self.source1.len().min(self.source2.len())
}
}
@@ -113,10 +111,10 @@ where
S2T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>;
type IntoIter = LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
LazyVecFrom2Iterator {
lazy: self,
source1: self.source1.iter(),
source2: self.source2.iter(),
@@ -134,6 +132,10 @@ where
S2I: StoredIndex,
S2T: StoredType,
{
fn version(&self) -> Version {
self.version()
}
fn name(&self) -> String {
self.name.clone()
}
@@ -146,19 +148,45 @@ where
self.source1.len().min(self.source2.len())
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
fn modified_time(&self) -> Result<std::time::Duration> {
Ok(self
.source1
.modified_time()?
.min(self.source2.modified_time()?))
}
}
impl<I, T, S1I, S1T, S2I, S2T> AnyIterableVec<I, T> for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T, S1I, S1T, S2I, S2T> AnyCollectableVec for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

96
crates/brk_computer/src/storage/vecs/vec/lazy3.rs → crates/brk_vec/src/variants/lazy3.rs
View File

@@ -1,7 +1,8 @@
use std::marker::PhantomData;
use brk_vec::{
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
};
pub type ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(
@@ -15,9 +16,9 @@ pub type ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(
pub struct LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
name: String,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source3: StoredVec<S3I, S3T>,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
source3: BoxedAnyIterableVec<S3I, S3T>,
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
phantom: PhantomData<I>,
}
@@ -36,9 +37,9 @@ where
pub fn init(
name: &str,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
source3: StoredVec<S3I, S3T>,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
source3: BoxedAnyIterableVec<S3I, S3T>,
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
) -> Self {
Self {
@@ -57,16 +58,16 @@ where
}
}
pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
pub struct LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
lazy: &'a LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>,
source1: StoredVecIterator<'a, S1I, S1T>,
source2: StoredVecIterator<'a, S2I, S2T>,
source3: StoredVecIterator<'a, S3I, S3T>,
source1: BoxedVecIterator<'a, S1I, S1T>,
source2: BoxedVecIterator<'a, S2I, S2T>,
source3: BoxedVecIterator<'a, S3I, S3T>,
index: usize,
}
impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator
for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
for LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType + 'a,
@@ -82,9 +83,9 @@ where
fn next(&mut self) -> Option<Self::Item> {
let opt = (self.lazy.compute)(
self.index,
&mut self.lazy.source1.iter(),
&mut self.lazy.source2.iter(),
&mut self.lazy.source3.iter(),
&mut *self.source1,
&mut *self.source2,
&mut *self.source3,
)
.map(|v| (I::from(self.index), Value::Owned(v)));
if opt.is_some() {
@@ -95,7 +96,7 @@ where
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> BaseVecIterator
for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
for LazyVecFrom3Iterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
@@ -113,7 +114,10 @@ where
#[inline]
fn len(&self) -> usize {
todo!();
self.source1
.len()
.min(self.source2.len())
.min(self.source3.len())
}
}
@@ -130,10 +134,10 @@ where
S3T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
type IntoIter = LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>;
fn into_iter(self) -> Self::IntoIter {
LazyVecIterator {
LazyVecFrom3Iterator {
lazy: self,
source1: self.source1.iter(),
source2: self.source2.iter(),
@@ -154,6 +158,10 @@ where
S3I: StoredIndex,
S3T: StoredType,
{
fn version(&self) -> Version {
self.version()
}
fn name(&self) -> String {
self.name.clone()
}
@@ -163,22 +171,58 @@ where
}
fn len(&self) -> usize {
self.source1.len().min(self.source2.len())
self.source1
.len()
.min(self.source2.len())
.min(self.source3.len())
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
fn modified_time(&self) -> Result<std::time::Duration> {
Ok(self
.source1
.modified_time()?
.min(self.source2.modified_time()?))
.min(self.source2.modified_time()?)
.min(self.source3.modified_time()?))
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>
for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyCollectableVec
for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

14
crates/brk_vec/src/variants/mod.rs
View File

@@ -1,5 +1,19 @@
mod compressed;
mod computed;
mod eager;
mod indexed;
mod lazy1;
mod lazy2;
mod lazy3;
mod raw;
mod stored;
pub use compressed::*;
pub use computed::*;
pub use eager::*;
pub use indexed::*;
pub use lazy1::*;
pub use lazy2::*;
pub use lazy3::*;
pub use raw::*;
pub use stored::*;

92
crates/brk_vec/src/variants/raw.rs
View File

@@ -4,6 +4,7 @@ use std::{
mem,
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use arc_swap::{ArcSwap, Guard};
@@ -11,8 +12,8 @@ use memmap2::Mmap;
use rayon::prelude::*;
use crate::{
BaseVecIterator, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice,
Value, Version,
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
Error, GenericStoredVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version,
};
#[derive(Debug)]
@@ -79,19 +80,16 @@ where
}
}
impl<I, T> DynamicVec for RawVec<I, T>
impl<I, T> GenericStoredVec<I, T> for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type I = I;
type T = T;
#[inline]
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let index = index * Self::SIZE_OF_T;
let slice = &mmap[index..(index + Self::SIZE_OF_T)];
Self::T::try_read_from_bytes(slice)
T::try_read_from_bytes(slice)
.map(|v| Some(v))
.map_err(Error::from)
}
@@ -119,28 +117,6 @@ where
fn path(&self) -> &Path {
self.pathbuf.as_path()
}
}
impl<I, T> GenericVec<I, T> for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
let stored_len = self.stored_len();
let from = from.unwrap_or_default();
let to = to.map_or(stored_len, |i| i.min(stored_len));
if from >= stored_len || from >= to {
return Ok(vec![]);
}
Ok(self
.iter_at_(from)
.take(to - from)
.map(|(_, v)| v.into_inner())
.collect::<Vec<_>>())
}
fn flush(&mut self) -> Result<()> {
let pushed_len = self.pushed_len();
@@ -188,10 +164,41 @@ where
Ok(())
}
// #[inline]
// fn version(&self) -> Version {
// self.version
// }
}
impl<I, T> AnyVec for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn version(&self) -> Version {
self.version
}
#[inline]
fn name(&self) -> String {
self.name_()
}
#[inline]
fn len(&self) -> usize {
self.len_()
}
#[inline]
fn modified_time(&self) -> Result<Duration> {
self.modified_time_()
}
#[inline]
fn index_type_to_string(&self) -> &str {
I::to_string()
}
}
impl<I, T> Clone for RawVec<I, T> {
@@ -270,3 +277,30 @@ where
}
}
}
impl<I, T> AnyIterableVec<I, T> for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T> AnyCollectableVec for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}

223
crates/brk_vec/src/variants/stored.rs Normal file
View File

@@ -0,0 +1,223 @@
use std::{path::Path, time::Duration};
use arc_swap::ArcSwap;
use memmap2::Mmap;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
Compressed, GenericStoredVec, Result, StoredIndex, StoredType, Value, Version,
};
use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator};
#[derive(Debug, Clone)]
pub enum StoredVec<I, T> {
Raw(RawVec<I, T>),
Compressed(CompressedVec<I, T>),
}
impl<I, T> StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result<Self> {
if *compressed {
Ok(Self::Compressed(CompressedVec::forced_import(
path, version,
)?))
} else {
Ok(Self::Raw(RawVec::forced_import(path, version)?))
}
}
}
impl<I, T> GenericStoredVec<I, T> for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn read_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
match self {
StoredVec::Raw(v) => v.read_(index, guard),
StoredVec::Compressed(v) => v.read_(index, guard),
}
}
#[inline]
fn mmap(&self) -> &ArcSwap<Mmap> {
match self {
StoredVec::Raw(v) => v.mmap(),
StoredVec::Compressed(v) => v.mmap(),
}
}
#[inline]
fn stored_len(&self) -> usize {
match self {
StoredVec::Raw(v) => v.stored_len(),
StoredVec::Compressed(v) => v.stored_len(),
}
}
#[inline]
fn pushed(&self) -> &[T] {
match self {
StoredVec::Raw(v) => v.pushed(),
StoredVec::Compressed(v) => v.pushed(),
}
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
match self {
StoredVec::Raw(v) => v.mut_pushed(),
StoredVec::Compressed(v) => v.mut_pushed(),
}
}
#[inline]
fn path(&self) -> &Path {
match self {
StoredVec::Raw(v) => v.path(),
StoredVec::Compressed(v) => v.path(),
}
}
fn flush(&mut self) -> Result<()> {
match self {
StoredVec::Raw(v) => v.flush(),
StoredVec::Compressed(v) => v.flush(),
}
}
fn truncate_if_needed(&mut self, index: I) -> Result<()> {
match self {
StoredVec::Raw(v) => v.truncate_if_needed(index),
StoredVec::Compressed(v) => v.truncate_if_needed(index),
}
}
}
impl<I, T> AnyVec for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn version(&self) -> Version {
match self {
StoredVec::Raw(v) => v.version(),
StoredVec::Compressed(v) => v.version(),
}
}
#[inline]
fn index_type_to_string(&self) -> &str {
I::to_string()
}
#[inline]
fn len(&self) -> usize {
self.pushed_len() + self.stored_len()
}
#[inline]
fn modified_time(&self) -> Result<Duration> {
match self {
StoredVec::Raw(v) => v.modified_time(),
StoredVec::Compressed(v) => v.modified_time(),
}
}
fn name(&self) -> String {
match self {
StoredVec::Raw(v) => v.name(),
StoredVec::Compressed(v) => v.name(),
}
}
}
#[derive(Debug)]
pub enum StoredVecIterator<'a, I, T> {
Raw(RawVecIterator<'a, I, T>),
Compressed(CompressedVecIterator<'a, I, T>),
}
impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T>
where
I: StoredIndex,
T: StoredType,
{
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Compressed(i) => i.next(),
Self::Raw(i) => i.next(),
}
}
}
impl<I, T> BaseVecIterator for StoredVecIterator<'_, I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn mut_index(&mut self) -> &mut usize {
match self {
Self::Compressed(iter) => iter.mut_index(),
Self::Raw(iter) => iter.mut_index(),
}
}
fn len(&self) -> usize {
match self {
Self::Compressed(i) => i.len(),
Self::Raw(i) => i.len(),
}
}
}
impl<'a, I, T> IntoIterator for &'a StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type Item = (I, Value<'a, T>);
type IntoIter = StoredVecIterator<'a, I, T>;
fn into_iter(self) -> Self::IntoIter {
match self {
StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()),
StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()),
}
}
}
impl<I, T> AnyIterableVec<I, T> for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
where
T: 'a,
{
Box::new(self.into_iter())
}
}
impl<I, T> AnyCollectableVec for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
CollectableVec::collect_range_serde_json(self, from, to)
}
}