Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-04 03:09:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: lazy part 4

Browse Source
This commit is contained in:
nym21
2025-05-03 17:28:48 +02:00
parent efa7294f59
commit 320c708e10
32 changed files with 326 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/brk_computer/src/storage/vecs/blocks.rs
View File

@@ -223,7 +223,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
vec![
self.height_to_interval.any_vec(),

4
crates/brk_computer/src/storage/vecs/grouped/builder.rs
View File

@@ -523,8 +523,8 @@ where
self.total.as_mut().unwrap()
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
let mut v: Vec<&dyn brk_vec::AnyStoredVec> = vec![];
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
let mut v: Vec<&dyn brk_vec::AnyVec> = vec![];
if let Some(first) = self.first.as_ref() {
v.push(first.any_vec());

4
crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed, Result, Version};
use brk_vec::{AnyVec, Compressed, Result, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
@@ -140,7 +140,7 @@ where
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
vec![self.dateindex.any_vec()],
self.dateindex_extra.any_vecs(),

4
crates/brk_computer/src/storage/vecs/grouped/from_height.rs
View File

@@ -5,7 +5,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
@@ -180,7 +180,7 @@ where
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height_extra.any_vecs(),

4
crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use brk_core::{DifficultyEpoch, Height};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed, Result, Version};
use brk_vec::{AnyVec, Compressed, Result, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
@@ -86,7 +86,7 @@ where
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
vec![self.height.any_vec()],
self.height_extra.any_vecs(),

4
crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs
View File

@@ -6,7 +6,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use crate::storage::{ComputedType, EagerVec, Indexes, indexes};
@@ -188,7 +188,7 @@ where
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height.any_vecs(),

4
crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Sats};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use crate::storage::{
EagerVec, marketprice,
@@ -145,7 +145,7 @@ impl ComputedValueVecsFromHeight {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
self.sats.any_vecs(),
self.bitcoin.any_vecs(),

4
crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use brk_core::{Bitcoin, Dollars, Sats, TxIndex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version};
use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version};
use crate::storage::{
EagerVec, marketprice,
@@ -147,7 +147,7 @@ impl ComputedValueVecsFromTxindex {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
self.sats.any_vecs(),
self.bitcoin.any_vecs(),

2
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -865,7 +865,7 @@ impl Vecs {
})
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
vec![
self.dateindex_to_date.any_vec(),
self.dateindex_to_dateindex.any_vec(),

2
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -1063,7 +1063,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
vec![
vec![
self.dateindex_to_close_in_cents.any_vec(),

2
crates/brk_computer/src/storage/vecs/mining.rs
View File

@@ -113,7 +113,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
self.indexes_to_difficulty.any_vecs(),
self.indexes_to_difficultyepoch.any_vecs(),

4
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, path::Path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyStoredVec, Compressed};
use brk_vec::{AnyVec, Compressed};
pub mod blocks;
pub mod grouped;
@@ -86,7 +86,7 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn AnyVec> {
[
self.indexes.any_vecs(),
self.blocks.any_vecs(),

22
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -6,7 +6,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, StoredIndex, VecIterator, Version};
use brk_vec::{AnyVec, Compressed, StoredIndex, VecIterator, Version};
use super::{
Computation, ComputedVec, ComputedVecFrom2, EagerVec, Indexes,
@@ -113,8 +113,9 @@ impl Vecs {
.add_total(),
)?,
inputindex_to_value: ComputedVec::forced_import_or_init_from_2(
path,
"inputindex_to_value",
computation,
&path.join("inputindex_to_value"),
Version::ZERO,
compressed,
indexer.vecs().outputindex_to_value.vec().clone(),
@@ -728,6 +729,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2ms_count.compute_all(
indexer,
indexes,
@@ -742,6 +744,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2pk33_count.compute_all(
indexer,
indexes,
@@ -756,6 +759,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2pk65_count.compute_all(
indexer,
indexes,
@@ -770,6 +774,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2pkh_count.compute_all(
indexer,
indexes,
@@ -784,6 +789,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2sh_count.compute_all(
indexer,
indexes,
@@ -798,6 +804,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2tr_count.compute_all(
indexer,
indexes,
@@ -812,6 +819,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2wpkh_count.compute_all(
indexer,
indexes,
@@ -826,6 +834,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_p2wsh_count.compute_all(
indexer,
indexes,
@@ -840,6 +849,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_opreturn_count.compute_all(
indexer,
indexes,
@@ -854,6 +864,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_unknownoutput_count.compute_all(
indexer,
indexes,
@@ -868,6 +879,7 @@ impl Vecs {
)
},
)?;
self.indexes_to_emptyoutput_count.compute_all(
indexer,
indexes,
@@ -886,13 +898,13 @@ impl Vecs {
Ok(())
}
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> {
[
vec![
self.inputindex_to_value.any_vec(),
self.txindex_to_is_coinbase.any_vec(),
// self.inputindex_to_value.any_vec(),
self.txindex_to_weight.any_vec(),
self.txindex_to_vsize.any_vec(),
self.txindex_to_weight.any_vec(),
],
self.indexes_to_tx_count.any_vecs(),
self.indexes_to_coinbase.any_vecs(),

15
crates/brk_computer/src/storage/vecs/vec/eager.rs
View File

@@ -4,6 +4,7 @@ use std::{
fmt::Debug,
ops::Add,
path::{Path, PathBuf},
time::Duration,
};
use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, TxIndex};
@@ -96,8 +97,12 @@ where
self.inner.is_empty()
}
fn file_name(&self) -> String {
self.inner.file_name()
pub fn name(&self) -> String {
self.inner.name()
}
pub fn modified_time(&self) -> Result<Duration> {
self.inner.modified_time()
}
pub fn vec(&self) -> &StoredVec<I, T> {
@@ -108,11 +113,11 @@ where
&mut self.inner
}
pub fn any_vec(&self) -> &dyn brk_vec::AnyStoredVec {
pub fn any_vec(&self) -> &dyn brk_vec::AnyVec {
&self.inner
}
pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyStoredVec {
pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyVec {
&mut self.inner
}
@@ -133,7 +138,7 @@ where
version.write(path.as_ref())?;
if self.is_empty() {
info!("Computing {}...", self.file_name())
info!("Computing {}...", self.name())
}
Ok(())

38
crates/brk_computer/src/storage/vecs/vec/lazy1.rs
View File

@@ -1,7 +1,7 @@
use std::marker::PhantomData;
use brk_vec::{
BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
};
pub type ComputeFrom1<T, S1I, S1T> =
@@ -9,6 +9,7 @@ pub type ComputeFrom1<T, S1I, S1T> =
#[derive(Clone)]
pub struct LazyVecFrom1<I, T, S1I, S1T> {
name: String,
version: Version,
source: StoredVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
@@ -23,11 +24,13 @@ where
S1T: StoredType,
{
pub fn init(
name: &str,
version: Version,
source: StoredVec<S1I, S1T>,
compute: ComputeFrom1<T, S1I, S1T>,
) -> Self {
Self {
name: name.to_owned(),
version,
source,
compute,
@@ -101,3 +104,36 @@ where
}
}
}
impl<I, T, S1I, S1T> AnyVec for LazyVecFrom1<I, T, S1I, S1T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
{
fn name(&self) -> String {
self.name.clone()
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
self.source.len()
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
self.source.modified_time()
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
}
}

43
crates/brk_computer/src/storage/vecs/vec/lazy2.rs
View File

@@ -1,7 +1,7 @@
use std::marker::PhantomData;
use brk_vec::{
BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
};
pub type ComputeFrom2<T, S1I, S1T, S2I, S2T> = for<'a> fn(
@@ -12,6 +12,7 @@ pub type ComputeFrom2<T, S1I, S1T, S2I, S2T> = for<'a> fn(
#[derive(Clone)]
pub struct LazyVecFrom2<I, T, S1I, S1T, S2I, S2T> {
name: String,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
@@ -29,12 +30,14 @@ where
S2T: StoredType,
{
pub fn init(
name: &str,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
compute: ComputeFrom2<T, S1I, S1T, S2I, S2T>,
) -> Self {
Self {
name: name.to_string(),
version,
source1,
source2,
@@ -121,3 +124,41 @@ where
}
}
}
impl<I, T, S1I, S1T, S2I, S2T> AnyVec for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
{
fn name(&self) -> String {
self.name.clone()
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
self.source1.len().min(self.source2.len())
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
Ok(self
.source1
.modified_time()?
.min(self.source2.modified_time()?))
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
}
}

45
crates/brk_computer/src/storage/vecs/vec/lazy3.rs
View File

@@ -1,7 +1,7 @@
use std::marker::PhantomData;
use brk_vec::{
BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version,
};
pub type ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(
@@ -13,6 +13,7 @@ pub type ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(
#[derive(Clone)]
pub struct LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
name: String,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
@@ -33,6 +34,7 @@ where
S3T: StoredType,
{
pub fn init(
name: &str,
version: Version,
source1: StoredVec<S1I, S1T>,
source2: StoredVec<S2I, S2T>,
@@ -40,6 +42,7 @@ where
compute: ComputeFrom3<T, S1I, S1T, S2I, S2T, S3I, S3T>,
) -> Self {
Self {
name: name.to_string(),
version,
source1,
source2,
@@ -139,3 +142,43 @@ where
}
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyVec for LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn name(&self) -> String {
self.name.clone()
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
self.source1.len().min(self.source2.len())
}
fn modified_time(&self) -> brk_vec::Result<std::time::Duration> {
Ok(self
.source1
.modified_time()?
.min(self.source2.modified_time()?))
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> brk_vec::Result<Vec<serde_json::Value>> {
todo!()
// self.
}
}

74
crates/brk_computer/src/storage/vecs/vec/mod.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::{path::Path, time::Duration};
use brk_exit::Exit;
use clap_derive::ValueEnum;
@@ -12,7 +12,9 @@ mod lazy3;
pub use _type::*;
use brk_core::StoredPhantom;
use brk_vec::{Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version};
use brk_vec::{
AnyVec, Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
};
pub use eager::*;
pub use lazy1::*;
pub use lazy2::*;
@@ -38,7 +40,7 @@ impl Computation {
}
#[derive(Clone)]
enum Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T> {
pub enum Dependencies<T, S1I, S1T, S2I, S2T, S3I, S3T> {
From1(StoredVec<S1I, S1T>, ComputeFrom1<T, S1I, S1T>),
From2(
(StoredVec<S1I, S1T>, StoredVec<S2I, S2T>),
@@ -70,7 +72,6 @@ pub enum ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T> {
LazyFrom1(LazyVecFrom1<I, T, S1I, S1T>),
LazyFrom2(LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>),
LazyFrom3(LazyVecFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T>),
// Lazy4
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
@@ -87,6 +88,7 @@ where
pub fn forced_import_or_init_from_1(
mode: Computation,
path: &Path,
name: &str,
version: Version,
compressed: Compressed,
source: StoredVec<S1I, S1T>,
@@ -97,13 +99,17 @@ where
vec: EagerVec::forced_import(path, version, compressed)?,
deps: Dependencies::From1(source, compute),
},
Computation::Lazy => Self::LazyFrom1(LazyVecFrom1::init(version, source, compute)),
Computation::Lazy => {
Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute))
}
})
}
#[allow(clippy::too_many_arguments)]
pub fn forced_import_or_init_from_2(
mode: Computation,
path: &Path,
name: &str,
mode: Computation,
version: Version,
compressed: Compressed,
source1: StoredVec<S1I, S1T>,
@@ -116,7 +122,7 @@ where
deps: Dependencies::From2((source1, source2), compute),
},
Computation::Lazy => {
Self::LazyFrom2(LazyVecFrom2::init(version, source1, source2, compute))
Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute))
}
})
}
@@ -125,6 +131,7 @@ where
pub fn forced_import_or_init_from_3(
mode: Computation,
path: &Path,
name: &str,
version: Version,
compressed: Compressed,
source1: StoredVec<S1I, S1T>,
@@ -138,7 +145,7 @@ where
deps: Dependencies::From3((source1, source2, source3), compute),
},
Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init(
version, source1, source2, source3, compute,
name, version, source1, source2, source3, compute,
)),
})
}
@@ -191,3 +198,54 @@ where
}
}
}
impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyVec for ComputedVec<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
where
I: StoredIndex,
T: StoredType,
S1I: StoredIndex,
S1T: StoredType,
S2I: StoredIndex,
S2T: StoredType,
S3I: StoredIndex,
S3T: StoredType,
{
fn name(&self) -> String {
match self {
ComputedVec::Eager { vec, .. } => vec.name(),
ComputedVec::LazyFrom1(v) => v.name(),
ComputedVec::LazyFrom2(v) => v.name(),
ComputedVec::LazyFrom3(v) => v.name(),
}
}
fn index_type_to_string(&self) -> &str {
I::to_string()
}
fn len(&self) -> usize {
match self {
ComputedVec::Eager { vec, .. } => vec.len(),
ComputedVec::LazyFrom1(v) => v.len(),
ComputedVec::LazyFrom2(v) => v.len(),
ComputedVec::LazyFrom3(v) => v.len(),
}
}
fn collect_range_serde_json(
&self,
from: Option<i64>,
to: Option<i64>,
) -> Result<Vec<serde_json::Value>> {
todo!()
}
fn modified_time(&self) -> Result<Duration> {
match self {
ComputedVec::Eager { vec, .. } => vec.modified_time(),
ComputedVec::LazyFrom1(v) => v.modified_time(),
ComputedVec::LazyFrom2(v) => v.modified_time(),
ComputedVec::LazyFrom3(v) => v.modified_time(),
}
}
}