Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-27 16:19:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

vec: rework part 4

Browse Source
This commit is contained in:
nym21
2025-04-10 15:55:26 +02:00
parent 0dd7e9359e
commit 139e93b2f0
16 changed files with 553 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
crates/brk_computer/src/storage/vecs/base.rs
View File

@@ -15,7 +15,7 @@ use brk_vec::{
const ONE_KIB: usize = 1024;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
const MAX_CACHE_SIZE: usize = 210 * ONE_MIB;
#[derive(Debug)]
pub struct ComputedVec<I, T>
@@ -24,7 +24,7 @@ where
T: StoredType,
{
computed_version: Option<Version>,
vec: StoredVec<I, T>,
inner: StoredVec<I, T>,
}
impl<I, T> ComputedVec<I, T>
@@ -43,7 +43,7 @@ where
Ok(Self {
computed_version: None,
vec,
inner: vec,
})
}
@@ -52,7 +52,7 @@ where
return Ok(());
}
exit.block();
self.vec.truncate_if_needed(index)?;
self.inner.truncate_if_needed(index)?;
exit.release();
Ok(())
}
@@ -67,11 +67,11 @@ where
if ord == Ordering::Greater {
self.safe_truncate_if_needed(index, exit)?;
}
self.vec.push(value);
self.inner.push(value);
}
}
if self.vec.pushed_len() * Self::SIZE_OF >= MAX_CACHE_SIZE {
if self.inner.pushed_len() * Self::SIZE_OF >= MAX_CACHE_SIZE {
self.safe_flush(exit)
} else {
Ok(())
@@ -83,52 +83,56 @@ where
return Ok(());
}
exit.block();
self.vec.flush()?;
self.inner.flush()?;
exit.release();
Ok(())
}
fn version(&self) -> Version {
self.vec.version()
self.inner.version()
}
pub fn len(&self) -> usize {
self.vec.len()
self.inner.len()
}
pub fn vec(&self) -> &StoredVec<I, T> {
&self.vec
&self.inner
}
pub fn mut_vec(&mut self) -> &mut StoredVec<I, T> {
&mut self.vec
&mut self.inner
}
pub fn any_vec(&self) -> &dyn brk_vec::AnyStoredVec {
&self.vec
&self.inner
}
pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyStoredVec {
&mut self.vec
&mut self.inner
}
pub fn get(&mut self, index: I) -> Result<Option<Value<T>>> {
self.vec.get(index)
pub fn cached_get(&mut self, index: I) -> Result<Option<Value<T>>> {
self.inner.cached_get(index)
}
pub fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
self.vec.collect_range(from, to)
pub fn collect_inclusive_range(&self, from: I, to: I) -> Result<Vec<T>> {
self.inner.collect_inclusive_range(from, to)
}
pub fn path(&self) -> &Path {
self.inner.path()
}
#[inline]
fn path_computed_version(&self) -> PathBuf {
self.vec.path().join("computed_version")
self.inner.path().join("computed_version")
}
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
let path = self.path_computed_version();
if version.validate(path.as_ref()).is_err() {
self.vec.reset()?;
self.inner.reset()?;
}
version.write(path.as_ref())?;
Ok(())
@@ -174,12 +178,12 @@ where
)?;
let index = max_from.min(
self.vec
.get_last()?
self.inner
.cached_get_last()?
.map_or_else(T::default, |v| v.into_inner()),
);
other.iter_from(index, |(v, i, ..)| {
if self.get(i).unwrap().is_none_or(|old_v| *old_v > v) {
if self.cached_get(i).unwrap().is_none_or(|old_v| *old_v > v) {
self.forced_push_at(i, v, exit)
} else {
Ok(())
@@ -207,7 +211,7 @@ where
let index = max_from.min(T::from(self.len()));
first_indexes.iter_from(index, |(value, first_index, ..)| {
let first_index = (first_index).to_usize()?;
let last_index = (last_indexes.get(value)?.unwrap()).to_usize()?;
let last_index = (last_indexes.cached_get(value)?.unwrap()).to_usize()?;
(first_index..last_index)
.try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
})?;
@@ -268,7 +272,7 @@ where
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.get(i)?.unwrap();
let last_index = last_indexes.cached_get(i)?.unwrap();
let count = (*last_index + 1_usize)
.checked_sub(first_index)
.unwrap_or_default();
@@ -298,7 +302,7 @@ where
self_to_other.iter_from(index, |(i, other, ..)| {
self.forced_push_at(
i,
T::from(other_to_self.get(other)?.unwrap().into_inner() == i),
T::from(other_to_self.cached_get(other)?.unwrap().into_inner() == i),
exit,
)
})?;
@@ -324,7 +328,7 @@ where
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(index, first_index, ..)| {
let last_index = last_indexes.get(index)?.unwrap();
let last_index = last_indexes.cached_get(index)?.unwrap();
let count = *last_index + 1_usize - first_index;
self.forced_push_at(index, count.into(), exit)
})?;
@@ -341,7 +345,7 @@ where
fn clone(&self) -> Self {
Self {
computed_version: self.computed_version,
vec: self.vec.clone(),
inner: self.inner.clone(),
}
}
}

48
crates/brk_computer/src/storage/vecs/grouped/builder.rs
View File

@@ -2,8 +2,7 @@ use std::path::Path;
use brk_exit::Exit;
use brk_vec::{
AnyStoredVec, Compressed, DynamicVec, GenericVec, Result, StoredIndex, StoredType, StoredVec,
Version,
Compressed, DynamicVec, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
};
use crate::storage::vecs::base::ComputedVec;
@@ -133,7 +132,7 @@ where
.checked_sub(1)
.map_or(T::from(0_usize), |prev_i| {
total_vec
.get(I::from(prev_i))
.cached_get(I::from(prev_i))
.unwrap()
.map_or(T::from(0_usize), |v| v.into_inner())
});
@@ -164,21 +163,24 @@ where
let index = self.starting_index(max_from);
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = *last_indexes.get(i).unwrap().unwrap();
let last_index = *last_indexes.cached_get(i)?.unwrap();
if let Some(first) = self.first.as_mut() {
let v = source.get(first_index).unwrap().unwrap().into_inner();
let v = source.cached_get(first_index)?.unwrap().into_inner();
first.forced_push_at(index, v, exit)?;
}
if let Some(last) = self.last.as_mut() {
let v = source.get(last_index).unwrap().unwrap().into_inner();
let v = source
.cached_get(last_index)
.inspect_err(|_| {
dbg!(last.path(), last_index);
})?
.unwrap()
.into_inner();
last.forced_push_at(index, v, exit)?;
}
let first_index = first_index.to_usize()?;
let last_index = last_index.to_usize()?;
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sorted = self.max.is_some()
@@ -191,8 +193,7 @@ where
let needs_values = needs_sorted || needs_average_sum_or_total;
if needs_values {
let mut values =
source.collect_range(Some(first_index as i64), Some(last_index as i64))?;
let mut values = source.collect_inclusive_range(first_index, last_index)?;
if needs_sorted {
values.sort_unstable();
@@ -251,7 +252,7 @@ where
T::from(0_usize),
|prev_i| {
total_vec
.get(I::from(prev_i))
.cached_get(I::from(prev_i))
.unwrap()
.unwrap()
.to_owned()
@@ -298,14 +299,14 @@ where
let index = self.starting_index(max_from);
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = *last_indexes.get(i).unwrap().unwrap();
let last_index = *last_indexes.cached_get(i).unwrap().unwrap();
if let Some(first) = self.first.as_mut() {
let v = source
.first
.as_mut()
.unwrap()
.get(first_index)
.cached_get(first_index)
.unwrap()
.unwrap()
.into_inner();
@@ -317,16 +318,13 @@ where
.last
.as_mut()
.unwrap()
.get(last_index)
.cached_get(last_index)
.unwrap()
.unwrap()
.into_inner();
last.forced_push_at(index, v, exit)?;
}
let first_index = Some(first_index.to_usize()? as i64);
let last_index = Some(last_index.to_usize()? as i64);
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sorted = self.max.is_some() || self.min.is_some();
@@ -339,7 +337,7 @@ where
.max
.as_ref()
.unwrap()
.collect_range(first_index, last_index)?;
.collect_inclusive_range(first_index, last_index)?;
values.sort_unstable();
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
}
@@ -349,7 +347,7 @@ where
.min
.as_ref()
.unwrap()
.collect_range(first_index, last_index)?;
.collect_inclusive_range(first_index, last_index)?;
values.sort_unstable();
min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
}
@@ -361,7 +359,7 @@ where
.average
.as_ref()
.unwrap()
.collect_range(first_index, last_index)?;
.collect_inclusive_range(first_index, last_index)?;
let len = values.len() as f64;
let total = values
.into_iter()
@@ -378,7 +376,7 @@ where
.sum
.as_ref()
.unwrap()
.collect_range(first_index, last_index)?;
.collect_inclusive_range(first_index, last_index)?;
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
@@ -390,7 +388,7 @@ where
T::from(0_usize),
|prev_i| {
total_vec
.get(I::from(prev_i))
.cached_get(I::from(prev_i))
.unwrap()
.unwrap()
.into_inner()
@@ -438,8 +436,8 @@ where
))
}
pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
let mut v: Vec<&dyn AnyStoredVec> = vec![];
pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
let mut v: Vec<&dyn brk_vec::AnyStoredVec> = vec![];
if let Some(first) = self.first.as_ref() {
v.push(first.any_vec());

63
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -345,7 +345,7 @@ impl Vecs {
|(h, d, s, ..)| {
let d = h
.decremented()
.and_then(|h| s.get(h).ok())
.and_then(|h| s.cached_get(h).ok())
.flatten()
.map_or(d, |prev_d| {
let prev_d = *prev_d;
@@ -365,7 +365,7 @@ impl Vecs {
let starting_dateindex = self
.height_to_dateindex
.get(starting_indexes.height.decremented().unwrap_or_default())?
.cached_get(starting_indexes.height.decremented().unwrap_or_default())?
.map_or_else(Default::default, |v| v.into_inner());
self.height_to_dateindex.compute_transform(
@@ -377,7 +377,7 @@ impl Vecs {
let starting_dateindex = if let Some(dateindex) = self
.height_to_dateindex
.get(starting_indexes.height.decremented().unwrap_or_default())?
.cached_get(starting_indexes.height.decremented().unwrap_or_default())?
.map(|v| v.into_inner())
{
starting_dateindex.min(dateindex)
@@ -450,7 +450,7 @@ impl Vecs {
let starting_weekindex = self
.dateindex_to_weekindex
.get(starting_dateindex)?
.cached_get(starting_dateindex)?
.map_or_else(Default::default, |v| v.into_inner());
self.dateindex_to_weekindex.compute_transform(
@@ -485,7 +485,12 @@ impl Vecs {
self.weekindex_to_timestamp.compute_transform(
starting_weekindex,
self.weekindex_to_first_dateindex.mut_vec(),
|(i, d, ..)| (i, *self.dateindex_to_timestamp.get(d).unwrap().unwrap()),
|(i, d, ..)| {
(
i,
*self.dateindex_to_timestamp.cached_get(d).unwrap().unwrap(),
)
},
exit,
)?;
@@ -493,7 +498,7 @@ impl Vecs {
let starting_monthindex = self
.dateindex_to_monthindex
.get(starting_dateindex)?
.cached_get(starting_dateindex)?
.map_or_else(Default::default, |v| v.into_inner());
self.dateindex_to_monthindex.compute_transform(
@@ -530,7 +535,12 @@ impl Vecs {
self.monthindex_to_timestamp.compute_transform(
starting_monthindex,
self.monthindex_to_first_dateindex.mut_vec(),
|(i, d, ..)| (i, *self.dateindex_to_timestamp.get(d).unwrap().unwrap()),
|(i, d, ..)| {
(
i,
*self.dateindex_to_timestamp.cached_get(d).unwrap().unwrap(),
)
},
exit,
)?;
@@ -538,7 +548,7 @@ impl Vecs {
let starting_quarterindex = self
.monthindex_to_quarterindex
.get(starting_monthindex)?
.cached_get(starting_monthindex)?
.map_or_else(Default::default, |v| v.into_inner());
self.monthindex_to_quarterindex.compute_transform(
@@ -575,7 +585,12 @@ impl Vecs {
self.quarterindex_to_timestamp.compute_transform(
starting_quarterindex,
self.quarterindex_to_first_monthindex.mut_vec(),
|(i, m, ..)| (i, *self.monthindex_to_timestamp.get(m).unwrap().unwrap()),
|(i, m, ..)| {
(
i,
*self.monthindex_to_timestamp.cached_get(m).unwrap().unwrap(),
)
},
exit,
)?;
@@ -583,7 +598,7 @@ impl Vecs {
let starting_yearindex = self
.monthindex_to_yearindex
.get(starting_monthindex)?
.cached_get(starting_monthindex)?
.map_or_else(Default::default, |v| v.into_inner());
self.monthindex_to_yearindex.compute_transform(
@@ -620,7 +635,12 @@ impl Vecs {
self.yearindex_to_timestamp.compute_transform(
starting_yearindex,
self.yearindex_to_first_monthindex.mut_vec(),
|(i, m, ..)| (i, *self.monthindex_to_timestamp.get(m).unwrap().unwrap()),
|(i, m, ..)| {
(
i,
*self.monthindex_to_timestamp.cached_get(m).unwrap().unwrap(),
)
},
exit,
)?;
@@ -628,7 +648,7 @@ impl Vecs {
let starting_decadeindex = self
.yearindex_to_decadeindex
.get(starting_yearindex)?
.cached_get(starting_yearindex)?
.map_or_else(Default::default, |v| v.into_inner());
self.yearindex_to_decadeindex.compute_transform(
@@ -663,7 +683,12 @@ impl Vecs {
self.decadeindex_to_timestamp.compute_transform(
starting_decadeindex,
self.decadeindex_to_first_yearindex.mut_vec(),
|(i, y, ..)| (i, *self.yearindex_to_timestamp.get(y).unwrap().unwrap()),
|(i, y, ..)| {
(
i,
*self.yearindex_to_timestamp.cached_get(y).unwrap().unwrap(),
)
},
exit,
)?;
@@ -671,7 +696,7 @@ impl Vecs {
let starting_difficultyepoch = self
.height_to_difficultyepoch
.get(starting_indexes.height)?
.cached_get(starting_indexes.height)?
.map_or_else(Default::default, |v| v.into_inner());
self.height_to_difficultyepoch.compute_transform(
@@ -709,7 +734,11 @@ impl Vecs {
|(i, h, ..)| {
(
i,
*indexer_vecs.height_to_timestamp.get(h).unwrap().unwrap(),
*indexer_vecs
.height_to_timestamp
.cached_get(h)
.unwrap()
.unwrap(),
)
},
exit,
@@ -719,7 +748,7 @@ impl Vecs {
let starting_halvingepoch = self
.height_to_halvingepoch
.get(starting_indexes.height)?
.cached_get(starting_indexes.height)?
.map_or_else(Default::default, |v| v.into_inner());
self.height_to_halvingepoch.compute_transform(
@@ -757,7 +786,7 @@ impl Vecs {
// |(i, h, ..)| {
// (
// i,
// *indexer_vecs.height_to_timestamp.get(h).unwrap().unwrap(),
// *indexer_vecs.height_to_timestamp.cached_get(h).unwrap().unwrap(),
// )
// },
// exit,

41
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -243,8 +243,9 @@ impl Vecs {
.get_height(
h,
t,
h.decremented()
.map(|prev_h| *height_to_timestamp.get(prev_h).unwrap().unwrap()),
h.decremented().map(|prev_h| {
*height_to_timestamp.cached_get(prev_h).unwrap().unwrap()
}),
)
.unwrap();
(h, ohlc)
@@ -470,7 +471,7 @@ impl Vecs {
.first
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
high: *self
@@ -479,7 +480,7 @@ impl Vecs {
.max
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
low: *self
@@ -488,7 +489,7 @@ impl Vecs {
.min
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
close,
@@ -516,7 +517,7 @@ impl Vecs {
.first
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
high: *self
@@ -525,7 +526,7 @@ impl Vecs {
.max
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
low: *self
@@ -534,7 +535,7 @@ impl Vecs {
.min
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
close,
@@ -562,7 +563,7 @@ impl Vecs {
.first
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
high: *self
@@ -571,7 +572,7 @@ impl Vecs {
.max
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
low: *self
@@ -580,7 +581,7 @@ impl Vecs {
.min
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
close,
@@ -608,7 +609,7 @@ impl Vecs {
.first
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
high: *self
@@ -617,7 +618,7 @@ impl Vecs {
.max
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
low: *self
@@ -626,7 +627,7 @@ impl Vecs {
.min
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
close,
@@ -654,7 +655,7 @@ impl Vecs {
.first
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
high: *self
@@ -663,7 +664,7 @@ impl Vecs {
.max
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
low: *self
@@ -672,7 +673,7 @@ impl Vecs {
.min
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
close,
@@ -704,7 +705,7 @@ impl Vecs {
.first
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
high: *self
@@ -713,7 +714,7 @@ impl Vecs {
.max
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
low: *self
@@ -722,7 +723,7 @@ impl Vecs {
.min
.as_mut()
.unwrap()
.get(i)
.cached_get(i)
.unwrap()
.unwrap(),
close,

6
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -182,8 +182,10 @@ impl Vecs {
|(txinindex, txoutindex, slf, other)| {
let value = if txoutindex == Txoutindex::COINBASE {
Sats::ZERO
} else if let Ok(Some(value)) =
indexer_vecs.txoutindex_to_value.mut_vec().get(txoutindex)
} else if let Ok(Some(value)) = indexer_vecs
.txoutindex_to_value
.mut_vec()
.cached_get(txoutindex)
{
*value
} else {