Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-30 20:28:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: wip

Browse Source
This commit is contained in:
nym21
2025-05-31 20:45:59 +02:00
parent cfc3081e8a
commit f976f672cf
34 changed files with 2109 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/lib.rs
+3
View File
@@ -47,12 +47,14 @@ impl Computer {
computation: Computation,
) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(
// TODO: Give self.path, join inside import
&self.path.join("vecs/computed"),
VERSION + Version::ZERO,
indexer,
self.fetcher.is_some(),
computation,
self.compressed,
indexer.keyspace(),
)?);
Ok(())
}
@@ -61,6 +63,7 @@ impl Computer {
/// Clone struct instead
pub fn import_stores(&mut self, indexer: &Indexer) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(
// TODO: Give self.path, join inside import
&self.path.join("stores"),
VERSION + Version::ZERO,
indexer.keyspace(),
crates/brk_computer/src/vecs/blocks.rs
+2 -1
View File
@@ -49,6 +49,7 @@ impl Vecs {
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
path,
"timestamp",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_first(),
@@ -132,7 +133,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.timeindexes_to_timestamp.compute(
self.timeindexes_to_timestamp.compute_all(
indexer,
indexes,
starting_indexes,
crates/brk_computer/src/vecs/fetched.rs
+42 -16
View File
@@ -169,6 +169,7 @@ impl Vecs {
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
path,
"open",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_first(),
@@ -176,6 +177,7 @@ impl Vecs {
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
path,
"high",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_max(),
@@ -183,6 +185,7 @@ impl Vecs {
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
path,
"low",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_min(),
@@ -190,6 +193,7 @@ impl Vecs {
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
path,
"close",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -197,6 +201,7 @@ impl Vecs {
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"open_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_first(),
@@ -204,6 +209,7 @@ impl Vecs {
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"high_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_max(),
@@ -211,6 +217,7 @@ impl Vecs {
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"low_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_min(),
@@ -218,6 +225,7 @@ impl Vecs {
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"close_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -463,7 +471,7 @@ impl Vecs {
exit,
)?;
self.timeindexes_to_close.compute(
self.timeindexes_to_close.compute_all(
indexer,
indexes,
starting_indexes,
@@ -478,7 +486,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_high.compute(
self.timeindexes_to_high.compute_all(
indexer,
indexes,
starting_indexes,
@@ -493,7 +501,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_low.compute(
self.timeindexes_to_low.compute_all(
indexer,
indexes,
starting_indexes,
@@ -508,7 +516,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_open.compute(
self.timeindexes_to_open.compute_all(
indexer,
indexes,
starting_indexes,
@@ -776,7 +784,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_open_in_sats.compute(
self.timeindexes_to_open_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -784,14 +792,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_open.dateindex,
self.timeindexes_to_open.dateindex.as_ref().unwrap(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
},
)?;
self.timeindexes_to_high_in_sats.compute(
self.timeindexes_to_high_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -799,14 +807,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_low.dateindex,
self.timeindexes_to_low.dateindex.as_ref().unwrap(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
},
)?;
self.timeindexes_to_low_in_sats.compute(
self.timeindexes_to_low_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -814,14 +822,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_high.dateindex,
self.timeindexes_to_high.dateindex.as_ref().unwrap(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
},
)?;
self.timeindexes_to_close_in_sats.compute(
self.timeindexes_to_close_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -829,7 +837,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_close.dateindex,
self.timeindexes_to_close.dateindex.as_ref().unwrap(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -856,12 +864,30 @@ impl Vecs {
exit,
)?;
let mut dateindex_first_iter = self.timeindexes_to_open_in_sats.dateindex.iter();
let mut dateindex_max_iter = self.timeindexes_to_high_in_sats.dateindex.iter();
let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
let mut dateindex_first_iter = self
.timeindexes_to_open_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
let mut dateindex_max_iter = self
.timeindexes_to_high_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
let mut dateindex_min_iter = self
.timeindexes_to_low_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
self.dateindex_to_ohlc_in_sats.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_close_in_sats.dateindex,
self.timeindexes_to_close_in_sats
.dateindex
.as_ref()
.unwrap(),
|(i, close, ..)| {
(
i,
crates/brk_computer/src/vecs/grouped/from_dateindex.rs
+56 -28
View File
@@ -5,7 +5,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, EagerVec};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec};
use crate::vecs::{Indexes, indexes};
@@ -16,7 +16,7 @@ pub struct ComputedVecsFromDateIndex<T>
where
T: ComputedType + PartialOrd,
{
pub dateindex: EagerVec<DateIndex, T>,
pub dateindex: Option<EagerVec<DateIndex, T>>,
pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
@@ -34,10 +34,16 @@ where
pub fn forced_import(
path: &Path,
name: &str,
compute_source: bool,
version: Version,
compressed: Compressed,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let dateindex = compute_source.then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, compressed)
.unwrap()
});
let dateindex_extra = ComputedVecBuilder::forced_import(
path,
name,
@@ -49,12 +55,7 @@ where
let options = options.remove_percentiles();
Ok(Self {
dateindex: EagerVec::forced_import(
path,
name,
version + VERSION + Version::ZERO,
compressed,
)?,
dateindex,
dateindex_extra,
weekindex: ComputedVecBuilder::forced_import(
path,
@@ -94,7 +95,7 @@ where
})
}
pub fn compute<F>(
pub fn compute_all<F>(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
@@ -112,14 +113,15 @@ where
) -> Result<()>,
{
compute(
&mut self.dateindex,
self.dateindex.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
self.compute_rest(indexes, starting_indexes, exit)
let dateindex: Option<&EagerVec<DateIndex, T>> = None;
self.compute_rest(indexes, starting_indexes, exit, dateindex)
}
pub fn compute_rest(
@@ -127,25 +129,49 @@ where
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
) -> color_eyre::Result<()> {
self.dateindex_extra
.extend(starting_indexes.dateindex, &self.dateindex, exit)?;
if let Some(dateindex) = dateindex {
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
&self.dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.weekindex.compute(
starting_indexes.weekindex,
dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
&self.dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
} else {
let dateindex = self.dateindex.as_ref().unwrap();
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
}
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
@@ -176,7 +202,9 @@ where
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![&self.dateindex as &dyn AnyCollectableVec],
self.dateindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.dateindex_extra.vecs(),
self.weekindex.vecs(),
self.monthindex.vecs(),
crates/brk_computer/src/vecs/grouped/from_height.rs
+5 -7
View File
@@ -17,7 +17,7 @@ pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
{
pub height: Option<Box<EagerVec<Height, T>>>,
pub height: Option<EagerVec<Height, T>>,
pub height_extra: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -45,10 +45,8 @@ where
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let height = compute_source.then(|| {
Box::new(
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, compressed)
.unwrap(),
)
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, compressed)
.unwrap()
});
let height_extra = ComputedVecBuilder::forced_import(
@@ -169,7 +167,7 @@ where
exit,
)?;
} else {
let height = self.height.as_ref().unwrap().as_ref();
let height = self.height.as_ref().unwrap();
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -238,7 +236,7 @@ where
[
self.height
.as_ref()
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs
+285 -129
View File
@@ -69,6 +69,7 @@ impl ComputedRatioVecsFromDateIndex {
ComputedVecsFromDateIndex::forced_import(
path,
name,
true,
version + VERSION,
compressed,
options,
@@ -78,6 +79,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -85,6 +87,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -92,6 +95,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1w_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1w_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -99,6 +103,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1m_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1m_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -106,6 +111,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1y_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -113,6 +119,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -120,6 +127,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_standard_deviation: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_standard_deviation"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -127,6 +135,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_9: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -134,6 +143,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -141,6 +151,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -148,6 +159,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -155,6 +167,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -162,6 +175,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -169,6 +183,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -176,6 +191,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -183,6 +199,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -190,6 +207,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -197,6 +215,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -204,6 +223,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -211,6 +231,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_9_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -218,6 +239,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -225,6 +247,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -232,6 +255,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -239,6 +263,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -246,6 +271,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -253,6 +279,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -260,6 +287,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -267,6 +295,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -274,6 +303,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -281,6 +311,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -288,6 +319,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -295,6 +327,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_zscore: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_zscore"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -320,10 +353,13 @@ impl ComputedRatioVecsFromDateIndex {
&Exit,
) -> Result<()>,
{
self.price
.as_mut()
.unwrap()
.compute(indexer, indexes, starting_indexes, exit, compute)?;
self.price.as_mut().unwrap().compute_all(
indexer,
indexes,
starting_indexes,
exit,
compute,
)?;
let date_to_price_opt: Option<&EagerVec<DateIndex, Dollars>> = None;
self.compute_rest(
@@ -349,9 +385,9 @@ impl ComputedRatioVecsFromDateIndex {
std::mem::transmute(&self.price.as_ref().unwrap().dateindex)
});
let closes = &fetched.timeindexes_to_close.dateindex;
let closes = fetched.timeindexes_to_close.dateindex.as_ref().unwrap();
self.ratio.compute(
self.ratio.compute_all(
indexer,
indexes,
starting_indexes,
@@ -376,7 +412,7 @@ impl ComputedRatioVecsFromDateIndex {
let min_ratio_date = DateIndex::try_from(Date::MIN_RATIO).unwrap();
self.ratio_sma.compute(
self.ratio_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -384,7 +420,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
usize::MAX,
exit,
Some(min_ratio_date),
@@ -392,7 +428,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1w_sma.compute(
self.ratio_1w_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -400,7 +436,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
7,
exit,
Some(min_ratio_date),
@@ -408,7 +444,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1m_sma.compute(
self.ratio_1m_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -416,7 +452,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
30,
exit,
Some(min_ratio_date),
@@ -424,7 +460,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1y_sma.compute(
self.ratio_1y_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -432,7 +468,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
365,
exit,
Some(min_ratio_date),
@@ -440,16 +476,17 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1y_sma_momentum_oscillator.compute(
self.ratio_1y_sma_momentum_oscillator.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
let mut ratio_1y_sma_iter = self.ratio_1y_sma.dateindex.into_iter();
let mut ratio_1y_sma_iter =
self.ratio_1y_sma.dateindex.as_ref().unwrap().into_iter();
v.compute_transform(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
|(i, ratio, ..)| {
(
i,
@@ -461,7 +498,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
let ratio_version = self.ratio.dateindex.version();
let ratio_version = self.ratio.dateindex.as_ref().unwrap().version();
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
@@ -478,7 +515,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap()
.min(starting_indexes.dateindex);
let mut sorted = self.ratio.dateindex.collect_range(
let mut sorted = self.ratio.dateindex.as_ref().unwrap().collect_range(
Some(min_ratio_date.unwrap_to_usize()),
Some(starting_dateindex.unwrap_to_usize()),
)?;
@@ -489,68 +526,116 @@ impl ComputedRatioVecsFromDateIndex {
// unreachable!();
// }
let mut sma_iter = self.ratio_sma.dateindex.into_iter();
let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
let nan = StoredF32::from(f32::NAN);
self.ratio
.dateindex
.as_ref()
.unwrap()
.iter_at(starting_dateindex)
.try_for_each(|(index, ratio)| -> Result<()> {
if index < min_ratio_date {
self.ratio_p0_1.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p0_5.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p1.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p99.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p0_1
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p0_5
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p1
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p99
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p99_5
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p99_9
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_standard_deviation
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p1sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p2sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p3sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_m1sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_m2sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_m3sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p1sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p2sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p3sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_m1sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_m2sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_m3sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
} else {
let ratio = ratio.into_inner();
let pos = sorted.binary_search(&ratio).unwrap_or_else(|pos| pos);
sorted.insert(pos, ratio);
self.ratio_p0_1.dateindex.forced_push_at(
self.ratio_p0_1.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.001),
exit,
)?;
self.ratio_p0_5.dateindex.forced_push_at(
self.ratio_p0_5.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.005),
exit,
)?;
self.ratio_p1.dateindex.forced_push_at(
self.ratio_p1.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.01),
exit,
)?;
self.ratio_p99.dateindex.forced_push_at(
self.ratio_p99.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.99),
exit,
)?;
self.ratio_p99_5.dateindex.forced_push_at(
index,
get_percentile(&sorted, 0.995),
exit,
)?;
self.ratio_p99_9.dateindex.forced_push_at(
index,
get_percentile(&sorted, 0.999),
exit,
)?;
self.ratio_p99_5
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, get_percentile(&sorted, 0.995), exit)?;
self.ratio_p99_9
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, get_percentile(&sorted, 0.999), exit)?;
let avg = sma_iter.unwrap_get_inner(index);
@@ -562,26 +647,40 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_standard_deviation
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, sd, exit)?;
self.ratio_p1sd
.dateindex
.forced_push_at(index, avg + sd, exit)?;
self.ratio_p2sd
.dateindex
.forced_push_at(index, avg + 2 * sd, exit)?;
self.ratio_p3sd
.dateindex
.forced_push_at(index, avg + 3 * sd, exit)?;
self.ratio_m1sd
.dateindex
.forced_push_at(index, avg - sd, exit)?;
self.ratio_m2sd
.dateindex
.forced_push_at(index, avg - 2 * sd, exit)?;
self.ratio_m3sd
.dateindex
.forced_push_at(index, avg - 3 * sd, exit)?;
self.ratio_p1sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + sd,
exit,
)?;
self.ratio_p2sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + 2 * sd,
exit,
)?;
self.ratio_p3sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + 3 * sd,
exit,
)?;
self.ratio_m1sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg - sd,
exit,
)?;
self.ratio_m2sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg - 2 * sd,
exit,
)?;
self.ratio_m3sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg - 3 * sd,
exit,
)?;
}
Ok(())
@@ -591,44 +690,96 @@ impl ComputedRatioVecsFromDateIndex {
.into_iter()
.try_for_each(|v| v.safe_flush(exit))?;
self.ratio_p99_9
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p99_5
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p99
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p1
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p0_5
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p0_1
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_standard_deviation
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p1sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p2sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p3sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_m1sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_m2sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_m3sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p99_9.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p99_5.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p99.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p1.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p0_5.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p0_1.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_standard_deviation.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p1sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p2sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p3sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_m1sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_m2sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_m3sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
let date_to_price = date_to_price_opt.unwrap_or_else(|| unsafe {
std::mem::transmute(&self.price.as_ref().unwrap().dateindex)
});
self.ratio_p99_as_price.compute(
self.ratio_p99_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p99.dateindex.into_iter();
let mut iter = self.ratio_p99.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -641,13 +792,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p99_5_as_price.compute(
self.ratio_p99_5_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p99_5.dateindex.into_iter();
let mut iter = self.ratio_p99_5.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -660,13 +811,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p99_9_as_price.compute(
self.ratio_p99_9_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p99_9.dateindex.into_iter();
let mut iter = self.ratio_p99_9.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -679,13 +830,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p1_as_price.compute(
self.ratio_p1_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p1.dateindex.into_iter();
let mut iter = self.ratio_p1.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -698,13 +849,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p0_5_as_price.compute(
self.ratio_p0_5_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p0_5.dateindex.into_iter();
let mut iter = self.ratio_p0_5.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -717,13 +868,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p0_1_as_price.compute(
self.ratio_p0_1_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p0_1.dateindex.into_iter();
let mut iter = self.ratio_p0_1.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -736,13 +887,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p1sd_as_price.compute(
self.ratio_p1sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p1sd.dateindex.into_iter();
let mut iter = self.ratio_p1sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -755,13 +906,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p2sd_as_price.compute(
self.ratio_p2sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p2sd.dateindex.into_iter();
let mut iter = self.ratio_p2sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -774,13 +925,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p3sd_as_price.compute(
self.ratio_p3sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p3sd.dateindex.into_iter();
let mut iter = self.ratio_p3sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -793,13 +944,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_m1sd_as_price.compute(
self.ratio_m1sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_m1sd.dateindex.into_iter();
let mut iter = self.ratio_m1sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -812,13 +963,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_m2sd_as_price.compute(
self.ratio_m2sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_m2sd.dateindex.into_iter();
let mut iter = self.ratio_m2sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -831,13 +982,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_m3sd_as_price.compute(
self.ratio_m3sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_m3sd.dateindex.into_iter();
let mut iter = self.ratio_m3sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -850,17 +1001,22 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_zscore.compute(
self.ratio_zscore.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut sma_iter = self.ratio_sma.dateindex.into_iter();
let mut sd_iter = self.ratio_standard_deviation.dateindex.into_iter();
let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
let mut sd_iter = self
.ratio_standard_deviation
.dateindex
.as_ref()
.unwrap()
.into_iter();
vec.compute_transform(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
|(i, ratio, ..)| {
let sma = sma_iter.unwrap_get_inner(i);
let sd = sd_iter.unwrap_get_inner(i);
@@ -876,19 +1032,19 @@ impl ComputedRatioVecsFromDateIndex {
fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec<DateIndex, StoredF32>> {
vec![
&mut self.ratio_standard_deviation.dateindex,
&mut self.ratio_p99_9.dateindex,
&mut self.ratio_p99_5.dateindex,
&mut self.ratio_p99.dateindex,
&mut self.ratio_p1.dateindex,
&mut self.ratio_p0_5.dateindex,
&mut self.ratio_p0_1.dateindex,
&mut self.ratio_p1sd.dateindex,
&mut self.ratio_p2sd.dateindex,
&mut self.ratio_p3sd.dateindex,
&mut self.ratio_m1sd.dateindex,
&mut self.ratio_m2sd.dateindex,
&mut self.ratio_m3sd.dateindex,
self.ratio_standard_deviation.dateindex.as_mut().unwrap(),
self.ratio_p99_9.dateindex.as_mut().unwrap(),
self.ratio_p99_5.dateindex.as_mut().unwrap(),
self.ratio_p99.dateindex.as_mut().unwrap(),
self.ratio_p1.dateindex.as_mut().unwrap(),
self.ratio_p0_5.dateindex.as_mut().unwrap(),
self.ratio_p0_1.dateindex.as_mut().unwrap(),
self.ratio_p1sd.dateindex.as_mut().unwrap(),
self.ratio_p2sd.dateindex.as_mut().unwrap(),
self.ratio_p3sd.dateindex.as_mut().unwrap(),
self.ratio_m1sd.dateindex.as_mut().unwrap(),
self.ratio_m2sd.dateindex.as_mut().unwrap(),
self.ratio_m3sd.dateindex.as_mut().unwrap(),
]
}
crates/brk_computer/src/vecs/grouped/value_from_height.rs
+2 -2
View File
@@ -127,14 +127,14 @@ impl ComputedValueVecsFromHeight {
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(
starting_indexes.height,
self.sats.height.as_ref().unwrap().as_ref(),
self.sats.height.as_ref().unwrap(),
exit,
)
},
)?;
}
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap().as_ref();
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
crates/brk_computer/src/vecs/market.rs
+170 -37
View File
File diff suppressed because it is too large Load Diff
crates/brk_computer/src/vecs/mining.rs
+4 -2
View File
@@ -41,6 +41,7 @@ impl Vecs {
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
path,
"difficultyepoch",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -48,6 +49,7 @@ impl Vecs {
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
path,
"halvingepoch",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -63,7 +65,7 @@ impl Vecs {
exit: &Exit,
) -> color_eyre::Result<()> {
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
self.indexes_to_difficultyepoch.compute(
self.indexes_to_difficultyepoch.compute_all(
indexer,
indexes,
starting_indexes,
@@ -87,7 +89,7 @@ impl Vecs {
)?;
let mut height_to_halvingepoch_iter = indexes.height_to_halvingepoch.into_iter();
self.indexes_to_halvingepoch.compute(
self.indexes_to_halvingepoch.compute_all(
indexer,
indexes,
starting_indexes,
crates/brk_computer/src/vecs/mod.rs
+3
View File
@@ -5,6 +5,7 @@ use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation};
use fjall::TransactionalKeyspace;
pub mod blocks;
pub mod constants;
@@ -40,6 +41,7 @@ impl Vecs {
fetch: bool,
computation: Computation,
compressed: Compressed,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -92,6 +94,7 @@ impl Vecs {
computation,
compressed,
fetched.as_ref(),
keyspace,
)?,
transactions: transactions::Vecs::forced_import(
path,
crates/brk_computer/src/vecs/stateful/cohort.rs
+499 -54
View File
@@ -5,20 +5,24 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_state::{CohortState, RealizedState};
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, VecIterator};
use brk_state::CohortState;
use brk_vec::{
AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, StoredIndex, VecIterator,
};
use fjall::TransactionalKeyspace;
use crate::vecs::{
Indexes, fetched,
grouped::{
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
StorableVecGeneatorOptions,
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromDateIndex,
ComputedVecsFromHeight, StorableVecGeneatorOptions,
},
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
starting_height: Height,
pub state: CohortState,
@@ -34,6 +38,16 @@ pub struct Vecs {
pub height_to_adjusted_value_created: Option<EagerVec<Height, Dollars>>,
pub height_to_value_destroyed: Option<EagerVec<Height, Dollars>>,
pub height_to_adjusted_value_destroyed: Option<EagerVec<Height, Dollars>>,
pub height_to_supply_in_profit: Option<EagerVec<Height, Sats>>,
pub height_to_supply_in_loss: Option<EagerVec<Height, Sats>>,
pub height_to_supply_even: Option<EagerVec<Height, Sats>>,
pub height_to_unrealized_profit: Option<EagerVec<Height, Dollars>>,
pub height_to_unrealized_loss: Option<EagerVec<Height, Dollars>>,
pub dateindex_to_supply_in_profit: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_supply_in_loss: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_supply_even: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_unrealized_profit: Option<EagerVec<DateIndex, Dollars>>,
pub dateindex_to_unrealized_loss: Option<EagerVec<DateIndex, Dollars>>,
pub dateindex_to_adjusted_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
pub dateindex_to_realized_cap_30d_change: Option<EagerVec<DateIndex, Dollars>>,
@@ -53,9 +67,15 @@ pub struct Vecs {
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_supply_in_profit: Option<ComputedVecsFromDateIndex<Sats>>,
pub indexes_to_supply_in_loss: Option<ComputedVecsFromDateIndex<Sats>>,
pub indexes_to_supply_even: Option<ComputedVecsFromDateIndex<Sats>>,
pub indexes_to_unrealized_profit: Option<ComputedVecsFromDateIndex<Dollars>>,
pub indexes_to_unrealized_loss: Option<ComputedVecsFromDateIndex<Dollars>>,
}
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
cohort_name: Option<&str>,
@@ -63,6 +83,8 @@ impl Vecs {
compressed: Compressed,
version: Version,
fetched: Option<&fetched::Vecs>,
keyspace: &TransactionalKeyspace,
stores_path: &Path,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
@@ -72,15 +94,163 @@ impl Vecs {
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
let mut state = CohortState::default();
if compute_dollars {
state.realized = Some(RealizedState::NAN);
}
let state = CohortState::default_and_import(
keyspace,
stores_path,
cohort_name.unwrap_or_default(),
version,
compute_dollars,
)?;
Ok(Self {
starting_height: Height::ZERO,
state,
height_to_supply_in_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_supply_in_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_supply_in_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("supply_in_profit"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply_even: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_even"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_supply_even: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_even"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_supply_even: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("supply_even"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply_in_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_supply_in_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_supply_in_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("supply_in_loss"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_unrealized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_unrealized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_unrealized_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("unrealized_profit"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("unrealized_loss"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
path,
@@ -346,6 +516,10 @@ impl Vecs {
pub fn starting_height(&self) -> Height {
[
self.state
.price_to_amount
.height()
.map_or(usize::MAX, |h| h.incremented().unwrap_to_usize()),
self.height_to_supply.len(),
self.height_to_utxo_count.len(),
self.height_to_realized_cap
@@ -369,6 +543,21 @@ impl Vecs {
self.height_to_adjusted_value_destroyed
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_in_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_in_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_even
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_unrealized_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_unrealized_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
]
.into_iter()
.map(Height::from)
@@ -483,6 +672,113 @@ impl Vecs {
.validate_computed_version_or_reset_file(
base_version + height_to_adjusted_value_destroyed_inner_version,
)?;
let height_to_supply_in_profit_inner_version = self
.height_to_supply_in_profit
.as_ref()
.unwrap()
.inner_version();
self.height_to_supply_in_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_supply_in_profit_inner_version,
)?;
let height_to_supply_in_loss_inner_version = self
.height_to_supply_in_loss
.as_ref()
.unwrap()
.inner_version();
self.height_to_supply_in_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_supply_in_loss_inner_version,
)?;
let height_to_supply_even_inner_version =
self.height_to_supply_even.as_ref().unwrap().inner_version();
self.height_to_supply_even
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_supply_even_inner_version,
)?;
let height_to_unrealized_profit_inner_version = self
.height_to_unrealized_profit
.as_ref()
.unwrap()
.inner_version();
self.height_to_unrealized_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_unrealized_profit_inner_version,
)?;
let height_to_unrealized_loss_inner_version = self
.height_to_unrealized_loss
.as_ref()
.unwrap()
.inner_version();
self.height_to_unrealized_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_unrealized_loss_inner_version,
)?;
let dateindex_to_supply_in_profit_inner_version = self
.dateindex_to_supply_in_profit
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_supply_in_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_supply_in_profit_inner_version,
)?;
let dateindex_to_supply_in_loss_inner_version = self
.dateindex_to_supply_in_loss
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_supply_in_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_supply_in_loss_inner_version,
)?;
let dateindex_to_supply_even_inner_version = self
.dateindex_to_supply_even
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_supply_even
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_supply_even_inner_version,
)?;
let dateindex_to_unrealized_profit_inner_version = self
.dateindex_to_unrealized_profit
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_unrealized_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_unrealized_profit_inner_version,
)?;
let dateindex_to_unrealized_loss_inner_version = self
.dateindex_to_unrealized_loss
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_unrealized_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_unrealized_loss_inner_version,
)?;
}
Ok(())
@@ -538,7 +834,70 @@ impl Vecs {
Ok(())
}
pub fn safe_flush_height_vecs(&mut self, exit: &Exit) -> Result<()> {
pub fn compute_then_force_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
exit: &Exit,
) -> Result<()> {
if let Some(height_price) = height_price {
let (height_unrealized_state, date_unrealized_state) = self
.state
.compute_unrealized_states(height_price, date_price.unwrap());
self.height_to_supply_even
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.supply_even, exit)?;
self.height_to_supply_in_profit
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.supply_in_profit, exit)?;
self.height_to_supply_in_loss
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.supply_in_loss, exit)?;
self.height_to_unrealized_profit
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.unrealized_profit, exit)?;
self.height_to_unrealized_loss
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.unrealized_loss, exit)?;
if let Some(date_unrealized_state) = date_unrealized_state {
let dateindex = dateindex.unwrap();
self.dateindex_to_supply_even
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.supply_even, exit)?;
self.dateindex_to_supply_in_profit
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.supply_in_profit, exit)?;
self.dateindex_to_supply_in_loss
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.supply_in_loss, exit)?;
self.dateindex_to_unrealized_profit
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.unrealized_profit, exit)?;
self.dateindex_to_unrealized_loss
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.unrealized_loss, exit)?;
}
}
Ok(())
}
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.height_to_supply.safe_flush(exit)?;
self.height_to_utxo_count.safe_flush(exit)?;
@@ -569,8 +928,50 @@ impl Vecs {
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_supply_in_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_supply_in_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_supply_even
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_unrealized_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_unrealized_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_supply_in_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_supply_in_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_supply_even
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_unrealized_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_unrealized_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
}
self.state.commit(height)?;
Ok(())
}
@@ -618,7 +1019,7 @@ impl Vecs {
vec.compute_divide(
starting_indexes.height,
self.height_to_realized_cap.as_ref().unwrap(),
&**self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
exit,
)
},
@@ -823,6 +1224,49 @@ impl Vecs {
.unwrap_last(),
exit,
)?;
self.indexes_to_supply_in_profit
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_supply_in_profit.as_ref().unwrap()),
)?;
self.indexes_to_supply_in_loss
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_supply_in_loss.as_ref().unwrap()),
)?;
self.indexes_to_supply_even.as_mut().unwrap().compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_supply_even.as_ref().unwrap()),
)?;
self.indexes_to_unrealized_profit
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_unrealized_profit.as_ref().unwrap()),
)?;
self.indexes_to_unrealized_loss
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_unrealized_loss.as_ref().unwrap()),
)?;
}
Ok(())
@@ -905,53 +1349,54 @@ impl Vecs {
self.dateindex_to_sell_side_risk_ratio
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_supply_in_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_supply_in_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_supply_even
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_unrealized_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_unrealized_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_supply_in_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_supply_in_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_supply_even
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_unrealized_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_unrealized_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.indexes_to_supply_in_profit
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_supply_in_loss
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_supply_even
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_unrealized_profit
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_unrealized_loss
.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
impl Clone for Vecs {
fn clone(&self) -> Self {
Self {
starting_height: self.starting_height,
state: CohortState::default(),
height_to_realized_cap: self.height_to_realized_cap.clone(),
height_to_supply: self.height_to_supply.clone(),
height_to_utxo_count: self.height_to_utxo_count.clone(),
height_to_realized_profit: self.height_to_realized_profit.clone(),
height_to_realized_loss: self.height_to_realized_loss.clone(),
height_to_value_created: self.height_to_value_created.clone(),
height_to_adjusted_value_created: self.height_to_adjusted_value_created.clone(),
height_to_value_destroyed: self.height_to_value_destroyed.clone(),
height_to_adjusted_value_destroyed: self.height_to_adjusted_value_destroyed.clone(),
indexes_to_supply: self.indexes_to_supply.clone(),
indexes_to_utxo_count: self.indexes_to_utxo_count.clone(),
indexes_to_realized_cap: self.indexes_to_realized_cap.clone(),
indexes_to_realized_profit: self.indexes_to_realized_profit.clone(),
indexes_to_realized_loss: self.indexes_to_realized_loss.clone(),
indexes_to_negative_realized_loss: self.indexes_to_negative_realized_loss.clone(),
indexes_to_value_created: self.indexes_to_value_created.clone(),
indexes_to_adjusted_value_created: self.indexes_to_adjusted_value_created.clone(),
indexes_to_value_destroyed: self.indexes_to_value_destroyed.clone(),
indexes_to_adjusted_value_destroyed: self.indexes_to_adjusted_value_destroyed.clone(),
dateindex_to_realized_cap_30d_change: self.dateindex_to_realized_cap_30d_change.clone(),
indexes_to_realized_value: self.indexes_to_realized_value.clone(),
indexes_to_net_realized_profit_and_loss: self
.indexes_to_net_realized_profit_and_loss
.clone(),
indexes_to_realized_price: self.indexes_to_realized_price.clone(),
dateindex_to_sell_side_risk_ratio: self.dateindex_to_sell_side_risk_ratio.clone(),
indexes_to_realized_price_extra: self.indexes_to_realized_price_extra.clone(),
dateindex_to_spent_output_profit_ratio: self
.dateindex_to_spent_output_profit_ratio
.clone(),
dateindex_to_adjusted_spent_output_profit_ratio: self
.dateindex_to_adjusted_spent_output_profit_ratio
.clone(),
}
}
}
crates/brk_computer/src/vecs/stateful/mod.rs
+234 -25
View File
@@ -1,12 +1,13 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_core::{Height, InputIndex, OutputIndex, OutputType, Sats, Version};
use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Compressed, Computation, EagerVec,
GenericStoredVec, StoredIndex, StoredVec, UnsafeSlice, VecIterator,
};
use fjall::TransactionalKeyspace;
use log::info;
use outputs::OutputCohorts;
use rayon::prelude::*;
@@ -47,12 +48,15 @@ impl Vecs {
_computation: Computation,
compressed: Compressed,
fetched: Option<&fetched::Vecs>,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
let mut states_path = path.to_owned();
states_path.pop();
states_path = states_path.join("states");
let mut root_path = path.to_owned();
root_path.pop();
let states_path = root_path.join("states");
root_path.pop();
let stores_path = root_path.join("stores");
Ok(Self {
chain_state: StoredVec::forced_import(
@@ -101,6 +105,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
by_term: OutputsByTerm {
short: cohort::Vecs::forced_import(
@@ -110,6 +116,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
long: cohort::Vecs::forced_import(
path,
@@ -118,6 +126,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_up_to: OutputsByUpTo {
@@ -128,6 +138,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1w: cohort::Vecs::forced_import(
path,
@@ -136,6 +148,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1m: cohort::Vecs::forced_import(
path,
@@ -144,6 +158,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2m: cohort::Vecs::forced_import(
path,
@@ -152,6 +168,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3m: cohort::Vecs::forced_import(
path,
@@ -160,6 +178,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4m: cohort::Vecs::forced_import(
path,
@@ -168,6 +188,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5m: cohort::Vecs::forced_import(
path,
@@ -176,6 +198,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6m: cohort::Vecs::forced_import(
path,
@@ -184,6 +208,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1y: cohort::Vecs::forced_import(
path,
@@ -192,6 +218,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2y: cohort::Vecs::forced_import(
path,
@@ -200,6 +228,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3y: cohort::Vecs::forced_import(
path,
@@ -208,6 +238,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4y: cohort::Vecs::forced_import(
path,
@@ -216,6 +248,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5y: cohort::Vecs::forced_import(
path,
@@ -224,6 +258,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6y: cohort::Vecs::forced_import(
path,
@@ -232,6 +268,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_7y: cohort::Vecs::forced_import(
path,
@@ -240,6 +278,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_8y: cohort::Vecs::forced_import(
path,
@@ -248,6 +288,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_10y: cohort::Vecs::forced_import(
path,
@@ -256,6 +298,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_15y: cohort::Vecs::forced_import(
path,
@@ -264,6 +308,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_from: OutputsByFrom {
@@ -274,6 +320,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1w: cohort::Vecs::forced_import(
path,
@@ -282,6 +330,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1m: cohort::Vecs::forced_import(
path,
@@ -290,6 +340,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2m: cohort::Vecs::forced_import(
path,
@@ -298,6 +350,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3m: cohort::Vecs::forced_import(
path,
@@ -306,6 +360,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4m: cohort::Vecs::forced_import(
path,
@@ -314,6 +370,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5m: cohort::Vecs::forced_import(
path,
@@ -322,6 +380,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6m: cohort::Vecs::forced_import(
path,
@@ -330,6 +390,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1y: cohort::Vecs::forced_import(
path,
@@ -338,6 +400,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2y: cohort::Vecs::forced_import(
path,
@@ -346,6 +410,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3y: cohort::Vecs::forced_import(
path,
@@ -354,6 +420,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4y: cohort::Vecs::forced_import(
path,
@@ -362,6 +430,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5y: cohort::Vecs::forced_import(
path,
@@ -370,6 +440,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6y: cohort::Vecs::forced_import(
path,
@@ -378,6 +450,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_7y: cohort::Vecs::forced_import(
path,
@@ -386,6 +460,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_8y: cohort::Vecs::forced_import(
path,
@@ -394,6 +470,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_10y: cohort::Vecs::forced_import(
path,
@@ -402,6 +480,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_15y: cohort::Vecs::forced_import(
path,
@@ -410,6 +490,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_range: OutputsByRange {
@@ -420,6 +502,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1w_to_1m: cohort::Vecs::forced_import(
path,
@@ -428,6 +512,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1m_to_3m: cohort::Vecs::forced_import(
path,
@@ -436,6 +522,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3m_to_6m: cohort::Vecs::forced_import(
path,
@@ -444,6 +532,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6m_to_1y: cohort::Vecs::forced_import(
path,
@@ -452,6 +542,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1y_to_2y: cohort::Vecs::forced_import(
path,
@@ -460,6 +552,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2y_to_3y: cohort::Vecs::forced_import(
path,
@@ -468,6 +562,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3y_to_4y: cohort::Vecs::forced_import(
path,
@@ -476,6 +572,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4y_to_5y: cohort::Vecs::forced_import(
path,
@@ -484,6 +582,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5y_to_7y: cohort::Vecs::forced_import(
path,
@@ -492,6 +592,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_7y_to_10y: cohort::Vecs::forced_import(
path,
@@ -500,6 +602,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_10y_to_15y: cohort::Vecs::forced_import(
path,
@@ -508,6 +612,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_epoch: OutputsByEpoch {
@@ -518,6 +624,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1: cohort::Vecs::forced_import(
path,
@@ -526,6 +634,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2: cohort::Vecs::forced_import(
path,
@@ -534,6 +644,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3: cohort::Vecs::forced_import(
path,
@@ -542,6 +654,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4: cohort::Vecs::forced_import(
path,
@@ -550,6 +664,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_size: OutputsBySize {
@@ -560,6 +676,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1sat_to_10sats: cohort::Vecs::forced_import(
path,
@@ -568,6 +686,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10sats_to_100sats: cohort::Vecs::forced_import(
path,
@@ -576,6 +696,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100sats_to_1_000sats: cohort::Vecs::forced_import(
path,
@@ -584,6 +706,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1_000sats_to_10_000sats: cohort::Vecs::forced_import(
path,
@@ -592,6 +716,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10_000sats_to_100_000sats: cohort::Vecs::forced_import(
path,
@@ -600,6 +726,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100_000sats_to_1_000_000sats: cohort::Vecs::forced_import(
path,
@@ -608,6 +736,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1_000_000sats_to_10_000_000sats: cohort::Vecs::forced_import(
path,
@@ -616,6 +746,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10_000_000sats_to_1btc: cohort::Vecs::forced_import(
path,
@@ -624,6 +756,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1btc_to_10btc: cohort::Vecs::forced_import(
path,
@@ -632,6 +766,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10btc_to_100btc: cohort::Vecs::forced_import(
path,
@@ -640,6 +776,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100btc_to_1_000btc: cohort::Vecs::forced_import(
path,
@@ -648,6 +786,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1_000btc_to_10_000btc: cohort::Vecs::forced_import(
path,
@@ -656,6 +796,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10_000btc_to_100_000btc: cohort::Vecs::forced_import(
path,
@@ -664,6 +806,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100_000btc: cohort::Vecs::forced_import(
path,
@@ -672,6 +816,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
// by_value: OutputsByValue {
@@ -775,6 +921,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2pk33: cohort::Vecs::forced_import(
path,
@@ -783,6 +931,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2pkh: cohort::Vecs::forced_import(
path,
@@ -791,6 +941,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2ms: cohort::Vecs::forced_import(
path,
@@ -799,6 +951,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2sh: cohort::Vecs::forced_import(
path,
@@ -807,6 +961,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
// opreturn: cohort::Vecs::forced_import(
// path,
@@ -815,6 +971,7 @@ impl Vecs {
// compressed,
// VERSION + Version::ZERO,
// fetched,
// keyspace
// )?,
p2wpkh: cohort::Vecs::forced_import(
path,
@@ -823,6 +980,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2wsh: cohort::Vecs::forced_import(
path,
@@ -831,6 +990,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2tr: cohort::Vecs::forced_import(
path,
@@ -839,6 +1000,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2a: cohort::Vecs::forced_import(
path,
@@ -847,6 +1010,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
empty: cohort::Vecs::forced_import(
path,
@@ -855,6 +1020,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
unknown: cohort::Vecs::forced_import(
path,
@@ -863,6 +1030,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
})
@@ -896,11 +1065,16 @@ impl Vecs {
.sats
.height
.as_ref()
.unwrap()
.as_ref();
let height_to_close = &fetched
.unwrap();
let height_to_close = fetched
.as_ref()
.map(|fetched| &fetched.chainindexes_to_close.height);
let dateindex_to_close = fetched
.as_ref()
.map(|fetched| fetched.timeindexes_to_close.dateindex.as_ref().unwrap());
let height_to_date_fixed = &indexes.height_to_date_fixed;
let dateindex_to_first_height = &indexes.dateindex_to_first_height;
let dateindex_to_height_count = &indexes.dateindex_to_height_count;
let inputindex_to_outputindex_mmap = inputindex_to_outputindex.mmap().load();
let outputindex_to_value_mmap = outputindex_to_value.mmap().load();
@@ -917,6 +1091,10 @@ impl Vecs {
// let mut outputindex_to_outputtype_iter_2 = outputindex_to_outputtype.into_iter();
let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter();
let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter();
let mut dateindex_to_close_iter = dateindex_to_close.as_ref().map(|v| v.into_iter());
let mut height_to_date_fixed_iter = height_to_date_fixed.into_iter();
let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter();
let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter();
let mut flat_vecs_ = self.utxos_vecs.as_mut_vec();
@@ -934,7 +1112,13 @@ impl Vecs {
+ height_to_unclaimed_rewards.version()
+ height_to_close
.as_ref()
.map_or(Version::ZERO, |v| v.version());
.map_or(Version::ZERO, |v| v.version())
+ dateindex_to_close
.as_ref()
.map_or(Version::ZERO, |v| v.version())
+ height_to_date_fixed.version()
+ dateindex_to_first_height.version()
+ dateindex_to_height_count.version();
flat_vecs_
.iter_mut()
@@ -1047,10 +1231,6 @@ impl Vecs {
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
// let sent_state = SentState::default();
// let received_state = ReceivedState::default();
// let realized_state = RealizedState::default();
let (mut height_to_sent, mut received) = thread::scope(|s| {
if chain_state_starting_height <= height {
s.spawn(|| {
@@ -1093,15 +1273,11 @@ impl Vecs {
.unwrap()
.into_inner();
// let input_height = *cached_txindex_to_height
// .entry(input_txindex)
// .or_insert_with(|| {
let height = txindex_to_height
.get_or_read(input_txindex, &txindex_to_height_mmap)
.unwrap()
.unwrap()
.into_inner();
// });
(height, value, input_type)
})
@@ -1204,8 +1380,9 @@ impl Vecs {
panic!("temp, just making sure")
}
self.utxos_vecs
.as_mut_vec()
let mut utxos_vecs = self.utxos_vecs.as_mut_vec();
utxos_vecs
.iter_mut()
.try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
@@ -1218,19 +1395,48 @@ impl Vecs {
self.height_to_opreturn_supply
.forced_push_at(height, opreturn_supply, exit)?;
let date = height_to_date_fixed_iter.unwrap_get_inner(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
let is_date_last_height =
date_first_height + Height::from(*date_height_count) == height;
let date_price = dateindex_to_close_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
utxos_vecs.par_iter_mut().try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})?;
if height != Height::ZERO && height.unwrap_to_usize() % 100_000 == 0 {
info!("Flushing...");
utxos_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
}
Ok(())
})?;
exit.block();
let mut flat_vecs_ = self.utxos_vecs.as_mut_vec();
info!("Flushing...");
// Flush rest of values
flat_vecs_
self.utxos_vecs
.as_mut_vec()
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_height_vecs(exit))?;
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
@@ -1246,9 +1452,12 @@ impl Vecs {
info!("Computing rest...");
// Compute other vecs from height vecs
flat_vecs_.par_iter_mut().try_for_each(|(_, v)| {
v.compute_rest(indexer, indexes, fetched, starting_indexes, exit)
})?;
self.utxos_vecs
.as_mut_vec()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest(indexer, indexes, fetched, starting_indexes, exit)
})?;
self.indexes_to_unspendable_supply.compute_rest(
indexer,
indexes,
crates/brk_computer/src/vecs/transactions.rs
+2 -12
View File
@@ -864,12 +864,7 @@ impl Vecs {
self.indexes_to_fee.sats.height.unwrap_sum().iter();
vec.compute_transform(
starting_indexes.height,
self.indexes_to_coinbase
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|(height, coinbase, ..)| {
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
(height, coinbase.checked_sub(fees).unwrap())
@@ -888,12 +883,7 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_transform(
starting_indexes.height,
self.indexes_to_subsidy
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
self.indexes_to_subsidy.sats.height.as_ref().unwrap(),
|(height, subsidy, ..)| {
let halving = HalvingEpoch::from(height);
let expected =
crates/brk_core/Cargo.toml
+2
View File
@@ -11,6 +11,7 @@ bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
byteview = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rapidhash = "1.4.0"
@@ -19,6 +20,7 @@ serde = { workspace = true }
serde_bytes = "0.11.17"
serde_derive = { workspace = true }
serde_json = { workspace = true }
thiserror = "2.0.12"
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
crates/brk_core/src/enums/error.rs
+30 -17
View File
@@ -1,20 +1,24 @@
use std::{
fmt::{self, Debug},
io,
time::SystemTimeError,
io, result, time,
};
use crate::Version;
pub type Result<T, E = Error> = std::result::Result<T, E>;
pub type Result<T, E = Error> = result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
SerdeJson(serde_json::Error),
Jiff(jiff::Error),
Fjall(fjall::Error),
SystemTimeError(time::SystemTimeError),
ZeroCopyError,
WrongEndian,
DifferentVersion { found: Version, expected: Version },
MmapsVecIsTooSmall,
IO(io::Error),
ZeroCopyError,
IndexTooHigh,
EmptyVec,
IndexTooLow,
@@ -24,17 +28,16 @@ pub enum Error {
UnsupportedUnflushedState,
RangeFromAfterTo(usize, usize),
DifferentCompressionMode,
SystemTimeError,
ToSerdeJsonValueError(serde_json::Error),
Jiff(jiff::Error),
WrongLength,
WrongAddressType,
UnindexableDate,
String(&'static str),
}
impl From<SystemTimeError> for Error {
fn from(_: SystemTimeError) -> Self {
Self::SystemTimeError
impl From<time::SystemTimeError> for Error {
fn from(value: time::SystemTimeError) -> Self {
Self::SystemTimeError(value)
}
}
@@ -50,6 +53,12 @@ impl From<jiff::Error> for Error {
}
}
impl From<fjall::Error> for Error {
fn from(value: fjall::Error) -> Self {
Self::Fjall(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
@@ -64,13 +73,20 @@ impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
impl From<serde_json::Error> for Error {
fn from(error: serde_json::Error) -> Self {
Self::ToSerdeJsonValueError(error)
Self::SerdeJson(error)
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::SystemTimeError(error) => Debug::fmt(&error, f),
Error::SerdeJson(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::Fjall(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "ZeroCopy error"),
Error::WrongEndian => write!(f, "Wrong endian"),
Error::DifferentVersion { found, expected } => {
write!(
@@ -79,7 +95,6 @@ impl fmt::Display for Error {
)
}
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::IndexTooLow => write!(f, "Index too low"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
@@ -91,19 +106,17 @@ impl fmt::Display for Error {
"Unsupported unflush state, please flush before using this function"
)
}
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
Error::SystemTimeError => write!(f, "SystemTimeError"),
Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(
f,
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
),
Error::String(s) => write!(f, "{s}"),
}
}
}
crates/brk_core/src/structs/dollars.rs
+30 -3
View File
@@ -4,11 +4,12 @@ use std::{
ops::{Add, AddAssign, Div, Mul},
};
use byteview::ByteView;
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{CheckedSub, Error, copy_first_8bytes};
use super::{Bitcoin, Cents, Close, Sats, StoredF32, StoredF64};
@@ -133,12 +134,18 @@ impl Div<Bitcoin> for Dollars {
impl Mul<Bitcoin> for Dollars {
type Output = Self;
fn mul(self, rhs: Bitcoin) -> Self::Output {
self * Sats::from(rhs)
}
}
impl Mul<Sats> for Dollars {
type Output = Self;
fn mul(self, rhs: Sats) -> Self::Output {
if self.is_nan() {
self
} else {
Self::from(Cents::from(
u128::from(Sats::from(rhs)) * u128::from(Cents::from(self))
/ u128::from(Sats::ONE_BTC),
u128::from(rhs) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
))
}
}
@@ -248,3 +255,23 @@ impl Ord for Dollars {
}
}
}
impl TryFrom<ByteView> for Dollars {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
let bytes = copy_first_8bytes(&value)?;
Ok(Self::from(f64::from_be_bytes(bytes)))
}
}
impl From<Dollars> for ByteView {
fn from(value: Dollars) -> Self {
Self::from(&value)
}
}
impl From<&Dollars> for ByteView {
fn from(value: &Dollars) -> Self {
Self::new(&value.to_be_bytes())
}
}
crates/brk_core/src/structs/sats.rs
+22 -1
View File
@@ -4,10 +4,11 @@ use std::{
};
use bitcoin::Amount;
use byteview::ByteView;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{CheckedSub, Error, copy_first_8bytes};
use super::{Bitcoin, Cents, Dollars, Height};
@@ -187,3 +188,23 @@ impl From<Sats> for u128 {
value.0 as u128
}
}
impl TryFrom<ByteView> for Sats {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
let bytes = copy_first_8bytes(&value)?;
Ok(Self::from(u64::from_be_bytes(bytes)))
}
}
impl From<&Sats> for ByteView {
fn from(value: &Sats) -> Self {
Self::new(&value.0.to_be_bytes())
}
}
impl From<Sats> for ByteView {
fn from(value: Sats) -> Self {
Self::from(&value)
}
}
crates/brk_core/src/utils/bytes.rs
+4 -2
View File
@@ -1,9 +1,11 @@
use crate::{Error, Result};
#[allow(clippy::result_unit_err)]
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(());
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
crates/brk_exit/src/lib.rs
+4
View File
@@ -57,6 +57,10 @@ impl Exit {
self.blocking.store(true, Ordering::SeqCst);
}
pub fn blocked(&self) -> bool {
self.blocking.load(Ordering::SeqCst)
}
pub fn release(&self) {
self.blocking.store(false, Ordering::SeqCst);
}
crates/brk_indexer/src/stores.rs
+13 -8
View File
@@ -1,8 +1,8 @@
use std::{fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
TxidPrefix, Value, Version,
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, Result,
TxIndex, TxidPrefix, Value, Version,
};
use brk_store::Store;
use brk_vec::AnyIterableVec;
@@ -37,26 +37,29 @@ impl Stores {
thread::scope(|scope| {
let addressbyteshash_to_outputtypeindex = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"addressbyteshash_to_outputtypeindex",
version + VERSION + Version::ZERO,
None,
)
});
let blockhashprefix_to_height = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"blockhashprefix_to_height",
version + VERSION + Version::ZERO,
None,
)
});
let txidprefix_to_txindex = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"txidprefix_to_txindex",
version + VERSION + Version::ZERO,
None,
)
});
@@ -285,8 +288,8 @@ impl Stores {
.unwrap()
}
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
thread::scope(|scope| -> fjall::Result<()> {
pub fn commit(&mut self, height: Height) -> Result<()> {
thread::scope(|scope| -> Result<()> {
let addressbyteshash_to_outputtypeindex_commit_handle =
scope.spawn(|| self.addressbyteshash_to_outputtypeindex.commit(height));
let blockhashprefix_to_height_commit_handle =
@@ -303,7 +306,9 @@ impl Stores {
Ok(())
})?;
self.keyspace.persist(PersistMode::SyncAll)
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn rotate_memtables(&self) {
crates/brk_server/Cargo.toml
+1 -1
View File
@@ -24,7 +24,7 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.72.1", features = ["codegen", "minifier"] }
oxc = { version = "0.72.2", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { workspace = true }
tower-http = { version = "0.6.4", features = ["compression-full", "trace"] }
crates/brk_state/Cargo.toml
+1
View File
@@ -10,6 +10,7 @@ repository.workspace = true
brk_core = { workspace = true }
brk_vec = { workspace = true }
brk_store = { workspace = true }
fjall = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
zerocopy = { workspace = true }
crates/brk_state/src/cohort.rs
+100 -12
View File
@@ -1,16 +1,47 @@
use brk_core::{Dollars, Sats};
use std::{cmp::Ordering, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Result, Sats, Version};
use brk_store::Store;
use fjall::TransactionalKeyspace;
use crate::UnrealizedState;
use super::{RealizedState, SupplyState};
#[derive(Default, Clone)]
#[derive(Clone)]
pub struct CohortState {
pub supply: SupplyState,
pub realized: Option<RealizedState>,
// pub price_to_amount: Store<Dollars, Sats>,
pub price_to_amount: Store<Dollars, Sats>,
}
impl CohortState {
pub fn default_and_import(
keyspace: &TransactionalKeyspace,
path: &Path,
name: &str,
version: Version,
compute_dollars: bool,
) -> Result<Self> {
Ok(Self {
supply: SupplyState::default(),
realized: compute_dollars.then_some(RealizedState::NAN),
price_to_amount: Store::import(
keyspace,
path,
&format!("{name}_price_to_amount"),
version + Version::TWO,
Some(None),
)?,
})
}
pub fn commit(&mut self, height: Height) -> Result<()> {
self.price_to_amount
.retain_or_del(|_, sats| *sats != Sats::ZERO);
self.price_to_amount.commit(height)
}
pub fn reset_single_iteration_values(&mut self) {
if let Some(realized) = self.realized.as_mut() {
realized.reset_single_iteration_values();
@@ -20,21 +51,27 @@ impl CohortState {
pub fn increment(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.increment(supply_state, price.unwrap());
let price = price.unwrap();
realized.increment(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) += supply_state.value;
}
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply -= supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.decrement(supply_state, price.unwrap());
let price = price.unwrap();
realized.decrement(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) -= supply_state.value;
}
}
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.receive(supply_state, price.unwrap());
let price = price.unwrap();
realized.receive(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) += supply_state.value;
}
}
@@ -47,12 +84,63 @@ impl CohortState {
) {
self.supply -= supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.send(
supply_state,
current_price.unwrap(),
prev_price.unwrap(),
older_than_hour,
);
let current_price = current_price.unwrap();
let prev_price = prev_price.unwrap();
realized.send(supply_state, current_price, prev_price, older_than_hour);
*self.price_to_amount.get_mut_or_default(&prev_price) -= supply_state.value;
}
}
pub fn compute_unrealized_states(
&self,
height_price: Dollars,
date_price: Option<Dollars>,
) -> (UnrealizedState, Option<UnrealizedState>) {
if self.price_to_amount.is_empty() {
return (
UnrealizedState::NAN,
date_price.map(|_| UnrealizedState::NAN),
);
}
let mut height_unrealized_state = UnrealizedState::ZERO;
let mut date_unrealized_state = date_price.map(|_| UnrealizedState::ZERO);
let update_state =
|price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
match price.cmp(&current_price) {
Ordering::Equal => {
state.supply_even += sats;
}
Ordering::Less => {
state.supply_in_profit += sats;
// if price > Dollars::ZERO {
// state.unrealized_profit +=
// current_price.checked_sub(price).unwrap() * sats;
// }
}
Ordering::Greater => {
state.supply_in_loss += sats;
// state.unrealized_loss += price.checked_sub(current_price).unwrap() * sats;
}
}
};
self.price_to_amount
.unordered_clone_iter()
.for_each(|(price, sats)| {
update_state(price, height_price, sats, &mut height_unrealized_state);
if let Some(date_price) = date_price {
update_state(
price,
date_price,
sats,
date_unrealized_state.as_mut().unwrap(),
)
}
});
(height_unrealized_state, date_unrealized_state)
}
}
crates/brk_state/src/lib.rs
+2
View File
@@ -5,11 +5,13 @@ mod realized;
// mod hot;
mod supply;
mod transacted;
mod unrealized;
pub use block::*;
pub use cohort::*;
pub use outputs::*;
pub use realized::*;
pub use unrealized::*;
// pub use hot::*;
pub use supply::*;
pub use transacted::*;
crates/brk_state/src/realized.rs
+5 -5
View File
@@ -1,6 +1,6 @@
use std::cmp::Ordering;
use brk_core::{Bitcoin, CheckedSub, Dollars};
use brk_core::{CheckedSub, Dollars};
use super::SupplyState;
@@ -52,12 +52,12 @@ impl RealizedState {
self.adj_value_destroyed = Dollars::ZERO;
}
let value = price * Bitcoin::from(supply_state.value);
let value = price * supply_state.value;
self.cap += value;
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) {
let value = price * Bitcoin::from(supply_state.value);
let value = price * supply_state.value;
self.cap = self.cap.checked_sub(value).unwrap();
}
@@ -72,8 +72,8 @@ impl RealizedState {
prev_price: Dollars,
older_than_hour: bool,
) {
let current_value = current_price * Bitcoin::from(supply_state.value);
let prev_value = prev_price * Bitcoin::from(supply_state.value);
let current_value = current_price * supply_state.value;
let prev_value = prev_price * supply_state.value;
self.value_created += current_value;
self.value_destroyed += prev_value;
crates/brk_state/src/unrealized.rs
+28
View File
@@ -0,0 +1,28 @@
use brk_core::{Dollars, Sats};
#[derive(Debug, Default, Clone)]
pub struct UnrealizedState {
pub supply_in_profit: Sats,
pub supply_even: Sats,
pub supply_in_loss: Sats,
pub unrealized_profit: Dollars,
pub unrealized_loss: Dollars,
}
impl UnrealizedState {
pub const NAN: Self = Self {
supply_in_profit: Sats::ZERO,
supply_even: Sats::ZERO,
supply_in_loss: Sats::ZERO,
unrealized_profit: Dollars::NAN,
unrealized_loss: Dollars::NAN,
};
pub const ZERO: Self = Self {
supply_in_profit: Sats::ZERO,
supply_even: Sats::ZERO,
supply_in_loss: Sats::ZERO,
unrealized_profit: Dollars::ZERO,
unrealized_loss: Dollars::ZERO,
};
}
crates/brk_store/src/lib.rs
+63 -16
View File
@@ -6,10 +6,10 @@ use std::{
path::Path,
};
use brk_core::{Height, Value, Version};
use brk_core::{Height, Result, Value, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace,
PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use zerocopy::{Immutable, IntoBytes};
@@ -25,29 +25,34 @@ pub struct Store<Key, Value> {
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
bloom_filter_bits: Option<Option<u8>>,
}
/// Use default if will read
const DEFAULT_BLOOM_FILTER_BITS: Option<u8> = Some(5);
const CHECK_COLLISISONS: bool = true;
const MAJOR_FJALL_VERSION: Version = Version::TWO;
impl<K, V> Store<K, V>
where
K: Debug + Clone + Into<ByteView> + Ord + Immutable + IntoBytes,
K: Debug + Clone + Into<ByteView> + TryFrom<ByteView> + Ord + Immutable + IntoBytes,
V: Debug + Clone + Into<ByteView> + TryFrom<ByteView>,
<K as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
{
pub fn import(
keyspace: TransactionalKeyspace,
keyspace: &TransactionalKeyspace,
path: &Path,
name: &str,
version: Version,
) -> color_eyre::Result<Self> {
bloom_filter_bits: Option<Option<u8>>,
) -> Result<Self> {
let (meta, partition) = StoreMeta::checked_open(
&keyspace,
keyspace,
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
|| {
Self::open_partition_handle(&keyspace, name).inspect_err(|e| {
Self::open_partition_handle(keyspace, name, bloom_filter_bits).inspect_err(|e| {
eprintln!("{e}");
eprintln!("Delete {path:?} and try again");
})
@@ -59,11 +64,12 @@ where
Ok(Self {
meta,
name: name.to_owned(),
keyspace,
keyspace: keyspace.clone(),
partition,
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
bloom_filter_bits,
})
}
@@ -77,6 +83,29 @@ where
}
}
pub fn get_mut_or_default(&mut self, key: &K) -> &mut V
where
V: Default,
{
self.puts.entry(key.clone()).or_insert_with(|| {
if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes()).unwrap() {
V::try_from(slice.as_bytes().into()).unwrap()
} else {
V::default()
}
})
}
pub fn unordered_clone_iter(&self) -> impl Iterator<Item = (K, V)> {
self.rtx
.iter(&self.partition)
.map(|res| res.unwrap())
.map(|(k, v)| (K::try_from(ByteView::from(k)).unwrap(), v))
.filter(|(k, _)| !self.puts.contains_key(k) && !self.dels.contains(k))
.map(|(k, v)| (k, V::try_from(ByteView::from(v)).unwrap()))
.chain(self.puts.iter().map(|(k, v)| (k.clone(), v.clone())))
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
if !self.dels.is_empty() {
@@ -102,6 +131,19 @@ where
}
}
pub fn retain_or_del<F>(&mut self, retain: F)
where
F: Fn(&K, &mut V) -> bool,
{
self.puts.retain(|k, v| {
let ret = retain(k, v);
if !ret {
self.dels.insert(k.clone());
}
ret
});
}
pub fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) && self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
@@ -169,20 +211,24 @@ where
fn open_partition_handle(
keyspace: &TransactionalKeyspace,
name: &str,
bloom_filter_bits: Option<Option<u8>>,
) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(
name,
PartitionCreateOptions::default()
.bloom_filter_bits(Some(5))
.max_memtable_size(8 * 1024 * 1024)
.manual_journal_persist(true),
)
keyspace
.open_partition(
name,
PartitionCreateOptions::default()
.bloom_filter_bits(bloom_filter_bits.unwrap_or(DEFAULT_BLOOM_FILTER_BITS))
.max_memtable_size(8 * 1024 * 1024)
.manual_journal_persist(true),
)
.map_err(|e| e.into())
}
pub fn reset_partition(&mut self) -> Result<()> {
self.keyspace.delete_partition(self.partition.clone())?;
self.keyspace.persist(PersistMode::SyncAll)?;
self.partition = Self::open_partition_handle(&self.keyspace, &self.name)?;
self.partition =
Self::open_partition_handle(&self.keyspace, &self.name, self.bloom_filter_bits)?;
Ok(())
}
}
@@ -201,6 +247,7 @@ where
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),
bloom_filter_bits: self.bloom_filter_bits,
}
}
}
crates/brk_store/src/meta.rs
+4 -4
View File
@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use brk_core::Version;
use brk_core::{Result, Version};
use fjall::{TransactionalKeyspace, TransactionalPartitionHandle};
use zerocopy::{FromBytes, IntoBytes};
@@ -23,9 +23,9 @@ impl StoreMeta {
path: &Path,
version: Version,
open_partition_handle: F,
) -> color_eyre::Result<(Self, TransactionalPartitionHandle)>
) -> Result<(Self, TransactionalPartitionHandle)>
where
F: Fn() -> fjall::Result<TransactionalPartitionHandle>,
F: Fn() -> Result<TransactionalPartitionHandle>,
{
fs::create_dir_all(path)?;
@@ -109,7 +109,7 @@ impl StoreMeta {
path.join("height")
}
fn read_length_(path: &Path) -> color_eyre::Result<usize> {
fn read_length_(path: &Path) -> Result<usize> {
Ok(fs::read(Self::path_length(path))
.map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default())
.unwrap_or_default())
crates/brk_vec/src/variants/eager.rs
+14 -4
View File
@@ -58,9 +58,14 @@ where
if exit.triggered() {
return Ok(());
}
exit.block();
let blocked = exit.blocked();
if !blocked {
exit.block();
}
self.inner.truncate_if_needed(index)?;
exit.release();
if !blocked {
exit.release();
}
Ok(())
}
@@ -89,9 +94,14 @@ where
if exit.triggered() {
return Ok(());
}
exit.block();
let blocked = exit.blocked();
if !blocked {
exit.block();
}
self.inner.flush()?;
exit.release();
if !blocked {
exit.release();
}
Ok(())
}