Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-27 16:19:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: utxos dataset part 1

Browse Source
This commit is contained in:
nym21
2025-05-16 23:33:19 +02:00
parent 8e6464dacb
commit 8a7003782b
25 changed files with 1058 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
crates/brk_computer/src/vecs/grouped/builder.rs
View File

@@ -18,18 +18,18 @@ where
I: StoredIndex,
T: ComputedType,
{
first: Option<EagerVec<I, T>>,
average: Option<EagerVec<I, T>>,
sum: Option<EagerVec<I, T>>,
max: Option<EagerVec<I, T>>,
_90p: Option<EagerVec<I, T>>,
_75p: Option<EagerVec<I, T>>,
median: Option<EagerVec<I, T>>,
_25p: Option<EagerVec<I, T>>,
_10p: Option<EagerVec<I, T>>,
min: Option<EagerVec<I, T>>,
last: Option<EagerVec<I, T>>,
total: Option<EagerVec<I, T>>,
first: Option<Box<EagerVec<I, T>>>,
average: Option<Box<EagerVec<I, T>>>,
sum: Option<Box<EagerVec<I, T>>>,
max: Option<Box<EagerVec<I, T>>>,
_90p: Option<Box<EagerVec<I, T>>>,
_75p: Option<Box<EagerVec<I, T>>>,
median: Option<Box<EagerVec<I, T>>>,
_25p: Option<Box<EagerVec<I, T>>>,
_10p: Option<Box<EagerVec<I, T>>>,
min: Option<Box<EagerVec<I, T>>>,
last: Option<Box<EagerVec<I, T>>>,
total: Option<Box<EagerVec<I, T>>>,
}
const VERSION: Version = Version::ZERO;
@@ -76,64 +76,120 @@ where
let s = Self {
first: options.first.then(|| {
EagerVec::forced_import(&maybe_prefix("first"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_prefix("first"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
last: options.last.then(|| {
EagerVec::forced_import(
&path.join(format!("{key}_to_{name}")),
version + Version::ZERO,
compressed,
Box::new(
EagerVec::forced_import(
&path.join(format!("{key}_to_{name}")),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
.unwrap()
}),
min: options.min.then(|| {
EagerVec::forced_import(&maybe_suffix("min"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("min"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
max: options.max.then(|| {
EagerVec::forced_import(&maybe_suffix("max"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("max"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
median: options.median.then(|| {
EagerVec::forced_import(
&maybe_suffix("median"),
version + Version::ZERO,
compressed,
Box::new(
EagerVec::forced_import(
&maybe_suffix("median"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
.unwrap()
}),
average: options.average.then(|| {
EagerVec::forced_import(
&maybe_suffix("average"),
version + Version::ZERO,
compressed,
Box::new(
EagerVec::forced_import(
&maybe_suffix("average"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
.unwrap()
}),
sum: options.sum.then(|| {
EagerVec::forced_import(&maybe_suffix("sum"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("sum"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
total: options.total.then(|| {
EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed)
.unwrap(),
)
}),
_90p: options._90p.then(|| {
EagerVec::forced_import(&maybe_suffix("90p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("90p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
_75p: options._75p.then(|| {
EagerVec::forced_import(&maybe_suffix("75p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("75p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
_25p: options._25p.then(|| {
EagerVec::forced_import(&maybe_suffix("25p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("25p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
_10p: options._10p.then(|| {
EagerVec::forced_import(&maybe_suffix("10p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("10p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
};
@@ -480,88 +536,88 @@ where
))
}
pub fn unwrap_first(&mut self) -> &mut EagerVec<I, T> {
self.first.as_mut().unwrap()
pub fn unwrap_first(&self) -> &EagerVec<I, T> {
self.first.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_average(&mut self) -> &mut EagerVec<I, T> {
self.average.as_mut().unwrap()
pub fn unwrap_average(&self) -> &EagerVec<I, T> {
self.average.as_ref().unwrap()
}
pub fn unwrap_sum(&mut self) -> &mut EagerVec<I, T> {
self.sum.as_mut().unwrap()
pub fn unwrap_sum(&self) -> &EagerVec<I, T> {
self.sum.as_ref().unwrap()
}
pub fn unwrap_max(&mut self) -> &mut EagerVec<I, T> {
self.max.as_mut().unwrap()
pub fn unwrap_max(&self) -> &EagerVec<I, T> {
self.max.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_90p(&mut self) -> &mut EagerVec<I, T> {
self._90p.as_mut().unwrap()
pub fn unwrap_90p(&self) -> &EagerVec<I, T> {
self._90p.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_75p(&mut self) -> &mut EagerVec<I, T> {
self._75p.as_mut().unwrap()
pub fn unwrap_75p(&self) -> &EagerVec<I, T> {
self._75p.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_median(&mut self) -> &mut EagerVec<I, T> {
self.median.as_mut().unwrap()
pub fn unwrap_median(&self) -> &EagerVec<I, T> {
self.median.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_25p(&mut self) -> &mut EagerVec<I, T> {
self._25p.as_mut().unwrap()
pub fn unwrap_25p(&self) -> &EagerVec<I, T> {
self._25p.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_10p(&mut self) -> &mut EagerVec<I, T> {
self._10p.as_mut().unwrap()
pub fn unwrap_10p(&self) -> &EagerVec<I, T> {
self._10p.as_ref().unwrap()
}
pub fn unwrap_min(&mut self) -> &mut EagerVec<I, T> {
self.min.as_mut().unwrap()
pub fn unwrap_min(&self) -> &EagerVec<I, T> {
self.min.as_ref().unwrap()
}
pub fn unwrap_last(&mut self) -> &mut EagerVec<I, T> {
self.last.as_mut().unwrap()
pub fn unwrap_last(&self) -> &EagerVec<I, T> {
self.last.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_total(&mut self) -> &mut EagerVec<I, T> {
self.total.as_mut().unwrap()
pub fn unwrap_total(&self) -> &EagerVec<I, T> {
self.total.as_ref().unwrap()
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
let mut v: Vec<&dyn AnyCollectableVec> = vec![];
if let Some(first) = self.first.as_ref() {
v.push(first);
v.push(first.as_ref());
}
if let Some(last) = self.last.as_ref() {
v.push(last);
v.push(last.as_ref());
}
if let Some(min) = self.min.as_ref() {
v.push(min);
v.push(min.as_ref());
}
if let Some(max) = self.max.as_ref() {
v.push(max);
v.push(max.as_ref());
}
if let Some(median) = self.median.as_ref() {
v.push(median);
v.push(median.as_ref());
}
if let Some(average) = self.average.as_ref() {
v.push(average);
v.push(average.as_ref());
}
if let Some(sum) = self.sum.as_ref() {
v.push(sum);
v.push(sum.as_ref());
}
if let Some(total) = self.total.as_ref() {
v.push(total);
v.push(total.as_ref());
}
if let Some(_90p) = self._90p.as_ref() {
v.push(_90p);
v.push(_90p.as_ref());
}
if let Some(_75p) = self._75p.as_ref() {
v.push(_75p);
v.push(_75p.as_ref());
}
if let Some(_25p) = self._25p.as_ref() {
v.push(_25p);
v.push(_25p.as_ref());
}
if let Some(_10p) = self._10p.as_ref() {
v.push(_10p);
v.push(_10p.as_ref());
}
v

16
crates/brk_computer/src/vecs/grouped/from_height.rs
View File

@@ -16,7 +16,7 @@ pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
{
pub height: Option<EagerVec<Height, T>>,
pub height: Option<Box<EagerVec<Height, T>>>,
pub height_extra: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -46,8 +46,14 @@ where
let version = VERSION + version;
let height = compute_source.then(|| {
EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&path.join(format!("height_to_{name}")),
version,
compressed,
)
.unwrap(),
)
});
let height_extra = ComputedVecBuilder::forced_import(
@@ -137,7 +143,7 @@ where
exit,
)?;
} else {
let height = self.height.as_ref().unwrap();
let height = self.height.as_ref().unwrap().as_ref();
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -206,7 +212,7 @@ where
[
self.height
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),

18
crates/brk_computer/src/vecs/grouped/from_txindex.rs
View File

@@ -19,7 +19,7 @@ pub struct ComputedVecsFromTxindex<T>
where
T: ComputedType + PartialOrd,
{
pub txindex: Option<EagerVec<TxIndex, T>>,
pub txindex: Option<Box<EagerVec<TxIndex, T>>>,
pub height: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -49,12 +49,14 @@ where
let version = VERSION + version;
let txindex = compute_source.then(|| {
EagerVec::forced_import(
&path.join(format!("txindex_to_{name}")),
version,
compressed,
Box::new(
EagerVec::forced_import(
&path.join(format!("txindex_to_{name}")),
version,
compressed,
)
.unwrap(),
)
.unwrap()
});
let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
@@ -132,7 +134,7 @@ where
exit,
)?;
} else {
let txindex = self.txindex.as_ref().unwrap();
let txindex = self.txindex.as_ref().unwrap().as_ref();
self.height.compute(
starting_indexes.height,
@@ -206,7 +208,7 @@ where
[
self.txindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
self.height.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),

93
crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs
View File

@@ -51,6 +51,7 @@ pub struct ComputedRatioVecsFromDateIndex {
pub ratio_m1sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_m2sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_m3sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_zscore: ComputedVecsFromDateIndex<StoredF32>,
}
const VERSION: Version = Version::ZERO;
@@ -75,217 +76,224 @@ impl ComputedRatioVecsFromDateIndex {
ratio: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1w_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1w_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1m_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1m_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1y_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_standard_deviation: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_standard_deviation"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_9: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_9_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_zscore: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_zscore"),
VERSION + version + Version::ZERO,
compressed,
options,
)?,
@@ -426,7 +434,6 @@ impl ComputedRatioVecsFromDateIndex {
)?;
let ratio_version = self.ratio.dateindex.version();
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
@@ -811,6 +818,27 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_zscore.compute(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut sma_iter = self.ratio_sma.dateindex.into_iter();
let mut sd_iter = self.ratio_standard_deviation.dateindex.into_iter();
vec.compute_transform(
starting_indexes.dateindex,
&self.ratio.dateindex,
|(i, ratio, ..)| {
let sma = sma_iter.unwrap_get_inner(i);
let sd = sd_iter.unwrap_get_inner(i);
(i, (ratio - sma) / sd)
},
exit,
)
},
)?;
Ok(())
}
@@ -866,6 +894,7 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_m1sd_as_price.vecs(),
self.ratio_m2sd_as_price.vecs(),
self.ratio_m3sd_as_price.vecs(),
self.ratio_zscore.vecs(),
]
.concat()
}

4
crates/brk_computer/src/vecs/grouped/value_from_height.rs
View File

@@ -129,14 +129,14 @@ impl ComputedValueVecsFromHeight {
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(
starting_indexes.height,
self.sats.height.as_ref().unwrap(),
self.sats.height.as_ref().unwrap().as_ref(),
exit,
)
},
)?;
}
let txindex = self.bitcoin.height.as_ref().unwrap();
let txindex = self.bitcoin.height.as_ref().unwrap().as_ref();
let price = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {