Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 23:29:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: utxos dataset part 1

Browse Source
This commit is contained in:
nym21
2025-05-16 23:33:19 +02:00
parent 8e6464dacb
commit 8a7003782b
25 changed files with 1058 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
crates/brk_computer/src/vecs/grouped/builder.rs
View File

@@ -18,18 +18,18 @@ where
I: StoredIndex,
T: ComputedType,
{
first: Option<EagerVec<I, T>>,
average: Option<EagerVec<I, T>>,
sum: Option<EagerVec<I, T>>,
max: Option<EagerVec<I, T>>,
_90p: Option<EagerVec<I, T>>,
_75p: Option<EagerVec<I, T>>,
median: Option<EagerVec<I, T>>,
_25p: Option<EagerVec<I, T>>,
_10p: Option<EagerVec<I, T>>,
min: Option<EagerVec<I, T>>,
last: Option<EagerVec<I, T>>,
total: Option<EagerVec<I, T>>,
first: Option<Box<EagerVec<I, T>>>,
average: Option<Box<EagerVec<I, T>>>,
sum: Option<Box<EagerVec<I, T>>>,
max: Option<Box<EagerVec<I, T>>>,
_90p: Option<Box<EagerVec<I, T>>>,
_75p: Option<Box<EagerVec<I, T>>>,
median: Option<Box<EagerVec<I, T>>>,
_25p: Option<Box<EagerVec<I, T>>>,
_10p: Option<Box<EagerVec<I, T>>>,
min: Option<Box<EagerVec<I, T>>>,
last: Option<Box<EagerVec<I, T>>>,
total: Option<Box<EagerVec<I, T>>>,
}
const VERSION: Version = Version::ZERO;
@@ -76,64 +76,120 @@ where
let s = Self {
first: options.first.then(|| {
EagerVec::forced_import(&maybe_prefix("first"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_prefix("first"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
last: options.last.then(|| {
EagerVec::forced_import(
&path.join(format!("{key}_to_{name}")),
version + Version::ZERO,
compressed,
Box::new(
EagerVec::forced_import(
&path.join(format!("{key}_to_{name}")),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
.unwrap()
}),
min: options.min.then(|| {
EagerVec::forced_import(&maybe_suffix("min"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("min"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
max: options.max.then(|| {
EagerVec::forced_import(&maybe_suffix("max"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("max"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
median: options.median.then(|| {
EagerVec::forced_import(
&maybe_suffix("median"),
version + Version::ZERO,
compressed,
Box::new(
EagerVec::forced_import(
&maybe_suffix("median"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
.unwrap()
}),
average: options.average.then(|| {
EagerVec::forced_import(
&maybe_suffix("average"),
version + Version::ZERO,
compressed,
Box::new(
EagerVec::forced_import(
&maybe_suffix("average"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
.unwrap()
}),
sum: options.sum.then(|| {
EagerVec::forced_import(&maybe_suffix("sum"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("sum"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
total: options.total.then(|| {
EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed)
.unwrap(),
)
}),
_90p: options._90p.then(|| {
EagerVec::forced_import(&maybe_suffix("90p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("90p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
_75p: options._75p.then(|| {
EagerVec::forced_import(&maybe_suffix("75p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("75p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
_25p: options._25p.then(|| {
EagerVec::forced_import(&maybe_suffix("25p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("25p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
_10p: options._10p.then(|| {
EagerVec::forced_import(&maybe_suffix("10p"), version + Version::ZERO, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&maybe_suffix("10p"),
version + Version::ZERO,
compressed,
)
.unwrap(),
)
}),
};
@@ -480,88 +536,88 @@ where
))
}
pub fn unwrap_first(&mut self) -> &mut EagerVec<I, T> {
self.first.as_mut().unwrap()
pub fn unwrap_first(&self) -> &EagerVec<I, T> {
self.first.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_average(&mut self) -> &mut EagerVec<I, T> {
self.average.as_mut().unwrap()
pub fn unwrap_average(&self) -> &EagerVec<I, T> {
self.average.as_ref().unwrap()
}
pub fn unwrap_sum(&mut self) -> &mut EagerVec<I, T> {
self.sum.as_mut().unwrap()
pub fn unwrap_sum(&self) -> &EagerVec<I, T> {
self.sum.as_ref().unwrap()
}
pub fn unwrap_max(&mut self) -> &mut EagerVec<I, T> {
self.max.as_mut().unwrap()
pub fn unwrap_max(&self) -> &EagerVec<I, T> {
self.max.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_90p(&mut self) -> &mut EagerVec<I, T> {
self._90p.as_mut().unwrap()
pub fn unwrap_90p(&self) -> &EagerVec<I, T> {
self._90p.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_75p(&mut self) -> &mut EagerVec<I, T> {
self._75p.as_mut().unwrap()
pub fn unwrap_75p(&self) -> &EagerVec<I, T> {
self._75p.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_median(&mut self) -> &mut EagerVec<I, T> {
self.median.as_mut().unwrap()
pub fn unwrap_median(&self) -> &EagerVec<I, T> {
self.median.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_25p(&mut self) -> &mut EagerVec<I, T> {
self._25p.as_mut().unwrap()
pub fn unwrap_25p(&self) -> &EagerVec<I, T> {
self._25p.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_10p(&mut self) -> &mut EagerVec<I, T> {
self._10p.as_mut().unwrap()
pub fn unwrap_10p(&self) -> &EagerVec<I, T> {
self._10p.as_ref().unwrap()
}
pub fn unwrap_min(&mut self) -> &mut EagerVec<I, T> {
self.min.as_mut().unwrap()
pub fn unwrap_min(&self) -> &EagerVec<I, T> {
self.min.as_ref().unwrap()
}
pub fn unwrap_last(&mut self) -> &mut EagerVec<I, T> {
self.last.as_mut().unwrap()
pub fn unwrap_last(&self) -> &EagerVec<I, T> {
self.last.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_total(&mut self) -> &mut EagerVec<I, T> {
self.total.as_mut().unwrap()
pub fn unwrap_total(&self) -> &EagerVec<I, T> {
self.total.as_ref().unwrap()
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
let mut v: Vec<&dyn AnyCollectableVec> = vec![];
if let Some(first) = self.first.as_ref() {
v.push(first);
v.push(first.as_ref());
}
if let Some(last) = self.last.as_ref() {
v.push(last);
v.push(last.as_ref());
}
if let Some(min) = self.min.as_ref() {
v.push(min);
v.push(min.as_ref());
}
if let Some(max) = self.max.as_ref() {
v.push(max);
v.push(max.as_ref());
}
if let Some(median) = self.median.as_ref() {
v.push(median);
v.push(median.as_ref());
}
if let Some(average) = self.average.as_ref() {
v.push(average);
v.push(average.as_ref());
}
if let Some(sum) = self.sum.as_ref() {
v.push(sum);
v.push(sum.as_ref());
}
if let Some(total) = self.total.as_ref() {
v.push(total);
v.push(total.as_ref());
}
if let Some(_90p) = self._90p.as_ref() {
v.push(_90p);
v.push(_90p.as_ref());
}
if let Some(_75p) = self._75p.as_ref() {
v.push(_75p);
v.push(_75p.as_ref());
}
if let Some(_25p) = self._25p.as_ref() {
v.push(_25p);
v.push(_25p.as_ref());
}
if let Some(_10p) = self._10p.as_ref() {
v.push(_10p);
v.push(_10p.as_ref());
}
v

16
crates/brk_computer/src/vecs/grouped/from_height.rs
View File

@@ -16,7 +16,7 @@ pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
{
pub height: Option<EagerVec<Height, T>>,
pub height: Option<Box<EagerVec<Height, T>>>,
pub height_extra: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -46,8 +46,14 @@ where
let version = VERSION + version;
let height = compute_source.then(|| {
EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)
.unwrap()
Box::new(
EagerVec::forced_import(
&path.join(format!("height_to_{name}")),
version,
compressed,
)
.unwrap(),
)
});
let height_extra = ComputedVecBuilder::forced_import(
@@ -137,7 +143,7 @@ where
exit,
)?;
} else {
let height = self.height.as_ref().unwrap();
let height = self.height.as_ref().unwrap().as_ref();
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -206,7 +212,7 @@ where
[
self.height
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),

18
crates/brk_computer/src/vecs/grouped/from_txindex.rs
View File

@@ -19,7 +19,7 @@ pub struct ComputedVecsFromTxindex<T>
where
T: ComputedType + PartialOrd,
{
pub txindex: Option<EagerVec<TxIndex, T>>,
pub txindex: Option<Box<EagerVec<TxIndex, T>>>,
pub height: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -49,12 +49,14 @@ where
let version = VERSION + version;
let txindex = compute_source.then(|| {
EagerVec::forced_import(
&path.join(format!("txindex_to_{name}")),
version,
compressed,
Box::new(
EagerVec::forced_import(
&path.join(format!("txindex_to_{name}")),
version,
compressed,
)
.unwrap(),
)
.unwrap()
});
let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
@@ -132,7 +134,7 @@ where
exit,
)?;
} else {
let txindex = self.txindex.as_ref().unwrap();
let txindex = self.txindex.as_ref().unwrap().as_ref();
self.height.compute(
starting_indexes.height,
@@ -206,7 +208,7 @@ where
[
self.txindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
self.height.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),

93
crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs
View File

@@ -51,6 +51,7 @@ pub struct ComputedRatioVecsFromDateIndex {
pub ratio_m1sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_m2sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_m3sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_zscore: ComputedVecsFromDateIndex<StoredF32>,
}
const VERSION: Version = Version::ZERO;
@@ -75,217 +76,224 @@ impl ComputedRatioVecsFromDateIndex {
ratio: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1w_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1w_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1m_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1m_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1y_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_standard_deviation: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_standard_deviation"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_9: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_9_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p99_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p0_1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_p3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_m3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd_as_price"),
VERSION + version + Version::TWO,
VERSION + version + Version::ZERO,
compressed,
options,
)?,
ratio_zscore: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_zscore"),
VERSION + version + Version::ZERO,
compressed,
options,
)?,
@@ -426,7 +434,6 @@ impl ComputedRatioVecsFromDateIndex {
)?;
let ratio_version = self.ratio.dateindex.version();
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
@@ -811,6 +818,27 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_zscore.compute(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut sma_iter = self.ratio_sma.dateindex.into_iter();
let mut sd_iter = self.ratio_standard_deviation.dateindex.into_iter();
vec.compute_transform(
starting_indexes.dateindex,
&self.ratio.dateindex,
|(i, ratio, ..)| {
let sma = sma_iter.unwrap_get_inner(i);
let sd = sd_iter.unwrap_get_inner(i);
(i, (ratio - sma) / sd)
},
exit,
)
},
)?;
Ok(())
}
@@ -866,6 +894,7 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_m1sd_as_price.vecs(),
self.ratio_m2sd_as_price.vecs(),
self.ratio_m3sd_as_price.vecs(),
self.ratio_zscore.vecs(),
]
.concat()
}

4
crates/brk_computer/src/vecs/grouped/value_from_height.rs
View File

@@ -129,14 +129,14 @@ impl ComputedValueVecsFromHeight {
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(
starting_indexes.height,
self.sats.height.as_ref().unwrap(),
self.sats.height.as_ref().unwrap().as_ref(),
exit,
)
},
)?;
}
let txindex = self.bitcoin.height.as_ref().unwrap();
let txindex = self.bitcoin.height.as_ref().unwrap().as_ref();
let price = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {

70
crates/brk_computer/src/vecs/indexes.rs
View File

@@ -11,7 +11,7 @@ use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CloneableAnyIterableVec, Compressed, Computation, ComputedVec,
ComputedVecFrom1, EagerVec, VecIterator, Version,
ComputedVecFrom1, ComputedVecFrom2, EagerVec, StoredIndex, VecIterator, Version,
};
#[derive(Clone)]
@@ -53,6 +53,7 @@ pub struct Vecs {
pub opreturnindex_to_opreturnindex:
ComputedVecFrom1<OpReturnIndex, OpReturnIndex, OpReturnIndex, TxIndex>,
pub outputindex_to_outputindex: ComputedVecFrom1<OutputIndex, OutputIndex, OutputIndex, Sats>,
pub outputindex_to_txindex: EagerVec<OutputIndex, TxIndex>,
pub p2aindex_to_p2aindex: ComputedVecFrom1<P2AIndex, P2AIndex, P2AIndex, P2ABytes>,
pub p2msindex_to_p2msindex: ComputedVecFrom1<P2MSIndex, P2MSIndex, P2MSIndex, TxIndex>,
pub p2pk33index_to_p2pk33index:
@@ -70,6 +71,10 @@ pub struct Vecs {
pub quarterindex_to_quarterindex:
ComputedVecFrom1<QuarterIndex, QuarterIndex, QuarterIndex, MonthIndex>,
pub txindex_to_height: EagerVec<TxIndex, Height>,
pub txindex_to_input_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, InputIndex, InputIndex, OutputIndex>,
pub txindex_to_output_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, OutputIndex, OutputIndex, Sats>,
pub txindex_to_txindex: ComputedVecFrom1<TxIndex, TxIndex, TxIndex, Txid>,
pub unknownoutputindex_to_unknownoutputindex:
ComputedVecFrom1<UnknownOutputIndex, UnknownOutputIndex, UnknownOutputIndex, TxIndex>,
@@ -121,6 +126,52 @@ impl Vecs {
|index, _| Some(index),
)?;
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
"txindex_to_input_count",
Version::ZERO,
compressed,
indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
indexer.vecs().inputindex_to_outputindex.boxed_clone(),
|index: TxIndex, txindex_to_first_inputindex_iter, inputindex_to_outputindex_iter| {
let txindex = index.unwrap_to_usize();
txindex_to_first_inputindex_iter
.next_at(txindex)
.map(|(_, start)| {
let start = usize::from(start.into_inner());
let end = txindex_to_first_inputindex_iter
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_inner()))
.unwrap_or_else(|| inputindex_to_outputindex_iter.len());
StoredUsize::from((start..end).count())
})
},
)?;
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
"txindex_to_output_count",
Version::ZERO,
compressed,
indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
indexer.vecs().outputindex_to_value.boxed_clone(),
|index: TxIndex, txindex_to_first_outputindex_iter, outputindex_to_value_iter| {
let txindex = index.unwrap_to_usize();
txindex_to_first_outputindex_iter
.next_at(txindex)
.map(|(_, start)| {
let start = usize::from(start.into_inner());
let end = txindex_to_first_outputindex_iter
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_inner()))
.unwrap_or_else(|| outputindex_to_value_iter.len());
StoredUsize::from((start..end).count())
})
},
)?;
let p2pk33index_to_p2pk33index = ComputedVec::forced_import_or_init_from_1(
computation,
path,
@@ -479,6 +530,8 @@ impl Vecs {
quarterindex_to_first_monthindex,
quarterindex_to_quarterindex,
txindex_to_txindex,
txindex_to_input_count,
txindex_to_output_count,
unknownoutputindex_to_unknownoutputindex,
weekindex_to_first_dateindex,
weekindex_to_weekindex,
@@ -545,6 +598,11 @@ impl Vecs {
Version::ZERO,
compressed,
)?,
outputindex_to_txindex: EagerVec::forced_import(
&path.join("outputindex_to_txindex"),
Version::ZERO,
compressed,
)?,
})
}
@@ -563,6 +621,13 @@ impl Vecs {
self.outputindex_to_outputindex
.compute_if_necessary(starting_indexes.outputindex, exit)?;
self.outputindex_to_txindex.compute_inverse_less_to_more(
starting_indexes.txindex,
&indexer_vecs.txindex_to_first_outputindex,
&self.txindex_to_output_count,
exit,
)?;
self.p2pk33index_to_p2pk33index
.compute_if_necessary(starting_indexes.p2pk33index, exit)?;
@@ -993,6 +1058,8 @@ impl Vecs {
&self.quarterindex_to_quarterindex,
&self.txindex_to_height,
&self.txindex_to_txindex,
&self.txindex_to_input_count,
&self.txindex_to_output_count,
&self.unknownoutputindex_to_unknownoutputindex,
&self.weekindex_to_dateindex_count,
&self.weekindex_to_first_dateindex,
@@ -1001,6 +1068,7 @@ impl Vecs {
&self.yearindex_to_first_monthindex,
&self.yearindex_to_monthindex_count,
&self.yearindex_to_yearindex,
&self.outputindex_to_txindex,
]
}
}

3
crates/brk_computer/src/vecs/mod.rs
View File

@@ -13,6 +13,7 @@ pub mod indexes;
pub mod market;
pub mod mining;
pub mod transactions;
pub mod utxos;
pub use indexes::Indexes;
@@ -24,6 +25,7 @@ pub struct Vecs {
pub mining: mining::Vecs,
pub market: market::Vecs,
pub transactions: transactions::Vecs,
// pub utxos: utxos::Vecs,
pub fetched: Option<fetched::Vecs>,
}
@@ -47,6 +49,7 @@ impl Vecs {
mining: mining::Vecs::forced_import(path, computation, compressed)?,
constants: constants::Vecs::forced_import(path, computation, compressed)?,
market: market::Vecs::forced_import(path, computation, compressed)?,
// utxos: utxos::Vecs::forced_import(path, computation, compressed)?,
transactions: transactions::Vecs::forced_import(
path,
indexer,

136
crates/brk_computer/src/vecs/transactions.rs
View File

@@ -9,7 +9,8 @@ use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Compressed, Computation,
ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, StoredIndex, Version,
ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, StoredIndex, VecIterator,
Version,
};
use super::{
@@ -73,18 +74,15 @@ pub struct Vecs {
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
pub inputindex_to_value:
ComputedVecFrom2<InputIndex, Sats, InputIndex, OutputIndex, OutputIndex, Sats>,
pub txindex_to_input_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, InputIndex, InputIndex, OutputIndex>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_is_coinbase: ComputedVecFrom2<TxIndex, bool, TxIndex, Height, Height, TxIndex>,
pub txindex_to_output_count:
ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, OutputIndex, OutputIndex, Sats>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
pub txindex_to_vsize: ComputedVecFrom1<TxIndex, StoredUsize, TxIndex, Weight>,
pub txindex_to_weight:
ComputedVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
pub txindex_to_fee: ComputedVecFrom2<TxIndex, Sats, TxIndex, Sats, TxIndex, Sats>,
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
impl Vecs {
@@ -196,29 +194,6 @@ impl Vecs {
},
)?;
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
"txindex_to_input_count",
Version::ZERO,
compressed,
indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
indexer.vecs().inputindex_to_outputindex.boxed_clone(),
|index: TxIndex, txindex_to_first_inputindex_iter, inputindex_to_outputindex_iter| {
let txindex = index.unwrap_to_usize();
txindex_to_first_inputindex_iter
.next_at(txindex)
.map(|(_, start)| {
let start = usize::from(start.into_inner());
let end = txindex_to_first_inputindex_iter
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_inner()))
.unwrap_or_else(|| inputindex_to_outputindex_iter.len());
StoredUsize::from((start..end).count())
})
},
)?;
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
@@ -226,7 +201,7 @@ impl Vecs {
Version::ZERO,
compressed,
indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
txindex_to_input_count.boxed_clone(),
indexes.txindex_to_input_count.boxed_clone(),
inputindex_to_value.boxed_clone(),
|index: TxIndex,
txindex_to_first_inputindex_iter,
@@ -268,29 +243,6 @@ impl Vecs {
// .add_total(),
// )?;
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
"txindex_to_output_count",
Version::ZERO,
compressed,
indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
indexer.vecs().outputindex_to_value.boxed_clone(),
|index: TxIndex, txindex_to_first_outputindex_iter, outputindex_to_value_iter| {
let txindex = index.unwrap_to_usize();
txindex_to_first_outputindex_iter
.next_at(txindex)
.map(|(_, start)| {
let start = usize::from(start.into_inner());
let end = txindex_to_first_outputindex_iter
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v.into_inner()))
.unwrap_or_else(|| outputindex_to_value_iter.len());
StoredUsize::from((start..end).count())
})
},
)?;
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
@@ -298,7 +250,7 @@ impl Vecs {
Version::ZERO,
compressed,
indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
txindex_to_output_count.boxed_clone(),
indexes.txindex_to_output_count.boxed_clone(),
indexer.vecs().outputindex_to_value.boxed_clone(),
|index: TxIndex,
txindex_to_first_outputindex_iter,
@@ -680,6 +632,14 @@ impl Vecs {
.add_sum()
.add_total(),
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
"utxo_count_bis",
true,
Version::TWO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
txindex_to_is_coinbase,
inputindex_to_value,
// indexes_to_input_value,
@@ -690,8 +650,6 @@ impl Vecs {
txindex_to_feerate,
txindex_to_vsize,
txindex_to_weight,
txindex_to_input_count,
txindex_to_output_count,
})
}
@@ -723,7 +681,7 @@ impl Vecs {
indexes,
starting_indexes,
exit,
Some(&self.txindex_to_input_count),
Some(&indexes.txindex_to_input_count),
)?;
self.indexes_to_output_count.compute_rest(
@@ -731,7 +689,7 @@ impl Vecs {
indexes,
starting_indexes,
exit,
Some(&self.txindex_to_output_count),
Some(&indexes.txindex_to_output_count),
)?;
let compute_indexes_to_tx_vany =
@@ -851,7 +809,7 @@ impl Vecs {
|vec, indexer, _, starting_indexes, exit| {
let mut txindex_to_first_outputindex_iter =
indexer.vecs().txindex_to_first_outputindex.iter();
let mut txindex_to_output_count_iter = self.txindex_to_output_count.iter();
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
vec.compute_transform(
starting_indexes.height,
@@ -886,7 +844,12 @@ impl Vecs {
self.indexes_to_fee.sats.height.unwrap_sum().iter();
vec.compute_transform(
starting_indexes.height,
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
self.indexes_to_coinbase
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
|(height, subsidy, ..)| {
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
(height, subsidy.checked_sub(fees).unwrap())
@@ -1076,6 +1039,58 @@ impl Vecs {
},
)?;
self.indexes_to_utxo_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
let mut input_count_iter = self
.indexes_to_input_count
.height
.unwrap_total()
.into_iter();
let mut opreturn_count_iter = self
.indexes_to_opreturn_count
.height_extra
.unwrap_total()
.into_iter();
v.compute_transform(
starting_indexes.height,
self.indexes_to_output_count.height.unwrap_total(),
|(h, output_count, ..)| {
let input_count = input_count_iter.unwrap_get_inner(h);
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
let block_count = usize::from(h + 1_usize);
// -1 > genesis output is unspendable
let mut utxo_count =
*output_count - (*input_count - block_count) - *opreturn_count - 1;
// txid dup: e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468
// Block 91_722 https://mempool.space/block/00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e
// Block 91_880 https://mempool.space/block/00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721
//
// txid dup: d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599
// Block 91_812 https://mempool.space/block/00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f
// Block 91_842 https://mempool.space/block/00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec
//
// Warning: Dups invalidate the previous coinbase according to
// https://chainquery.com/bitcoin-cli/gettxoutsetinfo
if h >= Height::new(91_842) {
utxo_count -= 1;
}
if h >= Height::new(91_880) {
utxo_count -= 1;
}
(h, StoredUsize::from(utxo_count))
},
exit,
)
},
)?;
Ok(())
}
@@ -1085,10 +1100,8 @@ impl Vecs {
&self.inputindex_to_value as &dyn AnyCollectableVec,
&self.txindex_to_fee,
&self.txindex_to_feerate,
&self.txindex_to_input_count,
&self.txindex_to_input_value,
&self.txindex_to_is_coinbase,
&self.txindex_to_output_count,
&self.txindex_to_output_value,
&self.txindex_to_vsize,
&self.txindex_to_weight,
@@ -1117,6 +1130,7 @@ impl Vecs {
self.indexes_to_tx_vsize.vecs(),
self.indexes_to_tx_weight.vecs(),
self.indexes_to_unknownoutput_count.vecs(),
self.indexes_to_utxo_count.vecs(),
]
.concat()
}

181
crates/brk_computer/src/vecs/utxos.rs Normal file
View File

@@ -0,0 +1,181 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, Compressed, Computation, EagerVec, StoredIndex,
VecIterator, Version,
};
use derive_deref::{Deref, DerefMut};
use crate::states::{CohortStates, Outputs};
use super::{
Indexes,
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
indexes, transactions,
};
#[derive(Clone, Deref, DerefMut)]
pub struct Vecs(Outputs<Vecs_>);
#[derive(Clone)]
pub struct Vecs_ {
pub height_to_realized_cap: EagerVec<Height, Dollars>,
pub indexes_to_realized_cap: ComputedVecsFromHeight<Dollars>,
pub height_to_supply: EagerVec<Height, Sats>,
pub indexes_to_supply: ComputedVecsFromHeight<Sats>,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
const VERSION: Version = Version::ZERO;
impl Vecs {
pub fn forced_import(
path: &Path,
_computation: Computation,
compressed: Compressed,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self(Outputs {
all: Vecs_ {
height_to_realized_cap: EagerVec::forced_import(
&path.join("height_to_realized_cap"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_realized_cap: ComputedVecsFromHeight::forced_import(
path,
"realized_cap",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
height_to_supply: EagerVec::forced_import(
&path.join("height_to_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_supply: ComputedVecsFromHeight::forced_import(
path,
"supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
height_to_utxo_count: EagerVec::forced_import(
&path.join("height_to_utxo_count"),
VERSION + Version::new(111),
compressed,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
"utxo_count",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
},
}))
}
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
transactions: &transactions::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
let indexer_vecs = indexer.vecs();
let height_to_first_outputindex = &indexer_vecs.height_to_first_outputindex;
let height_to_first_inputindex = &indexer_vecs.height_to_first_inputindex;
let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_last();
let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_last();
let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex;
let outputindex_to_value = &indexer_vecs.outputindex_to_value;
let txindex_to_height = &indexes.txindex_to_height;
let outputindex_to_txindex = &indexes.outputindex_to_txindex;
let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter();
let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter();
let mut height_to_output_count_iter = height_to_output_count.into_iter();
let mut height_to_input_count_iter = height_to_input_count.into_iter();
let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter();
let mut outputindex_to_value_iter = outputindex_to_value.into_iter();
let mut txindex_to_height_iter = txindex_to_height.into_iter();
let mut outputindex_to_txindex_iter = outputindex_to_txindex.into_iter();
let base_version = Version::ZERO
+ height_to_first_outputindex.version()
+ height_to_first_inputindex.version()
+ height_to_output_count.version()
+ height_to_input_count.version()
+ inputindex_to_outputindex.version()
+ outputindex_to_value.version()
+ txindex_to_height.version()
+ outputindex_to_txindex.version();
let height_to_realized_cap = &mut self.0.all.height_to_realized_cap;
let height_to_supply = &mut self.0.all.height_to_supply;
let height_to_utxo_count = &mut self.0.all.height_to_utxo_count;
height_to_realized_cap.validate_computed_version_or_reset_file(
base_version + height_to_realized_cap.inner_version(),
)?;
height_to_supply.validate_computed_version_or_reset_file(
base_version + height_to_supply.inner_version(),
)?;
height_to_utxo_count.validate_computed_version_or_reset_file(
base_version + height_to_utxo_count.inner_version(),
)?;
let starting_height = [
height_to_realized_cap.len(),
height_to_supply.len(),
height_to_utxo_count.len(),
]
.into_iter()
.map(Height::from)
.min()
.unwrap()
.min(starting_indexes.height);
let mut states = CohortStates::default();
if let Some(prev_height) = starting_height.checked_sub(Height::new(1)) {
states.realized_cap = height_to_realized_cap
.into_iter()
.unwrap_get_inner(prev_height);
states.supply = height_to_supply.into_iter().unwrap_get_inner(prev_height);
states.utxo_count = height_to_utxo_count
.into_iter()
.unwrap_get_inner(prev_height);
}
(starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
.map(Height::from)
.try_for_each(|height| -> color_eyre::Result<()> {
let first_outputindex = height_to_first_outputindex_iter.unwrap_get_inner(height);
let first_inputindex = height_to_first_inputindex_iter.unwrap_get_inner(height);
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
Ok(())
})?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
// [].concat()
vec![]
}
}