Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-05 03:39:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer + kibo: part 11

Browse Source
This commit is contained in:
nym21
2025-04-14 16:27:22 +02:00
parent 1c75ea046c
commit 942431e882
19 changed files with 1249 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
crates/brk_computer/src/storage/vecs/blocks.rs
View File

@@ -1,6 +1,6 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, StoredU32, StoredU64, StoredUsize, Timestamp, Weight};
use brk_core::{CheckedSub, Height, StoredU32, StoredU64, StoredUsize, Timestamp, Weight};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
@@ -8,15 +8,18 @@ use brk_vec::{Compressed, Version};
use super::{
Indexes,
base::ComputedVec,
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
indexes,
};
#[derive(Clone)]
pub struct Vecs {
pub height_to_interval: ComputedVec<Height, Timestamp>,
pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_block_weight: ComputedVecsFromHeight<Weight>,
pub height_to_vbytes: ComputedVec<Height, StoredU64>,
pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
}
@@ -26,10 +29,15 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
height_to_interval: ComputedVec::forced_import(
&path.join("height_to_interval"),
Version::ZERO,
compressed,
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
path,
"block_interval",
true,
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
@@ -61,10 +69,15 @@ impl Vecs {
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
)?,
height_to_vbytes: ComputedVec::forced_import(
&path.join("height_to_vbytes"),
Version::ZERO,
compressed,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
path,
"block_vbytes",
true,
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
@@ -79,30 +92,26 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.indexes_to_block_interval.compute_all(
indexer,
self.height_to_interval.compute_transform(
starting_indexes.height,
indexer.mut_vecs().height_to_timestamp.mut_vec(),
|(height, timestamp, _, height_to_timestamp)| {
let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
let prev_timestamp = *height_to_timestamp.cached_get(prev_h).unwrap().unwrap();
timestamp
.checked_sub(prev_timestamp)
.unwrap_or(Timestamp::ZERO)
});
(height, interval)
},
exit,
)?;
self.indexes_to_block_interval.compute_rest(
indexes,
starting_indexes,
exit,
|v, indexer, _, starting_indexes, exit| {
let indexer_vecs = indexer.mut_vecs();
v.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_timestamp.mut_vec(),
|(height, timestamp, _, height_to_timestamp)| {
let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
let prev_timestamp =
*height_to_timestamp.cached_get(prev_h).unwrap().unwrap();
timestamp
.checked_sub(prev_timestamp)
.unwrap_or(Timestamp::ZERO)
});
(height, interval)
},
exit,
)
},
Some(self.height_to_interval.mut_vec()),
)?;
self.indexes_to_block_count.compute_all(
@@ -113,7 +122,7 @@ impl Vecs {
|v, indexer, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
indexer.mut_vecs().height_to_block_weight.mut_vec(),
indexer.mut_vecs().height_to_weight.mut_vec(),
|(h, ..)| (h, StoredU32::from(1_u32)),
exit,
)
@@ -124,34 +133,33 @@ impl Vecs {
indexes,
starting_indexes,
exit,
Some(indexer.mut_vecs().height_to_block_weight.mut_vec()),
Some(indexer.mut_vecs().height_to_weight.mut_vec()),
)?;
self.indexes_to_block_size.compute_rest(
indexes,
starting_indexes,
exit,
Some(indexer.mut_vecs().height_to_block_size.mut_vec()),
Some(indexer.mut_vecs().height_to_total_size.mut_vec()),
)?;
self.indexes_to_block_vbytes.compute_all(
indexer,
self.height_to_vbytes.compute_transform(
starting_indexes.height,
indexer.mut_vecs().height_to_weight.mut_vec(),
|(h, w, ..)| {
(
h,
StoredU64::from(bitcoin::Weight::from(w).to_vbytes_floor()),
)
},
exit,
)?;
self.indexes_to_block_vbytes.compute_rest(
indexes,
starting_indexes,
exit,
|v, indexer, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.height,
indexer.mut_vecs().height_to_block_weight.mut_vec(),
|(h, w, ..)| {
(
h,
StoredU64::from(bitcoin::Weight::from(w).to_vbytes_floor()),
)
},
exit,
)
},
Some(self.height_to_vbytes.mut_vec()),
)?;
Ok(())
@@ -159,6 +167,10 @@ impl Vecs {
pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
[
vec![
self.height_to_interval.any_vec(),
self.height_to_vbytes.any_vec(),
],
self.indexes_to_block_interval.any_vecs(),
self.indexes_to_block_count.any_vecs(),
self.indexes_to_block_weight.any_vecs(),

4
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -12,8 +12,6 @@ use super::ComputedVec;
#[derive(Clone)]
pub struct Vecs {
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
pub dateindex_to_date: ComputedVec<Dateindex, Date>,
pub dateindex_to_dateindex: ComputedVec<Dateindex, Dateindex>,
pub dateindex_to_first_height: ComputedVec<Dateindex, Height>,
@@ -320,7 +318,7 @@ impl Vecs {
) -> color_eyre::Result<Indexes> {
let indexer_vecs = indexer.mut_vecs();
let height_count = indexer_vecs.height_to_block_size.len();
let height_count = indexer_vecs.height_to_total_size.len();
let txindexes_count = indexer_vecs.txindex_to_txid.len();
let txinindexes_count = indexer_vecs.txinindex_to_txoutindex.len();
let txoutindexes_count = indexer_vecs.txoutindex_to_addressindex.len();

1
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -19,7 +19,6 @@ use super::{
#[derive(Clone)]
pub struct Vecs {
// pub dateindex_to_close: ComputedVec<Dateindex, Close<Dollars>>,
pub dateindex_to_close_in_cents: ComputedVec<Dateindex, Close<Cents>>,
pub dateindex_to_high_in_cents: ComputedVec<Dateindex, High<Cents>>,
pub dateindex_to_low_in_cents: ComputedVec<Dateindex, Low<Cents>>,

237
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -7,7 +7,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{Compressed, DynamicVec, Version};
use brk_vec::{Compressed, DynamicVec, StoredIndex, Version};
use super::{
ComputedVec, Indexes,
@@ -17,22 +17,19 @@ use super::{
#[derive(Clone)]
pub struct Vecs {
// pub height_to_fee: ComputedVec<Txindex, Sats>,
// pub height_to_inputcount: ComputedVec<Height, u32>,
// pub height_to_maxfeerate: ComputedVec<Height, Feerate>,
// pub height_to_medianfeerate: ComputedVec<Height, Feerate>,
// pub height_to_minfeerate: ComputedVec<Height, Feerate>,
// pub height_to_outputcount: ComputedVec<Height, u32>,
// pub height_to_subsidy: ComputedVec<Height, u32>,
// pub height_to_totalfees: ComputedVec<Height, Sats>,
pub height_to_tx_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_fee: ComputedVecsFromTxindex<Sats>,
pub indexes_to_feerate: ComputedVecsFromTxindex<Feerate>,
pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_output_value: ComputedVecsFromTxindex<Sats>,
// pub txindex_to_is_v1: ComputedVec<Txindex, bool>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU32>,
// pub txindex_to_is_v2: ComputedVec<Txindex, bool>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU32>,
// pub txindex_to_is_v3: ComputedVec<Txindex, bool>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredUsize>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub txindex_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase: ComputedVec<Txindex, bool>,
pub txindex_to_output_count: ComputedVecsFromTxindex<StoredU64>,
@@ -40,6 +37,8 @@ pub struct Vecs {
pub txindex_to_weight: ComputedVec<Txindex, Weight>,
/// Value == 0 when Coinbase
pub txinindex_to_value: ComputedVec<Txinindex, Sats>,
pub indexes_to_subsidy: ComputedVecsFromHeight<Sats>,
pub indexes_to_coinbase: ComputedVecsFromHeight<Sats>,
}
impl Vecs {
@@ -47,42 +46,37 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
height_to_tx_count: ComputedVecsFromHeight::forced_import(
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
path,
"tx_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::ZERO)?,
// height_to_input_count: StorableVec::forced_import(
// &path.join("height_to_input_count"),
// Version::ZERO,
// )?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::ZERO)?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::ZERO)?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::ZERO)?,
// height_to_output_count: StorableVec::forced_import(
// &path.join("height_to_output_count"),
// Version::ZERO,
// )?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ZERO)?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::ZERO)?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::ZERO)?,
txindex_to_is_coinbase: ComputedVec::forced_import(
&path.join("txindex_to_is_coinbase"),
Version::ZERO,
compressed,
)?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::ZERO)?,
txindex_to_input_count: ComputedVecsFromTxindex::forced_import(
path,
"input_count",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
txindex_to_output_count: ComputedVecsFromTxindex::forced_import(
path,
@@ -90,15 +84,13 @@ impl Vecs {
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_average()
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
)?,
// txindex_to_output_value: ComputedVecsFromTxindex::forced_import(
// path,
// "output_value",
// Version::ZERO,
// compressed,
// StorableVecGeneatorOptions::default().add_sum().add_total(),
// )?,
txinindex_to_value: ComputedVec::forced_import(
&path.join("txinindex_to_value"),
Version::ZERO,
@@ -134,7 +126,10 @@ impl Vecs {
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_average()
.add_sum()
.add_total(),
)?,
indexes_to_output_value: ComputedVecsFromTxindex::forced_import(
path,
@@ -142,7 +137,10 @@ impl Vecs {
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_average()
.add_sum()
.add_total(),
)?,
indexes_to_fee: ComputedVecsFromTxindex::forced_import(
path,
@@ -150,7 +148,12 @@ impl Vecs {
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_total()
.add_percentiles()
.add_minmax()
.add_average(),
)?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
path,
@@ -173,6 +176,54 @@ impl Vecs {
Version::ZERO,
compressed,
)?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
path,
"tx_vsize",
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
.add_average(),
)?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
path,
"tx_weight",
false,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_minmax()
.add_average(),
)?,
indexes_to_subsidy: ComputedVecsFromHeight::forced_import(
path,
"subsidy",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_sum()
.add_total()
.add_minmax()
.add_average(),
)?,
indexes_to_coinbase: ComputedVecsFromHeight::forced_import(
path,
"coinbase",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_sum()
.add_total()
.add_percentiles()
.add_minmax()
.add_average(),
)?,
})
}
@@ -183,7 +234,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.height_to_tx_count.compute_all(
self.indexes_to_tx_count.compute_all(
indexer,
indexes,
starting_indexes,
@@ -228,14 +279,6 @@ impl Vecs {
},
)?;
// self.txindex_to_output_value.compute_rest(
// indexer,
// indexes,
// starting_indexes,
// exit,
// Some(indexer.mut_vecs().txoutindex_to_value.mut_vec()),
// )?;
let mut compute_indexes_to_tx_vany =
|indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredU32>, txversion| {
indexes_to_tx_vany.compute_all(
@@ -424,6 +467,94 @@ impl Vecs {
},
)?;
self.indexes_to_tx_weight.compute_rest(
indexer,
indexes,
starting_indexes,
exit,
Some(self.txindex_to_weight.mut_vec()),
)?;
self.indexes_to_tx_vsize.compute_rest(
indexer,
indexes,
starting_indexes,
exit,
Some(self.txindex_to_vsize.mut_vec()),
)?;
self.indexes_to_subsidy.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, indexer, indexes, starting_indexes, exit| {
let indexer_vecs = indexer.mut_vecs();
vec.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_first_txindex.mut_vec(),
|(height, txindex, ..)| {
let first_txoutindex = indexer_vecs
.txindex_to_first_txoutindex
.cached_get(txindex)
.unwrap()
.unwrap()
.into_inner()
.to_usize()
.unwrap();
let last_txoutindex = indexes
.txindex_to_last_txoutindex
.mut_vec()
.cached_get(txindex)
.unwrap()
.unwrap()
.into_inner()
.to_usize()
.unwrap();
let mut sats = Sats::ZERO;
(first_txoutindex..=last_txoutindex).for_each(|txoutindex| {
sats += indexer_vecs
.txoutindex_to_value
.cached_get_(txoutindex)
.unwrap()
.unwrap()
.into_inner();
});
(height, sats)
},
exit,
)
},
)?;
self.indexes_to_coinbase.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_transform(
starting_indexes.height,
self.indexes_to_subsidy.height.as_mut().unwrap().mut_vec(),
|(height, subsidy, ..)| {
let fees = self
.indexes_to_fee
.height
.sum
.as_mut()
.unwrap()
.mut_vec()
.cached_get(height)
.unwrap()
.unwrap()
.into_inner();
(height, subsidy.checked_sub(fees).unwrap())
},
exit,
)
},
)?;
Ok(())
}
@@ -435,14 +566,18 @@ impl Vecs {
self.txindex_to_weight.any_vec(),
self.txindex_to_vsize.any_vec(),
],
self.height_to_tx_count.any_vecs(),
self.indexes_to_tx_count.any_vecs(),
self.indexes_to_coinbase.any_vecs(),
self.indexes_to_fee.any_vecs(),
self.indexes_to_feerate.any_vecs(),
self.indexes_to_input_value.any_vecs(),
self.indexes_to_output_value.any_vecs(),
self.indexes_to_subsidy.any_vecs(),
self.indexes_to_tx_v1.any_vecs(),
self.indexes_to_tx_v2.any_vecs(),
self.indexes_to_tx_v3.any_vecs(),
self.indexes_to_fee.any_vecs(),
self.indexes_to_feerate.any_vecs(),
self.indexes_to_tx_vsize.any_vecs(),
self.indexes_to_tx_weight.any_vecs(),
self.txindex_to_input_count.any_vecs(),
self.txindex_to_output_count.any_vecs(),
]