diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 5cfff2724..4faea83fe 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -1,8 +1,9 @@ use std::{fs, path::Path}; -use brk_core::{CheckedSub, StoredU32, StoredUsize, Timestamp, Weight}; +use brk_core::{CheckedSub, StoredU32, StoredU64, StoredUsize, Timestamp, Weight}; use brk_exit::Exit; use brk_indexer::Indexer; +use brk_parser::bitcoin; use brk_vec::{AnyStorableVec, Compressed, Version}; use super::{ @@ -16,7 +17,7 @@ pub struct Vecs { pub indexes_to_block_interval: ComputedVecsFromHeight, pub indexes_to_block_count: ComputedVecsFromHeight, pub indexes_to_block_weight: ComputedVecsFromHeight, - // pub indexes_to_block_vbytes: ComputedVecsFromHeight<>, + pub indexes_to_block_vbytes: ComputedVecsFromHeight, pub indexes_to_block_size: ComputedVecsFromHeight, } @@ -60,6 +61,14 @@ impl Vecs { compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, + indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( + path, + "block_vbytes", + true, + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_sum().add_total(), + )?, }) } @@ -70,7 +79,7 @@ impl Vecs { starting_indexes: &Indexes, exit: &Exit, ) -> color_eyre::Result<()> { - self.indexes_to_block_interval.compute( + self.indexes_to_block_interval.compute_all( indexer, indexes, starting_indexes, @@ -95,57 +104,55 @@ impl Vecs { }, )?; - self.indexes_to_block_count.compute( + self.indexes_to_block_count.compute_all( indexer, indexes, starting_indexes, exit, |v, indexer, _, starting_indexes, exit| { - let indexer_vecs = indexer.mut_vecs(); - v.compute_transform( starting_indexes.height, - indexer_vecs.height_to_weight.mut_vec(), + indexer.mut_vecs().height_to_block_weight.mut_vec(), |(h, ..)| (h, StoredU32::from(1_u32)), exit, ) }, )?; - self.indexes_to_block_weight.compute( + self.indexes_to_block_weight.compute_rest( + indexes, + starting_indexes, + exit, + Some(indexer.mut_vecs().height_to_block_weight.mut_vec()), + )?; + + self.indexes_to_block_size.compute_rest( + indexes, + starting_indexes, + exit, + Some(indexer.mut_vecs().height_to_block_size.mut_vec()), + )?; + + self.indexes_to_block_vbytes.compute_all( indexer, indexes, starting_indexes, exit, |v, indexer, _, starting_indexes, exit| { - let indexer_vecs = indexer.mut_vecs(); - v.compute_transform( starting_indexes.height, - indexer_vecs.height_to_weight.mut_vec(), - |(h, w)| (h, w), + indexer.mut_vecs().height_to_block_weight.mut_vec(), + |(h, w, ..)| { + ( + h, + StoredU64::from(bitcoin::Weight::from(w).to_vbytes_floor()), + ) + }, exit, ) }, )?; - // self.indexes_to_block_size.compute( - // indexer, - // indexes, - // starting_indexes, - // exit, - // |v, indexer, _, starting_indexes, exit| { - // let indexer_vecs = indexer.mut_vecs(); - - // v.compute_transform( - // starting_indexes.height, - // indexer_vecs.height_to_weight.mut_vec(), - // |(h, ..)| (h, StoredU32::from(1_u32)), - // exit, - // ) - // }, - // )?; - Ok(()) } @@ -155,6 +162,7 @@ impl Vecs { self.indexes_to_block_count.any_vecs(), self.indexes_to_block_weight.any_vecs(), self.indexes_to_block_size.any_vecs(), + self.indexes_to_block_vbytes.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index bde5c005c..f6a0fd1bf 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -1,10 +1,9 @@ use std::path::Path; use brk_exit::Exit; -use brk_indexer::{Indexer, Indexes}; use brk_vec::{AnyStorableVec, Compressed, Result, StorableVec, StoredIndex, StoredType, Version}; -use crate::storage::vecs::{base::ComputedVec, indexes}; +use crate::storage::vecs::base::ComputedVec; use super::ComputedType; @@ -699,17 +698,3 @@ impl StorableVecGeneatorOptions { } } } - -pub enum Source<'a, F, I, T> -where - F: FnMut( - &mut ComputedVec, - &mut Indexer, - &mut indexes::Vecs, - &Indexes, - &Exit, - ) -> Result<()>, -{ - Compute(F), - Ref(&'a mut StorableVec), -} diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index 34df953c7..945d1083d 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -5,7 +5,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Compressed, Result, Version}; +use brk_vec::{AnyStorableVec, Compressed, Result, StorableVec, Version}; use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; @@ -67,7 +67,7 @@ where }) } - pub fn compute( + pub fn compute_all( &mut self, indexer: &mut Indexer, indexes: &mut indexes::Vecs, @@ -84,16 +84,34 @@ where &Exit, ) -> Result<()>, { - if let Some(height) = self.height.as_mut() { - compute(height, indexer, indexes, starting_indexes, exit)?; - } + compute( + self.height.as_mut().unwrap(), + indexer, + indexes, + starting_indexes, + exit, + )?; + + self.compute_rest(indexes, starting_indexes, exit, None)?; + + Ok(()) + } + + pub fn compute_rest( + &mut self, + indexes: &mut indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + height: Option<&mut StorableVec>, + ) -> color_eyre::Result<()> { + let height = height.unwrap_or_else(|| self.height.as_mut().unwrap().mut_vec()); self.height_extra - .extend(starting_indexes.height, self.height.mut_vec(), exit)?; + .extend(starting_indexes.height, height, exit)?; self.dateindex.compute( starting_indexes.dateindex, - self.height.mut_vec(), + height, indexes.dateindex_to_first_height.mut_vec(), indexes.dateindex_to_last_height.mut_vec(), exit, @@ -141,7 +159,7 @@ where self.difficultyepoch.compute( starting_indexes.difficultyepoch, - &mut self.height, + height, indexes.difficultyepoch_to_first_height.mut_vec(), indexes.difficultyepoch_to_last_height.mut_vec(), exit, @@ -152,7 +170,7 @@ where pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ - vec![self.height.any_vec()], + self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]), self.height_extra.any_vecs(), self.dateindex.any_vecs(), self.weekindex.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 814a1f2da..e1961beed 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -320,7 +320,7 @@ impl Vecs { ) -> color_eyre::Result { let indexer_vecs = indexer.mut_vecs(); - let height_count = indexer_vecs.height_to_size.len(); + let height_count = indexer_vecs.height_to_block_size.len(); let txindexes_count = indexer_vecs.txindex_to_txid.len(); let txinindexes_count = indexer_vecs.txinindex_to_txoutindex.len(); let txoutindexes_count = indexer_vecs.txoutindex_to_addressindex.len(); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index db826cab2..aa354c355 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -165,8 +165,8 @@ impl Indexer { .push_if_needed(height, block.header.difficulty_float())?; vecs.height_to_timestamp .push_if_needed(height, Timestamp::from(block.header.time))?; - vecs.height_to_size.push_if_needed(height, block.total_size())?; - vecs.height_to_weight.push_if_needed(height, block.weight().into())?; + vecs.height_to_block_size.push_if_needed(height, block.total_size().into())?; + vecs.height_to_block_weight.push_if_needed(height, block.weight().into())?; let inputs = block .txdata diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 66afccdbb..8c2bf4199 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -5,7 +5,7 @@ use brk_core::{ LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, - Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, + StoredUsize, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, }; use brk_vec::{AnyStorableVec, Compressed, Version}; use rayon::prelude::*; @@ -40,9 +40,9 @@ pub struct Vecs { pub height_to_first_txinindex: IndexedVec, pub height_to_first_txoutindex: IndexedVec, pub height_to_first_unknownindex: IndexedVec, - pub height_to_size: IndexedVec, + pub height_to_block_size: IndexedVec, pub height_to_timestamp: IndexedVec, - pub height_to_weight: IndexedVec, + pub height_to_block_weight: IndexedVec, pub multisigindex_to_height: IndexedVec, pub opreturnindex_to_height: IndexedVec, pub p2pk33index_to_height: IndexedVec, @@ -188,8 +188,8 @@ impl Vecs { Version::ZERO, compressed, )?, - height_to_size: IndexedVec::forced_import( - &path.join("height_to_size"), + height_to_block_size: IndexedVec::forced_import( + &path.join("height_to_block_size"), Version::ZERO, compressed, )?, @@ -198,8 +198,8 @@ impl Vecs { Version::ZERO, compressed, )?, - height_to_weight: IndexedVec::forced_import( - &path.join("height_to_weight"), + height_to_block_weight: IndexedVec::forced_import( + &path.join("height_to_block_weight"), Version::ZERO, compressed, )?, @@ -432,11 +432,11 @@ impl Vecs { .truncate_if_needed(height, saved_height)?; self.height_to_difficulty .truncate_if_needed(height, saved_height)?; - self.height_to_size + self.height_to_block_size .truncate_if_needed(height, saved_height)?; self.height_to_timestamp .truncate_if_needed(height, saved_height)?; - self.height_to_weight + self.height_to_block_weight .truncate_if_needed(height, saved_height)?; self.addressindex_to_addresstype @@ -632,9 +632,9 @@ impl Vecs { self.height_to_first_p2trindex.any_vec(), self.height_to_first_p2wpkhindex.any_vec(), self.height_to_first_p2wshindex.any_vec(), - self.height_to_size.any_vec(), + self.height_to_block_size.any_vec(), self.height_to_timestamp.any_vec(), - self.height_to_weight.any_vec(), + self.height_to_block_weight.any_vec(), self.p2pk33index_to_p2pk33addressbytes.any_vec(), self.p2pk65index_to_p2pk65addressbytes.any_vec(), self.p2pkhindex_to_p2pkhaddressbytes.any_vec(), @@ -694,9 +694,9 @@ impl Vecs { &mut self.height_to_first_p2trindex, &mut self.height_to_first_p2wpkhindex, &mut self.height_to_first_p2wshindex, - &mut self.height_to_size, + &mut self.height_to_block_size, &mut self.height_to_timestamp, - &mut self.height_to_weight, + &mut self.height_to_block_weight, &mut self.p2pk33index_to_p2pk33addressbytes, &mut self.p2pk65index_to_p2pk65addressbytes, &mut self.p2pkhindex_to_p2pkhaddressbytes, diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 213153470..76e260d74 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -10,7 +10,6 @@ use brk_parser::{ rpc::{self, RpcApi}, }; use brk_server::{Server, Website}; -use log::info; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/websites/kibo.money/scripts/options.js b/websites/kibo.money/scripts/options.js index a6526ab02..75dc5fec3 100644 --- a/websites/kibo.money/scripts/options.js +++ b/websites/kibo.money/scripts/options.js @@ -644,6 +644,167 @@ function initGroups() { * @returns {PartialOptionsTree} */ function createPartialOptions(colors) { + /** + * @typedef {"total-"} TotalPrefix + * @typedef {Extract} TotalVecId + * @typedef {"-sum"} SumSuffix + * @typedef {Extract} VecIdSum + * @typedef {"-average"} AverageSuffix + * @typedef {Extract} VecIdAverage + * @typedef {"-median"} MedianSuffix + * @typedef {Extract} VecIdMedian + * @typedef {"-90p"} _90pSuffix + * @typedef {Extract} VecId90p + * @typedef {"-75p"} _75pSuffix + * @typedef {Extract} VecId75p + * @typedef {"-25p"} _25pSuffix + * @typedef {Extract} VecId25p + * @typedef {"-10p"} _10pSuffix + * @typedef {Extract} VecId10p + * @typedef {"-max"} MaxSuffix + * @typedef {Extract} VecIdMax + * @typedef {"-min"} MinSuffix + * @typedef {Extract} VecIdMin + * @typedef {VecId extends infer X + ? X extends string + ? `${X}${SumSuffix}` extends VecIdSum + ? `${TotalPrefix}${X}` extends TotalVecId + ? X + : never + : never + : never + : never} BaseTotalSumVecId + * @typedef {VecId extends infer X + ? X extends string + ? `${X}${MinSuffix}` extends VecIdMin + ? `${X}${MaxSuffix}` extends VecIdMax + ? X + : never + : never + : never + : never} MinMaxVecId + * @typedef {VecId extends infer X + ? X extends string + ? `${X}${AverageSuffix}` extends VecIdAverage + ? X + : never + : never + : never} AverageVecId + * @typedef {VecId extends infer X + ? X extends string + ? `${X}${MedianSuffix}` extends VecIdMedian + ? X + : never + : never + : never} MedianVecId + * @typedef {VecId extends infer X + ? X extends string + ? `${X}${_90pSuffix}` extends VecId90p + ? `${X}${_75pSuffix}` extends VecId75p + ? `${X}${_25pSuffix}` extends VecId25p + ? `${X}${_10pSuffix}` extends VecId10p + ? X + : never + : never + : never + : never + : never + : never} PercentilesVecId + * @typedef {AverageVecId & MinMaxVecId & MedianVecId & PercentilesVecId} AverageMinMaxPercentilesVecId + */ + + /** + * @template {BaseTotalSumVecId} T + * @param {Object} args + * @param {string} args.name + * @param {string} args.title + * @param {T} args.key + */ + function createBaseSumTotal({ name, title, key }) { + return /** @satisfies {PartialChartOption} */ ({ + name, + title, + bottom: [ + { key, title: name, color: colors.bitcoin }, + { + key: `${key}-sum`, + title: "Sum", + color: colors.bitcoin, + }, + { + key: `total-${key}`, + title: "Total", + color: colors.offBitcoin, + defaultActive: false, + }, + ], + }); + } + + /** + * @template {AverageMinMaxPercentilesVecId} T + * @param {Object} args + * @param {string} args.name + * @param {string} args.title + * @param {T} args.key + */ + function createBaseAverageMinMaxPercentiles({ name, title, key }) { + return /** @satisfies {PartialChartOption} */ ({ + name, + title, + bottom: [ + { key, title: name, color: colors.bitcoin }, + { + key: `${key}-average`, + title: "Average", + color: colors.orange, + }, + { + key: `${key}-median`, + title: "Median", + color: colors.amber, + defaultActive: false, + }, + { + key: `${key}-75p`, + title: "75p", + color: colors.red, + defaultActive: false, + }, + { + key: `${key}-25p`, + title: "25p", + color: colors.yellow, + defaultActive: false, + }, + { + key: `${key}-90p`, + title: "90p", + color: colors.rose, + defaultActive: false, + }, + { + key: `${key}-10p`, + title: "10p", + color: colors.lime, + defaultActive: false, + }, + { + key: `${key}-max`, + title: "Max", + color: colors.pink, + defaultActive: false, + }, + { + key: `${key}-min`, + title: "Min", + color: colors.green, + defaultActive: false, + }, + ], + }); + } + return [ { name: "Charts", @@ -667,82 +828,31 @@ function createPartialOptions(colors) { { name: "Blocks", tree: [ - { + createBaseSumTotal({ name: "Count", title: "Block Count", - bottom: [ - { key: "block-count", title: "Count", color: colors.bitcoin }, - { - key: "block-count-sum", - title: "Sum", - color: colors.bitcoin, - }, - { - key: "total-block-count", - title: "Total", - color: colors.offBitcoin, - defaultActive: false, - }, - ], - }, - { + key: "block-count", + }), + createBaseAverageMinMaxPercentiles({ name: "Interval", title: "Block Interval", - bottom: [ - { - key: "block-interval", - title: "Interval", - color: colors.bitcoin, - }, - { - key: "block-interval-average", - title: "Average", - color: colors.orange, - }, - { - key: "block-interval-median", - title: "Median", - color: colors.amber, - defaultActive: false, - }, - { - key: "block-interval-75p", - title: "75p", - color: colors.red, - defaultActive: false, - }, - { - key: "block-interval-25p", - title: "25p", - color: colors.yellow, - defaultActive: false, - }, - { - key: "block-interval-90p", - title: "90p", - color: colors.rose, - defaultActive: false, - }, - { - key: "block-interval-10p", - title: "10p", - color: colors.lime, - defaultActive: false, - }, - { - key: "block-interval-max", - title: "Max", - color: colors.pink, - defaultActive: false, - }, - { - key: "block-interval-min", - title: "Min", - color: colors.green, - defaultActive: false, - }, - ], - }, + key: "block-interval", + }), + createBaseSumTotal({ + name: "Size", + title: "Block Size", + key: "block-size", + }), + createBaseSumTotal({ + name: "Weight", + title: "Block Weight", + key: "block-weight", + }), + createBaseSumTotal({ + name: "Vbytes", + title: "Block Virtual Bytes", + key: "block-vbytes", + }), ], }, { @@ -751,55 +861,21 @@ function createPartialOptions(colors) { { name: "Inputs", tree: [ - { + createBaseSumTotal({ name: "Count", title: "Transaction Input Count", - bottom: [ - { - key: "input-count", - title: "Count", - color: colors.bitcoin, - }, - { - key: "input-count-sum", - title: "Sum", - color: colors.bitcoin, - }, - { - key: "total-input-count", - title: "Total", - color: colors.offBitcoin, - defaultActive: false, - }, - ], - }, + key: "input-count", + }), ], }, { name: "Outputs", tree: [ - { + createBaseSumTotal({ name: "Count", title: "Transaction Output Count", - bottom: [ - { - key: "output-count", - title: "Count", - color: colors.bitcoin, - }, - { - key: "output-count-sum", - title: "Sum", - color: colors.bitcoin, - }, - { - key: "total-output-count", - title: "Total", - color: colors.offBitcoin, - defaultActive: false, - }, - ], - }, + key: "output-count", + }), ], }, ], @@ -1162,6 +1238,12 @@ export function initOptions({ anyPartial.unit = "Satoshis"; } else if (key.includes("count")) { anyPartial.unit = "Count"; + } else if (key.includes("-size")) { + anyPartial.unit = "Megabytes"; + } else if (key.includes("-weight")) { + anyPartial.unit = "Weight"; + } else if (key.includes("-vbytes")) { + anyPartial.unit = "Virtual Bytes"; } else { console.log(anyPartial); throw Error("Unit not set"); diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index 38cf85af1..56f39e982 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -75,6 +75,8 @@ export function createVecIdToIndexes() { "block-interval-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "block-size": [Height], "block-size-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "block-vbytes": [Height], + "block-vbytes-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "block-weight": [Height], "block-weight-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], blockhash: [Height], @@ -142,10 +144,10 @@ export function createVecIdToIndexes() { quarterindex: [Monthindex, Quarterindex], "real-date": [Height], "sats-per-dollar": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], - size: [Height], timestamp: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch, Halvingepoch], "total-block-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-block-size": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-block-vbytes": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-block-weight": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-input-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-output-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], @@ -155,7 +157,6 @@ export function createVecIdToIndexes() { txversion: [Txindex], value: [Txoutindex], weekindex: [Dateindex, Weekindex], - weight: [Height], yearindex: [Monthindex, Yearindex], } }