computer: part 7

This commit is contained in:
nym21
2025-04-07 15:48:00 +02:00
parent 4c7e9fbee2
commit cd69ec4fa3
9 changed files with 278 additions and 185 deletions
+37 -29
View File
@@ -1,8 +1,9 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, StoredU32, StoredUsize, Timestamp, Weight};
use brk_core::{CheckedSub, StoredU32, StoredU64, StoredUsize, Timestamp, Weight};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{AnyStorableVec, Compressed, Version};
use super::{
@@ -16,7 +17,7 @@ pub struct Vecs {
pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_block_weight: ComputedVecsFromHeight<Weight>,
// pub indexes_to_block_vbytes: ComputedVecsFromHeight<>,
pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
}
@@ -60,6 +61,14 @@ impl Vecs {
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
path,
"block_vbytes",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
)?,
})
}
@@ -70,7 +79,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.indexes_to_block_interval.compute(
self.indexes_to_block_interval.compute_all(
indexer,
indexes,
starting_indexes,
@@ -95,57 +104,55 @@ impl Vecs {
},
)?;
self.indexes_to_block_count.compute(
self.indexes_to_block_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, _, starting_indexes, exit| {
let indexer_vecs = indexer.mut_vecs();
v.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_weight.mut_vec(),
indexer.mut_vecs().height_to_block_weight.mut_vec(),
|(h, ..)| (h, StoredU32::from(1_u32)),
exit,
)
},
)?;
self.indexes_to_block_weight.compute(
self.indexes_to_block_weight.compute_rest(
indexes,
starting_indexes,
exit,
Some(indexer.mut_vecs().height_to_block_weight.mut_vec()),
)?;
self.indexes_to_block_size.compute_rest(
indexes,
starting_indexes,
exit,
Some(indexer.mut_vecs().height_to_block_size.mut_vec()),
)?;
self.indexes_to_block_vbytes.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, indexer, _, starting_indexes, exit| {
let indexer_vecs = indexer.mut_vecs();
v.compute_transform(
starting_indexes.height,
indexer_vecs.height_to_weight.mut_vec(),
|(h, w)| (h, w),
indexer.mut_vecs().height_to_block_weight.mut_vec(),
|(h, w, ..)| {
(
h,
StoredU64::from(bitcoin::Weight::from(w).to_vbytes_floor()),
)
},
exit,
)
},
)?;
// self.indexes_to_block_size.compute(
// indexer,
// indexes,
// starting_indexes,
// exit,
// |v, indexer, _, starting_indexes, exit| {
// let indexer_vecs = indexer.mut_vecs();
// v.compute_transform(
// starting_indexes.height,
// indexer_vecs.height_to_weight.mut_vec(),
// |(h, ..)| (h, StoredU32::from(1_u32)),
// exit,
// )
// },
// )?;
Ok(())
}
@@ -155,6 +162,7 @@ impl Vecs {
self.indexes_to_block_count.any_vecs(),
self.indexes_to_block_weight.any_vecs(),
self.indexes_to_block_size.any_vecs(),
self.indexes_to_block_vbytes.any_vecs(),
]
.concat()
}
@@ -1,10 +1,9 @@
use std::path::Path;
use brk_exit::Exit;
use brk_indexer::{Indexer, Indexes};
use brk_vec::{AnyStorableVec, Compressed, Result, StorableVec, StoredIndex, StoredType, Version};
use crate::storage::vecs::{base::ComputedVec, indexes};
use crate::storage::vecs::base::ComputedVec;
use super::ComputedType;
@@ -699,17 +698,3 @@ impl StorableVecGeneatorOptions {
}
}
}
pub enum Source<'a, F, I, T>
where
F: FnMut(
&mut ComputedVec<I, T>,
&mut Indexer,
&mut indexes::Vecs,
&Indexes,
&Exit,
) -> Result<()>,
{
Compute(F),
Ref(&'a mut StorableVec<I, T>),
}
@@ -5,7 +5,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, Compressed, Result, Version};
use brk_vec::{AnyStorableVec, Compressed, Result, StorableVec, Version};
use crate::storage::vecs::{Indexes, base::ComputedVec, indexes};
@@ -67,7 +67,7 @@ where
})
}
pub fn compute<F>(
pub fn compute_all<F>(
&mut self,
indexer: &mut Indexer,
indexes: &mut indexes::Vecs,
@@ -84,16 +84,34 @@ where
&Exit,
) -> Result<()>,
{
if let Some(height) = self.height.as_mut() {
compute(height, indexer, indexes, starting_indexes, exit)?;
}
compute(
self.height.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
self.compute_rest(indexes, starting_indexes, exit, None)?;
Ok(())
}
pub fn compute_rest(
&mut self,
indexes: &mut indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
height: Option<&mut StorableVec<Height, T>>,
) -> color_eyre::Result<()> {
let height = height.unwrap_or_else(|| self.height.as_mut().unwrap().mut_vec());
self.height_extra
.extend(starting_indexes.height, self.height.mut_vec(), exit)?;
.extend(starting_indexes.height, height, exit)?;
self.dateindex.compute(
starting_indexes.dateindex,
self.height.mut_vec(),
height,
indexes.dateindex_to_first_height.mut_vec(),
indexes.dateindex_to_last_height.mut_vec(),
exit,
@@ -141,7 +159,7 @@ where
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
&mut self.height,
height,
indexes.difficultyepoch_to_first_height.mut_vec(),
indexes.difficultyepoch_to_last_height.mut_vec(),
exit,
@@ -152,7 +170,7 @@ where
pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
[
vec![self.height.any_vec()],
self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
self.height_extra.any_vecs(),
self.dateindex.any_vecs(),
self.weekindex.any_vecs(),
@@ -320,7 +320,7 @@ impl Vecs {
) -> color_eyre::Result<Indexes> {
let indexer_vecs = indexer.mut_vecs();
let height_count = indexer_vecs.height_to_size.len();
let height_count = indexer_vecs.height_to_block_size.len();
let txindexes_count = indexer_vecs.txindex_to_txid.len();
let txinindexes_count = indexer_vecs.txinindex_to_txoutindex.len();
let txoutindexes_count = indexer_vecs.txoutindex_to_addressindex.len();
+2 -2
View File
@@ -165,8 +165,8 @@ impl Indexer {
.push_if_needed(height, block.header.difficulty_float())?;
vecs.height_to_timestamp
.push_if_needed(height, Timestamp::from(block.header.time))?;
vecs.height_to_size.push_if_needed(height, block.total_size())?;
vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
vecs.height_to_block_size.push_if_needed(height, block.total_size().into())?;
vecs.height_to_block_weight.push_if_needed(height, block.weight().into())?;
let inputs = block
.txdata
+13 -13
View File
@@ -5,7 +5,7 @@ use brk_core::{
LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes,
P2PK65index, P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes,
P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats,
Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
StoredUsize, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
};
use brk_vec::{AnyStorableVec, Compressed, Version};
use rayon::prelude::*;
@@ -40,9 +40,9 @@ pub struct Vecs {
pub height_to_first_txinindex: IndexedVec<Height, Txinindex>,
pub height_to_first_txoutindex: IndexedVec<Height, Txoutindex>,
pub height_to_first_unknownindex: IndexedVec<Height, Unknownindex>,
pub height_to_size: IndexedVec<Height, usize>,
pub height_to_block_size: IndexedVec<Height, StoredUsize>,
pub height_to_timestamp: IndexedVec<Height, Timestamp>,
pub height_to_weight: IndexedVec<Height, Weight>,
pub height_to_block_weight: IndexedVec<Height, Weight>,
pub multisigindex_to_height: IndexedVec<Multisigindex, Height>,
pub opreturnindex_to_height: IndexedVec<Opreturnindex, Height>,
pub p2pk33index_to_height: IndexedVec<P2PK33index, Height>,
@@ -188,8 +188,8 @@ impl Vecs {
Version::ZERO,
compressed,
)?,
height_to_size: IndexedVec::forced_import(
&path.join("height_to_size"),
height_to_block_size: IndexedVec::forced_import(
&path.join("height_to_block_size"),
Version::ZERO,
compressed,
)?,
@@ -198,8 +198,8 @@ impl Vecs {
Version::ZERO,
compressed,
)?,
height_to_weight: IndexedVec::forced_import(
&path.join("height_to_weight"),
height_to_block_weight: IndexedVec::forced_import(
&path.join("height_to_block_weight"),
Version::ZERO,
compressed,
)?,
@@ -432,11 +432,11 @@ impl Vecs {
.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty
.truncate_if_needed(height, saved_height)?;
self.height_to_size
self.height_to_block_size
.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp
.truncate_if_needed(height, saved_height)?;
self.height_to_weight
self.height_to_block_weight
.truncate_if_needed(height, saved_height)?;
self.addressindex_to_addresstype
@@ -632,9 +632,9 @@ impl Vecs {
self.height_to_first_p2trindex.any_vec(),
self.height_to_first_p2wpkhindex.any_vec(),
self.height_to_first_p2wshindex.any_vec(),
self.height_to_size.any_vec(),
self.height_to_block_size.any_vec(),
self.height_to_timestamp.any_vec(),
self.height_to_weight.any_vec(),
self.height_to_block_weight.any_vec(),
self.p2pk33index_to_p2pk33addressbytes.any_vec(),
self.p2pk65index_to_p2pk65addressbytes.any_vec(),
self.p2pkhindex_to_p2pkhaddressbytes.any_vec(),
@@ -694,9 +694,9 @@ impl Vecs {
&mut self.height_to_first_p2trindex,
&mut self.height_to_first_p2wpkhindex,
&mut self.height_to_first_p2wshindex,
&mut self.height_to_size,
&mut self.height_to_block_size,
&mut self.height_to_timestamp,
&mut self.height_to_weight,
&mut self.height_to_block_weight,
&mut self.p2pk33index_to_p2pk33addressbytes,
&mut self.p2pk65index_to_p2pk65addressbytes,
&mut self.p2pkhindex_to_p2pkhaddressbytes,
-1
View File
@@ -10,7 +10,6 @@ use brk_parser::{
rpc::{self, RpcApi},
};
use brk_server::{Server, Website};
use log::info;
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
+194 -112
View File
@@ -644,6 +644,167 @@ function initGroups() {
* @returns {PartialOptionsTree}
*/
function createPartialOptions(colors) {
/**
* @typedef {"total-"} TotalPrefix
* @typedef {Extract<VecId, `${TotalPrefix}${string}`>} TotalVecId
* @typedef {"-sum"} SumSuffix
* @typedef {Extract<VecId, `${string}${SumSuffix}`>} VecIdSum
* @typedef {"-average"} AverageSuffix
* @typedef {Extract<VecId, `${string}${AverageSuffix}`>} VecIdAverage
* @typedef {"-median"} MedianSuffix
* @typedef {Extract<VecId, `${string}${MedianSuffix}`>} VecIdMedian
* @typedef {"-90p"} _90pSuffix
* @typedef {Extract<VecId, `${string}${_90pSuffix}`>} VecId90p
* @typedef {"-75p"} _75pSuffix
* @typedef {Extract<VecId, `${string}${_75pSuffix}`>} VecId75p
* @typedef {"-25p"} _25pSuffix
* @typedef {Extract<VecId, `${string}${_25pSuffix}`>} VecId25p
* @typedef {"-10p"} _10pSuffix
* @typedef {Extract<VecId, `${string}${_10pSuffix}`>} VecId10p
* @typedef {"-max"} MaxSuffix
* @typedef {Extract<VecId, `${string}${MaxSuffix}`>} VecIdMax
* @typedef {"-min"} MinSuffix
* @typedef {Extract<VecId, `${string}${MinSuffix}`>} VecIdMin
* @typedef {VecId extends infer X
? X extends string
? `${X}${SumSuffix}` extends VecIdSum
? `${TotalPrefix}${X}` extends TotalVecId
? X
: never
: never
: never
: never} BaseTotalSumVecId
* @typedef {VecId extends infer X
? X extends string
? `${X}${MinSuffix}` extends VecIdMin
? `${X}${MaxSuffix}` extends VecIdMax
? X
: never
: never
: never
: never} MinMaxVecId
* @typedef {VecId extends infer X
? X extends string
? `${X}${AverageSuffix}` extends VecIdAverage
? X
: never
: never
: never} AverageVecId
* @typedef {VecId extends infer X
? X extends string
? `${X}${MedianSuffix}` extends VecIdMedian
? X
: never
: never
: never} MedianVecId
* @typedef {VecId extends infer X
? X extends string
? `${X}${_90pSuffix}` extends VecId90p
? `${X}${_75pSuffix}` extends VecId75p
? `${X}${_25pSuffix}` extends VecId25p
? `${X}${_10pSuffix}` extends VecId10p
? X
: never
: never
: never
: never
: never
: never} PercentilesVecId
* @typedef {AverageVecId & MinMaxVecId & MedianVecId & PercentilesVecId} AverageMinMaxPercentilesVecId
*/
/**
* @template {BaseTotalSumVecId} T
* @param {Object} args
* @param {string} args.name
* @param {string} args.title
* @param {T} args.key
*/
function createBaseSumTotal({ name, title, key }) {
return /** @satisfies {PartialChartOption} */ ({
name,
title,
bottom: [
{ key, title: name, color: colors.bitcoin },
{
key: `${key}-sum`,
title: "Sum",
color: colors.bitcoin,
},
{
key: `total-${key}`,
title: "Total",
color: colors.offBitcoin,
defaultActive: false,
},
],
});
}
/**
* @template {AverageMinMaxPercentilesVecId} T
* @param {Object} args
* @param {string} args.name
* @param {string} args.title
* @param {T} args.key
*/
function createBaseAverageMinMaxPercentiles({ name, title, key }) {
return /** @satisfies {PartialChartOption} */ ({
name,
title,
bottom: [
{ key, title: name, color: colors.bitcoin },
{
key: `${key}-average`,
title: "Average",
color: colors.orange,
},
{
key: `${key}-median`,
title: "Median",
color: colors.amber,
defaultActive: false,
},
{
key: `${key}-75p`,
title: "75p",
color: colors.red,
defaultActive: false,
},
{
key: `${key}-25p`,
title: "25p",
color: colors.yellow,
defaultActive: false,
},
{
key: `${key}-90p`,
title: "90p",
color: colors.rose,
defaultActive: false,
},
{
key: `${key}-10p`,
title: "10p",
color: colors.lime,
defaultActive: false,
},
{
key: `${key}-max`,
title: "Max",
color: colors.pink,
defaultActive: false,
},
{
key: `${key}-min`,
title: "Min",
color: colors.green,
defaultActive: false,
},
],
});
}
return [
{
name: "Charts",
@@ -667,82 +828,31 @@ function createPartialOptions(colors) {
{
name: "Blocks",
tree: [
{
createBaseSumTotal({
name: "Count",
title: "Block Count",
bottom: [
{ key: "block-count", title: "Count", color: colors.bitcoin },
{
key: "block-count-sum",
title: "Sum",
color: colors.bitcoin,
},
{
key: "total-block-count",
title: "Total",
color: colors.offBitcoin,
defaultActive: false,
},
],
},
{
key: "block-count",
}),
createBaseAverageMinMaxPercentiles({
name: "Interval",
title: "Block Interval",
bottom: [
{
key: "block-interval",
title: "Interval",
color: colors.bitcoin,
},
{
key: "block-interval-average",
title: "Average",
color: colors.orange,
},
{
key: "block-interval-median",
title: "Median",
color: colors.amber,
defaultActive: false,
},
{
key: "block-interval-75p",
title: "75p",
color: colors.red,
defaultActive: false,
},
{
key: "block-interval-25p",
title: "25p",
color: colors.yellow,
defaultActive: false,
},
{
key: "block-interval-90p",
title: "90p",
color: colors.rose,
defaultActive: false,
},
{
key: "block-interval-10p",
title: "10p",
color: colors.lime,
defaultActive: false,
},
{
key: "block-interval-max",
title: "Max",
color: colors.pink,
defaultActive: false,
},
{
key: "block-interval-min",
title: "Min",
color: colors.green,
defaultActive: false,
},
],
},
key: "block-interval",
}),
createBaseSumTotal({
name: "Size",
title: "Block Size",
key: "block-size",
}),
createBaseSumTotal({
name: "Weight",
title: "Block Weight",
key: "block-weight",
}),
createBaseSumTotal({
name: "Vbytes",
title: "Block Virtual Bytes",
key: "block-vbytes",
}),
],
},
{
@@ -751,55 +861,21 @@ function createPartialOptions(colors) {
{
name: "Inputs",
tree: [
{
createBaseSumTotal({
name: "Count",
title: "Transaction Input Count",
bottom: [
{
key: "input-count",
title: "Count",
color: colors.bitcoin,
},
{
key: "input-count-sum",
title: "Sum",
color: colors.bitcoin,
},
{
key: "total-input-count",
title: "Total",
color: colors.offBitcoin,
defaultActive: false,
},
],
},
key: "input-count",
}),
],
},
{
name: "Outputs",
tree: [
{
createBaseSumTotal({
name: "Count",
title: "Transaction Output Count",
bottom: [
{
key: "output-count",
title: "Count",
color: colors.bitcoin,
},
{
key: "output-count-sum",
title: "Sum",
color: colors.bitcoin,
},
{
key: "total-output-count",
title: "Total",
color: colors.offBitcoin,
defaultActive: false,
},
],
},
key: "output-count",
}),
],
},
],
@@ -1162,6 +1238,12 @@ export function initOptions({
anyPartial.unit = "Satoshis";
} else if (key.includes("count")) {
anyPartial.unit = "Count";
} else if (key.includes("-size")) {
anyPartial.unit = "Megabytes";
} else if (key.includes("-weight")) {
anyPartial.unit = "Weight";
} else if (key.includes("-vbytes")) {
anyPartial.unit = "Virtual Bytes";
} else {
console.log(anyPartial);
throw Error("Unit not set");
@@ -75,6 +75,8 @@ export function createVecIdToIndexes() {
"block-interval-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"block-size": [Height],
"block-size-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"block-vbytes": [Height],
"block-vbytes-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"block-weight": [Height],
"block-weight-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
blockhash: [Height],
@@ -142,10 +144,10 @@ export function createVecIdToIndexes() {
quarterindex: [Monthindex, Quarterindex],
"real-date": [Height],
"sats-per-dollar": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
size: [Height],
timestamp: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch, Halvingepoch],
"total-block-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"total-block-size": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"total-block-vbytes": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"total-block-weight": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"total-input-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
"total-output-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
@@ -155,7 +157,6 @@ export function createVecIdToIndexes() {
txversion: [Txindex],
value: [Txoutindex],
weekindex: [Dateindex, Weekindex],
weight: [Height],
yearindex: [Monthindex, Yearindex],
}
}