diff --git a/README.md b/README.md index 0f27b6102..9c042bf57 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@

> **WARNING** -> +> > This project is still a work in progress and while it's much better in many ways than its previous version ([kibo v0.5](https://github.com/kibo-money/kibo)), it doesn't yet include all of those datasets. If you're interested in having everything right now, please use the latter until feature parity is achieved. > > The explorer part (mempool.space/electrs) is also not viable just yet. @@ -58,6 +58,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible For more information visit: [`brk_cli`](https://crates.io/crates/brk_cli) - **[Crates](https://crates.io/crates/brk)** \ Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility. + PRs are welcome, especially if their goal is to introduce additional datasets. The primary goal of this project is to be fully-featured and accessible for everyone, regardless of their background or financial situation - whether that person is an enthusiast, researcher, miner, analyst, or simply curious. diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index 73cfee2db..0bdbd1eac 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -6,7 +6,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, Compressed, Result, Version}; +use brk_vec::{AnyStorableVec, Compressed, Result, StorableVec, Version}; use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; @@ -17,7 +17,7 @@ pub struct ComputedVecsFromTxindex where T: ComputedType + PartialOrd, { - pub txindex: ComputedVec, + pub txindex: Option>, pub txindex_extra: ComputedVecBuilder, pub height: ComputedVecBuilder, pub dateindex: ComputedVecBuilder, @@ -38,15 +38,19 @@ where pub fn forced_import( path: &Path, name: &str, + compute_source: bool, version: Version, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let txindex = ComputedVec::forced_import( - &path.join(format!("txindex_to_{name}")), - version, - compressed, - )?; + let txindex = compute_source.then(|| { + ComputedVec::forced_import( + &path.join(format!("txindex_to_{name}")), + version, + compressed, + ) + .unwrap() + }); let txindex_extra = ComputedVecBuilder::forced_import( path, @@ -75,7 +79,7 @@ where }) } - pub fn compute( + pub fn compute_all( &mut self, indexer: &mut Indexer, indexes: &mut indexes::Vecs, @@ -92,14 +96,35 @@ where &Exit, ) -> Result<()>, { - compute(&mut self.txindex, indexer, indexes, starting_indexes, exit)?; + compute( + self.txindex.as_mut().unwrap(), + indexer, + indexes, + starting_indexes, + exit, + )?; + + self.compute_rest(indexer, indexes, starting_indexes, exit, None)?; + + Ok(()) + } + + pub fn compute_rest( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + txindex: Option<&mut StorableVec>, + ) -> color_eyre::Result<()> { + let txindex = txindex.unwrap_or_else(|| self.txindex.as_mut().unwrap().mut_vec()); self.txindex_extra - .extend(starting_indexes.txindex, self.txindex.mut_vec(), exit)?; + .extend(starting_indexes.txindex, txindex, exit)?; self.height.compute( starting_indexes.height, - self.txindex.mut_vec(), + txindex, indexer.mut_vecs().height_to_first_txindex.mut_vec(), indexes.height_to_last_txindex.mut_vec(), exit, @@ -166,7 +191,7 @@ where pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ - vec![self.txindex.any_vec()], + self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]), self.txindex_extra.any_vecs(), self.height.any_vecs(), self.dateindex.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 2c630c89a..063112dec 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -1,13 +1,13 @@ use std::{fs, path::Path}; -use brk_core::{StoredU64, Txindex}; +use brk_core::{Sats, StoredU64, Txindex, Txinindex, Txoutindex}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; use super::{ ComputedVec, Indexes, - grouped::{ComputedVecsFromTxindex, StorableVecGeneatorOptions}, + grouped::{ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions}, indexes, }; @@ -21,15 +21,21 @@ pub struct Vecs { // pub height_to_outputcount: ComputedVec, // pub height_to_subsidy: ComputedVec, // pub height_to_totalfees: ComputedVec, - // pub height_to_txcount: ComputedVec, // pub txindex_to_fee: ComputedVec, - pub txindex_to_is_coinbase: ComputedVec, // pub txindex_to_feerate: ComputedVec, - pub txindex_to_input_count: ComputedVecsFromTxindex, // pub txindex_to_input_sum: ComputedVec, - pub txindex_to_output_count: ComputedVecsFromTxindex, // pub txindex_to_output_sum: ComputedVec, + // pub txindex_to_output_value: ComputedVecsFromTxindex, + // pub txindex_to_version_1: ComputedVecsFromTxindex, + // pub txindex_to_version_2: ComputedVecsFromTxindex, + // pub txindex_to_version_3: ComputedVecsFromTxindex, // pub txinindex_to_value: ComputedVec, + pub height_to_tx_count: ComputedVecsFromHeight, + pub txindex_to_input_count: ComputedVecsFromTxindex, + pub txindex_to_is_coinbase: ComputedVec, + pub txindex_to_output_count: ComputedVecsFromTxindex, + /// Value == 0 when Coinbase + pub txinindex_to_value: ComputedVec, } impl Vecs { @@ -37,6 +43,14 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { + height_to_tx_count: ComputedVecsFromHeight::forced_import( + path, + "tx_count", + true, + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_sum().add_total(), + )?, // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::ZERO)?, // height_to_input_count: StorableVec::forced_import( // &path.join("height_to_input_count"), @@ -65,6 +79,7 @@ impl Vecs { txindex_to_input_count: ComputedVecsFromTxindex::forced_import( path, "input_count", + true, Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), @@ -72,15 +87,23 @@ impl Vecs { txindex_to_output_count: ComputedVecsFromTxindex::forced_import( path, "output_count", + true, Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, - // txinindex_to_value: StorableVec::forced_import( - // &path.join("txinindex_to_value"), + // txindex_to_output_value: ComputedVecsFromTxindex::forced_import( + // path, + // "output_value", // Version::ZERO, // compressed, + // StorableVecGeneatorOptions::default().add_sum().add_total(), // )?, + txinindex_to_value: ComputedVec::forced_import( + &path.join("txinindex_to_value"), + Version::ZERO, + compressed, + )?, }) } @@ -91,7 +114,22 @@ impl Vecs { starting_indexes: &Indexes, exit: &Exit, ) -> color_eyre::Result<()> { - self.txindex_to_input_count.compute( + self.height_to_tx_count.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, indexer, indexes, starting_indexes, exit| { + v.compute_count_from_indexes( + starting_indexes.height, + indexer.mut_vecs().height_to_first_txindex.mut_vec(), + indexes.height_to_last_txindex.mut_vec(), + exit, + ) + }, + )?; + + self.txindex_to_input_count.compute_all( indexer, indexes, starting_indexes, @@ -106,7 +144,7 @@ impl Vecs { }, )?; - self.txindex_to_output_count.compute( + self.txindex_to_output_count.compute_all( indexer, indexes, starting_indexes, @@ -121,6 +159,14 @@ impl Vecs { }, )?; + // self.txindex_to_output_value.compute_rest( + // indexer, + // indexes, + // starting_indexes, + // exit, + // Some(indexer.mut_vecs().txoutindex_to_value.mut_vec()), + // )?; + let indexer_vecs = indexer.mut_vecs(); self.txindex_to_is_coinbase.compute_is_first_ordered( @@ -130,23 +176,26 @@ impl Vecs { exit, )?; - // self.txinindex_to_value.compute_transform( - // starting_indexes.txinindex, - // indexer_vecs.txinindex_to_txoutindex.mut_vec(), - // |(txinindex, txoutindex, slf, other)| { - // let value = - // if let Ok(Some(value)) = indexer_vecs.txoutindex_to_value.read(txoutindex) { - // *value - // } else { - // dbg!(txinindex, txoutindex, slf.len(), other.len()); - // panic!() - // }; - // (txinindex, value) - // }, - // exit, - // )?; + self.txinindex_to_value.compute_transform( + starting_indexes.txinindex, + indexer_vecs.txinindex_to_txoutindex.mut_vec(), + |(txinindex, txoutindex, slf, other)| { + let value = if txoutindex == Txoutindex::COINBASE { + Sats::ZERO + } else if let Ok(Some(value)) = + indexer_vecs.txoutindex_to_value.mut_vec().get(txoutindex) + { + *value + } else { + dbg!(txinindex, txoutindex, slf.len(), other.len()); + panic!() + }; + (txinindex, value) + }, + exit, + )?; - // self.vecs.txindex_to_fee.compute_transform( + // self.txindex_to_fee.compute_transform( // &mut self.vecs.txindex_to_height, // &mut indexer.vecs().height_to_first_txindex, // )?; @@ -156,7 +205,11 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ - vec![self.txindex_to_is_coinbase.any_vec()], + vec![ + self.txindex_to_is_coinbase.any_vec(), + self.txinindex_to_value.any_vec(), + ], + self.height_to_tx_count.any_vecs(), self.txindex_to_output_count.any_vecs(), self.txindex_to_input_count.any_vecs(), ] diff --git a/crates/brk_core/src/structs/stored_u64.rs b/crates/brk_core/src/structs/stored_u64.rs index 365276b8f..26b52adfa 100644 --- a/crates/brk_core/src/structs/stored_u64.rs +++ b/crates/brk_core/src/structs/stored_u64.rs @@ -6,7 +6,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; -use super::{Txinindex, Txoutindex}; +use super::{Txindex, Txinindex, Txoutindex}; #[derive( Debug, @@ -80,6 +80,12 @@ impl From for f64 { } } +impl From for StoredU64 { + fn from(value: Txindex) -> Self { + Self(*value as u64) + } +} + impl From for StoredU64 { fn from(value: Txinindex) -> Self { Self(*value) diff --git a/websites/kibo.money/scripts/options.js b/websites/kibo.money/scripts/options.js index 75dc5fec3..f1274c39f 100644 --- a/websites/kibo.money/scripts/options.js +++ b/websites/kibo.money/scripts/options.js @@ -18,7 +18,7 @@ * "Transactions" | * "US Dollars" | * "Virtual Bytes" | - * "Weight" + * "Weight Units" * } Unit * * @typedef {Object} BaseSeriesBlueprint @@ -810,23 +810,28 @@ function createPartialOptions(colors) { name: "Charts", tree: [ { - name: "btc/usd", - title: "Bitcoin Price in US Dollars", - }, - { - name: "usd/sats", - title: "Satoshis Per US Dollar", - unit: "Satoshis", - bottom: [ + name: "Price", + tree: [ { - key: "sats-per-dollar", - title: "Satoshis", - color: colors.bitcoin, + name: "btc/usd", + title: "Bitcoin Price in US Dollars", + }, + { + name: "usd/sats", + title: "Satoshis Per US Dollar", + unit: "Satoshis", + bottom: [ + { + key: "sats-per-dollar", + title: "Satoshis", + color: colors.bitcoin, + }, + ], }, ], }, { - name: "Blocks", + name: "Block", tree: [ createBaseSumTotal({ name: "Count", @@ -856,28 +861,33 @@ function createPartialOptions(colors) { ], }, { - name: "Transactions", + name: "Transaction", tree: [ - { - name: "Inputs", - tree: [ - createBaseSumTotal({ - name: "Count", - title: "Transaction Input Count", - key: "input-count", - }), - ], - }, - { - name: "Outputs", - tree: [ - createBaseSumTotal({ - name: "Count", - title: "Transaction Output Count", - key: "output-count", - }), - ], - }, + createBaseSumTotal({ + name: "Count", + title: "Transaction Count", + key: "tx-count", + }), + ], + }, + { + name: "Input", + tree: [ + createBaseSumTotal({ + name: "Count", + title: "Transaction Input Count", + key: "input-count", + }), + ], + }, + { + name: "Output", + tree: [ + createBaseSumTotal({ + name: "Count", + title: "Transaction Output Count", + key: "output-count", + }), ], }, ], @@ -934,7 +944,7 @@ function createPartialOptions(colors) { url: () => "https://status.kibo.money/", }, { - name: "Crate", + name: "Crates", url: () => "https://crates.io/crates/brk", }, ], @@ -1241,7 +1251,7 @@ export function initOptions({ } else if (key.includes("-size")) { anyPartial.unit = "Megabytes"; } else if (key.includes("-weight")) { - anyPartial.unit = "Weight"; + anyPartial.unit = "Weight Units"; } else if (key.includes("-vbytes")) { anyPartial.unit = "Virtual Bytes"; } else { diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index 56f39e982..6d2190ec4 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -152,10 +152,13 @@ export function createVecIdToIndexes() { "total-input-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-output-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-size": [Txindex], + "total-tx-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "tx-count": [Height], + "tx-count-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], txid: [Txindex], txoutindex: [Txinindex], txversion: [Txindex], - value: [Txoutindex], + value: [Txinindex, Txoutindex], weekindex: [Dateindex, Weekindex], yearindex: [Monthindex, Yearindex], }