diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index 4526ece6a..5e393c309 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -61,7 +61,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { loop { let block_count = rpc.get_block_count()?; - info!("{block_count} blocks found."); + info!("{} blocks found.", block_count + 1); let starting_indexes = indexer.index(&parser, rpc, &exit)?; diff --git a/crates/brk_computer/src/storage/stores.rs b/crates/brk_computer/src/storage/stores.rs index 8bf8aaf06..d7cc4adee 100644 --- a/crates/brk_computer/src/storage/stores.rs +++ b/crates/brk_computer/src/storage/stores.rs @@ -13,9 +13,9 @@ pub struct Stores { impl Stores { pub fn import(path: &Path) -> color_eyre::Result { let address_to_utxos_received = - Store::import(&path.join("address_to_utxos_received"), Version::ONE)?; + Store::import(&path.join("address_to_utxos_received"), Version::ZERO)?; let address_to_utxos_spent = - Store::import(&path.join("address_to_utxos_spent"), Version::ONE)?; + Store::import(&path.join("address_to_utxos_spent"), Version::ZERO)?; Ok(Self { address_to_utxos_received, diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index cee0c60f3..8ae0c1a28 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -31,7 +31,7 @@ where version: Version, compressed: Compressed, ) -> brk_vec::Result { - let vec = brk_vec::StorableVec::forced_import(path, version, compressed)?; + let vec = StorableVec::forced_import(path, version, compressed)?; Ok(Self { computed_version: None, @@ -88,11 +88,11 @@ where self.vec.len() } - pub fn vec(&self) -> &brk_vec::StorableVec { + pub fn vec(&self) -> &StorableVec { &self.vec } - pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec { + pub fn mut_vec(&mut self) -> &mut StorableVec { &mut self.vec } @@ -129,14 +129,14 @@ where pub fn compute_transform( &mut self, max_from: A, - other: &mut brk_vec::StorableVec, + other: &mut StorableVec, mut t: F, exit: &Exit, ) -> Result<()> where A: StoredIndex, B: StoredType, - F: FnMut((A, B, &mut Self, &mut brk_vec::StorableVec)) -> (I, T), + F: FnMut((A, B, &mut Self, &mut StorableVec)) -> (I, T), { self.validate_computed_version_or_reset_file( Version::ZERO + self.version() + other.version(), @@ -154,7 +154,7 @@ where pub fn compute_inverse_more_to_less( &mut self, max_from: T, - other: &mut brk_vec::StorableVec, + other: &mut StorableVec, exit: &Exit, ) -> Result<()> where @@ -181,8 +181,8 @@ where pub fn compute_inverse_less_to_more( &mut self, max_from: T, - first_indexes: &mut brk_vec::StorableVec, - last_indexes: &mut brk_vec::StorableVec, + first_indexes: &mut StorableVec, + last_indexes: &mut StorableVec, exit: &Exit, ) -> Result<()> where @@ -207,7 +207,7 @@ where pub fn compute_last_index_from_first( &mut self, max_from: I, - first_indexes: &mut brk_vec::StorableVec, + first_indexes: &mut StorableVec, final_len: usize, exit: &Exit, ) -> Result<()> @@ -242,8 +242,8 @@ where pub fn compute_count_from_indexes( &mut self, max_from: I, - first_indexes: &mut brk_vec::StorableVec, - last_indexes: &mut brk_vec::StorableVec, + first_indexes: &mut StorableVec, + last_indexes: &mut StorableVec, exit: &Exit, ) -> Result<()> where @@ -270,8 +270,8 @@ where pub fn compute_is_first_ordered( &mut self, max_from: I, - self_to_other: &mut brk_vec::StorableVec, - other_to_self: &mut brk_vec::StorableVec, + self_to_other: &mut StorableVec, + other_to_self: &mut StorableVec, exit: &Exit, ) -> Result<()> where @@ -294,8 +294,8 @@ where pub fn compute_sum_from_indexes( &mut self, max_from: I, - first_indexes: &mut brk_vec::StorableVec, - last_indexes: &mut brk_vec::StorableVec, + first_indexes: &mut StorableVec, + last_indexes: &mut StorableVec, exit: &Exit, ) -> Result<()> where diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index b66007784..5cfff2724 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -1,6 +1,6 @@ use std::{fs, path::Path}; -use brk_core::{CheckedSub, CounterU32, Timestamp}; +use brk_core::{CheckedSub, StoredU32, StoredUsize, Timestamp, Weight}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; @@ -14,7 +14,10 @@ use super::{ #[derive(Clone)] pub struct Vecs { pub indexes_to_block_interval: ComputedVecsFromHeight, - pub indexes_to_block_count: ComputedVecsFromHeight, + pub indexes_to_block_count: ComputedVecsFromHeight, + pub indexes_to_block_weight: ComputedVecsFromHeight, + // pub indexes_to_block_vbytes: ComputedVecsFromHeight<>, + pub indexes_to_block_size: ComputedVecsFromHeight, } impl Vecs { @@ -25,7 +28,8 @@ impl Vecs { indexes_to_block_interval: ComputedVecsFromHeight::forced_import( path, "block_interval", - Version::ONE, + true, + Version::ZERO, compressed, StorableVecGeneatorOptions::default() .add_percentiles() @@ -35,7 +39,24 @@ impl Vecs { indexes_to_block_count: ComputedVecsFromHeight::forced_import( path, "block_count", - Version::ONE, + true, + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_sum().add_total(), + )?, + indexes_to_block_weight: ComputedVecsFromHeight::forced_import( + path, + "block_weight", + false, + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_sum().add_total(), + )?, + indexes_to_block_size: ComputedVecsFromHeight::forced_import( + path, + "block_size", + false, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, @@ -85,12 +106,46 @@ impl Vecs { v.compute_transform( starting_indexes.height, indexer_vecs.height_to_weight.mut_vec(), - |(h, ..)| (h, CounterU32::from(1_u32)), + |(h, ..)| (h, StoredU32::from(1_u32)), exit, ) }, )?; + self.indexes_to_block_weight.compute( + indexer, + indexes, + starting_indexes, + exit, + |v, indexer, _, starting_indexes, exit| { + let indexer_vecs = indexer.mut_vecs(); + + v.compute_transform( + starting_indexes.height, + indexer_vecs.height_to_weight.mut_vec(), + |(h, w)| (h, w), + exit, + ) + }, + )?; + + // self.indexes_to_block_size.compute( + // indexer, + // indexes, + // starting_indexes, + // exit, + // |v, indexer, _, starting_indexes, exit| { + // let indexer_vecs = indexer.mut_vecs(); + + // v.compute_transform( + // starting_indexes.height, + // indexer_vecs.height_to_weight.mut_vec(), + // |(h, ..)| (h, StoredU32::from(1_u32)), + // exit, + // ) + // }, + // )?; + Ok(()) } @@ -98,6 +153,8 @@ impl Vecs { [ self.indexes_to_block_interval.any_vecs(), self.indexes_to_block_count.any_vecs(), + self.indexes_to_block_weight.any_vecs(), + self.indexes_to_block_size.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index db58aed8f..bde5c005c 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -1,9 +1,10 @@ use std::path::Path; use brk_exit::Exit; +use brk_indexer::{Indexer, Indexes}; use brk_vec::{AnyStorableVec, Compressed, Result, StorableVec, StoredIndex, StoredType, Version}; -use crate::storage::vecs::base::ComputedVec; +use crate::storage::vecs::{base::ComputedVec, indexes}; use super::ComputedType; @@ -66,48 +67,48 @@ where let s = Self { first: options.first.then(|| { - ComputedVec::forced_import(&maybe_prefix("first"), Version::ONE, compressed) + ComputedVec::forced_import(&maybe_prefix("first"), Version::ZERO, compressed) .unwrap() }), last: options.last.then(|| { ComputedVec::forced_import( &path.join(format!("{key}_to_{name}")), - Version::ONE, + Version::ZERO, compressed, ) .unwrap() }), min: options.min.then(|| { - ComputedVec::forced_import(&maybe_suffix("min"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("min"), Version::ZERO, compressed).unwrap() }), max: options.max.then(|| { - ComputedVec::forced_import(&maybe_suffix("max"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("max"), Version::ZERO, compressed).unwrap() }), median: options.median.then(|| { - ComputedVec::forced_import(&maybe_suffix("median"), Version::ONE, compressed) + ComputedVec::forced_import(&maybe_suffix("median"), Version::ZERO, compressed) .unwrap() }), average: options.average.then(|| { - ComputedVec::forced_import(&maybe_suffix("average"), Version::ONE, compressed) + ComputedVec::forced_import(&maybe_suffix("average"), Version::ZERO, compressed) .unwrap() }), sum: options.sum.then(|| { - ComputedVec::forced_import(&maybe_suffix("sum"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("sum"), Version::ZERO, compressed).unwrap() }), total: options.total.then(|| { - ComputedVec::forced_import(&prefix("total"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&prefix("total"), Version::ZERO, compressed).unwrap() }), _90p: options._90p.then(|| { - ComputedVec::forced_import(&maybe_suffix("90p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("90p"), Version::ZERO, compressed).unwrap() }), _75p: options._75p.then(|| { - ComputedVec::forced_import(&maybe_suffix("75p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("75p"), Version::ZERO, compressed).unwrap() }), _25p: options._25p.then(|| { - ComputedVec::forced_import(&maybe_suffix("25p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("25p"), Version::ZERO, compressed).unwrap() }), _10p: options._10p.then(|| { - ComputedVec::forced_import(&maybe_suffix("10p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&maybe_suffix("10p"), Version::ZERO, compressed).unwrap() }), }; @@ -154,9 +155,9 @@ where pub fn compute( &mut self, max_from: I, - source: &mut ComputedVec, - first_indexes: &mut brk_vec::StorableVec, - last_indexes: &mut brk_vec::StorableVec, + source: &mut StorableVec, + first_indexes: &mut StorableVec, + last_indexes: &mut StorableVec, exit: &Exit, ) -> Result<()> where @@ -276,8 +277,8 @@ where &mut self, max_from: I, source: &mut ComputedVecBuilder, - first_indexes: &mut brk_vec::StorableVec, - last_indexes: &mut brk_vec::StorableVec, + first_indexes: &mut StorableVec, + last_indexes: &mut StorableVec, exit: &Exit, ) -> Result<()> where @@ -698,3 +699,17 @@ impl StorableVecGeneatorOptions { } } } + +pub enum Source<'a, F, I, T> +where + F: FnMut( + &mut ComputedVec, + &mut Indexer, + &mut indexes::Vecs, + &Indexes, + &Exit, + ) -> Result<()>, +{ + Compute(F), + Ref(&'a mut StorableVec), +} diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs index d586010a8..5b8566465 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs @@ -85,7 +85,7 @@ where self.weekindex.compute( starting_indexes.weekindex, - &mut self.dateindex, + self.dateindex.mut_vec(), indexes.weekindex_to_first_dateindex.mut_vec(), indexes.weekindex_to_last_dateindex.mut_vec(), exit, @@ -93,7 +93,7 @@ where self.monthindex.compute( starting_indexes.monthindex, - &mut self.dateindex, + self.dateindex.mut_vec(), indexes.monthindex_to_first_dateindex.mut_vec(), indexes.monthindex_to_last_dateindex.mut_vec(), exit, diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index 1efcc8e57..34df953c7 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -16,7 +16,7 @@ pub struct ComputedVecsFromHeight where T: ComputedType + PartialOrd, { - pub height: ComputedVec, + pub height: Option>, pub height_extra: ComputedVecBuilder, pub dateindex: ComputedVecBuilder, pub weekindex: ComputedVecBuilder, @@ -36,15 +36,15 @@ where pub fn forced_import( path: &Path, name: &str, + compute_source: bool, version: Version, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let height = ComputedVec::forced_import( - &path.join(format!("height_to_{name}")), - version, - compressed, - )?; + let height = compute_source.then(|| { + ComputedVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed) + .unwrap() + }); let height_extra = ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?; @@ -84,14 +84,16 @@ where &Exit, ) -> Result<()>, { - compute(&mut self.height, indexer, indexes, starting_indexes, exit)?; + if let Some(height) = self.height.as_mut() { + compute(height, indexer, indexes, starting_indexes, exit)?; + } self.height_extra .extend(starting_indexes.height, self.height.mut_vec(), exit)?; self.dateindex.compute( starting_indexes.dateindex, - &mut self.height, + self.height.mut_vec(), indexes.dateindex_to_first_height.mut_vec(), indexes.dateindex_to_last_height.mut_vec(), exit, diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs index 2a40a80e9..5c260d77e 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -75,7 +75,7 @@ where self.difficultyepoch.compute( starting_indexes.difficultyepoch, - &mut self.height, + self.height.mut_vec(), indexes.difficultyepoch_to_first_height.mut_vec(), indexes.difficultyepoch_to_last_height.mut_vec(), exit, diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index d05a7dcbf..73cfee2db 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -99,7 +99,7 @@ where self.height.compute( starting_indexes.height, - &mut self.txindex, + self.txindex.mut_vec(), indexer.mut_vecs().height_to_first_txindex.mut_vec(), indexes.height_to_last_txindex.mut_vec(), exit, diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index b45c564e4..814a1f2da 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -71,242 +71,242 @@ impl Vecs { Ok(Self { dateindex_to_date: ComputedVec::forced_import( &path.join("dateindex_to_date"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_dateindex: ComputedVec::forced_import( &path.join("dateindex_to_dateindex"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_first_height: ComputedVec::forced_import( &path.join("dateindex_to_first_height"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_last_height: ComputedVec::forced_import( &path.join("dateindex_to_last_height"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_real_date: ComputedVec::forced_import( &path.join("height_to_real_date"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_fixed_date: ComputedVec::forced_import( &path.join("height_to_fixed_date"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_dateindex: ComputedVec::forced_import( &path.join("height_to_dateindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_height: ComputedVec::forced_import( &path.join("height_to_height"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_last_txindex: ComputedVec::forced_import( &path.join("height_to_last_txindex"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_last_txinindex: ComputedVec::forced_import( &path.join("txindex_to_last_txinindex"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_last_txoutindex: ComputedVec::forced_import( &path.join("txindex_to_last_txoutindex"), - Version::ONE, + Version::ZERO, compressed, )?, difficultyepoch_to_first_height: ComputedVec::forced_import( &path.join("difficultyepoch_to_first_height"), - Version::ONE, + Version::ZERO, compressed, )?, difficultyepoch_to_last_height: ComputedVec::forced_import( &path.join("difficultyepoch_to_last_height"), - Version::ONE, + Version::ZERO, compressed, )?, halvingepoch_to_first_height: ComputedVec::forced_import( &path.join("halvingepoch_to_first_height"), - Version::ONE, + Version::ZERO, compressed, )?, halvingepoch_to_last_height: ComputedVec::forced_import( &path.join("halvingepoch_to_last_height"), - Version::ONE, + Version::ZERO, compressed, )?, weekindex_to_first_dateindex: ComputedVec::forced_import( &path.join("weekindex_to_first_dateindex"), - Version::ONE, + Version::ZERO, compressed, )?, weekindex_to_last_dateindex: ComputedVec::forced_import( &path.join("weekindex_to_last_dateindex"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_first_dateindex: ComputedVec::forced_import( &path.join("monthindex_to_first_dateindex"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_last_dateindex: ComputedVec::forced_import( &path.join("monthindex_to_last_dateindex"), - Version::ONE, + Version::ZERO, compressed, )?, yearindex_to_first_monthindex: ComputedVec::forced_import( &path.join("yearindex_to_first_monthindex"), - Version::ONE, + Version::ZERO, compressed, )?, yearindex_to_last_monthindex: ComputedVec::forced_import( &path.join("yearindex_to_last_monthindex"), - Version::ONE, + Version::ZERO, compressed, )?, decadeindex_to_first_yearindex: ComputedVec::forced_import( &path.join("decadeindex_to_first_yearindex"), - Version::ONE, + Version::ZERO, compressed, )?, decadeindex_to_last_yearindex: ComputedVec::forced_import( &path.join("decadeindex_to_last_yearindex"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_weekindex: ComputedVec::forced_import( &path.join("dateindex_to_weekindex"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_monthindex: ComputedVec::forced_import( &path.join("dateindex_to_monthindex"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_yearindex: ComputedVec::forced_import( &path.join("monthindex_to_yearindex"), - Version::ONE, + Version::ZERO, compressed, )?, yearindex_to_decadeindex: ComputedVec::forced_import( &path.join("yearindex_to_decadeindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_difficultyepoch: ComputedVec::forced_import( &path.join("height_to_difficultyepoch"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_halvingepoch: ComputedVec::forced_import( &path.join("height_to_halvingepoch"), - Version::ONE, + Version::ZERO, compressed, )?, weekindex_to_weekindex: ComputedVec::forced_import( &path.join("weekindex_to_weekindex"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_monthindex: ComputedVec::forced_import( &path.join("monthindex_to_monthindex"), - Version::ONE, + Version::ZERO, compressed, )?, yearindex_to_yearindex: ComputedVec::forced_import( &path.join("yearindex_to_yearindex"), - Version::ONE, + Version::ZERO, compressed, )?, decadeindex_to_decadeindex: ComputedVec::forced_import( &path.join("decadeindex_to_decadeindex"), - Version::ONE, + Version::ZERO, compressed, )?, difficultyepoch_to_difficultyepoch: ComputedVec::forced_import( &path.join("difficultyepoch_to_difficultyepoch"), - Version::ONE, + Version::ZERO, compressed, )?, halvingepoch_to_halvingepoch: ComputedVec::forced_import( &path.join("halvingepoch_to_halvingepoch"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_timestamp: ComputedVec::forced_import( &path.join("dateindex_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, decadeindex_to_timestamp: ComputedVec::forced_import( &path.join("decadeindex_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, difficultyepoch_to_timestamp: ComputedVec::forced_import( &path.join("difficultyepoch_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, halvingepoch_to_timestamp: ComputedVec::forced_import( &path.join("halvingepoch_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_timestamp: ComputedVec::forced_import( &path.join("monthindex_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, weekindex_to_timestamp: ComputedVec::forced_import( &path.join("weekindex_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, yearindex_to_timestamp: ComputedVec::forced_import( &path.join("yearindex_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_fixed_timestamp: ComputedVec::forced_import( &path.join("height_to_fixed_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_quarterindex: ComputedVec::forced_import( &path.join("monthindex_to_quarterindex"), - Version::ONE, + Version::ZERO, compressed, )?, quarterindex_to_first_monthindex: ComputedVec::forced_import( &path.join("quarterindex_to_first_monthindex"), - Version::ONE, + Version::ZERO, compressed, )?, quarterindex_to_last_monthindex: ComputedVec::forced_import( &path.join("quarterindex_to_last_monthindex"), - Version::ONE, + Version::ZERO, compressed, )?, quarterindex_to_quarterindex: ComputedVec::forced_import( &path.join("quarterindex_to_quarterindex"), - Version::ONE, + Version::ZERO, compressed, )?, quarterindex_to_timestamp: ComputedVec::forced_import( &path.join("quarterindex_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, }) diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index b77202991..c3946299e 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -63,163 +63,163 @@ impl Vecs { Ok(Self { dateindex_to_ohlc_in_cents: ComputedVec::forced_import( &fetched_path.join("dateindex_to_ohlc_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_ohlc: ComputedVec::forced_import( &path.join("dateindex_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_close_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_close_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_high_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_high_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_low_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_low_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, dateindex_to_open_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_open_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_ohlc_in_cents: ComputedVec::forced_import( &fetched_path.join("height_to_ohlc_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_ohlc: ComputedVec::forced_import( &path.join("height_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_close_in_cents: ComputedVec::forced_import( &path.join("height_to_close_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_high_in_cents: ComputedVec::forced_import( &path.join("height_to_high_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_low_in_cents: ComputedVec::forced_import( &path.join("height_to_low_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_open_in_cents: ComputedVec::forced_import( &path.join("height_to_open_in_cents"), - Version::ONE, + Version::ZERO, compressed, )?, timeindexes_to_open: ComputedVecsFromDateindex::forced_import( path, "open", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_first(), )?, timeindexes_to_high: ComputedVecsFromDateindex::forced_import( path, "high", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_max(), )?, timeindexes_to_low: ComputedVecsFromDateindex::forced_import( path, "low", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_min(), )?, timeindexes_to_close: ComputedVecsFromDateindex::forced_import( path, "close", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_last(), )?, timeindexes_to_sats_per_dollar: ComputedVecsFromDateindex::forced_import( path, "sats_per_dollar", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_last(), )?, chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( path, "open", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_first(), )?, chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( path, "high", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_max(), )?, chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( path, "low", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_min(), )?, chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( path, "close", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_last(), )?, chainindexes_to_sats_per_dollar: ComputedVecsFromHeightStrict::forced_import( path, "sats_per_dollar", - Version::ONE, + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_last(), )?, weekindex_to_ohlc: ComputedVec::forced_import( &path.join("weekindex_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, difficultyepoch_to_ohlc: ComputedVec::forced_import( &path.join("difficultyepoch_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, monthindex_to_ohlc: ComputedVec::forced_import( &path.join("monthindex_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, quarterindex_to_ohlc: ComputedVec::forced_import( &path.join("quarterindex_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, yearindex_to_ohlc: ComputedVec::forced_import( &path.join("yearindex_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, - // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ONE, compressed)?, + // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ZERO, compressed)?, decadeindex_to_ohlc: ComputedVec::forced_import( &path.join("decadeindex_to_ohlc"), - Version::ONE, + Version::ZERO, compressed, )?, }) diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 143bc8392..2c630c89a 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -1,6 +1,6 @@ use std::{fs, path::Path}; -use brk_core::{CounterU64, Txindex}; +use brk_core::{StoredU64, Txindex}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; @@ -25,10 +25,10 @@ pub struct Vecs { // pub txindex_to_fee: ComputedVec, pub txindex_to_is_coinbase: ComputedVec, // pub txindex_to_feerate: ComputedVec, - pub txindex_to_inputs_count: ComputedVecsFromTxindex, - // pub txindex_to_inputs_sum: ComputedVec, - pub txindex_to_outputs_count: ComputedVecsFromTxindex, - // pub txindex_to_outputs_sum: ComputedVec, + pub txindex_to_input_count: ComputedVecsFromTxindex, + // pub txindex_to_input_sum: ComputedVec, + pub txindex_to_output_count: ComputedVecsFromTxindex, + // pub txindex_to_output_sum: ComputedVec, // pub txinindex_to_value: ComputedVec, } @@ -37,48 +37,48 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::ONE)?, + // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::ZERO)?, // height_to_input_count: StorableVec::forced_import( // &path.join("height_to_input_count"), - // Version::ONE, + // Version::ZERO, // )?, - // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::ONE)?, - // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::ONE)?, - // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::ONE)?, + // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::ZERO)?, + // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::ZERO)?, + // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::ZERO)?, // height_to_output_count: StorableVec::forced_import( // &path.join("height_to_output_count"), - // Version::ONE, + // Version::ZERO, // )?, - // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ONE)?, - // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::ONE)?, - // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::ONE)?, + // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ZERO)?, + // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::ZERO)?, + // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::ZERO)?, // txindex_to_fee: StorableVec::forced_import( // &path.join("txindex_to_fee"), - // Version::ONE, + // Version::ZERO, // )?, txindex_to_is_coinbase: ComputedVec::forced_import( &path.join("txindex_to_is_coinbase"), - Version::ONE, + Version::ZERO, compressed, )?, - // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::ONE)?, - txindex_to_inputs_count: ComputedVecsFromTxindex::forced_import( + // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::ZERO)?, + txindex_to_input_count: ComputedVecsFromTxindex::forced_import( path, - "inputs_count", - Version::ONE, + "input_count", + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, - txindex_to_outputs_count: ComputedVecsFromTxindex::forced_import( + txindex_to_output_count: ComputedVecsFromTxindex::forced_import( path, - "outputs_count", - Version::ONE, + "output_count", + Version::ZERO, compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, // txinindex_to_value: StorableVec::forced_import( // &path.join("txinindex_to_value"), - // Version::ONE, + // Version::ZERO, // compressed, // )?, }) @@ -91,7 +91,7 @@ impl Vecs { starting_indexes: &Indexes, exit: &Exit, ) -> color_eyre::Result<()> { - self.txindex_to_inputs_count.compute( + self.txindex_to_input_count.compute( indexer, indexes, starting_indexes, @@ -106,7 +106,7 @@ impl Vecs { }, )?; - self.txindex_to_outputs_count.compute( + self.txindex_to_output_count.compute( indexer, indexes, starting_indexes, @@ -121,20 +121,6 @@ impl Vecs { }, )?; - // self.txindex_to_inputs_count.compute_count_from_indexes( - // starting_indexes.txindex, - // indexer_vecs.txindex_to_first_txinindex.mut_vec(), - // indexes.txindex_to_last_txinindex.mut_vec(), - // exit, - // )?; - - // self.txindex_to_outputs_count.compute_count_from_indexes( - // starting_indexes.txindex, - // indexer_vecs.txindex_to_first_txoutindex.mut_vec(), - // indexes.txindex_to_last_txoutindex.mut_vec(), - // exit, - // )?; - let indexer_vecs = indexer.mut_vecs(); self.txindex_to_is_coinbase.compute_is_first_ordered( @@ -171,8 +157,8 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ vec![self.txindex_to_is_coinbase.any_vec()], - self.txindex_to_outputs_count.any_vecs(), - self.txindex_to_inputs_count.any_vecs(), + self.txindex_to_output_count.any_vecs(), + self.txindex_to_input_count.any_vecs(), ] .concat() } diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_core/src/structs/mod.rs index 780c04f2e..c0ea3a7a6 100644 --- a/crates/brk_core/src/structs/mod.rs +++ b/crates/brk_core/src/structs/mod.rs @@ -7,8 +7,6 @@ mod bitcoin; mod blockhash; mod cents; mod compressed; -mod counter_u32; -mod counter_u64; mod date; mod dateindex; mod decadeindex; @@ -22,6 +20,9 @@ mod monthindex; mod ohlc; mod quarterindex; mod sats; +mod stored_u32; +mod stored_u64; +mod stored_usize; mod timestamp; mod txid; mod txindex; @@ -44,8 +45,6 @@ pub use bitcoin::*; pub use blockhash::*; pub use cents::*; pub use compressed::*; -pub use counter_u32::*; -pub use counter_u64::*; pub use date::*; pub use dateindex::*; pub use decadeindex::*; @@ -59,6 +58,9 @@ pub use monthindex::*; pub use ohlc::*; pub use quarterindex::*; pub use sats::*; +pub use stored_u32::*; +pub use stored_u64::*; +pub use stored_usize::*; pub use timestamp::*; pub use txid::*; pub use txindex::*; diff --git a/crates/brk_core/src/structs/counter_u32.rs b/crates/brk_core/src/structs/stored_u32.rs similarity index 77% rename from crates/brk_core/src/structs/counter_u32.rs rename to crates/brk_core/src/structs/stored_u32.rs index 43c8c4495..8fb595cc9 100644 --- a/crates/brk_core/src/structs/counter_u32.rs +++ b/crates/brk_core/src/structs/stored_u32.rs @@ -21,9 +21,9 @@ use crate::CheckedSub; KnownLayout, Serialize, )] -pub struct CounterU32(u32); +pub struct StoredU32(u32); -impl CounterU32 { +impl StoredU32 { pub const ZERO: Self = Self(0); pub fn new(counter: u32) -> Self { @@ -31,39 +31,39 @@ impl CounterU32 { } } -impl From for CounterU32 { +impl From for StoredU32 { fn from(value: u32) -> Self { Self(value) } } -impl From for CounterU32 { +impl From for StoredU32 { fn from(value: usize) -> Self { Self(value as u32) } } -impl CheckedSub for CounterU32 { +impl CheckedSub for StoredU32 { fn checked_sub(self, rhs: Self) -> Option { self.0.checked_sub(rhs.0).map(Self) } } -impl Div for CounterU32 { +impl Div for StoredU32 { type Output = Self; fn div(self, rhs: usize) -> Self::Output { Self(self.0 / rhs as u32) } } -impl Add for CounterU32 { +impl Add for StoredU32 { type Output = Self; fn add(self, rhs: Self) -> Self::Output { Self(self.0 + rhs.0) } } -impl From for CounterU32 { +impl From for StoredU32 { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { panic!() @@ -72,8 +72,8 @@ impl From for CounterU32 { } } -impl From for f64 { - fn from(value: CounterU32) -> Self { +impl From for f64 { + fn from(value: StoredU32) -> Self { value.0 as f64 } } diff --git a/crates/brk_core/src/structs/counter_u64.rs b/crates/brk_core/src/structs/stored_u64.rs similarity index 76% rename from crates/brk_core/src/structs/counter_u64.rs rename to crates/brk_core/src/structs/stored_u64.rs index 6df79bfac..365276b8f 100644 --- a/crates/brk_core/src/structs/counter_u64.rs +++ b/crates/brk_core/src/structs/stored_u64.rs @@ -23,9 +23,9 @@ use super::{Txinindex, Txoutindex}; KnownLayout, Serialize, )] -pub struct CounterU64(u64); +pub struct StoredU64(u64); -impl CounterU64 { +impl StoredU64 { pub const ZERO: Self = Self(0); pub fn new(counter: u64) -> Self { @@ -33,39 +33,39 @@ impl CounterU64 { } } -impl From for CounterU64 { +impl From for StoredU64 { fn from(value: u64) -> Self { Self(value) } } -impl From for CounterU64 { +impl From for StoredU64 { fn from(value: usize) -> Self { Self(value as u64) } } -impl CheckedSub for CounterU64 { +impl CheckedSub for StoredU64 { fn checked_sub(self, rhs: Self) -> Option { self.0.checked_sub(rhs.0).map(Self) } } -impl Div for CounterU64 { +impl Div for StoredU64 { type Output = Self; fn div(self, rhs: usize) -> Self::Output { Self(self.0 / rhs as u64) } } -impl Add for CounterU64 { +impl Add for StoredU64 { type Output = Self; fn add(self, rhs: Self) -> Self::Output { Self(self.0 + rhs.0) } } -impl From for CounterU64 { +impl From for StoredU64 { fn from(value: f64) -> Self { if value < 0.0 || value > u32::MAX as f64 { panic!() @@ -74,19 +74,19 @@ impl From for CounterU64 { } } -impl From for f64 { - fn from(value: CounterU64) -> Self { +impl From for f64 { + fn from(value: StoredU64) -> Self { value.0 as f64 } } -impl From for CounterU64 { +impl From for StoredU64 { fn from(value: Txinindex) -> Self { Self(*value) } } -impl From for CounterU64 { +impl From for StoredU64 { fn from(value: Txoutindex) -> Self { Self(*value) } diff --git a/crates/brk_core/src/structs/stored_usize.rs b/crates/brk_core/src/structs/stored_usize.rs new file mode 100644 index 000000000..4a7cd7d96 --- /dev/null +++ b/crates/brk_core/src/structs/stored_usize.rs @@ -0,0 +1,73 @@ +use std::ops::{Add, Div}; + +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::CheckedSub; + +#[derive( + Debug, + Deref, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] +pub struct StoredUsize(usize); + +impl StoredUsize { + pub const ZERO: Self = Self(0); + + pub fn new(counter: usize) -> Self { + Self(counter) + } +} + +impl From for StoredUsize { + fn from(value: usize) -> Self { + Self(value) + } +} + +impl CheckedSub for StoredUsize { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} + +impl Div for StoredUsize { + type Output = Self; + fn div(self, rhs: usize) -> Self::Output { + Self(self.0 / rhs) + } +} + +impl Add for StoredUsize { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl From for StoredUsize { + fn from(value: f64) -> Self { + if value < 0.0 || value > u32::MAX as f64 { + panic!() + } + Self(value as usize) + } +} + +impl From for f64 { + fn from(value: StoredUsize) -> Self { + value.0 as f64 + } +} diff --git a/crates/brk_core/src/structs/weight.rs b/crates/brk_core/src/structs/weight.rs index 0b086e94d..991915a95 100644 --- a/crates/brk_core/src/structs/weight.rs +++ b/crates/brk_core/src/structs/weight.rs @@ -1,8 +1,24 @@ +use std::ops::{Add, Div}; + use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)] +#[derive( + Debug, + Deref, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Immutable, + IntoBytes, + KnownLayout, + FromBytes, + Serialize, +)] pub struct Weight(u64); impl From for Weight { @@ -16,3 +32,35 @@ impl From for bitcoin::Weight { Self::from_wu(*value) } } + +impl From for Weight { + fn from(value: usize) -> Self { + Self(value as u64) + } +} + +impl From for Weight { + fn from(value: f64) -> Self { + Self(value as u64) + } +} + +impl From for f64 { + fn from(value: Weight) -> Self { + value.0 as f64 + } +} + +impl Add for Weight { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl Div for Weight { + type Output = Self; + fn div(self, rhs: usize) -> Self::Output { + Self::from(self.0 as usize / rhs) + } +} diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index d2d66d20c..aeb4b1b4b 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -5,9 +5,10 @@ use brk_core::{ Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; +use brk_vec::{Result, StoredIndex, StoredType, Value}; use color_eyre::eyre::ContextCompat; -use crate::{Stores, Vecs}; +use crate::{IndexedVec, Stores, Vecs}; #[derive(Debug, Default, Clone)] pub struct Indexes { @@ -65,13 +66,7 @@ impl Indexes { .push_if_needed(height, self.p2wpkhindex)?; vecs.height_to_first_p2wshindex .push_if_needed(height, self.p2wshindex)?; - Ok(()) - } - pub fn push_future_if_needed(&mut self, vecs: &mut Vecs) -> brk_vec::Result<()> { - self.height.increment(); - self.push_if_needed(vecs)?; - self.height.decrement(); Ok(()) } } @@ -111,32 +106,121 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { .unwrap_or(starting_height); Ok(Self { - addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?, - emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?, + addressindex: *starting_index( + &vecs.height_to_first_addressindex, + &vecs.addressindex_to_height, + height, + )? + .context("")?, + emptyindex: *starting_index( + &vecs.height_to_first_emptyindex, + &vecs.emptyindex_to_height, + height, + )? + .context("")?, height, - multisigindex: *vecs - .height_to_first_multisigindex - .get(height)? - .context("")?, - opreturnindex: *vecs - .height_to_first_opreturnindex - .get(height)? - .context("")?, - p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?, - p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?, - p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?, - p2shindex: *vecs.height_to_first_p2shindex.get(height)?.context("")?, - p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?, - p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?, - p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?, - pushonlyindex: *vecs - .height_to_first_pushonlyindex - .get(height)? - .context("")?, - txindex: *vecs.height_to_first_txindex.get(height)?.context("")?, - txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?, - txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?, - unknownindex: *vecs.height_to_first_unknownindex.get(height)?.context("")?, + multisigindex: *starting_index( + &vecs.height_to_first_multisigindex, + &vecs.multisigindex_to_height, + height, + )? + .context("")?, + opreturnindex: *starting_index( + &vecs.height_to_first_opreturnindex, + &vecs.opreturnindex_to_height, + height, + )? + .context("")?, + p2pk33index: *starting_index( + &vecs.height_to_first_p2pk33index, + &vecs.p2pk33index_to_height, + height, + )? + .context("")?, + p2pk65index: *starting_index( + &vecs.height_to_first_p2pk65index, + &vecs.p2pk65index_to_height, + height, + )? + .context("")?, + p2pkhindex: *starting_index( + &vecs.height_to_first_p2pkhindex, + &vecs.p2pkhindex_to_height, + height, + )? + .context("")?, + p2shindex: *starting_index( + &vecs.height_to_first_p2shindex, + &vecs.p2shindex_to_height, + height, + )? + .context("")?, + p2trindex: *starting_index( + &vecs.height_to_first_p2trindex, + &vecs.p2trindex_to_height, + height, + )? + .context("")?, + p2wpkhindex: *starting_index( + &vecs.height_to_first_p2wpkhindex, + &vecs.p2wpkhindex_to_height, + height, + )? + .context("")?, + p2wshindex: *starting_index( + &vecs.height_to_first_p2wshindex, + &vecs.p2wshindex_to_height, + height, + )? + .context("")?, + pushonlyindex: *starting_index( + &vecs.height_to_first_pushonlyindex, + &vecs.pushonlyindex_to_height, + height, + )? + .context("")?, + txindex: *starting_index( + &vecs.height_to_first_txindex, + &vecs.txindex_to_height, + height, + )? + .context("")?, + txinindex: *starting_index( + &vecs.height_to_first_txinindex, + &vecs.txinindex_to_height, + height, + )? + .context("")?, + txoutindex: *starting_index( + &vecs.height_to_first_txoutindex, + &vecs.txoutindex_to_height, + height, + )? + .context("")?, + unknownindex: *starting_index( + &vecs.height_to_first_unknownindex, + &vecs.unknownindex_to_height, + height, + )? + .context("")?, }) } } + +pub fn starting_index<'a, I>( + height_to_index: &'a IndexedVec, + index_to_height: &'a IndexedVec, + starting_height: Height, +) -> Result>> +where + I: StoredType + StoredIndex + From, +{ + if height_to_index + .height() + .is_ok_and(|h| h + 1_u32 == starting_height) + { + Ok(Some(Value::Owned(I::from(index_to_height.len())))) + } else { + height_to_index.get(starting_height) + } +} diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index c138d6d09..db826cab2 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -31,7 +31,7 @@ pub use stores::*; pub use vecs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1000; -const COLLISIONS_CHECKED_UP_TO: u32 = 888_000; +const COLLISIONS_CHECKED_UP_TO: u32 = 890_000; #[derive(Clone)] pub struct Indexer { @@ -101,6 +101,7 @@ impl Indexer { let vecs = self.vecs.as_mut().unwrap(); let stores = self.stores.as_mut().unwrap(); + // Cloned because we want to return starting indexes for the computer let mut idxs = starting_indexes.clone(); let start = Some(idxs.height); @@ -153,6 +154,8 @@ impl Indexer { return Err(eyre!("Collision, expect prefix to need be set yet")); } + idxs.push_if_needed(vecs)?; + stores .blockhash_prefix_to_height .insert_if_needed(blockhash_prefix, height, height); @@ -462,6 +465,9 @@ impl Indexer { vecs.txoutindex_to_value.push_if_needed(txoutindex, sats)?; + vecs.txoutindex_to_height + .push_if_needed(txoutindex, height)?; + let mut addressindex = idxs.addressindex; let mut addresshash = None; @@ -481,18 +487,55 @@ impl Indexer { idxs.addressindex.increment(); let addresstypeindex = match addresstype { - Addresstype::Empty => idxs.emptyindex.copy_then_increment(), - Addresstype::Multisig => idxs.multisigindex.copy_then_increment(), - Addresstype::OpReturn => idxs.opreturnindex.copy_then_increment(), - Addresstype::PushOnly => idxs.pushonlyindex.copy_then_increment(), - Addresstype::Unknown => idxs.unknownindex.copy_then_increment(), - Addresstype::P2PK65 => idxs.p2pk65index.copy_then_increment(), - Addresstype::P2PK33 => idxs.p2pk33index.copy_then_increment(), - Addresstype::P2PKH => idxs.p2pkhindex.copy_then_increment(), - Addresstype::P2SH => idxs.p2shindex.copy_then_increment(), - Addresstype::P2WPKH => idxs.p2wpkhindex.copy_then_increment(), - Addresstype::P2WSH => idxs.p2wshindex.copy_then_increment(), - Addresstype::P2TR => idxs.p2trindex.copy_then_increment(), + Addresstype::Empty => { + vecs.emptyindex_to_height + .push_if_needed(idxs.emptyindex, height)?; + idxs.emptyindex.copy_then_increment() + }, + Addresstype::Multisig => { + vecs.multisigindex_to_height.push_if_needed(idxs.multisigindex, height)?; + idxs.multisigindex.copy_then_increment() + }, + Addresstype::OpReturn => { + vecs.opreturnindex_to_height.push_if_needed(idxs.opreturnindex, height)?; + idxs.opreturnindex.copy_then_increment() + }, + Addresstype::PushOnly => { + vecs.pushonlyindex_to_height.push_if_needed(idxs.pushonlyindex, height)?; + idxs.pushonlyindex.copy_then_increment() + }, + Addresstype::Unknown => { + vecs.unknownindex_to_height.push_if_needed(idxs.unknownindex, height)?; + idxs.unknownindex.copy_then_increment() + }, + Addresstype::P2PK65 => { + vecs.p2pk65index_to_height.push_if_needed(idxs.p2pk65index, height)?; + idxs.p2pk65index.copy_then_increment() + }, + Addresstype::P2PK33 => { + vecs.p2pk33index_to_height.push_if_needed(idxs.p2pk33index, height)?; + idxs.p2pk33index.copy_then_increment() + }, + Addresstype::P2PKH => { + vecs.p2pkhindex_to_height.push_if_needed(idxs.p2pkhindex, height)?; + idxs.p2pkhindex.copy_then_increment() + }, + Addresstype::P2SH => { + vecs.p2shindex_to_height.push_if_needed(idxs.p2shindex, height)?; + idxs.p2shindex.copy_then_increment() + }, + Addresstype::P2WPKH => { + vecs.p2wpkhindex_to_height.push_if_needed(idxs.p2wpkhindex, height)?; + idxs.p2wpkhindex.copy_then_increment() + }, + Addresstype::P2WSH => { + vecs.p2wshindex_to_height.push_if_needed(idxs.p2wshindex, height)?; + idxs.p2wshindex.copy_then_increment() + }, + Addresstype::P2TR => { + vecs.p2trindex_to_height.push_if_needed(idxs.p2trindex, height)?; + idxs.p2trindex.copy_then_increment() + }, }; vecs.addressindex_to_addresstype @@ -580,6 +623,10 @@ impl Indexer { vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?; + vecs.txinindex_to_height + .push_if_needed(txinindex, height)?; + + Ok(()) })?; @@ -668,8 +715,6 @@ impl Indexer { idxs.txinindex += Txinindex::from(inputs_len); idxs.txoutindex += Txoutindex::from(outputs_len); - idxs.push_future_if_needed(vecs)?; - export_if_needed(stores, vecs, height, false, exit)?; Ok(()) diff --git a/crates/brk_indexer/src/stores/mod.rs b/crates/brk_indexer/src/stores/mod.rs index abaac46eb..ecda0e5b6 100644 --- a/crates/brk_indexer/src/stores/mod.rs +++ b/crates/brk_indexer/src/stores/mod.rs @@ -27,11 +27,11 @@ impl Stores { pub fn import(path: &Path) -> color_eyre::Result { thread::scope(|scope| { let addresshash_to_addressindex = scope - .spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::ONE)); + .spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::ZERO)); let blockhash_prefix_to_height = scope - .spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::ONE)); + .spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::ZERO)); let txid_prefix_to_txindex = - scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::ONE)); + scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::ZERO)); Ok(Self { addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?, diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_indexer/src/vecs/base.rs index fd94b769f..7199ed0c9 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_indexer/src/vecs/base.rs @@ -33,7 +33,7 @@ where version: Version, compressed: Compressed, ) -> brk_vec::Result { - let mut vec = brk_vec::StorableVec::forced_import(path, version, compressed)?; + let mut vec = StorableVec::forced_import(path, version, compressed)?; vec.enable_large_cache(); @@ -107,7 +107,7 @@ where Ok(()) } Ordering::Less => { - dbg!(index, value); + dbg!(index, value, self.vec.len(), self.path_height()); Err(Error::IndexTooHigh) } } @@ -126,11 +126,11 @@ where self.vec.flush() } - pub fn vec(&self) -> &brk_vec::StorableVec { + pub fn vec(&self) -> &StorableVec { &self.vec } - pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec { + pub fn mut_vec(&mut self) -> &mut StorableVec { &mut self.vec } diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 2791bf927..66afccdbb 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -21,17 +21,13 @@ pub struct Vecs { pub addressindex_to_addresstype: IndexedVec, pub addressindex_to_addresstypeindex: IndexedVec, pub addressindex_to_height: IndexedVec, + pub emptyindex_to_height: IndexedVec, pub height_to_blockhash: IndexedVec, pub height_to_difficulty: IndexedVec, pub height_to_first_addressindex: IndexedVec, pub height_to_first_emptyindex: IndexedVec, pub height_to_first_multisigindex: IndexedVec, pub height_to_first_opreturnindex: IndexedVec, - pub height_to_first_pushonlyindex: IndexedVec, - pub height_to_first_txindex: IndexedVec, - pub height_to_first_txinindex: IndexedVec, - pub height_to_first_txoutindex: IndexedVec, - pub height_to_first_unknownindex: IndexedVec, pub height_to_first_p2pk33index: IndexedVec, pub height_to_first_p2pk65index: IndexedVec, pub height_to_first_p2pkhindex: IndexedVec, @@ -39,29 +35,47 @@ pub struct Vecs { pub height_to_first_p2trindex: IndexedVec, pub height_to_first_p2wpkhindex: IndexedVec, pub height_to_first_p2wshindex: IndexedVec, + pub height_to_first_pushonlyindex: IndexedVec, + pub height_to_first_txindex: IndexedVec, + pub height_to_first_txinindex: IndexedVec, + pub height_to_first_txoutindex: IndexedVec, + pub height_to_first_unknownindex: IndexedVec, pub height_to_size: IndexedVec, pub height_to_timestamp: IndexedVec, pub height_to_weight: IndexedVec, + pub multisigindex_to_height: IndexedVec, + pub opreturnindex_to_height: IndexedVec, + pub p2pk33index_to_height: IndexedVec, pub p2pk33index_to_p2pk33addressbytes: IndexedVec, + pub p2pk65index_to_height: IndexedVec, pub p2pk65index_to_p2pk65addressbytes: IndexedVec, + pub p2pkhindex_to_height: IndexedVec, pub p2pkhindex_to_p2pkhaddressbytes: IndexedVec, + pub p2shindex_to_height: IndexedVec, pub p2shindex_to_p2shaddressbytes: IndexedVec, + pub p2trindex_to_height: IndexedVec, pub p2trindex_to_p2traddressbytes: IndexedVec, + pub p2wpkhindex_to_height: IndexedVec, pub p2wpkhindex_to_p2wpkhaddressbytes: IndexedVec, + pub p2wshindex_to_height: IndexedVec, pub p2wshindex_to_p2wshaddressbytes: IndexedVec, + pub pushonlyindex_to_height: IndexedVec, + pub txindex_to_base_size: IndexedVec, pub txindex_to_first_txinindex: IndexedVec, pub txindex_to_first_txoutindex: IndexedVec, pub txindex_to_height: IndexedVec, - pub txindex_to_locktime: IndexedVec, - pub txindex_to_txid: IndexedVec, - pub txindex_to_base_size: IndexedVec, - pub txindex_to_total_size: IndexedVec, pub txindex_to_is_explicitly_rbf: IndexedVec, + pub txindex_to_locktime: IndexedVec, + pub txindex_to_total_size: IndexedVec, + pub txindex_to_txid: IndexedVec, pub txindex_to_txversion: IndexedVec, + pub txinindex_to_height: IndexedVec, /// If txoutindex == Txoutindex MAX then is it's coinbase pub txinindex_to_txoutindex: IndexedVec, pub txoutindex_to_addressindex: IndexedVec, + pub txoutindex_to_height: IndexedVec, pub txoutindex_to_value: IndexedVec, + pub unknownindex_to_height: IndexedVec, } impl Vecs { @@ -71,217 +85,287 @@ impl Vecs { Ok(Self { addressindex_to_addresstype: IndexedVec::forced_import( &path.join("addressindex_to_addresstype"), - Version::ONE, + Version::ZERO, compressed, )?, addressindex_to_addresstypeindex: IndexedVec::forced_import( &path.join("addressindex_to_addresstypeindex"), - Version::ONE, + Version::ZERO, compressed, )?, addressindex_to_height: IndexedVec::forced_import( &path.join("addressindex_to_height"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_blockhash: IndexedVec::forced_import( &path.join("height_to_blockhash"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, height_to_difficulty: IndexedVec::forced_import( &path.join("height_to_difficulty"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_addressindex: IndexedVec::forced_import( &path.join("height_to_first_addressindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_emptyindex: IndexedVec::forced_import( &path.join("height_to_first_emptyindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_multisigindex: IndexedVec::forced_import( &path.join("height_to_first_multisigindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_opreturnindex: IndexedVec::forced_import( &path.join("height_to_first_opreturnindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_pushonlyindex: IndexedVec::forced_import( &path.join("height_to_first_pushonlyindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_txindex: IndexedVec::forced_import( &path.join("height_to_first_txindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_txinindex: IndexedVec::forced_import( &path.join("height_to_first_txinindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_txoutindex: IndexedVec::forced_import( &path.join("height_to_first_txoutindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_unknownindex: IndexedVec::forced_import( &path.join("height_to_first_unkownindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2pk33index: IndexedVec::forced_import( &path.join("height_to_first_p2pk33index"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2pk65index: IndexedVec::forced_import( &path.join("height_to_first_p2pk65index"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2pkhindex: IndexedVec::forced_import( &path.join("height_to_first_p2pkhindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2shindex: IndexedVec::forced_import( &path.join("height_to_first_p2shindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2trindex: IndexedVec::forced_import( &path.join("height_to_first_p2trindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2wpkhindex: IndexedVec::forced_import( &path.join("height_to_first_p2wpkhindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_first_p2wshindex: IndexedVec::forced_import( &path.join("height_to_first_p2wshindex"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_size: IndexedVec::forced_import( &path.join("height_to_size"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_timestamp: IndexedVec::forced_import( &path.join("height_to_timestamp"), - Version::ONE, + Version::ZERO, compressed, )?, height_to_weight: IndexedVec::forced_import( &path.join("height_to_weight"), - Version::ONE, + Version::ZERO, compressed, )?, p2pk33index_to_p2pk33addressbytes: IndexedVec::forced_import( &path.join("p2pk33index_to_p2pk33addressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, p2pk65index_to_p2pk65addressbytes: IndexedVec::forced_import( &path.join("p2pk65index_to_p2pk65addressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, p2pkhindex_to_p2pkhaddressbytes: IndexedVec::forced_import( &path.join("p2pkhindex_to_p2pkhaddressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, p2shindex_to_p2shaddressbytes: IndexedVec::forced_import( &path.join("p2shindex_to_p2shaddressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, p2trindex_to_p2traddressbytes: IndexedVec::forced_import( &path.join("p2trindex_to_p2traddressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, p2wpkhindex_to_p2wpkhaddressbytes: IndexedVec::forced_import( &path.join("p2wpkhindex_to_p2wpkhaddressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, p2wshindex_to_p2wshaddressbytes: IndexedVec::forced_import( &path.join("p2wshindex_to_p2wshaddressbytes"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, txindex_to_first_txinindex: IndexedVec::forced_import( &path.join("txindex_to_first_txinindex"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_first_txoutindex: IndexedVec::forced_import( &path.join("txindex_to_first_txoutindex"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, txindex_to_height: IndexedVec::forced_import( &path.join("txindex_to_height"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_locktime: IndexedVec::forced_import( &path.join("txindex_to_locktime"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_txid: IndexedVec::forced_import( &path.join("txindex_to_txid"), - Version::ONE, + Version::ZERO, Compressed::NO, )?, txindex_to_base_size: IndexedVec::forced_import( &path.join("txindex_to_base_size"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_total_size: IndexedVec::forced_import( &path.join("txindex_to_total_size"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_is_explicitly_rbf: IndexedVec::forced_import( &path.join("txindex_to_is_explicitly_rbf"), - Version::ONE, + Version::ZERO, compressed, )?, txindex_to_txversion: IndexedVec::forced_import( &path.join("txindex_to_txversion"), - Version::ONE, + Version::ZERO, compressed, )?, txinindex_to_txoutindex: IndexedVec::forced_import( &path.join("txinindex_to_txoutindex"), - Version::ONE, + Version::ZERO, compressed, )?, txoutindex_to_addressindex: IndexedVec::forced_import( &path.join("txoutindex_to_addressindex"), - Version::ONE, + Version::ZERO, compressed, )?, txoutindex_to_value: IndexedVec::forced_import( &path.join("txoutindex_to_value"), - Version::ONE, + Version::ZERO, + compressed, + )?, + emptyindex_to_height: IndexedVec::forced_import( + &path.join("emptyindex_to_height"), + Version::ZERO, + compressed, + )?, + multisigindex_to_height: IndexedVec::forced_import( + &path.join("multisigindex_to_height"), + Version::ZERO, + compressed, + )?, + opreturnindex_to_height: IndexedVec::forced_import( + &path.join("opreturnindex_to_height"), + Version::ZERO, + compressed, + )?, + pushonlyindex_to_height: IndexedVec::forced_import( + &path.join("pushonlyindex_to_height"), + Version::ZERO, + compressed, + )?, + txinindex_to_height: IndexedVec::forced_import( + &path.join("txinindex_to_height"), + Version::ZERO, + compressed, + )?, + txoutindex_to_height: IndexedVec::forced_import( + &path.join("txoutindex_to_height"), + Version::ZERO, + compressed, + )?, + unknownindex_to_height: IndexedVec::forced_import( + &path.join("unknownindex_to_height"), + Version::ZERO, + compressed, + )?, + p2pk33index_to_height: IndexedVec::forced_import( + &path.join("p2pk33index_to_height"), + Version::ZERO, + compressed, + )?, + p2pk65index_to_height: IndexedVec::forced_import( + &path.join("p2pk65index_to_height"), + Version::ZERO, + compressed, + )?, + p2pkhindex_to_height: IndexedVec::forced_import( + &path.join("p2pkhindex_to_height"), + Version::ZERO, + compressed, + )?, + p2shindex_to_height: IndexedVec::forced_import( + &path.join("p2shindex_to_height"), + Version::ZERO, + compressed, + )?, + p2trindex_to_height: IndexedVec::forced_import( + &path.join("p2trindex_to_height"), + Version::ZERO, + compressed, + )?, + p2wpkhindex_to_height: IndexedVec::forced_import( + &path.join("p2wpkhindex_to_height"), + Version::ZERO, + compressed, + )?, + p2wshindex_to_height: IndexedVec::forced_import( + &path.join("p2wshindex_to_height"), + Version::ZERO, compressed, )?, }) @@ -290,8 +374,26 @@ impl Vecs { pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> { let saved_height = starting_indexes.height.decremented().unwrap_or_default(); - // We don't want to override the starting indexes so we cut from n + 1 - let height = starting_indexes.height.incremented(); + // Now we can cut everything that's out of date + let &Indexes { + addressindex, + height, + p2pk33index, + p2pk65index, + p2pkhindex, + p2shindex, + p2trindex, + p2wpkhindex, + p2wshindex, + txindex, + txinindex, + txoutindex, + unknownindex, + pushonlyindex, + opreturnindex, + multisigindex, + emptyindex, + } = starting_indexes; self.height_to_first_addressindex .truncate_if_needed(height, saved_height)?; @@ -326,23 +428,6 @@ impl Vecs { self.height_to_first_unknownindex .truncate_if_needed(height, saved_height)?; - // Now we can cut everything that's out of date - let &Indexes { - addressindex, - height, - p2pk33index, - p2pk65index, - p2pkhindex, - p2shindex, - p2trindex, - p2wpkhindex, - p2wshindex, - txindex, - txinindex, - txoutindex, - .. - } = starting_indexes; - self.height_to_blockhash .truncate_if_needed(height, saved_height)?; self.height_to_difficulty @@ -403,6 +488,35 @@ impl Vecs { self.txoutindex_to_value .truncate_if_needed(txoutindex, saved_height)?; + self.emptyindex_to_height + .truncate_if_needed(emptyindex, saved_height)?; + self.multisigindex_to_height + .truncate_if_needed(multisigindex, saved_height)?; + self.opreturnindex_to_height + .truncate_if_needed(opreturnindex, saved_height)?; + self.pushonlyindex_to_height + .truncate_if_needed(pushonlyindex, saved_height)?; + self.txinindex_to_height + .truncate_if_needed(txinindex, saved_height)?; + self.txoutindex_to_height + .truncate_if_needed(txoutindex, saved_height)?; + self.unknownindex_to_height + .truncate_if_needed(unknownindex, saved_height)?; + self.p2pk33index_to_height + .truncate_if_needed(p2pk33index, saved_height)?; + self.p2pk65index_to_height + .truncate_if_needed(p2pk65index, saved_height)?; + self.p2pkhindex_to_height + .truncate_if_needed(p2pkhindex, saved_height)?; + self.p2shindex_to_height + .truncate_if_needed(p2shindex, saved_height)?; + self.p2trindex_to_height + .truncate_if_needed(p2trindex, saved_height)?; + self.p2wpkhindex_to_height + .truncate_if_needed(p2wpkhindex, saved_height)?; + self.p2wshindex_to_height + .truncate_if_needed(p2wshindex, saved_height)?; + Ok(()) } @@ -540,6 +654,20 @@ impl Vecs { self.txinindex_to_txoutindex.any_vec(), self.txoutindex_to_addressindex.any_vec(), self.txoutindex_to_value.any_vec(), + self.emptyindex_to_height.any_vec(), + self.multisigindex_to_height.any_vec(), + self.opreturnindex_to_height.any_vec(), + self.pushonlyindex_to_height.any_vec(), + self.txinindex_to_height.any_vec(), + self.txoutindex_to_height.any_vec(), + self.unknownindex_to_height.any_vec(), + self.p2pk33index_to_height.any_vec(), + self.p2pk65index_to_height.any_vec(), + self.p2pkhindex_to_height.any_vec(), + self.p2shindex_to_height.any_vec(), + self.p2trindex_to_height.any_vec(), + self.p2wpkhindex_to_height.any_vec(), + self.p2wshindex_to_height.any_vec(), ] } @@ -588,6 +716,20 @@ impl Vecs { &mut self.txinindex_to_txoutindex, &mut self.txoutindex_to_addressindex, &mut self.txoutindex_to_value, + &mut self.emptyindex_to_height, + &mut self.multisigindex_to_height, + &mut self.opreturnindex_to_height, + &mut self.pushonlyindex_to_height, + &mut self.txinindex_to_height, + &mut self.txoutindex_to_height, + &mut self.unknownindex_to_height, + &mut self.p2pk33index_to_height, + &mut self.p2pk65index_to_height, + &mut self.p2pkhindex_to_height, + &mut self.p2shindex_to_height, + &mut self.p2trindex_to_height, + &mut self.p2wpkhindex_to_height, + &mut self.p2wshindex_to_height, ] } } diff --git a/crates/brk_query/src/index.rs b/crates/brk_query/src/index.rs index 1a44ee37f..b2398c9dd 100644 --- a/crates/brk_query/src/index.rs +++ b/crates/brk_query/src/index.rs @@ -24,10 +24,15 @@ pub enum Index { Decadeindex, Difficultyepoch, Halvingepoch, + Emptyindex, + Multisigindex, + Opreturnindex, + Pushonlyindex, + Unknownindex, } impl Index { - pub fn all() -> [Self; 20] { + pub fn all() -> [Self; 25] { [ Self::Height, Self::Dateindex, @@ -49,6 +54,11 @@ impl Index { Self::Txindex, Self::Txinindex, Self::Txoutindex, + Self::Emptyindex, + Self::Multisigindex, + Self::Opreturnindex, + Self::Pushonlyindex, + Self::Unknownindex, ] } @@ -75,6 +85,11 @@ impl Index { Self::P2TRindex => &["p2tr", "p2trindex"], Self::P2WPKHindex => &["p2wpkh", "p2wpkhindex"], Self::P2WSHindex => &["p2wsh", "p2wshindex"], + Self::Emptyindex => &["empty", "emptyindex"], + Self::Multisigindex => &["multisig", "multisigindex"], + Self::Opreturnindex => &["opreturn", "opreturnindex"], + Self::Pushonlyindex => &["pushonly", "pushonlyindex"], + Self::Unknownindex => &["unknown", "unknownindex"], } } @@ -122,6 +137,12 @@ impl TryFrom<&str> for Index { v if (Self::Difficultyepoch).possible_values().contains(&v) => Self::Difficultyepoch, v if (Self::Halvingepoch).possible_values().contains(&v) => Self::Halvingepoch, v if (Self::Quarterindex).possible_values().contains(&v) => Self::Quarterindex, + v if (Self::Quarterindex).possible_values().contains(&v) => Self::Quarterindex, + v if (Self::Emptyindex).possible_values().contains(&v) => Self::Emptyindex, + v if (Self::Multisigindex).possible_values().contains(&v) => Self::Multisigindex, + v if (Self::Opreturnindex).possible_values().contains(&v) => Self::Opreturnindex, + v if (Self::Pushonlyindex).possible_values().contains(&v) => Self::Pushonlyindex, + v if (Self::Unknownindex).possible_values().contains(&v) => Self::Unknownindex, _ => return Err(eyre!("Bad index")), }) } diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 3a72ec3e8..213153470 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -60,14 +60,10 @@ pub fn main() -> color_eyre::Result<()> { loop { let block_count = rpc.get_block_count()?; - info!("{block_count} blocks found."); - let starting_indexes = indexer.index(&parser, rpc, &exit)?; computer.compute(&mut indexer, starting_indexes, &exit)?; - info!("Waiting for new blocks..."); - while block_count == rpc.get_block_count()? { sleep(Duration::from_secs(1)) } diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index b06c012bc..bc17f3549 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -7,7 +7,7 @@ fn main() -> Result<(), Box> { { let mut vec: StorableVec = - StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?; + StorableVec::forced_import(Path::new("./vec"), Version::ZERO, Compressed::YES)?; (0..21_u32).for_each(|v| { vec.push(v); @@ -21,7 +21,7 @@ fn main() -> Result<(), Box> { { let mut vec: StorableVec = - StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?; + StorableVec::forced_import(Path::new("./vec"), Version::ZERO, Compressed::YES)?; dbg!(vec.get(0)?); dbg!(vec.get(0)?); @@ -43,7 +43,7 @@ fn main() -> Result<(), Box> { { let mut vec: StorableVec = - StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?; + StorableVec::forced_import(Path::new("./vec"), Version::ZERO, Compressed::YES)?; vec.enable_large_cache(); diff --git a/websites/kibo.money/scripts/options.js b/websites/kibo.money/scripts/options.js index a7ff9b951..a6526ab02 100644 --- a/websites/kibo.money/scripts/options.js +++ b/websites/kibo.money/scripts/options.js @@ -675,7 +675,7 @@ function createPartialOptions(colors) { { key: "block-count-sum", title: "Sum", - color: colors.orange, + color: colors.bitcoin, }, { key: "total-block-count", @@ -745,6 +745,65 @@ function createPartialOptions(colors) { }, ], }, + { + name: "Transactions", + tree: [ + { + name: "Inputs", + tree: [ + { + name: "Count", + title: "Transaction Input Count", + bottom: [ + { + key: "input-count", + title: "Count", + color: colors.bitcoin, + }, + { + key: "input-count-sum", + title: "Sum", + color: colors.bitcoin, + }, + { + key: "total-input-count", + title: "Total", + color: colors.offBitcoin, + defaultActive: false, + }, + ], + }, + ], + }, + { + name: "Outputs", + tree: [ + { + name: "Count", + title: "Transaction Output Count", + bottom: [ + { + key: "output-count", + title: "Count", + color: colors.bitcoin, + }, + { + key: "output-count-sum", + title: "Sum", + color: colors.bitcoin, + }, + { + key: "total-output-count", + title: "Total", + color: colors.offBitcoin, + defaultActive: false, + }, + ], + }, + ], + }, + ], + }, ], }, { diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index 7d9b65daf..38cf85af1 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -22,8 +22,13 @@ /** @typedef {17} Txindex */ /** @typedef {18} Txinindex */ /** @typedef {19} Txoutindex */ +/** @typedef {20} Emptyindex */ +/** @typedef {21} Multisigindex */ +/** @typedef {22} Opreturnindex */ +/** @typedef {23} Pushonlyindex */ +/** @typedef {24} Unknownindex */ -/** @typedef {Height | Dateindex | Weekindex | Difficultyepoch | Monthindex | Quarterindex | Yearindex | Decadeindex | Halvingepoch | Addressindex | P2PK33index | P2PK65index | P2PKHindex | P2SHindex | P2TRindex | P2WPKHindex | P2WSHindex | Txindex | Txinindex | Txoutindex} Index */ +/** @typedef {Height | Dateindex | Weekindex | Difficultyepoch | Monthindex | Quarterindex | Yearindex | Decadeindex | Halvingepoch | Addressindex | P2PK33index | P2PK65index | P2PKHindex | P2SHindex | P2TRindex | P2WPKHindex | P2WSHindex | Txindex | Txinindex | Txoutindex | Emptyindex | Multisigindex | Opreturnindex | Pushonlyindex | Unknownindex} Index */ export function createVecIdToIndexes() { const Height = /** @satisfies {Height} */ (0); @@ -46,6 +51,11 @@ export function createVecIdToIndexes() { const Txindex = /** @satisfies {Txindex} */ (17); const Txinindex = /** @satisfies {Txinindex} */ (18); const Txoutindex = /** @satisfies {Txoutindex} */ (19); + const Emptyindex = /** @satisfies {Emptyindex} */ (20); + const Multisigindex = /** @satisfies {Multisigindex} */ (21); + const Opreturnindex = /** @satisfies {Opreturnindex} */ (22); + const Pushonlyindex = /** @satisfies {Pushonlyindex} */ (23); + const Unknownindex = /** @satisfies {Unknownindex} */ (24); return { addressindex: [Txoutindex], @@ -63,6 +73,10 @@ export function createVecIdToIndexes() { "block-interval-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "block-interval-median": [Dateindex], "block-interval-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "block-size": [Height], + "block-size-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "block-weight": [Height], + "block-weight-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], blockhash: [Height], close: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "close-in-cents": [Dateindex, Height], @@ -94,11 +108,11 @@ export function createVecIdToIndexes() { "fixed-date": [Height], "fixed-timestamp": [Height], halvingepoch: [Height, Halvingepoch], - height: [Addressindex, Height, Txindex], + height: [Addressindex, Height, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Txindex, Txinindex, Txoutindex, Emptyindex, Multisigindex, Opreturnindex, Pushonlyindex, Unknownindex], high: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "high-in-cents": [Dateindex, Height], - "inputs-count": [Txindex], - "inputs-count-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "input-count": [Txindex], + "input-count-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "is-coinbase": [Txindex], "is-explicitly-rbf": [Txindex], "last-dateindex": [Weekindex, Monthindex], @@ -116,8 +130,8 @@ export function createVecIdToIndexes() { "ohlc-in-cents": [Dateindex, Height], open: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "open-in-cents": [Dateindex, Height], - "outputs-count": [Txindex], - "outputs-count-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "output-count": [Txindex], + "output-count-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], p2pk33addressbytes: [P2PK33index], p2pk65addressbytes: [P2PK65index], p2pkhaddressbytes: [P2PKHindex], @@ -131,8 +145,10 @@ export function createVecIdToIndexes() { size: [Height], timestamp: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch, Halvingepoch], "total-block-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], - "total-inputs-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], - "total-outputs-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-block-size": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-block-weight": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-input-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-output-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-size": [Txindex], txid: [Txindex], txoutindex: [Txinindex],