diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index 8fcc8e535..fb182205f 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -249,11 +249,12 @@ where let index = max_from.min(I::from(self.len())); let one = T::from(1); let mut prev_index: Option = None; - first_indexes.iter_from(index, |(i, v, ..)| { + first_indexes.iter_from(index, |(index, v, ..)| { if let Some(prev_index) = prev_index.take() { - self.forced_push_at(prev_index, v.checked_sub(one).unwrap(), exit)?; + let value = v.checked_sub(one).unwrap(); + self.forced_push_at(prev_index, value, exit)?; } - prev_index.replace(i); + prev_index.replace(index); Ok(()) })?; if let Some(prev_index) = prev_index { @@ -276,7 +277,65 @@ where ) -> Result<()> where T: From, - T2: StoredType + Copy + Add + CheckedSub + TryInto + Default, + T2: StoredType + + StoredIndex + + Copy + + Add + + CheckedSub + + TryInto + + Default, + >::Error: error::Error + 'static, + { + let opt: Option bool>> = None; + self.compute_filtered_count_from_indexes_(max_from, first_indexes, last_indexes, opt, exit) + } + + pub fn compute_filtered_count_from_indexes( + &mut self, + max_from: I, + first_indexes: &mut StoredVec, + last_indexes: &mut StoredVec, + filter: F, + exit: &Exit, + ) -> Result<()> + where + T: From, + T2: StoredType + + StoredIndex + + Copy + + Add + + CheckedSub + + TryInto + + Default, + >::Error: error::Error + 'static, + F: FnMut(T2) -> bool, + { + self.compute_filtered_count_from_indexes_( + max_from, + first_indexes, + last_indexes, + Some(Box::new(filter)), + exit, + ) + } + + fn compute_filtered_count_from_indexes_( + &mut self, + max_from: I, + first_indexes: &mut StoredVec, + last_indexes: &mut StoredVec, + mut filter: Option bool + '_>>, + exit: &Exit, + ) -> Result<()> + where + T: From, + T2: StoredType + + StoredIndex + + Copy + + Add + + CheckedSub + + TryInto + + Default, >::Error: error::Error + 'static, { self.validate_computed_version_or_reset_file( @@ -285,11 +344,14 @@ where let index = max_from.min(I::from(self.len())); first_indexes.iter_from(index, |(i, first_index, ..)| { - let last_index = last_indexes.cached_get(i)?.unwrap(); - let count = (*last_index + 1_usize) - .checked_sub(first_index) - .unwrap_or_default(); - self.forced_push_at(i, count.into(), exit) + let last_index = last_indexes.cached_get(i)?.unwrap().into_inner(); + let range = first_index.to_usize().unwrap()..=last_index.to_usize().unwrap(); + let count = if let Some(filter) = filter.as_mut() { + range.into_iter().filter(|i| filter(T2::from(*i))).count() + } else { + range.count() + }; + self.forced_push_at(i, T::from(T2::from(count)), exit) })?; self.safe_flush(exit) diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index 5e2f82402..ea3fbc7ec 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -4,6 +4,7 @@ use brk_exit::Exit; use brk_vec::{ Compressed, DynamicVec, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version, }; +use color_eyre::eyre::ContextCompat; use crate::storage::vecs::base::ComputedVec; @@ -162,8 +163,8 @@ where { let index = self.starting_index(max_from); - first_indexes.iter_from(index, |(i, first_index, ..)| { - let last_index = *last_indexes.cached_get(i)?.unwrap(); + first_indexes.iter_from(index, |(i, first_index, first_indexes)| { + let last_index = last_indexes.cached_get(i)?.unwrap().into_inner(); if let Some(first) = self.first.as_mut() { let v = source.cached_get(first_index)?.unwrap().into_inner(); @@ -199,7 +200,27 @@ where values.sort_unstable(); if let Some(max) = self.max.as_mut() { - max.forced_push_at(i, values.last().unwrap().clone(), exit)?; + max.forced_push_at( + i, + values + .last() + .context("expect some") + .inspect_err(|_| { + dbg!( + &values, + max.path(), + first_indexes.path(), + first_index, + last_indexes.path(), + last_index, + source.len(), + source.path() + ); + }) + .unwrap() + .clone(), + exit, + )?; } if let Some(_90p) = self._90p.as_mut() { diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 38c4984a4..b1ba2f3e3 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -363,9 +363,11 @@ impl Vecs { exit, )?; + let decremented_starting_height = starting_indexes.height.decremented().unwrap_or_default(); + let starting_dateindex = self .height_to_dateindex - .cached_get(starting_indexes.height.decremented().unwrap_or_default())? + .cached_get(decremented_starting_height)? .map_or_else(Default::default, |v| v.into_inner()); self.height_to_dateindex.compute_transform( @@ -377,7 +379,7 @@ impl Vecs { let starting_dateindex = if let Some(dateindex) = self .height_to_dateindex - .cached_get(starting_indexes.height.decremented().unwrap_or_default())? + .cached_get(decremented_starting_height)? .map(|v| v.into_inner()) { starting_dateindex.min(dateindex) @@ -696,7 +698,7 @@ impl Vecs { let starting_difficultyepoch = self .height_to_difficultyepoch - .cached_get(starting_indexes.height)? + .cached_get(decremented_starting_height)? .map_or_else(Default::default, |v| v.into_inner()); self.height_to_difficultyepoch.compute_transform( @@ -748,7 +750,7 @@ impl Vecs { let starting_halvingepoch = self .height_to_halvingepoch - .cached_get(starting_indexes.height)? + .cached_get(decremented_starting_height)? .map_or_else(Default::default, |v| v.into_inner()); self.height_to_halvingepoch.compute_transform( diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index d4e76348d..9fd7c1142 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -1,6 +1,6 @@ use std::{fs, path::Path}; -use brk_core::{Sats, StoredU8, StoredU32, StoredU64, Txindex, Txinindex, Txoutindex}; +use brk_core::{Sats, StoredU32, StoredU64, TxVersion, Txindex, Txinindex, Txoutindex}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{Compressed, DynamicVec, Version}; @@ -26,9 +26,6 @@ pub struct Vecs { // pub txindex_to_input_sum: ComputedVec, // pub txindex_to_output_sum: ComputedVec, // pub txindex_to_output_value: ComputedVecsFromTxindex, - pub txindex_to_v1: ComputedVec, - pub txindex_to_v2: ComputedVec, - pub txindex_to_v3: ComputedVec, pub indexes_to_tx_v1: ComputedVecsFromHeight, pub indexes_to_tx_v2: ComputedVecsFromHeight, pub indexes_to_tx_v3: ComputedVecsFromHeight, @@ -107,21 +104,6 @@ impl Vecs { Version::ZERO, compressed, )?, - txindex_to_v1: ComputedVec::forced_import( - &path.join("txindex_to_v1"), - Version::ZERO, - compressed, - )?, - txindex_to_v2: ComputedVec::forced_import( - &path.join("txindex_to_v2"), - Version::ZERO, - compressed, - )?, - txindex_to_v3: ComputedVec::forced_import( - &path.join("txindex_to_v3"), - Version::ZERO, - compressed, - )?, indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( path, "tx_v1", @@ -209,6 +191,37 @@ impl Vecs { // Some(indexer.mut_vecs().txoutindex_to_value.mut_vec()), // )?; + let mut compute_indexes_to_tx_vany = + |indexes_to_tx_vany: &mut ComputedVecsFromHeight, txversion| { + indexes_to_tx_vany.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |vec, indexer, indexes, starting_indexes, exit| { + let indexer_vecs = indexer.mut_vecs(); + vec.compute_filtered_count_from_indexes( + starting_indexes.height, + indexer_vecs.height_to_first_txindex.mut_vec(), + indexes.height_to_last_txindex.mut_vec(), + |txindex| { + let v = indexer_vecs + .txindex_to_txversion + .cached_get(txindex) + .unwrap() + .unwrap() + .into_inner(); + v == txversion + }, + exit, + ) + }, + ) + }; + compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v1, TxVersion::ONE)?; + compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?; + compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?; + let indexer_vecs = indexer.mut_vecs(); self.txindex_to_is_coinbase.compute_is_first_ordered( @@ -218,51 +231,6 @@ impl Vecs { exit, )?; - self.txindex_to_v1.compute_transform( - starting_indexes.txindex, - indexer_vecs.txindex_to_txversion.mut_vec(), - |(i, v, ..)| (i, StoredU8::from(v)), - exit, - )?; - // self.indexes_to_tx_v1.compute_all( - // indexer, - // indexes, - // starting_indexes, - // exit, - // |vec, indexer, indexes, indexes, exit| { - // vec.compute_transform( - // starting_indexes.height, - // indexer.mut_vecs().txindex_to_txversion.mut_vec(), - // || {}, - // exit, - // )?; - // }, - // )?; - self.txindex_to_v2.compute_transform( - starting_indexes.txindex, - indexer_vecs.txindex_to_txversion.mut_vec(), - |(i, v, ..)| (i, StoredU8::from(v)), - exit, - )?; - // self.indexes_to_tx_v1.compute_rest( - // starting_indexes.txindex, - // indexer_vecs.txindex_to_txversion.mut_vec(), - // |(i, v, ..)| (i, StoredU8::from(v)), - // exit, - // )?; - self.txindex_to_v3.compute_transform( - starting_indexes.txindex, - indexer_vecs.txindex_to_txversion.mut_vec(), - |(i, v, ..)| (i, StoredU8::from(v)), - exit, - )?; - // self.indexes_to_tx_v1.compute_rest( - // starting_indexes.txindex, - // indexer_vecs.txindex_to_txversion.mut_vec(), - // |(i, v, ..)| (i, StoredU8::from(v)), - // exit, - // )?; - self.txinindex_to_value.compute_transform( starting_indexes.txinindex, indexer_vecs.txinindex_to_txoutindex.mut_vec(), @@ -297,9 +265,6 @@ impl Vecs { vec![ self.txindex_to_is_coinbase.any_vec(), self.txinindex_to_value.any_vec(), - self.txindex_to_v1.any_vec(), - self.txindex_to_v2.any_vec(), - self.txindex_to_v3.any_vec(), ], self.height_to_tx_count.any_vecs(), self.txindex_to_output_count.any_vecs(), diff --git a/crates/brk_core/src/structs/txindex.rs b/crates/brk_core/src/structs/txindex.rs index 3f8ed0bb7..43c6c80cd 100644 --- a/crates/brk_core/src/structs/txindex.rs +++ b/crates/brk_core/src/structs/txindex.rs @@ -7,6 +7,8 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Error}; +use super::StoredU32; + #[derive( Debug, PartialEq, @@ -97,3 +99,9 @@ impl From for ByteView { Self::new(value.as_bytes()) } } + +impl From for StoredU32 { + fn from(value: Txindex) -> Self { + Self::from(value.0) + } +} diff --git a/crates/brk_core/src/structs/txversion.rs b/crates/brk_core/src/structs/txversion.rs index 9a8f91b34..8a271f180 100644 --- a/crates/brk_core/src/structs/txversion.rs +++ b/crates/brk_core/src/structs/txversion.rs @@ -4,9 +4,29 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::StoredU8; -#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)] +#[derive( + Debug, + Deref, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Immutable, + IntoBytes, + KnownLayout, + FromBytes, + Serialize, +)] pub struct TxVersion(i32); +impl TxVersion { + pub const ONE: Self = Self(1); + pub const TWO: Self = Self(2); + pub const THREE: Self = Self(3); +} + impl From for TxVersion { fn from(value: bitcoin::transaction::Version) -> Self { Self(value.0) diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index f7ec58872..355113dcd 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -122,6 +122,14 @@ where StoredVec::Compressed(v) => v.mut_pushed(), } } + + #[inline] + fn path(&self) -> &Path { + match self { + StoredVec::Raw(v) => v.path(), + StoredVec::Compressed(v) => v.path(), + } + } } impl GenericVec for StoredVec @@ -160,14 +168,6 @@ where } } - #[inline] - fn path(&self) -> &Path { - match self { - StoredVec::Raw(v) => v.path(), - StoredVec::Compressed(v) => v.path(), - } - } - #[inline] fn version(&self) -> Version { match self { diff --git a/crates/brk_vec/src/traits/dynamic.rs b/crates/brk_vec/src/traits/dynamic.rs index fd541b7d6..0f35e4de3 100644 --- a/crates/brk_vec/src/traits/dynamic.rs +++ b/crates/brk_vec/src/traits/dynamic.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{path::Path, sync::Arc}; use arc_swap::{ArcSwap, Guard}; use memmap2::Mmap; @@ -113,4 +113,6 @@ pub trait DynamicVec: Send + Sync { Err(Error::IndexTooLow) } } + + fn path(&self) -> &Path; } diff --git a/crates/brk_vec/src/traits/generic.rs b/crates/brk_vec/src/traits/generic.rs index ac6074517..b958bd631 100644 --- a/crates/brk_vec/src/traits/generic.rs +++ b/crates/brk_vec/src/traits/generic.rs @@ -61,8 +61,7 @@ where #[inline] fn reset(&mut self) -> Result<()> { - self.file_write_all(&[])?; - Ok(()) + self.file_truncate_and_write_all(0, &[]) } fn new_mmap(file: File) -> Result> { @@ -174,8 +173,6 @@ where Ok(self.collect_range_axum_json(from, to)?.into_response()) } - fn path(&self) -> &Path; - #[inline] fn path_vec(&self) -> PathBuf { Self::path_vec_(self.path()) diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 24ead9dc1..21e545abe 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -278,6 +278,11 @@ where fn mut_pushed(&mut self) -> &mut Vec { self.inner.mut_pushed() } + + #[inline] + fn path(&self) -> &Path { + self.inner.path() + } } impl GenericVec for CompressedVec @@ -441,6 +446,15 @@ where Ok(()) } + fn reset(&mut self) -> Result<()> { + let mut pages_meta = (**self.pages_meta.load()).clone(); + pages_meta.truncate(0); + pages_meta.write()?; + self.pages_meta.store(Arc::new(pages_meta)); + self.reset_caches(); + self.file_truncate_and_write_all(0, &[]) + } + fn truncate_if_needed(&mut self, index: I) -> Result<()> { let index = index.to_usize()?; @@ -453,12 +467,12 @@ where return Ok(()); } + let mut pages_meta = (**self.pages_meta.load()).clone(); + let page_index = Self::index_to_page_index(index); let guard = self.guard().as_ref().unwrap(); - let mut pages_meta = (**self.pages_meta.load()).clone(); - let values = self.decode_page(page_index, guard)?; let mut buf = vec![]; @@ -490,11 +504,6 @@ where Ok(()) } - #[inline] - fn path(&self) -> &Path { - self.inner.path() - } - #[inline] fn version(&self) -> Version { self.inner.version() diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 4a58d91ae..fd59d709a 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -114,6 +114,11 @@ where fn mut_pushed(&mut self) -> &mut Vec { &mut self.pushed } + + #[inline] + fn path(&self) -> &Path { + self.pathbuf.as_path() + } } impl GenericVec for RawVec @@ -211,11 +216,6 @@ where Ok(()) } - #[inline] - fn path(&self) -> &Path { - self.pathbuf.as_path() - } - #[inline] fn version(&self) -> Version { self.version diff --git a/websites/kibo.money/index.html b/websites/kibo.money/index.html index 3dc986890..da326a15d 100644 --- a/websites/kibo.money/index.html +++ b/websites/kibo.money/index.html @@ -877,9 +877,10 @@ background-image: linear-gradient( to bottom, transparent, + var(--background-color) 90%, var(--background-color) ); - height: 24rem; + height: 21rem; bottom: 0; left: 0; right: 0; diff --git a/websites/kibo.money/scripts/options.js b/websites/kibo.money/scripts/options.js index a1e851bb8..fcc1bd7ed 100644 --- a/websites/kibo.money/scripts/options.js +++ b/websites/kibo.money/scripts/options.js @@ -17,6 +17,7 @@ * "Seconds" | * "Transactions" | * "US Dollars" | + * "Version" | * "Virtual Bytes" | * "Weight Units" * } Unit @@ -868,6 +869,26 @@ function createPartialOptions(colors) { title: "Transaction Count", key: "tx-count", }), + { + name: "Version", + tree: [ + createBaseSumTotal({ + name: "1", + title: "Transaction V1 Count", + key: "tx-v1", + }), + createBaseSumTotal({ + name: "2", + title: "Transaction V2 Count", + key: "tx-v2", + }), + createBaseSumTotal({ + name: "3", + title: "Transaction V3 Count", + key: "tx-v3", + }), + ], + }, ], }, { @@ -1254,6 +1275,8 @@ export function initOptions({ anyPartial.unit = "Weight Units"; } else if (key.includes("-vbytes")) { anyPartial.unit = "Virtual Bytes"; + } else if (key.match(/v[0-9]/g)) { + anyPartial.unit = "Version"; } else { console.log(anyPartial); throw Error("Unit not set"); diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index 3a89fe428..fe7b41e68 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -167,9 +167,6 @@ export function createVecIdToIndexes() { txid: [Txindex], txoutindex: [Txinindex], txversion: [Txindex], - v1: [Txindex], - v2: [Txindex], - v3: [Txindex], value: [Txinindex, Txoutindex], weekindex: [Dateindex, Weekindex], yearindex: [Monthindex, Yearindex],