From 5ab61973565a4ad0daf22fb0b033f3e61b788ea7 Mon Sep 17 00:00:00 2001 From: nym21 Date: Wed, 23 Apr 2025 22:36:10 +0200 Subject: [PATCH] computer: init lazy/eager --- Cargo.lock | 88 +++--- .../brk_computer/src/storage/vecs/blocks.rs | 11 +- .../src/storage/vecs/grouped/builder.rs | 124 +++------ .../storage/vecs/grouped/from_dateindex.rs | 10 +- .../src/storage/vecs/grouped/from_height.rs | 10 +- .../vecs/grouped/from_height_strict.rs | 15 +- .../src/storage/vecs/grouped/from_txindex.rs | 10 +- .../src/storage/vecs/grouped/mod.rs | 2 - .../storage/vecs/grouped/value_from_height.rs | 5 +- .../vecs/grouped/value_from_txindex.rs | 5 +- .../brk_computer/src/storage/vecs/indexes.rs | 258 +++++++++--------- .../src/storage/vecs/marketprice.rs | 106 +++---- crates/brk_computer/src/storage/vecs/mod.rs | 6 +- .../src/storage/vecs/transactions.rs | 18 +- .../{grouped/stored_type.rs => vec/_type.rs} | 0 .../storage/vecs/{base.rs => vec/eager.rs} | 12 +- .../brk_computer/src/storage/vecs/vec/lazy.rs | 76 ++++++ .../brk_computer/src/storage/vecs/vec/mod.rs | 7 + crates/brk_indexer/examples/main.rs | 2 +- crates/brk_indexer/src/lib.rs | 2 +- crates/brk_server/Cargo.toml | 2 +- websites/kibo.money/scripts/options.js | 18 -- 22 files changed, 405 insertions(+), 382 deletions(-) rename crates/brk_computer/src/storage/vecs/{grouped/stored_type.rs => vec/_type.rs} (100%) rename crates/brk_computer/src/storage/vecs/{base.rs => vec/eager.rs} (98%) create mode 100644 crates/brk_computer/src/storage/vecs/vec/lazy.rs create mode 100644 crates/brk_computer/src/storage/vecs/vec/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ebf9700eb..9345e9f2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1204,9 +1204,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", @@ -1791,9 +1791,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339134dd7e4bb36c2d0e97601fe4193b37649f729e94e7c26667feecd439b27e" +checksum = "409ba57c4752b7fab9609af78790aadeb541316933d1525484dad877988622f6" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1834,9 +1834,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c3dd872f5f819775c28163977399a1dd4f2177f3745dcb1bfd4214bae27e43" +checksum = "43fe5f8d34c351e3c90b512046dbfff2ff87c8a1927ebe6945f1cdefcc3219d0" dependencies = [ "allocator-api2", "bumpalo", @@ -1848,9 +1848,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1bebb401e36d73f35400ee9609ab6372b457543ce7bb1ef17c922235f24ba7b" +checksum = "5ad2e9c3fb4b409b14fd791fbbf6ced9a81363364beedd48059bbb8f42bf4987" dependencies = [ "bitflags", "cow-utils", @@ -1865,9 +1865,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b997c2a787321814d6e92f212486b4d590af4dbfff5c2265d3b0386e016eafe" +checksum = "578b69ed39d814cb8396674e17a0f94f1c02bc25d226b265c2913bd6e84afffb" dependencies = [ "proc-macro2", "quote", @@ -1876,9 +1876,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ee363bd1ff2000ae2a1052bb4d68e827e62f76e0139449ad1301ad9adc6ac4" +checksum = "f437ca959bf931d5facbc6df77c7491a4aec32df5b9f647ffa19c52055a7dc56" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1888,9 +1888,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77cb606168f768c747885ae48272287a028de5374db865c29d6d307d3b601a97" +checksum = "ccecf0bce4034ebf17e7edb6f214fa0e6f8d134d1a3fb98408d6bcbef024a3a1" dependencies = [ "bitflags", "itertools", @@ -1903,9 +1903,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afef88d94fe9eee073332e3f908bd429baa989300d83152dab10dad99bd2a66" +checksum = "e3361ed40dd6b4b6a7a4b44c175aa6beea68eefed68bc32a28684a0d1a00bc39" dependencies = [ "bitflags", "cow-utils", @@ -1924,15 +1924,15 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235fe6251a3f06813e77b45f3cd9c7ec5d0a23741c33f6c308b5af393d5d2409" +checksum = "0d2d64d2e627796f73e37ccb51f1f87cd9de612245ef3f790051855587d3734e" [[package]] name = "oxc_diagnostics" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd0f62519773d942baf1a17474e53b535182182f2cb41319ad1c2782e34310d" +checksum = "96ece1486240cfeec516eaaa7332c96decb3a756d72e504bb63393507996c39d" dependencies = [ "cow-utils", "oxc-miette", @@ -1940,9 +1940,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d073fdd2a3e1adb63731faab3c3861d0f9244cc15e88453d78460a4b2a604ca" +checksum = "0d7d5ddf1cd783ed1a930ac6eedd4f73c6a5359beaabad7ee8333e366816291c" dependencies = [ "cow-utils", "num-bigint", @@ -1954,9 +1954,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df015cfd58d084b27d049281835094556a4469db3e2ac45b0676336e0aeed8" +checksum = "62de94b10df7ab86a379f01b75076f6a762fc91a001678108318203fc0805ad9" [[package]] name = "oxc_index" @@ -1966,9 +1966,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfc3363726f4ade77ba4d9e56cc38cc306436feee7b72e1c69436fb30c02b9b" +checksum = "8d691c6609bca852ae1e0571de564de3b4d110a46c9877aa029e9dba56b76d1c" dependencies = [ "fixedbitset", "itertools", @@ -1983,9 +1983,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095580f12a6277d4cb532310ba95cc6d1f719b6c59341fb392b71e37d1207e6" +checksum = "294aa5790cc0beae7018d99dc5b389d373725ef7ea7ecef5d0ed18794b3414b3" dependencies = [ "cow-utils", "oxc_allocator", @@ -2005,9 +2005,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d54fd003e5347dcb5ccdcc96cb283bf32a648ff291912b5a6e5e718b3359935" +checksum = "5b7defa49cc20f9e3c346674b06752cacaacce053cde69a04f0f85f31e00820f" dependencies = [ "bitflags", "cow-utils", @@ -2028,9 +2028,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6e8058355e43a2388112eddf73031d8749b40255ad6f3a824e35fe0c6addec" +checksum = "14d37e8585630a167c4a2b121dcdab52bf13b71a6bd64260efe797956bd4f6d0" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -2044,9 +2044,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9f297db174c99b43e1b268b0960dcc588767093575cc39bb5517196a7a21d3" +checksum = "895130e4f197b77d9b54deffccc2ce8a9d977435c022a49ddecb1f467b946854" dependencies = [ "itertools", "oxc_allocator", @@ -2080,9 +2080,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69575b94d8fdac097784abf48a5e43ee38375aa377f8e9bb8bc80f2ef3d2269c" +checksum = "8350c982e6284f41668ebd3c714de22280a19f617c3cb6117bbc4e16afc6b13a" dependencies = [ "compact_str", "oxc-miette", @@ -2093,9 +2093,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c4940ed3e38ce51fb6bc3e880edb815182bdcdc19c6dc110857625aac9c9a7" +checksum = "7e85d9197e08e176d6f58a3885d125738c46a69145410e82b3a32f676ddd5546" dependencies = [ "bitflags", "cow-utils", @@ -2114,9 +2114,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f8868dcb4ac2cd1e08dadbf34cd4fd9bbf36b70ef64dfa3376e1042a9f705a3" +checksum = "ac43e668dcd3a0b17cf4f889feac5321e56430a0550205df2e85b3e9314f6caa" dependencies = [ "compact_str", "itoa", @@ -2384,7 +2384,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -2459,7 +2459,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -2956,9 +2956,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index d48c3c044..2f6dd8426 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -7,19 +7,18 @@ use brk_parser::bitcoin; use brk_vec::{Compressed, Version}; use super::{ - Indexes, - base::ComputedVec, + EagerVec, Indexes, grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions}, indexes, }; #[derive(Clone)] pub struct Vecs { - pub height_to_interval: ComputedVec, + pub height_to_interval: EagerVec, pub indexes_to_block_interval: ComputedVecsFromHeight, pub indexes_to_block_count: ComputedVecsFromHeight, pub indexes_to_block_weight: ComputedVecsFromHeight, - pub height_to_vbytes: ComputedVec, + pub height_to_vbytes: EagerVec, pub indexes_to_block_vbytes: ComputedVecsFromHeight, pub indexes_to_block_size: ComputedVecsFromHeight, } @@ -29,7 +28,7 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - height_to_interval: ComputedVec::forced_import( + height_to_interval: EagerVec::forced_import( &path.join("height_to_interval"), Version::ZERO, compressed, @@ -69,7 +68,7 @@ impl Vecs { compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, - height_to_vbytes: ComputedVec::forced_import( + height_to_vbytes: EagerVec::forced_import( &path.join("height_to_vbytes"), Version::ZERO, compressed, diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index cf8a2119b..da4730d39 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -6,9 +6,7 @@ use brk_vec::{ }; use color_eyre::eyre::ContextCompat; -use crate::storage::vecs::base::ComputedVec; - -use super::ComputedType; +use crate::storage::{ComputedType, EagerVec}; #[derive(Clone, Debug)] pub struct ComputedVecBuilder @@ -16,18 +14,18 @@ where I: StoredIndex, T: ComputedType, { - first: Option>, - average: Option>, - sum: Option>, - max: Option>, - _90p: Option>, - _75p: Option>, - median: Option>, - _25p: Option>, - _10p: Option>, - min: Option>, - last: Option>, - total: Option>, + first: Option>, + average: Option>, + sum: Option>, + max: Option>, + _90p: Option>, + _75p: Option>, + median: Option>, + _25p: Option>, + _10p: Option>, + min: Option>, + last: Option>, + total: Option>, } const VERSION: Version = Version::ZERO; @@ -74,15 +72,11 @@ where let s = Self { first: options.first.then(|| { - ComputedVec::forced_import( - &maybe_prefix("first"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_prefix("first"), version + Version::ZERO, compressed) + .unwrap() }), last: options.last.then(|| { - ComputedVec::forced_import( + EagerVec::forced_import( &path.join(format!("{key}_to_{name}")), version + Version::ZERO, compressed, @@ -90,23 +84,15 @@ where .unwrap() }), min: options.min.then(|| { - ComputedVec::forced_import( - &maybe_suffix("min"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("min"), version + Version::ZERO, compressed) + .unwrap() }), max: options.max.then(|| { - ComputedVec::forced_import( - &maybe_suffix("max"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("max"), version + Version::ZERO, compressed) + .unwrap() }), median: options.median.then(|| { - ComputedVec::forced_import( + EagerVec::forced_import( &maybe_suffix("median"), version + Version::ZERO, compressed, @@ -114,7 +100,7 @@ where .unwrap() }), average: options.average.then(|| { - ComputedVec::forced_import( + EagerVec::forced_import( &maybe_suffix("average"), version + Version::ZERO, compressed, @@ -122,48 +108,28 @@ where .unwrap() }), sum: options.sum.then(|| { - ComputedVec::forced_import( - &maybe_suffix("sum"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("sum"), version + Version::ZERO, compressed) + .unwrap() }), total: options.total.then(|| { - ComputedVec::forced_import(&prefix("total"), version + Version::ZERO, compressed) + EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed) .unwrap() }), _90p: options._90p.then(|| { - ComputedVec::forced_import( - &maybe_suffix("90p"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("90p"), version + Version::ZERO, compressed) + .unwrap() }), _75p: options._75p.then(|| { - ComputedVec::forced_import( - &maybe_suffix("75p"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("75p"), version + Version::ZERO, compressed) + .unwrap() }), _25p: options._25p.then(|| { - ComputedVec::forced_import( - &maybe_suffix("25p"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("25p"), version + Version::ZERO, compressed) + .unwrap() }), _10p: options._10p.then(|| { - ComputedVec::forced_import( - &maybe_suffix("10p"), - version + Version::ZERO, - compressed, - ) - .unwrap() + EagerVec::forced_import(&maybe_suffix("10p"), version + Version::ZERO, compressed) + .unwrap() }), }; @@ -476,40 +442,40 @@ where )) } - pub fn unwrap_first(&mut self) -> &mut ComputedVec { + pub fn unwrap_first(&mut self) -> &mut EagerVec { self.first.as_mut().unwrap() } - pub fn unwrap_average(&mut self) -> &mut ComputedVec { + pub fn unwrap_average(&mut self) -> &mut EagerVec { self.average.as_mut().unwrap() } - pub fn unwrap_sum(&mut self) -> &mut ComputedVec { + pub fn unwrap_sum(&mut self) -> &mut EagerVec { self.sum.as_mut().unwrap() } - pub fn unwrap_max(&mut self) -> &mut ComputedVec { + pub fn unwrap_max(&mut self) -> &mut EagerVec { self.max.as_mut().unwrap() } - pub fn unwrap_90p(&mut self) -> &mut ComputedVec { + pub fn unwrap_90p(&mut self) -> &mut EagerVec { self._90p.as_mut().unwrap() } - pub fn unwrap_75p(&mut self) -> &mut ComputedVec { + pub fn unwrap_75p(&mut self) -> &mut EagerVec { self._75p.as_mut().unwrap() } - pub fn unwrap_median(&mut self) -> &mut ComputedVec { + pub fn unwrap_median(&mut self) -> &mut EagerVec { self.median.as_mut().unwrap() } - pub fn unwrap_25p(&mut self) -> &mut ComputedVec { + pub fn unwrap_25p(&mut self) -> &mut EagerVec { self._25p.as_mut().unwrap() } - pub fn unwrap_10p(&mut self) -> &mut ComputedVec { + pub fn unwrap_10p(&mut self) -> &mut EagerVec { self._10p.as_mut().unwrap() } - pub fn unwrap_min(&mut self) -> &mut ComputedVec { + pub fn unwrap_min(&mut self) -> &mut EagerVec { self.min.as_mut().unwrap() } - pub fn unwrap_last(&mut self) -> &mut ComputedVec { + pub fn unwrap_last(&mut self) -> &mut EagerVec { self.last.as_mut().unwrap() } - pub fn unwrap_total(&mut self) -> &mut ComputedVec { + pub fn unwrap_total(&mut self) -> &mut EagerVec { self.total.as_mut().unwrap() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs index 0639cc0f4..e37bbca16 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs @@ -5,16 +5,16 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed, Result, Version}; -use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; +use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromDateindex where T: ComputedType + PartialOrd, { - pub dateindex: ComputedVec, + pub dateindex: EagerVec, pub dateindex_extra: ComputedVecBuilder, pub weekindex: ComputedVecBuilder, pub monthindex: ComputedVecBuilder, @@ -49,7 +49,7 @@ where let options = options.remove_percentiles(); Ok(Self { - dateindex: ComputedVec::forced_import( + dateindex: EagerVec::forced_import( &path.join(format!("dateindex_to_{name}")), version, compressed, @@ -79,7 +79,7 @@ where ) -> color_eyre::Result<()> where F: FnMut( - &mut ComputedVec, + &mut EagerVec, &mut Indexer, &mut indexes::Vecs, &Indexes, diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index 99e442af5..018babb18 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -7,16 +7,16 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; -use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; +use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeight where T: ComputedType + PartialOrd, { - pub height: Option>, + pub height: Option>, pub height_extra: ComputedVecBuilder, pub dateindex: ComputedVecBuilder, pub weekindex: ComputedVecBuilder, @@ -46,7 +46,7 @@ where let version = VERSION + version; let height = compute_source.then(|| { - ComputedVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed) + EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed) .unwrap() }); @@ -95,7 +95,7 @@ where ) -> color_eyre::Result<()> where F: FnMut( - &mut ComputedVec, + &mut EagerVec, &mut Indexer, &mut indexes::Vecs, &Indexes, diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs index 98251b12e..06275ae5a 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -5,16 +5,16 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed, Result, Version}; -use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; +use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeightStrict where T: ComputedType + PartialOrd, { - pub height: ComputedVec, + pub height: EagerVec, pub height_extra: ComputedVecBuilder, pub difficultyepoch: ComputedVecBuilder, // TODO: pub halvingepoch: StorableVecGeneator, @@ -36,11 +36,8 @@ where ) -> color_eyre::Result { let version = VERSION + version; - let height = ComputedVec::forced_import( - &path.join(format!("height_to_{name}")), - version, - compressed, - )?; + let height = + EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)?; let height_extra = ComputedVecBuilder::forced_import( path, @@ -72,7 +69,7 @@ where ) -> color_eyre::Result<()> where F: FnMut( - &mut ComputedVec, + &mut EagerVec, &mut Indexer, &mut indexes::Vecs, &Indexes, diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index ad9a9d167..4eef7ab64 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -8,16 +8,16 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; -use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; +use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; -use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromTxindex where T: ComputedType + PartialOrd, { - pub txindex: Option>, + pub txindex: Option>, pub height: ComputedVecBuilder, pub dateindex: ComputedVecBuilder, pub weekindex: ComputedVecBuilder, @@ -47,7 +47,7 @@ where let version = VERSION + version; let txindex = compute_source.then(|| { - ComputedVec::forced_import( + EagerVec::forced_import( &path.join(format!("txindex_to_{name}")), version, compressed, @@ -91,7 +91,7 @@ where ) -> color_eyre::Result<()> where F: FnMut( - &mut ComputedVec, + &mut EagerVec, &mut Indexer, &mut indexes::Vecs, &Indexes, diff --git a/crates/brk_computer/src/storage/vecs/grouped/mod.rs b/crates/brk_computer/src/storage/vecs/grouped/mod.rs index 3ebd53122..54861f84d 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/mod.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/mod.rs @@ -3,7 +3,6 @@ mod from_dateindex; mod from_height; mod from_height_strict; mod from_txindex; -mod stored_type; mod value_from_height; mod value_from_txindex; @@ -12,6 +11,5 @@ pub use from_dateindex::*; pub use from_height::*; pub use from_height_strict::*; pub use from_txindex::*; -pub use stored_type::*; pub use value_from_height::*; pub use value_from_txindex::*; diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs index aabf3d38f..caa75f05a 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs @@ -6,8 +6,7 @@ use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; use crate::storage::{ - base::ComputedVec, - marketprice, + EagerVec, marketprice, vecs::{Indexes, indexes}, }; @@ -74,7 +73,7 @@ impl ComputedValueVecsFromHeight { ) -> color_eyre::Result<()> where F: FnMut( - &mut ComputedVec, + &mut EagerVec, &mut Indexer, &mut indexes::Vecs, &Indexes, diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs index d17ab2300..8f327b984 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs @@ -6,8 +6,7 @@ use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; use crate::storage::{ - base::ComputedVec, - marketprice, + EagerVec, marketprice, vecs::{Indexes, indexes}, }; @@ -74,7 +73,7 @@ impl ComputedValueVecsFromTxindex { ) -> color_eyre::Result<()> where F: FnMut( - &mut ComputedVec, + &mut EagerVec, &mut Indexer, &mut indexes::Vecs, &Indexes, diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 8fb04272d..7d722ab7c 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -10,74 +10,74 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{Compressed, Version}; -use super::ComputedVec; +use super::EagerVec; #[derive(Clone)] pub struct Vecs { - pub addressindex_to_addressindex: ComputedVec, - pub dateindex_to_date: ComputedVec, - pub dateindex_to_dateindex: ComputedVec, - pub dateindex_to_first_height: ComputedVec, - pub dateindex_to_last_height: ComputedVec, - pub dateindex_to_monthindex: ComputedVec, - pub dateindex_to_timestamp: ComputedVec, - pub dateindex_to_weekindex: ComputedVec, - pub decadeindex_to_decadeindex: ComputedVec, - pub decadeindex_to_first_yearindex: ComputedVec, - pub decadeindex_to_last_yearindex: ComputedVec, - pub decadeindex_to_timestamp: ComputedVec, - pub difficultyepoch_to_difficultyepoch: ComputedVec, - pub difficultyepoch_to_first_height: ComputedVec, - pub difficultyepoch_to_last_height: ComputedVec, - pub difficultyepoch_to_timestamp: ComputedVec, - pub emptyindex_to_emptyindex: ComputedVec, - pub halvingepoch_to_first_height: ComputedVec, - pub halvingepoch_to_halvingepoch: ComputedVec, - pub halvingepoch_to_last_height: ComputedVec, - pub halvingepoch_to_timestamp: ComputedVec, - pub height_to_dateindex: ComputedVec, - pub height_to_difficultyepoch: ComputedVec, - pub height_to_fixed_date: ComputedVec, - pub height_to_fixed_timestamp: ComputedVec, - pub height_to_halvingepoch: ComputedVec, - pub height_to_height: ComputedVec, - pub height_to_last_txindex: ComputedVec, - pub height_to_real_date: ComputedVec, - pub monthindex_to_first_dateindex: ComputedVec, - pub monthindex_to_last_dateindex: ComputedVec, - pub monthindex_to_monthindex: ComputedVec, - pub monthindex_to_quarterindex: ComputedVec, - pub monthindex_to_timestamp: ComputedVec, - pub monthindex_to_yearindex: ComputedVec, - pub multisigindex_to_multisigindex: ComputedVec, - pub opreturnindex_to_opreturnindex: ComputedVec, - pub p2pk33index_to_p2pk33index: ComputedVec, - pub p2pk65index_to_p2pk65index: ComputedVec, - pub p2pkhindex_to_p2pkhindex: ComputedVec, - pub p2shindex_to_p2shindex: ComputedVec, - pub p2trindex_to_p2trindex: ComputedVec, - pub p2wpkhindex_to_p2wpkhindex: ComputedVec, - pub p2wshindex_to_p2wshindex: ComputedVec, - pub pushonlyindex_to_pushonlyindex: ComputedVec, - pub quarterindex_to_first_monthindex: ComputedVec, - pub quarterindex_to_last_monthindex: ComputedVec, - pub quarterindex_to_quarterindex: ComputedVec, - pub quarterindex_to_timestamp: ComputedVec, - pub txindex_to_last_txinindex: ComputedVec, - pub txindex_to_last_txoutindex: ComputedVec, - pub txindex_to_txindex: ComputedVec, - pub txinindex_to_txinindex: ComputedVec, - pub txoutindex_to_txoutindex: ComputedVec, - pub unknownindex_to_unknownindex: ComputedVec, - pub weekindex_to_first_dateindex: ComputedVec, - pub weekindex_to_last_dateindex: ComputedVec, - pub weekindex_to_timestamp: ComputedVec, - pub weekindex_to_weekindex: ComputedVec, - pub yearindex_to_decadeindex: ComputedVec, - pub yearindex_to_first_monthindex: ComputedVec, - pub yearindex_to_last_monthindex: ComputedVec, - pub yearindex_to_timestamp: ComputedVec, - pub yearindex_to_yearindex: ComputedVec, + pub addressindex_to_addressindex: EagerVec, + pub dateindex_to_date: EagerVec, + pub dateindex_to_dateindex: EagerVec, + pub dateindex_to_first_height: EagerVec, + pub dateindex_to_last_height: EagerVec, + pub dateindex_to_monthindex: EagerVec, + pub dateindex_to_timestamp: EagerVec, + pub dateindex_to_weekindex: EagerVec, + pub decadeindex_to_decadeindex: EagerVec, + pub decadeindex_to_first_yearindex: EagerVec, + pub decadeindex_to_last_yearindex: EagerVec, + pub decadeindex_to_timestamp: EagerVec, + pub difficultyepoch_to_difficultyepoch: EagerVec, + pub difficultyepoch_to_first_height: EagerVec, + pub difficultyepoch_to_last_height: EagerVec, + pub difficultyepoch_to_timestamp: EagerVec, + pub emptyindex_to_emptyindex: EagerVec, + pub halvingepoch_to_first_height: EagerVec, + pub halvingepoch_to_halvingepoch: EagerVec, + pub halvingepoch_to_last_height: EagerVec, + pub halvingepoch_to_timestamp: EagerVec, + pub height_to_dateindex: EagerVec, + pub height_to_difficultyepoch: EagerVec, + pub height_to_fixed_date: EagerVec, + pub height_to_fixed_timestamp: EagerVec, + pub height_to_halvingepoch: EagerVec, + pub height_to_height: EagerVec, + pub height_to_last_txindex: EagerVec, + pub height_to_real_date: EagerVec, + pub monthindex_to_first_dateindex: EagerVec, + pub monthindex_to_last_dateindex: EagerVec, + pub monthindex_to_monthindex: EagerVec, + pub monthindex_to_quarterindex: EagerVec, + pub monthindex_to_timestamp: EagerVec, + pub monthindex_to_yearindex: EagerVec, + pub multisigindex_to_multisigindex: EagerVec, + pub opreturnindex_to_opreturnindex: EagerVec, + pub p2pk33index_to_p2pk33index: EagerVec, + pub p2pk65index_to_p2pk65index: EagerVec, + pub p2pkhindex_to_p2pkhindex: EagerVec, + pub p2shindex_to_p2shindex: EagerVec, + pub p2trindex_to_p2trindex: EagerVec, + pub p2wpkhindex_to_p2wpkhindex: EagerVec, + pub p2wshindex_to_p2wshindex: EagerVec, + pub pushonlyindex_to_pushonlyindex: EagerVec, + pub quarterindex_to_first_monthindex: EagerVec, + pub quarterindex_to_last_monthindex: EagerVec, + pub quarterindex_to_quarterindex: EagerVec, + pub quarterindex_to_timestamp: EagerVec, + pub txindex_to_last_txinindex: EagerVec, + pub txindex_to_last_txoutindex: EagerVec, + pub txindex_to_txindex: EagerVec, + pub txinindex_to_txinindex: EagerVec, + pub txoutindex_to_txoutindex: EagerVec, + pub unknownindex_to_unknownindex: EagerVec, + pub weekindex_to_first_dateindex: EagerVec, + pub weekindex_to_last_dateindex: EagerVec, + pub weekindex_to_timestamp: EagerVec, + pub weekindex_to_weekindex: EagerVec, + pub yearindex_to_decadeindex: EagerVec, + pub yearindex_to_first_monthindex: EagerVec, + pub yearindex_to_last_monthindex: EagerVec, + pub yearindex_to_timestamp: EagerVec, + pub yearindex_to_yearindex: EagerVec, } impl Vecs { @@ -85,322 +85,322 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - dateindex_to_date: ComputedVec::forced_import( + dateindex_to_date: EagerVec::forced_import( &path.join("dateindex_to_date"), Version::ZERO, compressed, )?, - dateindex_to_dateindex: ComputedVec::forced_import( + dateindex_to_dateindex: EagerVec::forced_import( &path.join("dateindex_to_dateindex"), Version::ZERO, compressed, )?, - dateindex_to_first_height: ComputedVec::forced_import( + dateindex_to_first_height: EagerVec::forced_import( &path.join("dateindex_to_first_height"), Version::ZERO, compressed, )?, - dateindex_to_last_height: ComputedVec::forced_import( + dateindex_to_last_height: EagerVec::forced_import( &path.join("dateindex_to_last_height"), Version::ZERO, compressed, )?, - height_to_real_date: ComputedVec::forced_import( + height_to_real_date: EagerVec::forced_import( &path.join("height_to_real_date"), Version::ZERO, compressed, )?, - height_to_fixed_date: ComputedVec::forced_import( + height_to_fixed_date: EagerVec::forced_import( &path.join("height_to_fixed_date"), Version::ZERO, compressed, )?, - height_to_dateindex: ComputedVec::forced_import( + height_to_dateindex: EagerVec::forced_import( &path.join("height_to_dateindex"), Version::ZERO, compressed, )?, - height_to_height: ComputedVec::forced_import( + height_to_height: EagerVec::forced_import( &path.join("height_to_height"), Version::ZERO, compressed, )?, - height_to_last_txindex: ComputedVec::forced_import( + height_to_last_txindex: EagerVec::forced_import( &path.join("height_to_last_txindex"), Version::ZERO, compressed, )?, - txindex_to_last_txinindex: ComputedVec::forced_import( + txindex_to_last_txinindex: EagerVec::forced_import( &path.join("txindex_to_last_txinindex"), Version::ZERO, compressed, )?, - txindex_to_last_txoutindex: ComputedVec::forced_import( + txindex_to_last_txoutindex: EagerVec::forced_import( &path.join("txindex_to_last_txoutindex"), Version::ZERO, compressed, )?, - difficultyepoch_to_first_height: ComputedVec::forced_import( + difficultyepoch_to_first_height: EagerVec::forced_import( &path.join("difficultyepoch_to_first_height"), Version::ZERO, compressed, )?, - difficultyepoch_to_last_height: ComputedVec::forced_import( + difficultyepoch_to_last_height: EagerVec::forced_import( &path.join("difficultyepoch_to_last_height"), Version::ZERO, compressed, )?, - halvingepoch_to_first_height: ComputedVec::forced_import( + halvingepoch_to_first_height: EagerVec::forced_import( &path.join("halvingepoch_to_first_height"), Version::ZERO, compressed, )?, - halvingepoch_to_last_height: ComputedVec::forced_import( + halvingepoch_to_last_height: EagerVec::forced_import( &path.join("halvingepoch_to_last_height"), Version::ZERO, compressed, )?, - weekindex_to_first_dateindex: ComputedVec::forced_import( + weekindex_to_first_dateindex: EagerVec::forced_import( &path.join("weekindex_to_first_dateindex"), Version::ZERO, compressed, )?, - weekindex_to_last_dateindex: ComputedVec::forced_import( + weekindex_to_last_dateindex: EagerVec::forced_import( &path.join("weekindex_to_last_dateindex"), Version::ZERO, compressed, )?, - monthindex_to_first_dateindex: ComputedVec::forced_import( + monthindex_to_first_dateindex: EagerVec::forced_import( &path.join("monthindex_to_first_dateindex"), Version::ZERO, compressed, )?, - monthindex_to_last_dateindex: ComputedVec::forced_import( + monthindex_to_last_dateindex: EagerVec::forced_import( &path.join("monthindex_to_last_dateindex"), Version::ZERO, compressed, )?, - yearindex_to_first_monthindex: ComputedVec::forced_import( + yearindex_to_first_monthindex: EagerVec::forced_import( &path.join("yearindex_to_first_monthindex"), Version::ZERO, compressed, )?, - yearindex_to_last_monthindex: ComputedVec::forced_import( + yearindex_to_last_monthindex: EagerVec::forced_import( &path.join("yearindex_to_last_monthindex"), Version::ZERO, compressed, )?, - decadeindex_to_first_yearindex: ComputedVec::forced_import( + decadeindex_to_first_yearindex: EagerVec::forced_import( &path.join("decadeindex_to_first_yearindex"), Version::ZERO, compressed, )?, - decadeindex_to_last_yearindex: ComputedVec::forced_import( + decadeindex_to_last_yearindex: EagerVec::forced_import( &path.join("decadeindex_to_last_yearindex"), Version::ZERO, compressed, )?, - dateindex_to_weekindex: ComputedVec::forced_import( + dateindex_to_weekindex: EagerVec::forced_import( &path.join("dateindex_to_weekindex"), Version::ZERO, compressed, )?, - dateindex_to_monthindex: ComputedVec::forced_import( + dateindex_to_monthindex: EagerVec::forced_import( &path.join("dateindex_to_monthindex"), Version::ZERO, compressed, )?, - monthindex_to_yearindex: ComputedVec::forced_import( + monthindex_to_yearindex: EagerVec::forced_import( &path.join("monthindex_to_yearindex"), Version::ZERO, compressed, )?, - yearindex_to_decadeindex: ComputedVec::forced_import( + yearindex_to_decadeindex: EagerVec::forced_import( &path.join("yearindex_to_decadeindex"), Version::ZERO, compressed, )?, - height_to_difficultyepoch: ComputedVec::forced_import( + height_to_difficultyepoch: EagerVec::forced_import( &path.join("height_to_difficultyepoch"), Version::ZERO, compressed, )?, - height_to_halvingepoch: ComputedVec::forced_import( + height_to_halvingepoch: EagerVec::forced_import( &path.join("height_to_halvingepoch"), Version::ZERO, compressed, )?, - weekindex_to_weekindex: ComputedVec::forced_import( + weekindex_to_weekindex: EagerVec::forced_import( &path.join("weekindex_to_weekindex"), Version::ZERO, compressed, )?, - monthindex_to_monthindex: ComputedVec::forced_import( + monthindex_to_monthindex: EagerVec::forced_import( &path.join("monthindex_to_monthindex"), Version::ZERO, compressed, )?, - yearindex_to_yearindex: ComputedVec::forced_import( + yearindex_to_yearindex: EagerVec::forced_import( &path.join("yearindex_to_yearindex"), Version::ZERO, compressed, )?, - decadeindex_to_decadeindex: ComputedVec::forced_import( + decadeindex_to_decadeindex: EagerVec::forced_import( &path.join("decadeindex_to_decadeindex"), Version::ZERO, compressed, )?, - difficultyepoch_to_difficultyepoch: ComputedVec::forced_import( + difficultyepoch_to_difficultyepoch: EagerVec::forced_import( &path.join("difficultyepoch_to_difficultyepoch"), Version::ZERO, compressed, )?, - halvingepoch_to_halvingepoch: ComputedVec::forced_import( + halvingepoch_to_halvingepoch: EagerVec::forced_import( &path.join("halvingepoch_to_halvingepoch"), Version::ZERO, compressed, )?, - dateindex_to_timestamp: ComputedVec::forced_import( + dateindex_to_timestamp: EagerVec::forced_import( &path.join("dateindex_to_timestamp"), Version::ZERO, compressed, )?, - decadeindex_to_timestamp: ComputedVec::forced_import( + decadeindex_to_timestamp: EagerVec::forced_import( &path.join("decadeindex_to_timestamp"), Version::ZERO, compressed, )?, - difficultyepoch_to_timestamp: ComputedVec::forced_import( + difficultyepoch_to_timestamp: EagerVec::forced_import( &path.join("difficultyepoch_to_timestamp"), Version::ZERO, compressed, )?, - halvingepoch_to_timestamp: ComputedVec::forced_import( + halvingepoch_to_timestamp: EagerVec::forced_import( &path.join("halvingepoch_to_timestamp"), Version::ZERO, compressed, )?, - monthindex_to_timestamp: ComputedVec::forced_import( + monthindex_to_timestamp: EagerVec::forced_import( &path.join("monthindex_to_timestamp"), Version::ZERO, compressed, )?, - weekindex_to_timestamp: ComputedVec::forced_import( + weekindex_to_timestamp: EagerVec::forced_import( &path.join("weekindex_to_timestamp"), Version::ZERO, compressed, )?, - yearindex_to_timestamp: ComputedVec::forced_import( + yearindex_to_timestamp: EagerVec::forced_import( &path.join("yearindex_to_timestamp"), Version::ZERO, compressed, )?, - height_to_fixed_timestamp: ComputedVec::forced_import( + height_to_fixed_timestamp: EagerVec::forced_import( &path.join("height_to_fixed_timestamp"), Version::ZERO, compressed, )?, - monthindex_to_quarterindex: ComputedVec::forced_import( + monthindex_to_quarterindex: EagerVec::forced_import( &path.join("monthindex_to_quarterindex"), Version::ZERO, compressed, )?, - quarterindex_to_first_monthindex: ComputedVec::forced_import( + quarterindex_to_first_monthindex: EagerVec::forced_import( &path.join("quarterindex_to_first_monthindex"), Version::ZERO, compressed, )?, - quarterindex_to_last_monthindex: ComputedVec::forced_import( + quarterindex_to_last_monthindex: EagerVec::forced_import( &path.join("quarterindex_to_last_monthindex"), Version::ZERO, compressed, )?, - quarterindex_to_quarterindex: ComputedVec::forced_import( + quarterindex_to_quarterindex: EagerVec::forced_import( &path.join("quarterindex_to_quarterindex"), Version::ZERO, compressed, )?, - quarterindex_to_timestamp: ComputedVec::forced_import( + quarterindex_to_timestamp: EagerVec::forced_import( &path.join("quarterindex_to_timestamp"), Version::ZERO, compressed, )?, - p2pk33index_to_p2pk33index: ComputedVec::forced_import( + p2pk33index_to_p2pk33index: EagerVec::forced_import( &path.join("p2pk33index_to_p2pk33index"), Version::ZERO, compressed, )?, - p2pk65index_to_p2pk65index: ComputedVec::forced_import( + p2pk65index_to_p2pk65index: EagerVec::forced_import( &path.join("p2pk65index_to_p2pk65index"), Version::ZERO, compressed, )?, - p2pkhindex_to_p2pkhindex: ComputedVec::forced_import( + p2pkhindex_to_p2pkhindex: EagerVec::forced_import( &path.join("p2pkhindex_to_p2pkhindex"), Version::ZERO, compressed, )?, - p2shindex_to_p2shindex: ComputedVec::forced_import( + p2shindex_to_p2shindex: EagerVec::forced_import( &path.join("p2shindex_to_p2shindex"), Version::ZERO, compressed, )?, - p2trindex_to_p2trindex: ComputedVec::forced_import( + p2trindex_to_p2trindex: EagerVec::forced_import( &path.join("p2trindex_to_p2trindex"), Version::ZERO, compressed, )?, - p2wpkhindex_to_p2wpkhindex: ComputedVec::forced_import( + p2wpkhindex_to_p2wpkhindex: EagerVec::forced_import( &path.join("p2wpkhindex_to_p2wpkhindex"), Version::ZERO, compressed, )?, - p2wshindex_to_p2wshindex: ComputedVec::forced_import( + p2wshindex_to_p2wshindex: EagerVec::forced_import( &path.join("p2wshindex_to_p2wshindex"), Version::ZERO, compressed, )?, - txindex_to_txindex: ComputedVec::forced_import( + txindex_to_txindex: EagerVec::forced_import( &path.join("txindex_to_txindex"), Version::ZERO, compressed, )?, - txinindex_to_txinindex: ComputedVec::forced_import( + txinindex_to_txinindex: EagerVec::forced_import( &path.join("txinindex_to_txinindex"), Version::ZERO, compressed, )?, - emptyindex_to_emptyindex: ComputedVec::forced_import( + emptyindex_to_emptyindex: EagerVec::forced_import( &path.join("emptyindex_to_emptyindex"), Version::ZERO, compressed, )?, - multisigindex_to_multisigindex: ComputedVec::forced_import( + multisigindex_to_multisigindex: EagerVec::forced_import( &path.join("multisigindex_to_multisigindex"), Version::ZERO, compressed, )?, - opreturnindex_to_opreturnindex: ComputedVec::forced_import( + opreturnindex_to_opreturnindex: EagerVec::forced_import( &path.join("opreturnindex_to_opreturnindex"), Version::ZERO, compressed, )?, - pushonlyindex_to_pushonlyindex: ComputedVec::forced_import( + pushonlyindex_to_pushonlyindex: EagerVec::forced_import( &path.join("pushonlyindex_to_pushonlyindex"), Version::ZERO, compressed, )?, - unknownindex_to_unknownindex: ComputedVec::forced_import( + unknownindex_to_unknownindex: EagerVec::forced_import( &path.join("unknownindex_to_unknownindex"), Version::ZERO, compressed, )?, - addressindex_to_addressindex: ComputedVec::forced_import( + addressindex_to_addressindex: EagerVec::forced_import( &path.join("addressindex_to_addressindex"), Version::ZERO, compressed, )?, - txoutindex_to_txoutindex: ComputedVec::forced_import( + txoutindex_to_txoutindex: EagerVec::forced_import( &path.join("txoutindex_to_txoutindex"), Version::ZERO, compressed, diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index f1548df09..ff5f3ebb1 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -10,7 +10,7 @@ use brk_indexer::Indexer; use brk_vec::{Compressed, Version}; use super::{ - ComputedVec, Indexes, + EagerVec, Indexes, grouped::{ ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions, }, @@ -19,20 +19,20 @@ use super::{ #[derive(Clone)] pub struct Vecs { - pub dateindex_to_close_in_cents: ComputedVec>, - pub dateindex_to_high_in_cents: ComputedVec>, - pub dateindex_to_low_in_cents: ComputedVec>, - pub dateindex_to_ohlc: ComputedVec, - pub dateindex_to_ohlc_in_sats: ComputedVec, - pub dateindex_to_ohlc_in_cents: ComputedVec, - pub dateindex_to_open_in_cents: ComputedVec>, - pub height_to_close_in_cents: ComputedVec>, - pub height_to_high_in_cents: ComputedVec>, - pub height_to_low_in_cents: ComputedVec>, - pub height_to_ohlc: ComputedVec, - pub height_to_ohlc_in_sats: ComputedVec, - pub height_to_ohlc_in_cents: ComputedVec, - pub height_to_open_in_cents: ComputedVec>, + pub dateindex_to_close_in_cents: EagerVec>, + pub dateindex_to_high_in_cents: EagerVec>, + pub dateindex_to_low_in_cents: EagerVec>, + pub dateindex_to_ohlc: EagerVec, + pub dateindex_to_ohlc_in_sats: EagerVec, + pub dateindex_to_ohlc_in_cents: EagerVec, + pub dateindex_to_open_in_cents: EagerVec>, + pub height_to_close_in_cents: EagerVec>, + pub height_to_high_in_cents: EagerVec>, + pub height_to_low_in_cents: EagerVec>, + pub height_to_ohlc: EagerVec, + pub height_to_ohlc_in_sats: EagerVec, + pub height_to_ohlc_in_cents: EagerVec, + pub height_to_open_in_cents: EagerVec>, pub timeindexes_to_close: ComputedVecsFromDateindex>, pub timeindexes_to_high: ComputedVecsFromDateindex>, pub timeindexes_to_low: ComputedVecsFromDateindex>, @@ -49,20 +49,20 @@ pub struct Vecs { pub chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict>, pub chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict>, pub chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict>, - pub weekindex_to_ohlc: ComputedVec, - pub weekindex_to_ohlc_in_sats: ComputedVec, - pub difficultyepoch_to_ohlc: ComputedVec, - pub difficultyepoch_to_ohlc_in_sats: ComputedVec, - pub monthindex_to_ohlc: ComputedVec, - pub monthindex_to_ohlc_in_sats: ComputedVec, - pub quarterindex_to_ohlc: ComputedVec, - pub quarterindex_to_ohlc_in_sats: ComputedVec, - pub yearindex_to_ohlc: ComputedVec, - pub yearindex_to_ohlc_in_sats: ComputedVec, + pub weekindex_to_ohlc: EagerVec, + pub weekindex_to_ohlc_in_sats: EagerVec, + pub difficultyepoch_to_ohlc: EagerVec, + pub difficultyepoch_to_ohlc_in_sats: EagerVec, + pub monthindex_to_ohlc: EagerVec, + pub monthindex_to_ohlc_in_sats: EagerVec, + pub quarterindex_to_ohlc: EagerVec, + pub quarterindex_to_ohlc_in_sats: EagerVec, + pub yearindex_to_ohlc: EagerVec, + pub yearindex_to_ohlc_in_sats: EagerVec, // pub halvingepoch_to_ohlc: StorableVec, // pub halvingepoch_to_ohlc_in_sats: StorableVec, - pub decadeindex_to_ohlc: ComputedVec, - pub decadeindex_to_ohlc_in_sats: ComputedVec, + pub decadeindex_to_ohlc: EagerVec, + pub decadeindex_to_ohlc_in_sats: EagerVec, } const VERSION: Version = Version::ZERO; @@ -78,72 +78,72 @@ impl Vecs { fetched_path = fetched_path.join("fetched/vecs"); Ok(Self { - dateindex_to_ohlc_in_cents: ComputedVec::forced_import( + dateindex_to_ohlc_in_cents: EagerVec::forced_import( &fetched_path.join("dateindex_to_ohlc_in_cents"), Version::ZERO, compressed, )?, - dateindex_to_ohlc: ComputedVec::forced_import( + dateindex_to_ohlc: EagerVec::forced_import( &path.join("dateindex_to_ohlc"), Version::ZERO, compressed, )?, - dateindex_to_ohlc_in_sats: ComputedVec::forced_import( + dateindex_to_ohlc_in_sats: EagerVec::forced_import( &path.join("dateindex_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - dateindex_to_close_in_cents: ComputedVec::forced_import( + dateindex_to_close_in_cents: EagerVec::forced_import( &path.join("dateindex_to_close_in_cents"), Version::ZERO, compressed, )?, - dateindex_to_high_in_cents: ComputedVec::forced_import( + dateindex_to_high_in_cents: EagerVec::forced_import( &path.join("dateindex_to_high_in_cents"), Version::ZERO, compressed, )?, - dateindex_to_low_in_cents: ComputedVec::forced_import( + dateindex_to_low_in_cents: EagerVec::forced_import( &path.join("dateindex_to_low_in_cents"), Version::ZERO, compressed, )?, - dateindex_to_open_in_cents: ComputedVec::forced_import( + dateindex_to_open_in_cents: EagerVec::forced_import( &path.join("dateindex_to_open_in_cents"), Version::ZERO, compressed, )?, - height_to_ohlc_in_cents: ComputedVec::forced_import( + height_to_ohlc_in_cents: EagerVec::forced_import( &fetched_path.join("height_to_ohlc_in_cents"), Version::ZERO, compressed, )?, - height_to_ohlc: ComputedVec::forced_import( + height_to_ohlc: EagerVec::forced_import( &path.join("height_to_ohlc"), Version::ZERO, compressed, )?, - height_to_ohlc_in_sats: ComputedVec::forced_import( + height_to_ohlc_in_sats: EagerVec::forced_import( &path.join("height_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - height_to_close_in_cents: ComputedVec::forced_import( + height_to_close_in_cents: EagerVec::forced_import( &path.join("height_to_close_in_cents"), Version::ZERO, compressed, )?, - height_to_high_in_cents: ComputedVec::forced_import( + height_to_high_in_cents: EagerVec::forced_import( &path.join("height_to_high_in_cents"), Version::ZERO, compressed, )?, - height_to_low_in_cents: ComputedVec::forced_import( + height_to_low_in_cents: EagerVec::forced_import( &path.join("height_to_low_in_cents"), Version::ZERO, compressed, )?, - height_to_open_in_cents: ComputedVec::forced_import( + height_to_open_in_cents: EagerVec::forced_import( &path.join("height_to_open_in_cents"), Version::ZERO, compressed, @@ -260,63 +260,63 @@ impl Vecs { compressed, StorableVecGeneatorOptions::default().add_last(), )?, - weekindex_to_ohlc: ComputedVec::forced_import( + weekindex_to_ohlc: EagerVec::forced_import( &path.join("weekindex_to_ohlc"), Version::ZERO, compressed, )?, - weekindex_to_ohlc_in_sats: ComputedVec::forced_import( + weekindex_to_ohlc_in_sats: EagerVec::forced_import( &path.join("weekindex_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - difficultyepoch_to_ohlc: ComputedVec::forced_import( + difficultyepoch_to_ohlc: EagerVec::forced_import( &path.join("difficultyepoch_to_ohlc"), Version::ZERO, compressed, )?, - difficultyepoch_to_ohlc_in_sats: ComputedVec::forced_import( + difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import( &path.join("difficultyepoch_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - monthindex_to_ohlc: ComputedVec::forced_import( + monthindex_to_ohlc: EagerVec::forced_import( &path.join("monthindex_to_ohlc"), Version::ZERO, compressed, )?, - monthindex_to_ohlc_in_sats: ComputedVec::forced_import( + monthindex_to_ohlc_in_sats: EagerVec::forced_import( &path.join("monthindex_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - quarterindex_to_ohlc: ComputedVec::forced_import( + quarterindex_to_ohlc: EagerVec::forced_import( &path.join("quarterindex_to_ohlc"), Version::ZERO, compressed, )?, - quarterindex_to_ohlc_in_sats: ComputedVec::forced_import( + quarterindex_to_ohlc_in_sats: EagerVec::forced_import( &path.join("quarterindex_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - yearindex_to_ohlc: ComputedVec::forced_import( + yearindex_to_ohlc: EagerVec::forced_import( &path.join("yearindex_to_ohlc"), Version::ZERO, compressed, )?, - yearindex_to_ohlc_in_sats: ComputedVec::forced_import( + yearindex_to_ohlc_in_sats: EagerVec::forced_import( &path.join("yearindex_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ZERO, compressed)?, - decadeindex_to_ohlc: ComputedVec::forced_import( + decadeindex_to_ohlc: EagerVec::forced_import( &path.join("decadeindex_to_ohlc"), Version::ZERO, compressed, )?, - decadeindex_to_ohlc_in_sats: ComputedVec::forced_import( + decadeindex_to_ohlc_in_sats: EagerVec::forced_import( &path.join("decadeindex_to_ohlc_in_sats"), VERSION + VERSION_IN_SATS + Version::ZERO, compressed, diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 1ba7b1802..04a060007 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -5,15 +5,15 @@ use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed}; -pub mod base; pub mod blocks; pub mod grouped; pub mod indexes; pub mod marketprice; pub mod transactions; +pub mod vec; -use base::*; -use indexes::*; +pub use indexes::*; +pub use vec::*; #[derive(Clone)] pub struct Vecs { diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index aec977184..a572b71d7 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -10,7 +10,7 @@ use brk_parser::bitcoin; use brk_vec::{Compressed, DynamicVec, StoredIndex, Version}; use super::{ - ComputedVec, Indexes, + EagerVec, Indexes, grouped::{ ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions, @@ -34,12 +34,12 @@ pub struct Vecs { pub indexes_to_tx_vsize: ComputedVecsFromTxindex, pub indexes_to_tx_weight: ComputedVecsFromTxindex, pub txindex_to_input_count: ComputedVecsFromTxindex, - pub txindex_to_is_coinbase: ComputedVec, + pub txindex_to_is_coinbase: EagerVec, pub txindex_to_output_count: ComputedVecsFromTxindex, - pub txindex_to_vsize: ComputedVec, - pub txindex_to_weight: ComputedVec, + pub txindex_to_vsize: EagerVec, + pub txindex_to_weight: EagerVec, /// Value == 0 when Coinbase - pub txinindex_to_value: ComputedVec, + pub txinindex_to_value: EagerVec, pub indexes_to_subsidy: ComputedValueVecsFromHeight, pub indexes_to_coinbase: ComputedValueVecsFromHeight, } @@ -67,7 +67,7 @@ impl Vecs { .add_total(), )?, // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ZERO)?, - txindex_to_is_coinbase: ComputedVec::forced_import( + txindex_to_is_coinbase: EagerVec::forced_import( &path.join("txindex_to_is_coinbase"), Version::ZERO, compressed, @@ -98,7 +98,7 @@ impl Vecs { .add_sum() .add_total(), )?, - txinindex_to_value: ComputedVec::forced_import( + txinindex_to_value: EagerVec::forced_import( &path.join("txinindex_to_value"), Version::ZERO, compressed, @@ -174,12 +174,12 @@ impl Vecs { .add_minmax() .add_average(), )?, - txindex_to_weight: ComputedVec::forced_import( + txindex_to_weight: EagerVec::forced_import( &path.join("txindex_to_weight"), Version::ZERO, compressed, )?, - txindex_to_vsize: ComputedVec::forced_import( + txindex_to_vsize: EagerVec::forced_import( &path.join("txindex_to_vsize"), Version::ZERO, compressed, diff --git a/crates/brk_computer/src/storage/vecs/grouped/stored_type.rs b/crates/brk_computer/src/storage/vecs/vec/_type.rs similarity index 100% rename from crates/brk_computer/src/storage/vecs/grouped/stored_type.rs rename to crates/brk_computer/src/storage/vecs/vec/_type.rs diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/vec/eager.rs similarity index 98% rename from crates/brk_computer/src/storage/vecs/base.rs rename to crates/brk_computer/src/storage/vecs/vec/eager.rs index a10c3b904..38b52849d 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/vec/eager.rs @@ -18,7 +18,7 @@ const ONE_MIB: usize = ONE_KIB * ONE_KIB; const MAX_CACHE_SIZE: usize = 210 * ONE_MIB; #[derive(Debug)] -pub struct ComputedVec +pub struct EagerVec where I: StoredIndex, T: StoredType, @@ -27,7 +27,7 @@ where inner: StoredVec, } -impl ComputedVec +impl EagerVec where I: StoredIndex, T: StoredType, @@ -443,7 +443,7 @@ where } } -impl ComputedVec +impl EagerVec where I: StoredIndex, { @@ -467,7 +467,7 @@ where } } -impl ComputedVec { +impl EagerVec { pub fn compute_from_bitcoin( &mut self, max_from: Height, @@ -490,7 +490,7 @@ impl ComputedVec { } } -impl ComputedVec { +impl EagerVec { pub fn compute_from_bitcoin( &mut self, max_from: Txindex, @@ -515,7 +515,7 @@ impl ComputedVec { } } -impl Clone for ComputedVec +impl Clone for EagerVec where I: StoredIndex, T: StoredType, diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy.rs b/crates/brk_computer/src/storage/vecs/vec/lazy.rs new file mode 100644 index 000000000..b27a65e19 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/vec/lazy.rs @@ -0,0 +1,76 @@ +use core::error; +use std::{ + cmp::Ordering, + fmt::Debug, + ops::Add, + path::{Path, PathBuf}, +}; + +use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, Txindex}; +use brk_exit::Exit; +use brk_vec::{ + Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version, +}; +use log::info; + +const ONE_KIB: usize = 1024; +const ONE_MIB: usize = ONE_KIB * ONE_KIB; +const MAX_CACHE_SIZE: usize = 210 * ONE_MIB; + +#[derive(Debug, Clone, Copy)] +enum Mode { + Lazy, + Eager, +} + +#[derive(Debug)] +pub struct LazyVec +where + I: StoredIndex, + T: StoredType, +{ + inner: StoredVec, +} + +impl LazyVec +where + I: StoredIndex, + T: StoredType, +{ + const SIZE_OF: usize = size_of::(); + + fn version(&self) -> Version { + self.inner.version() + } + + pub fn len(&self) -> usize { + self.inner.len() + } + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn unwrap_cached_get(&mut self, index: I) -> Option { + self.inner.unwrap_cached_get(index) + } + #[inline] + pub fn double_unwrap_cached_get(&mut self, index: I) -> T { + self.inner.double_unwrap_cached_get(index) + } + + pub fn collect_inclusive_range(&self, from: I, to: I) -> Result> { + self.inner.collect_inclusive_range(from, to) + } +} + +impl Clone for LazyVec +where + I: StoredIndex, + T: StoredType, +{ + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/mod.rs b/crates/brk_computer/src/storage/vecs/vec/mod.rs new file mode 100644 index 000000000..c7e9b78ce --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/vec/mod.rs @@ -0,0 +1,7 @@ +mod _type; +mod eager; +// mod lazy; + +pub use _type::*; +pub use eager::*; +// pub use lazy::*; diff --git a/crates/brk_indexer/examples/main.rs b/crates/brk_indexer/examples/main.rs index 2302f5ea2..849ccd39f 100644 --- a/crates/brk_indexer/examples/main.rs +++ b/crates/brk_indexer/examples/main.rs @@ -24,7 +24,7 @@ fn main() -> color_eyre::Result<()> { let outputs = Path::new("../../_outputs"); - let mut indexer = Indexer::new(outputs.join("indexed").to_owned(), true, true)?; + let mut indexer = Indexer::new(outputs.join("indexed").to_owned(), false, false)?; indexer.import_stores()?; indexer.import_vecs()?; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 8b8050e58..7b41dae9d 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -31,7 +31,7 @@ pub use stores::*; pub use vecs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1000; -const COLLISIONS_CHECKED_UP_TO: u32 = 890_000; +const COLLISIONS_CHECKED_UP_TO: u32 = 893_000; #[derive(Clone)] pub struct Indexer { diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index dda8db847..329bc2be6 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -21,7 +21,7 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.65.0", features = ["codegen", "minifier"] } +oxc = { version = "0.66.0", features = ["codegen", "minifier"] } serde = { workspace = true } tokio = { version = "1.44.2", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full", "trace"] } diff --git a/websites/kibo.money/scripts/options.js b/websites/kibo.money/scripts/options.js index 66be3f0f7..9df7cc9ca 100644 --- a/websites/kibo.money/scripts/options.js +++ b/websites/kibo.money/scripts/options.js @@ -443,39 +443,21 @@ function createPartialOptions(colors) { name: "Versions", title: "Transaction Versions", bottom: [ - // { - // name: "1", - // title: "Transaction V1 Count", - // bottom: [ createBaseSeries({ key: "tx-v1", name: "v1 Count", }), ...createSumTotalSeries({ concat: "tx-v1", name: "v1" }), - // ], - // }, - // { - // name: "2", - // title: "Transaction V2 Count", - // bottom: [ createBaseSeries({ key: "tx-v2", name: "v2 Count", }), ...createSumTotalSeries({ concat: "tx-v2", name: "v2" }), - // ], - // }, - // { - // name: "3", - // title: "Transaction V3 Count", - // bottom: [ createBaseSeries({ key: "tx-v3", name: "v3 Count", }), ...createSumTotalSeries({ concat: "tx-v3", name: "v3" }), - // ], - // }, ], }, ],