diff --git a/Cargo.lock b/Cargo.lock index 183d6f23e..672605d8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,13 @@ version = 4 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -187,9 +181,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core", "bytes", @@ -241,17 +235,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -416,6 +410,7 @@ dependencies = [ "brk_query", "brk_server", "clap", + "clap_derive", "color-eyre", "log", "serde", @@ -434,9 +429,12 @@ dependencies = [ "brk_logger", "brk_parser", "brk_vec", + "clap", + "clap_derive", "color-eyre", "fjall", "log", + "serde", ] [[package]] @@ -453,7 +451,9 @@ dependencies = [ "rlimit", "serde", "serde_bytes", + "serde_derive", "zerocopy 0.8.25", + "zerocopy-derive 0.8.25", ] [[package]] @@ -530,6 +530,7 @@ dependencies = [ "brk_indexer", "brk_vec", "clap", + "clap_derive", "color-eyre", "derive_deref", "serde", @@ -552,6 +553,7 @@ dependencies = [ "brk_parser", "brk_query", "clap", + "clap_derive", "color-eyre", "jiff", "log", @@ -575,6 +577,7 @@ dependencies = [ "serde", "serde_json", "zerocopy 0.8.25", + "zerocopy-derive 0.8.25", "zstd", ] @@ -662,9 +665,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.20" +version = "1.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" +checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0" dependencies = [ "jobserver", "libc", @@ -713,7 +716,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", - "clap_derive", ] [[package]] @@ -748,27 +750,27 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "color-eyre" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" +checksum = "e6e1761c0e16f8883bbbb8ce5990867f4f06bf11a0253da6495a04ce4b6ef0ec" dependencies = [ "backtrace", "color-spantrace", "eyre", "indenter", "once_cell", - "owo-colors 3.5.0", + "owo-colors", "tracing-error", ] [[package]] name = "color-spantrace" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" +checksum = "2ddd8d5bfda1e11a501d0a7303f3bfed9aa632ebdb859be40d0fd70478ed70d5" dependencies = [ "once_cell", - "owo-colors 3.5.0", + "owo-colors", "tracing-core", "tracing-error", ] @@ -1143,7 +1145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "miniz_oxide 0.8.8", + "miniz_oxide", ] [[package]] @@ -1243,9 +1245,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "guardian" @@ -1267,9 +1269,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" dependencies = [ "allocator-api2", "equivalent", @@ -1447,7 +1449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "serde", ] @@ -1492,9 +1494,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a064218214dc6a10fbae5ec5fa888d80c45d611aba169222fc272072bf7aef6" +checksum = "27e77966151130221b079bcec80f1f34a9e414fa489d99152a201c07fd2182bc" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1507,9 +1509,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254" +checksum = "97265751f8a9a4228476f2fc17874a9e7e70e96b893368e42619880fe143b48a" dependencies = [ "proc-macro2", "quote", @@ -1675,15 +1677,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.8" @@ -1772,9 +1765,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -1791,12 +1784,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "owo-colors" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" - [[package]] name = "owo-colors" version = "4.2.0" @@ -1828,7 +1815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c278d00ecc50ee84aba4768a7ab74eb325dff4dca8c0581495b850d53480ba" dependencies = [ "cfg-if", - "owo-colors 4.2.0", + "owo-colors", "oxc-miette-derive", "textwrap", "thiserror", @@ -1854,7 +1841,7 @@ checksum = "8d5e92c9020da9b683bcf06bec4895e618e5d86a290c6e8a805faa7fde701116" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "oxc_data_structures", "rustc-hash", "simdutf8", @@ -2219,7 +2206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a98c6720655620a521dcc722d0ad66cd8afd5d86e34a89ef691c50b7b24de06" dependencies = [ "fixedbitset", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "indexmap 2.9.0", "serde", ] @@ -2352,7 +2339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287e56aac5a2b4fb25a6fb050961d157635924c8696305a5c937a76f29841a0f" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] [[package]] @@ -2501,9 +2488,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ "bitflags", "errno", @@ -3438,9 +3425,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb8234a863ea0e8cd7284fcdd4f145233eb00fee02bbdd9861aec44e6477bc5" +checksum = "d9fb597c990f03753e08d3c29efbfcf2019a003b4bf4ba19225c158e1549f0f3" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index cf631f6d4..53d330419 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ panic = "abort" inherits = "release" [workspace.dependencies] -axum = "0.8.3" +axum = "0.8.4" bitcoin = { version = "0.32.5", features = ["serde"] } bitcoincore-rpc = "0.19.0" brk_cli = { version = "0", path = "crates/brk_cli" } @@ -31,18 +31,21 @@ brk_query = { version = "0", path = "crates/brk_query" } brk_server = { version = "0", path = "crates/brk_server" } brk_vec = { version = "0", path = "crates/brk_vec" } byteview = "0.6.1" -clap = { version = "4.5.37", features = ["derive", "string"] } -color-eyre = "0.6.3" +clap = { version = "4.5.37", features = ["string"] } +clap_derive = "4.5.32" +color-eyre = "0.6.4" derive_deref = "1.1.1" fjall = "2.10.0" -jiff = "0.2.10" +jiff = "0.2.11" log = { version = "0.4.27" } minreq = { version = "2.13.4", features = ["https", "serde_json"] } rayon = "1.10.0" -serde = { version = "1.0.219", features = ["derive"] } +serde = { version = "1.0.219" } +serde_derive = "1.0.219" serde_json = { version = "1.0.140", features = ["float_roundtrip"] } tabled = "0.19.0" -zerocopy = { version = "0.8.25", features = ["derive"] } +zerocopy = { version = "0.8.25" } +zerocopy-derive = "0.8.25" [workspace.metadata.release] shared-version = true diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 47c889df4..87b87dd1d 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -16,7 +16,8 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_query = { workspace = true } brk_server = { workspace = true } -clap = { workspace = true, features = ["string"] } +clap = { workspace = true } +clap_derive = { workspace = true } color-eyre = { workspace = true } log = { workspace = true } serde = { workspace = true } diff --git a/crates/brk_cli/src/lib.rs b/crates/brk_cli/src/lib.rs index 9b293a3d4..810c65bec 100644 --- a/crates/brk_cli/src/lib.rs +++ b/crates/brk_cli/src/lib.rs @@ -2,7 +2,8 @@ use std::fs; use brk_core::{dot_brk_log_path, dot_brk_path}; use brk_query::Params as QueryArgs; -use clap::{Parser, Subcommand}; +use clap::Parser; +use clap_derive::{Parser, Subcommand}; use query::query; use run::{RunConfig, run}; diff --git a/crates/brk_cli/src/query.rs b/crates/brk_cli/src/query.rs index 489142d26..86786e550 100644 --- a/crates/brk_cli/src/query.rs +++ b/crates/brk_cli/src/query.rs @@ -14,7 +14,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> { indexer.import_vecs()?; let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed); - computer.import_vecs(&indexer)?; + computer.import_vecs(&indexer, config.computation())?; let query = Query::build(&indexer, &computer); diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index c8be9f5f9..16dcd1023 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -5,14 +5,14 @@ use std::{ time::Duration, }; -use brk_computer::Computer; +use brk_computer::{Computation, Computer}; use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path}; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::rpc::{self, Auth, Client, RpcApi}; use brk_server::{Server, Website, tokio}; -use clap::{Parser, ValueEnum}; +use clap_derive::{Parser, ValueEnum}; use color_eyre::eyre::eyre; use log::info; use serde::{Deserialize, Serialize}; @@ -34,7 +34,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed); computer.import_stores(&indexer)?; - computer.import_vecs(&indexer)?; + computer.import_vecs(&indexer, config.computation())?; tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -123,6 +123,10 @@ pub struct RunConfig { #[arg(short, long)] mode: Option, + /// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved + #[arg(short, long)] + computation: Option, + /// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved #[arg(short, long, value_name = "BOOL")] compressed: Option, @@ -413,6 +417,10 @@ impl RunConfig { .then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap()) } + pub fn computation(&self) -> Computation { + self.computation.unwrap_or_default() + } + pub fn compressed(&self) -> bool { self.compressed.is_none_or(|b| b) } diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 5afc56a7d..39d20acf6 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -14,6 +14,9 @@ brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_vec = { workspace = true } +clap = { workspace = true } +clap_derive = { workspace = true } color-eyre = { workspace = true } fjall = { workspace = true } log = { workspace = true } +serde = { workspace = true } diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index f7685d3ba..32287f2f8 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -1,6 +1,6 @@ use std::path::Path; -use brk_computer::Computer; +use brk_computer::{Computation, Computer}; use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_fetcher::Fetcher; @@ -34,7 +34,7 @@ pub fn main() -> color_eyre::Result<()> { let mut computer = Computer::new(outputs_dir, Some(fetcher), compressed); computer.import_stores(&indexer)?; - computer.import_vecs(&indexer)?; + computer.import_vecs(&indexer, Computation::Lazy)?; let starting_indexes = indexer.index(&parser, rpc, &exit)?; diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 9170f274b..be1c0e683 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -14,6 +14,7 @@ mod storage; use brk_vec::Compressed; use log::info; +pub use storage::Computation; use storage::{Stores, Vecs}; #[derive(Clone)] @@ -36,11 +37,16 @@ impl Computer { } } - pub fn import_vecs(&mut self, indexer: &Indexer) -> color_eyre::Result<()> { + pub fn import_vecs( + &mut self, + indexer: &Indexer, + computation: Computation, + ) -> color_eyre::Result<()> { self.vecs = Some(Vecs::import( &self.path.join("vecs/computed"), indexer, self.fetcher.is_some(), + computation, self.compressed, )?); Ok(()) diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 71d9e1050..71749d7fe 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -223,7 +223,7 @@ impl Vecs { Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { [ vec![ self.height_to_interval.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index 7ffd5189d..424fc1499 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -190,7 +190,7 @@ where self.monthindex.any_vecs(), self.quarterindex.any_vecs(), self.yearindex.any_vecs(), - // self.halvingepoch.as_any_vecs(), + // self.halvingepoch.any_vecs(), self.decadeindex.any_vecs(), ] .concat() diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs index 98127b792..19b9a8b9a 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -91,7 +91,7 @@ where vec![self.height.any_vec()], self.height_extra.any_vecs(), self.difficultyepoch.any_vecs(), - // self.halvingepoch.as_any_vecs(), + // self.halvingepoch.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index e9a290416..78abf3e68 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -198,7 +198,7 @@ where self.monthindex.any_vecs(), self.quarterindex.any_vecs(), self.yearindex.any_vecs(), - // self.halvingepoch.as_any_vecs(), + // self.halvingepoch.any_vecs(), self.decadeindex.any_vecs(), ] .concat() diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index b99d5e278..f3f9e29f0 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -865,7 +865,7 @@ impl Vecs { }) } - pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { vec![ self.dateindex_to_date.any_vec(), self.dateindex_to_dateindex.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index ba1dcb447..b66505363 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -1063,7 +1063,7 @@ impl Vecs { Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { vec![ vec![ self.dateindex_to_close_in_cents.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/mining.rs b/crates/brk_computer/src/storage/vecs/mining.rs index 736fe7d61..e939d4fb0 100644 --- a/crates/brk_computer/src/storage/vecs/mining.rs +++ b/crates/brk_computer/src/storage/vecs/mining.rs @@ -113,7 +113,7 @@ impl Vecs { Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { [ self.indexes_to_difficulty.any_vecs(), self.indexes_to_difficultyepoch.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 6bc24fb10..a192d31e5 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -30,6 +30,7 @@ impl Vecs { path: &Path, indexer: &Indexer, fetch: bool, + computation: Computation, compressed: Compressed, ) -> color_eyre::Result { fs::create_dir_all(path)?; @@ -38,7 +39,13 @@ impl Vecs { blocks: blocks::Vecs::forced_import(path, compressed)?, indexes: indexes::Vecs::forced_import(path, compressed)?, mining: mining::Vecs::forced_import(path, compressed)?, - transactions: transactions::Vecs::forced_import(path, indexer, compressed, fetch)?, + transactions: transactions::Vecs::forced_import( + path, + indexer, + computation, + compressed, + fetch, + )?, marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()), }) } @@ -79,15 +86,13 @@ impl Vecs { Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { [ - self.indexes.as_any_vecs(), - self.blocks.as_any_vecs(), - self.mining.as_any_vecs(), - self.transactions.as_any_vecs(), - self.marketprice - .as_ref() - .map_or(vec![], |v| v.as_any_vecs()), + self.indexes.any_vecs(), + self.blocks.any_vecs(), + self.mining.any_vecs(), + self.transactions.any_vecs(), + self.marketprice.as_ref().map_or(vec![], |v| v.any_vecs()), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 17a93d848..2b3070a7b 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -9,7 +9,7 @@ use brk_parser::bitcoin; use brk_vec::{Compressed, StoredIndex, VecIterator, Version}; use super::{ - EagerVec, Indexes, LazyVec, + Computation, ComputedVec, ComputedVecFrom2, EagerVec, Indexes, grouped::{ ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions, @@ -47,7 +47,8 @@ pub struct Vecs { pub indexes_to_tx_vsize: ComputedVecsFromTxindex, pub indexes_to_tx_weight: ComputedVecsFromTxindex, pub indexes_to_unknownoutput_count: ComputedVecsFromHeight, - pub inputindex_to_value: LazyVec, + pub inputindex_to_value: + ComputedVecFrom2, pub indexes_to_input_count: ComputedVecsFromTxindex, pub txindex_to_is_coinbase: EagerVec, pub indexes_to_output_count: ComputedVecsFromTxindex, @@ -59,6 +60,7 @@ impl Vecs { pub fn forced_import( path: &Path, indexer: &Indexer, + computation: Computation, compressed: Compressed, compute_dollars: bool, ) -> color_eyre::Result { @@ -110,8 +112,11 @@ impl Vecs { .add_sum() .add_total(), )?, - inputindex_to_value: LazyVec::init( + inputindex_to_value: ComputedVec::forced_import_or_init_from_2( + computation, + &path.join("inputindex_to_value"), Version::ZERO, + compressed, indexer.vecs().outputindex_to_value.vec().clone(), indexer.vecs().inputindex_to_outputindex.vec().clone(), |index, outputindex_to_value_iter, inputindex_to_outputindex_iter| { @@ -131,7 +136,7 @@ impl Vecs { }, ) }, - ), + )?, indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( path, "tx_v1", @@ -536,22 +541,22 @@ impl Vecs { )?; // let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter(); - // self.inputindex_to_value.compute_transform( - // starting_indexes.inputindex, - // indexer.vecs().inputindex_to_outputindex.vec(), - // |(inputindex, outputindex, ..)| { - // let value = if outputindex == OutputIndex::COINBASE { - // Sats::ZERO - // } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { - // value.into_inner() - // } else { - // dbg!(inputindex, outputindex); - // panic!() - // }; - // (inputindex, value) - // }, - // exit, - // )?; + self.inputindex_to_value.compute_if_necessary( + starting_indexes.inputindex, + // indexer.vecs().inputindex_to_outputindex.vec(), + // |(inputindex, outputindex, ..)| { + // let value = if outputindex == OutputIndex::COINBASE { + // Sats::ZERO + // } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { + // value.into_inner() + // } else { + // dbg!(inputindex, outputindex); + // panic!() + // }; + // (inputindex, value) + // }, + exit, + )?; self.indexes_to_output_value.compute_all( indexer, @@ -881,7 +886,7 @@ impl Vecs { Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { [ vec![ self.txindex_to_is_coinbase.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/vec/eager.rs b/crates/brk_computer/src/storage/vecs/vec/eager.rs index e95bdd77b..7c7db8a49 100644 --- a/crates/brk_computer/src/storage/vecs/vec/eager.rs +++ b/crates/brk_computer/src/storage/vecs/vec/eager.rs @@ -18,12 +18,8 @@ const ONE_KIB: usize = 1024; const ONE_MIB: usize = ONE_KIB * ONE_KIB; const MAX_CACHE_SIZE: usize = 210 * ONE_MIB; -#[derive(Debug)] -pub struct EagerVec -where - I: StoredIndex, - T: StoredType, -{ +#[derive(Debug, Clone)] +pub struct EagerVec { computed_version: Option, inner: StoredVec, } @@ -89,8 +85,8 @@ where Ok(()) } - fn version(&self) -> Version { - self.inner.version() + pub fn version(&self) -> Version { + self.computed_version.unwrap() } pub fn len(&self) -> usize { @@ -147,28 +143,42 @@ where self.into_iter() } + pub fn compute_to( + &mut self, + max_from: I, + to: usize, + version: Version, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + F: FnMut(I) -> (I, T), + { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.inner.version() + version, + )?; + + let index = max_from.min(I::from(self.len())); + (index.to_usize()?..to).try_for_each(|i| { + let (i, v) = t(I::from(i)); + self.forced_push_at(i, v, exit) + })?; + + self.safe_flush(exit) + } + pub fn compute_range( &mut self, max_from: I, other: &StoredVec, - mut t: F, + t: F, exit: &Exit, ) -> Result<()> where A: StoredType, F: FnMut(I) -> (I, T), { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + other.version(), - )?; - - let index = max_from.min(I::from(self.len())); - (index.to_usize()?..other.len()).try_for_each(|i| { - let (i, v) = t(I::from(i)); - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) + self.compute_to(max_from, other.len(), other.version(), t, exit) } pub fn compute_transform( @@ -184,7 +194,7 @@ where F: FnMut((A, B, &Self)) -> (I, T), { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + other.version(), + Version::ZERO + self.inner.version() + other.version(), )?; let index = max_from.min(A::from(self.len())); @@ -207,7 +217,7 @@ where T: StoredIndex, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + other.version(), + Version::ZERO + self.inner.version() + other.version(), )?; let index = max_from.min( @@ -241,7 +251,10 @@ where T: StoredIndex, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + first_indexes.version() + indexes_count.version(), + Version::ZERO + + self.inner.version() + + first_indexes.version() + + indexes_count.version(), )?; let mut indexes_count_iter = indexes_count.iter(); @@ -333,7 +346,7 @@ where >::Error: error::Error + 'static, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + first_indexes.version(), // + last_indexes.version(), + Version::ZERO + self.inner.version() + first_indexes.version(), // + last_indexes.version(), )?; let mut other_iter = first_indexes.iter(); @@ -371,7 +384,10 @@ where A: StoredIndex + StoredType, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + self_to_other.version() + other_to_self.version(), + Version::ZERO + + self.inner.version() + + self_to_other.version() + + other_to_self.version(), )?; let mut other_to_self_iter = other_to_self.iter(); @@ -400,7 +416,10 @@ where T2: StoredIndex + StoredType, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + first_indexes.version() + indexes_count.version(), + Version::ZERO + + self.inner.version() + + first_indexes.version() + + indexes_count.version(), )?; let mut indexes_count_iter = indexes_count.iter(); @@ -434,7 +453,7 @@ where exit: &Exit, ) -> Result<()> { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + sats.version(), + Version::ZERO + self.inner.version() + sats.version(), )?; let index = max_from.min(I::from(self.len())); @@ -456,7 +475,7 @@ impl EagerVec { exit: &Exit, ) -> Result<()> { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + bitcoin.version(), + Version::ZERO + self.inner.version() + bitcoin.version(), )?; let mut price_iter = price.iter(); @@ -481,7 +500,7 @@ impl EagerVec { exit: &Exit, ) -> Result<()> { self.validate_computed_version_or_reset_file( - Version::ZERO + self.version() + bitcoin.version(), + Version::ZERO + self.inner.version() + bitcoin.version(), )?; let mut i_to_height_iter = i_to_height.iter(); @@ -498,19 +517,6 @@ impl EagerVec { } } -impl Clone for EagerVec -where - I: StoredIndex, - T: StoredType, -{ - fn clone(&self) -> Self { - Self { - computed_version: self.computed_version, - inner: self.inner.clone(), - } - } -} - impl<'a, I, T> IntoIterator for &'a EagerVec where I: StoredIndex, diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy1.rs b/crates/brk_computer/src/storage/vecs/vec/lazy1.rs new file mode 100644 index 000000000..9c67f11e1 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/vec/lazy1.rs @@ -0,0 +1,103 @@ +use std::marker::PhantomData; + +use brk_vec::{ + BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, +}; + +pub type ComputeFrom1 = + for<'a> fn(usize, &mut dyn BaseVecIterator)>) -> Option; + +#[derive(Clone)] +pub struct LazyVecFrom1 { + version: Version, + source: StoredVec, + compute: ComputeFrom1, + phantom: PhantomData, +} + +impl LazyVecFrom1 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, +{ + pub fn init( + version: Version, + source: StoredVec, + compute: ComputeFrom1, + ) -> Self { + Self { + version, + source, + compute, + phantom: PhantomData, + } + } + + fn version(&self) -> Version { + self.version + } +} + +pub struct LazyVecIterator<'a, I, T, S1I, S1T> { + lazy: &'a LazyVecFrom1, + source: StoredVecIterator<'a, S1I, S1T>, + index: usize, +} + +impl<'a, I, T, S1I, S1T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T> +where + I: StoredIndex, + T: StoredType + 'a, + S1I: StoredIndex, + S1T: StoredType, +{ + type Item = (I, Value<'a, T>); + + fn next(&mut self) -> Option { + let opt = (self.lazy.compute)(self.index, &mut self.lazy.source.iter()) + .map(|v| (I::from(self.index), Value::Owned(v))); + if opt.is_some() { + self.index += 1; + } + opt + } +} + +impl BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T> +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, +{ + #[inline] + fn mut_index(&mut self) -> &mut usize { + &mut self.index + } + + #[inline] + fn len(&self) -> usize { + todo!(); + } +} + +impl<'a, I, T, S1I, S1T> IntoIterator for &'a LazyVecFrom1 +where + I: StoredIndex, + T: StoredType + 'a, + S1I: StoredIndex, + S1T: StoredType, +{ + type Item = (I, Value<'a, T>); + type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T>; + + fn into_iter(self) -> Self::IntoIter { + LazyVecIterator { + lazy: self, + source: self.source.iter(), + index: 0, + } + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy.rs b/crates/brk_computer/src/storage/vecs/vec/lazy2.rs similarity index 54% rename from crates/brk_computer/src/storage/vecs/vec/lazy.rs rename to crates/brk_computer/src/storage/vecs/vec/lazy2.rs index 4c1c2a08c..d4561661a 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy.rs +++ b/crates/brk_computer/src/storage/vecs/vec/lazy2.rs @@ -4,40 +4,22 @@ use brk_vec::{ BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, }; -// LazyVec owns SourceVecs -// -// Functions: -// init() -// version() -// iter() -// len() ? -// -// When .iter() called convert SourcesVecs into iterators -// iter owns source iters -// call compute function to convert index and source iters to T +pub type ComputeFrom2 = for<'a> fn( + usize, + &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, +) -> Option; #[derive(Clone)] -pub struct LazyVec -where - I: StoredIndex, - T: StoredType, - S1I: StoredIndex, - S1T: StoredType, - S2I: StoredIndex, - S2T: StoredType, -{ +pub struct LazyVecFrom2 { version: Version, source1: StoredVec, source2: StoredVec, - compute: for<'a> fn( - usize, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, - ) -> Option, + compute: ComputeFrom2, phantom: PhantomData, } -impl LazyVec +impl LazyVecFrom2 where I: StoredIndex, T: StoredType, @@ -50,11 +32,7 @@ where version: Version, source1: StoredVec, source2: StoredVec, - compute: for<'a> fn( - usize, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, - ) -> Option, + compute: ComputeFrom2, ) -> Self { Self { version, @@ -68,59 +46,15 @@ where fn version(&self) -> Version { self.version } - - fn len(&self) -> usize { - todo!() - } } -pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> -where - I: StoredIndex, - T: StoredType, - S1I: StoredIndex, - S1T: StoredType, - S2I: StoredIndex, - S2T: StoredType, -{ - lazy: &'a LazyVec, +pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> { + lazy: &'a LazyVecFrom2, source1: StoredVecIterator<'a, S1I, S1T>, source2: StoredVecIterator<'a, S2I, S2T>, index: usize, } -impl LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T> -where - I: StoredIndex, - T: StoredType, - S1I: StoredIndex, - S1T: StoredType, - S2I: StoredIndex, - S2T: StoredType, -{ - #[inline] - pub fn set(&mut self, i: I) -> &mut Self { - self.index = i.unwrap_to_usize(); - self - } - - #[inline] - pub fn set_(&mut self, i: usize) { - self.index = i; - } - - #[inline] - pub fn get(&mut self, i: I) -> Option> { - self.set(i).next().map(|(_, v)| v) - } - - #[inline] - pub fn get_(&mut self, i: usize) -> Option> { - self.set_(i); - self.next().map(|(_, v)| v) - } -} - impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> where I: StoredIndex, @@ -163,11 +97,10 @@ where #[inline] fn len(&self) -> usize { todo!(); - // self.vec.len() } } -impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVec +impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVecFrom2 where I: StoredIndex, T: StoredType + 'a, diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy3.rs b/crates/brk_computer/src/storage/vecs/vec/lazy3.rs new file mode 100644 index 000000000..0a6baf8ef --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/vec/lazy3.rs @@ -0,0 +1,141 @@ +use std::marker::PhantomData; + +use brk_vec::{ + BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, +}; + +pub type ComputeFrom3 = for<'a> fn( + usize, + &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, + &mut dyn BaseVecIterator)>, +) -> Option; + +#[derive(Clone)] +pub struct LazyVecFrom3 { + version: Version, + source1: StoredVec, + source2: StoredVec, + source3: StoredVec, + compute: ComputeFrom3, + phantom: PhantomData, +} + +impl LazyVecFrom3 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + pub fn init( + version: Version, + source1: StoredVec, + source2: StoredVec, + source3: StoredVec, + compute: ComputeFrom3, + ) -> Self { + Self { + version, + source1, + source2, + source3, + compute, + phantom: PhantomData, + } + } + + fn version(&self) -> Version { + self.version + } +} + +pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { + lazy: &'a LazyVecFrom3, + source1: StoredVecIterator<'a, S1I, S1T>, + source2: StoredVecIterator<'a, S2I, S2T>, + source3: StoredVecIterator<'a, S3I, S3T>, + index: usize, +} + +impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator + for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> +where + I: StoredIndex, + T: StoredType + 'a, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + type Item = (I, Value<'a, T>); + + fn next(&mut self) -> Option { + let opt = (self.lazy.compute)( + self.index, + &mut self.lazy.source1.iter(), + &mut self.lazy.source2.iter(), + &mut self.lazy.source3.iter(), + ) + .map(|v| (I::from(self.index), Value::Owned(v))); + if opt.is_some() { + self.index += 1; + } + opt + } +} + +impl BaseVecIterator + for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + #[inline] + fn mut_index(&mut self) -> &mut usize { + &mut self.index + } + + #[inline] + fn len(&self) -> usize { + todo!(); + } +} + +impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator + for &'a LazyVecFrom3 +where + I: StoredIndex, + T: StoredType + 'a, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + type Item = (I, Value<'a, T>); + type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; + + fn into_iter(self) -> Self::IntoIter { + LazyVecIterator { + lazy: self, + source1: self.source1.iter(), + source2: self.source2.iter(), + source3: self.source3.iter(), + index: 0, + } + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/mod.rs b/crates/brk_computer/src/storage/vecs/vec/mod.rs index 59af2d5b4..db26330ea 100644 --- a/crates/brk_computer/src/storage/vecs/vec/mod.rs +++ b/crates/brk_computer/src/storage/vecs/vec/mod.rs @@ -1,13 +1,193 @@ +use std::path::Path; + +use brk_exit::Exit; +use clap_derive::ValueEnum; +use serde::{Deserialize, Serialize}; + mod _type; mod eager; -mod lazy; +mod lazy1; +mod lazy2; +mod lazy3; pub use _type::*; +use brk_core::StoredPhantom; +use brk_vec::{Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version}; pub use eager::*; -pub use lazy::*; +pub use lazy1::*; +pub use lazy2::*; +pub use lazy3::*; -#[derive(Debug, Clone, Copy)] -enum Mode { - Lazy, +#[derive( + Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum, +)] +pub enum Computation { Eager, + #[default] + Lazy, +} + +impl Computation { + pub fn eager(&self) -> bool { + *self == Self::Eager + } + + pub fn lazy(&self) -> bool { + *self == Self::Lazy + } +} + +#[derive(Clone)] +enum Dependencies { + From1(StoredVec, ComputeFrom1), + From2( + (StoredVec, StoredVec), + ComputeFrom2, + ), + From3( + ( + StoredVec, + StoredVec, + StoredVec, + ), + ComputeFrom3, + ), +} + +pub type ComputedVecFrom1 = + ComputedVec; +pub type ComputedVecFrom2 = + ComputedVec; +pub type ComputedVecFrom3 = + ComputedVec; + +#[derive(Clone)] +pub enum ComputedVec { + Eager { + vec: EagerVec, + deps: Dependencies, + }, + LazyFrom1(LazyVecFrom1), + LazyFrom2(LazyVecFrom2), + LazyFrom3(LazyVecFrom3), + // Lazy4 +} + +impl ComputedVec +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + pub fn forced_import_or_init_from_1( + mode: Computation, + path: &Path, + version: Version, + compressed: Compressed, + source: StoredVec, + compute: ComputeFrom1, + ) -> brk_vec::Result { + Ok(match mode { + Computation::Eager => Self::Eager { + vec: EagerVec::forced_import(path, version, compressed)?, + deps: Dependencies::From1(source, compute), + }, + Computation::Lazy => Self::LazyFrom1(LazyVecFrom1::init(version, source, compute)), + }) + } + + pub fn forced_import_or_init_from_2( + mode: Computation, + path: &Path, + version: Version, + compressed: Compressed, + source1: StoredVec, + source2: StoredVec, + compute: ComputeFrom2, + ) -> brk_vec::Result { + Ok(match mode { + Computation::Eager => Self::Eager { + vec: EagerVec::forced_import(path, version, compressed)?, + deps: Dependencies::From2((source1, source2), compute), + }, + Computation::Lazy => { + Self::LazyFrom2(LazyVecFrom2::init(version, source1, source2, compute)) + } + }) + } + + #[allow(clippy::too_many_arguments)] + pub fn forced_import_or_init_from_3( + mode: Computation, + path: &Path, + version: Version, + compressed: Compressed, + source1: StoredVec, + source2: StoredVec, + source3: StoredVec, + compute: ComputeFrom3, + ) -> brk_vec::Result { + Ok(match mode { + Computation::Eager => Self::Eager { + vec: EagerVec::forced_import(path, version, compressed)?, + deps: Dependencies::From3((source1, source2, source3), compute), + }, + Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init( + version, source1, source2, source3, compute, + )), + }) + } + + pub fn compute_if_necessary(&mut self, max_from: I, exit: &Exit) -> Result<()> { + let (vec, dependencies) = if let ComputedVec::Eager { + vec, + deps: dependencies, + } = self + { + (vec, dependencies) + } else { + return Ok(()); + }; + + match dependencies { + Dependencies::From1(source, compute) => { + let version = source.version(); + let mut iter = source.iter(); + let t = |i: I| { + compute(i.unwrap_to_usize(), &mut iter) + .map(|v| (i, v)) + .unwrap() + }; + vec.compute_to(max_from, 1, version, t, exit) + } + Dependencies::From2((source1, source2), compute) => { + let version = source1.version() + source2.version(); + let mut iter1 = source1.iter(); + let mut iter2 = source2.iter(); + let t = |i: I| { + compute(i.unwrap_to_usize(), &mut iter1, &mut iter2) + .map(|v| (i, v)) + .unwrap() + }; + vec.compute_to(max_from, 1, version, t, exit) + } + Dependencies::From3((source1, source2, source3), compute) => { + let version = source1.version() + source2.version() + source3.version(); + let mut iter1 = source1.iter(); + let mut iter2 = source2.iter(); + let mut iter3 = source3.iter(); + let t = |i: I| { + compute(i.unwrap_to_usize(), &mut iter1, &mut iter2, &mut iter3) + .map(|v| (i, v)) + .unwrap() + }; + vec.compute_to(max_from, 1, version, t, exit) + } + } + } } diff --git a/crates/brk_core/Cargo.toml b/crates/brk_core/Cargo.toml index a55b56e2c..6ab376ed7 100644 --- a/crates/brk_core/Cargo.toml +++ b/crates/brk_core/Cargo.toml @@ -16,8 +16,10 @@ log = { workspace = true } rapidhash = "1.4.0" rlimit = "0.10.2" serde = { workspace = true } +serde_derive = { workspace = true } serde_bytes = "0.11.17" zerocopy = { workspace = true } +zerocopy-derive = { workspace = true } [package.metadata.cargo-machete] ignored = ["serde_bytes"] diff --git a/crates/brk_core/src/structs/_addressindex.rs b/crates/brk_core/src/structs/_addressindex.rs index cdd4b0592..b705719e5 100644 --- a/crates/brk_core/src/structs/_addressindex.rs +++ b/crates/brk_core/src/structs/_addressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use byteview::ByteView; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::Error; diff --git a/crates/brk_core/src/structs/_addressindexoutputindex.rs b/crates/brk_core/src/structs/_addressindexoutputindex.rs index c85062b1f..7a7147ef4 100644 --- a/crates/brk_core/src/structs/_addressindexoutputindex.rs +++ b/crates/brk_core/src/structs/_addressindexoutputindex.rs @@ -1,5 +1,5 @@ use byteview::ByteView; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::Error; diff --git a/crates/brk_core/src/structs/_addresstype.rs b/crates/brk_core/src/structs/_addresstype.rs index 6d0247e9d..fb1705148 100644 --- a/crates/brk_core/src/structs/_addresstype.rs +++ b/crates/brk_core/src/structs/_addresstype.rs @@ -1,5 +1,5 @@ use serde::Serialize; -use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; +use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; use super::OutputType; diff --git a/crates/brk_core/src/structs/addressbytes.rs b/crates/brk_core/src/structs/addressbytes.rs index cafa0ba60..d22cf485f 100644 --- a/crates/brk_core/src/structs/addressbytes.rs +++ b/crates/brk_core/src/structs/addressbytes.rs @@ -8,7 +8,7 @@ use bitcoin::{ }; use derive_deref::{Deref, DerefMut}; use serde::{Serialize, Serializer}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::Error; diff --git a/crates/brk_core/src/structs/addressbyteshash.rs b/crates/brk_core/src/structs/addressbyteshash.rs index 303f225e8..35bcb19d1 100644 --- a/crates/brk_core/src/structs/addressbyteshash.rs +++ b/crates/brk_core/src/structs/addressbyteshash.rs @@ -2,7 +2,8 @@ use std::hash::Hasher; use byteview::ByteView; use derive_deref::Deref; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::Error; diff --git a/crates/brk_core/src/structs/bitcoin.rs b/crates/brk_core/src/structs/bitcoin.rs index 8020a47dc..9dff9e90c 100644 --- a/crates/brk_core/src/structs/bitcoin.rs +++ b/crates/brk_core/src/structs/bitcoin.rs @@ -1,7 +1,7 @@ use std::ops::{Add, Div, Mul}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Sats; diff --git a/crates/brk_core/src/structs/blockhash.rs b/crates/brk_core/src/structs/blockhash.rs index f1375219a..52576b3c1 100644 --- a/crates/brk_core/src/structs/blockhash.rs +++ b/crates/brk_core/src/structs/blockhash.rs @@ -4,7 +4,7 @@ use bitcoin::hashes::Hash; use bitcoincore_rpc::{Client, RpcApi}; use derive_deref::Deref; use serde::{Serialize, Serializer}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Height; diff --git a/crates/brk_core/src/structs/blockhashprefix.rs b/crates/brk_core/src/structs/blockhashprefix.rs index 345d90a1f..6b7d30d19 100644 --- a/crates/brk_core/src/structs/blockhashprefix.rs +++ b/crates/brk_core/src/structs/blockhashprefix.rs @@ -1,6 +1,7 @@ use byteview::ByteView; use derive_deref::Deref; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Error, copy_first_8bytes}; diff --git a/crates/brk_core/src/structs/cents.rs b/crates/brk_core/src/structs/cents.rs index 7aa06256d..7cb1a3d43 100644 --- a/crates/brk_core/src/structs/cents.rs +++ b/crates/brk_core/src/structs/cents.rs @@ -1,7 +1,7 @@ use std::ops::{Add, Div}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Dollars; diff --git a/crates/brk_core/src/structs/date.rs b/crates/brk_core/src/structs/date.rs index a8f252ee3..d4e745d9d 100644 --- a/crates/brk_core/src/structs/date.rs +++ b/crates/brk_core/src/structs/date.rs @@ -1,6 +1,6 @@ use jiff::{Span, civil::Date as Date_, tz::TimeZone}; use serde::{Serialize, Serializer}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{DateIndex, Timestamp}; diff --git a/crates/brk_core/src/structs/dateindex.rs b/crates/brk_core/src/structs/dateindex.rs index 2d248e02a..47d927374 100644 --- a/crates/brk_core/src/structs/dateindex.rs +++ b/crates/brk_core/src/structs/dateindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use serde::Serialize; // use color_eyre::eyre::eyre; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Error}; diff --git a/crates/brk_core/src/structs/decadeindex.rs b/crates/brk_core/src/structs/decadeindex.rs index 013e07f2d..b0f50984c 100644 --- a/crates/brk_core/src/structs/decadeindex.rs +++ b/crates/brk_core/src/structs/decadeindex.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/difficultyepoch.rs b/crates/brk_core/src/structs/difficultyepoch.rs index fb6963392..2c898267e 100644 --- a/crates/brk_core/src/structs/difficultyepoch.rs +++ b/crates/brk_core/src/structs/difficultyepoch.rs @@ -4,7 +4,7 @@ use std::{ }; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs index d4568e70d..32a014a33 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_core/src/structs/dollars.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div, Mul}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Bitcoin, Cents, Sats}; diff --git a/crates/brk_core/src/structs/feerate.rs b/crates/brk_core/src/structs/feerate.rs index ea502e0e0..dfd34f33e 100644 --- a/crates/brk_core/src/structs/feerate.rs +++ b/crates/brk_core/src/structs/feerate.rs @@ -1,7 +1,7 @@ use std::ops::{Add, Div}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Sats, StoredUsize}; diff --git a/crates/brk_core/src/structs/halvingepoch.rs b/crates/brk_core/src/structs/halvingepoch.rs index a989a4c19..dfe31e4d4 100644 --- a/crates/brk_core/src/structs/halvingepoch.rs +++ b/crates/brk_core/src/structs/halvingepoch.rs @@ -4,7 +4,7 @@ use std::{ }; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index 23cae0d5a..c66d02bf4 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -5,7 +5,8 @@ use std::{ use bitcoincore_rpc::{Client, RpcApi}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/inputindex.rs b/crates/brk_core/src/structs/inputindex.rs index c9b98c554..be518e442 100644 --- a/crates/brk_core/src/structs/inputindex.rs +++ b/crates/brk_core/src/structs/inputindex.rs @@ -2,7 +2,7 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/monthindex.rs b/crates/brk_core/src/structs/monthindex.rs index 5bc68e9f8..fd96bc9cb 100644 --- a/crates/brk_core/src/structs/monthindex.rs +++ b/crates/brk_core/src/structs/monthindex.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/ohlc.rs b/crates/brk_core/src/structs/ohlc.rs index e48ac4481..6684cc724 100644 --- a/crates/brk_core/src/structs/ohlc.rs +++ b/crates/brk_core/src/structs/ohlc.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::{Serialize, Serializer, ser::SerializeTuple}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Cents, Dollars, Sats}; diff --git a/crates/brk_core/src/structs/outputindex.rs b/crates/brk_core/src/structs/outputindex.rs index 6dabbf294..df860d56a 100644 --- a/crates/brk_core/src/structs/outputindex.rs +++ b/crates/brk_core/src/structs/outputindex.rs @@ -2,7 +2,7 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/outputtype.rs b/crates/brk_core/src/structs/outputtype.rs index f6e3d7f6a..8345a55eb 100644 --- a/crates/brk_core/src/structs/outputtype.rs +++ b/crates/brk_core/src/structs/outputtype.rs @@ -1,6 +1,6 @@ use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; use serde::Serialize; -use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; +use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; #[derive( Debug, diff --git a/crates/brk_core/src/structs/outputtypeindex.rs b/crates/brk_core/src/structs/outputtypeindex.rs index e5a302b6d..523c61213 100644 --- a/crates/brk_core/src/structs/outputtypeindex.rs +++ b/crates/brk_core/src/structs/outputtypeindex.rs @@ -3,7 +3,8 @@ use std::ops::Add; use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Error}; diff --git a/crates/brk_core/src/structs/quarterindex.rs b/crates/brk_core/src/structs/quarterindex.rs index e15726878..5f4e31863 100644 --- a/crates/brk_core/src/structs/quarterindex.rs +++ b/crates/brk_core/src/structs/quarterindex.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/rawlocktime.rs b/crates/brk_core/src/structs/rawlocktime.rs index d4263763f..986592749 100644 --- a/crates/brk_core/src/structs/rawlocktime.rs +++ b/crates/brk_core/src/structs/rawlocktime.rs @@ -1,6 +1,6 @@ use bitcoin::absolute::LockTime; use serde::Serialize; -use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; +use zerocopy_derive::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; #[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)] pub struct RawLockTime(u32); diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index dd36ed37f..920f0deed 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -5,7 +5,7 @@ use std::{ use bitcoin::Amount; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/stored_f64.rs b/crates/brk_core/src/structs/stored_f64.rs index 9369b5ecb..356c82f25 100644 --- a/crates/brk_core/src/structs/stored_f64.rs +++ b/crates/brk_core/src/structs/stored_f64.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/stored_u32.rs b/crates/brk_core/src/structs/stored_u32.rs index c656e5863..4b0608a2c 100644 --- a/crates/brk_core/src/structs/stored_u32.rs +++ b/crates/brk_core/src/structs/stored_u32.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/stored_u64.rs b/crates/brk_core/src/structs/stored_u64.rs index 93c0b2808..b5427647e 100644 --- a/crates/brk_core/src/structs/stored_u64.rs +++ b/crates/brk_core/src/structs/stored_u64.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/stored_u8.rs b/crates/brk_core/src/structs/stored_u8.rs index 69d8fde16..70a27246c 100644 --- a/crates/brk_core/src/structs/stored_u8.rs +++ b/crates/brk_core/src/structs/stored_u8.rs @@ -2,11 +2,14 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; +pub type StoredPhantom = StoredU8; + #[derive( + Default, Debug, Deref, Clone, @@ -77,3 +80,16 @@ impl From for f64 { value.0 as f64 } } + +impl Add for StoredU8 { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0.checked_add(rhs as u8).unwrap()) + } +} + +impl From for usize { + fn from(value: StoredU8) -> Self { + value.0 as usize + } +} diff --git a/crates/brk_core/src/structs/stored_usize.rs b/crates/brk_core/src/structs/stored_usize.rs index db68c8fe8..26c209b16 100644 --- a/crates/brk_core/src/structs/stored_usize.rs +++ b/crates/brk_core/src/structs/stored_usize.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs index ced075237..6cec46034 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_core/src/structs/timestamp.rs @@ -3,7 +3,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use jiff::{civil::date, tz::TimeZone}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/txid.rs b/crates/brk_core/src/structs/txid.rs index 0b0475c0e..2b7748665 100644 --- a/crates/brk_core/src/structs/txid.rs +++ b/crates/brk_core/src/structs/txid.rs @@ -3,7 +3,7 @@ use std::{fmt, mem}; use bitcoin::hashes::Hash; use derive_deref::Deref; use serde::{Serialize, Serializer}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct Txid([u8; 32]); diff --git a/crates/brk_core/src/structs/txidprefix.rs b/crates/brk_core/src/structs/txidprefix.rs index 9341b0d10..7107090ab 100644 --- a/crates/brk_core/src/structs/txidprefix.rs +++ b/crates/brk_core/src/structs/txidprefix.rs @@ -1,6 +1,7 @@ use byteview::ByteView; use derive_deref::Deref; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Error, copy_first_8bytes}; diff --git a/crates/brk_core/src/structs/txindex.rs b/crates/brk_core/src/structs/txindex.rs index 2095713c4..6bc798a4f 100644 --- a/crates/brk_core/src/structs/txindex.rs +++ b/crates/brk_core/src/structs/txindex.rs @@ -3,7 +3,8 @@ use std::ops::{Add, AddAssign}; use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{CheckedSub, Error}; diff --git a/crates/brk_core/src/structs/txversion.rs b/crates/brk_core/src/structs/txversion.rs index ed649eebe..2eb5ab4af 100644 --- a/crates/brk_core/src/structs/txversion.rs +++ b/crates/brk_core/src/structs/txversion.rs @@ -1,6 +1,6 @@ use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::StoredU8; diff --git a/crates/brk_core/src/structs/weekindex.rs b/crates/brk_core/src/structs/weekindex.rs index dc7250d5a..18fb844c1 100644 --- a/crates/brk_core/src/structs/weekindex.rs +++ b/crates/brk_core/src/structs/weekindex.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_core/src/structs/weight.rs b/crates/brk_core/src/structs/weight.rs index 7ec7bd5c8..5c56ceb6f 100644 --- a/crates/brk_core/src/structs/weight.rs +++ b/crates/brk_core/src/structs/weight.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div}; use derive_deref::Deref; use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive( Debug, diff --git a/crates/brk_core/src/structs/yearindex.rs b/crates/brk_core/src/structs/yearindex.rs index 3c378a9be..9e304657c 100644 --- a/crates/brk_core/src/structs/yearindex.rs +++ b/crates/brk_core/src/structs/yearindex.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::CheckedSub; diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 5dbdb88f7..c841be780 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -442,20 +442,20 @@ impl Vecs { } pub fn flush(&mut self, height: Height) -> Result<()> { - self.as_mut_any_vecs() + self.mut_any_vecs() .into_par_iter() .try_for_each(|vec| vec.flush(height)) } pub fn starting_height(&mut self) -> Height { - self.as_mut_any_vecs() + self.mut_any_vecs() .into_iter() .map(|vec| vec.height().map(Height::incremented).unwrap_or_default()) .min() .unwrap() } - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { vec![ self.emptyoutputindex_to_txindex.any_vec(), self.height_to_blockhash.any_vec(), @@ -504,7 +504,7 @@ impl Vecs { ] } - fn as_mut_any_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> { + fn mut_any_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> { vec![ &mut self.emptyoutputindex_to_txindex, &mut self.height_to_blockhash, diff --git a/crates/brk_query/Cargo.toml b/crates/brk_query/Cargo.toml index 068f56748..fe99255e2 100644 --- a/crates/brk_query/Cargo.toml +++ b/crates/brk_query/Cargo.toml @@ -11,6 +11,7 @@ brk_computer = { workspace = true } brk_indexer = { workspace = true } brk_vec = { workspace = true } clap = { workspace = true } +clap_derive = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } serde = { workspace = true } diff --git a/crates/brk_query/examples/main.rs b/crates/brk_query/examples/main.rs index 21e5e5412..0c7af83a7 100644 --- a/crates/brk_query/examples/main.rs +++ b/crates/brk_query/examples/main.rs @@ -1,6 +1,6 @@ use std::path::Path; -use brk_computer::Computer; +use brk_computer::{Computation, Computer}; use brk_indexer::Indexer; use brk_query::{Index, Query}; @@ -15,7 +15,7 @@ pub fn main() -> color_eyre::Result<()> { indexer.import_vecs()?; let mut computer = Computer::new(outputs_dir, None, compressed); - computer.import_vecs(&indexer)?; + computer.import_vecs(&indexer, Computation::Lazy)?; let query = Query::build(&indexer, &computer); diff --git a/crates/brk_query/src/format.rs b/crates/brk_query/src/format.rs index 137dd7cae..78279d02d 100644 --- a/crates/brk_query/src/format.rs +++ b/crates/brk_query/src/format.rs @@ -1,4 +1,4 @@ -use clap::ValueEnum; +use clap_derive::ValueEnum; use color_eyre::eyre::eyre; use serde::Deserialize; diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index dc27f83f3..ffff4226e 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -34,13 +34,13 @@ impl<'a> Query<'a> { indexer .vecs() - .as_any_vecs() + .any_vecs() .into_iter() .for_each(|vec| vec_trees.insert(vec)); computer .vecs() - .as_any_vecs() + .any_vecs() .into_iter() .for_each(|vec| vec_trees.insert(vec)); diff --git a/crates/brk_query/src/params.rs b/crates/brk_query/src/params.rs index 167c86467..7221b8532 100644 --- a/crates/brk_query/src/params.rs +++ b/crates/brk_query/src/params.rs @@ -1,4 +1,5 @@ -use clap::{Parser, builder::PossibleValuesParser}; +use clap::builder::PossibleValuesParser; +use clap_derive::Parser; use serde::Deserialize; use serde_with::{OneOrMany, formats::PreferOne, serde_as}; diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 73af84e5d..42294d7fd 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -17,6 +17,7 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_query = { workspace = true } clap = { workspace = true } +clap_derive = { workspace = true } color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 8c50767e2..b04bbadd1 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -1,6 +1,6 @@ use std::{path::Path, thread::sleep, time::Duration}; -use brk_computer::Computer; +use brk_computer::{Computation, Computer}; use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_fetcher::Fetcher; @@ -40,7 +40,7 @@ pub fn main() -> color_eyre::Result<()> { let mut computer = Computer::new(outputs_dir, fetcher, compressed); computer.import_stores(&indexer)?; - computer.import_vecs(&indexer)?; + computer.import_vecs(&indexer, Computation::Lazy)?; tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/brk_server/src/files/website.rs b/crates/brk_server/src/files/website.rs index 747eab345..0bd8ae21c 100644 --- a/crates/brk_server/src/files/website.rs +++ b/crates/brk_server/src/files/website.rs @@ -1,4 +1,4 @@ -use clap::ValueEnum; +use clap_derive::ValueEnum; use serde::{Deserialize, Serialize}; #[derive( diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index d7bc59001..734a1d506 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -16,4 +16,5 @@ rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } zerocopy = { workspace = true } +zerocopy-derive = { workspace = true } zstd = "0.13.3" diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 3bb1cc684..90524dea7 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -19,11 +19,7 @@ pub use traits::*; use variants::*; #[derive(Debug, Clone)] -pub enum StoredVec -where - I: StoredIndex, - T: StoredType, -{ +pub enum StoredVec { Raw(RawVec), Compressed(CompressedVec), } @@ -209,11 +205,7 @@ where } #[derive(Debug)] -pub enum StoredVecIterator<'a, I, T> -where - I: StoredIndex, - T: StoredType, -{ +pub enum StoredVecIterator<'a, I, T> { Raw(RawVecIterator<'a, I, T>), Compressed(CompressedVecIterator<'a, I, T>), } diff --git a/crates/brk_vec/src/structs/compressed_page_meta.rs b/crates/brk_vec/src/structs/compressed_page_meta.rs index 6f0ee9911..a0b18cf7e 100644 --- a/crates/brk_vec/src/structs/compressed_page_meta.rs +++ b/crates/brk_vec/src/structs/compressed_page_meta.rs @@ -1,4 +1,4 @@ -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive(Debug, Clone, IntoBytes, Immutable, FromBytes, KnownLayout)] pub struct CompressedPageMetadata { diff --git a/crates/brk_vec/src/structs/length.rs b/crates/brk_vec/src/structs/length.rs index 5634443d4..7f7542f53 100644 --- a/crates/brk_vec/src/structs/length.rs +++ b/crates/brk_vec/src/structs/length.rs @@ -5,7 +5,8 @@ use std::{ path::Path, }; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Error, Result}; diff --git a/crates/brk_vec/src/structs/version.rs b/crates/brk_vec/src/structs/version.rs index c1940f205..1939cd9f0 100644 --- a/crates/brk_vec/src/structs/version.rs +++ b/crates/brk_vec/src/structs/version.rs @@ -5,7 +5,8 @@ use std::{ path::Path, }; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Error, Result}; diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 69e97350d..3b7f2016b 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -359,11 +359,7 @@ where } } -impl Clone for CompressedVec -where - I: StoredIndex, - T: StoredType, -{ +impl Clone for CompressedVec { fn clone(&self) -> Self { Self { inner: self.inner.clone(), diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index f370481ba..95fc66fae 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -194,11 +194,7 @@ where } } -impl Clone for RawVec -where - I: StoredIndex, - T: StoredType, -{ +impl Clone for RawVec { fn clone(&self) -> Self { Self { version: self.version,