diff --git a/Cargo.lock b/Cargo.lock index c0d7e79ca..cd60e19b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,7 +494,7 @@ dependencies = [ "brk_server", "brk_store", "brk_structs", - "brk_vecs", + "vecdb", ] [[package]] @@ -521,7 +521,6 @@ dependencies = [ "brk_logger", "brk_parser", "brk_server", - "brk_vecs", "clap", "clap_derive", "color-eyre", @@ -530,6 +529,7 @@ dependencies = [ "serde", "tokio", "toml", + "vecdb", "zip", ] @@ -545,11 +545,11 @@ dependencies = [ "brk_logger", "brk_parser", "brk_structs", - "brk_vecs", "derive_deref", "log", "rayon", "serde", + "vecdb", "zerocopy", "zerocopy-derive", ] @@ -559,11 +559,11 @@ name = "brk_error" version = "0.0.85" dependencies = [ "bitcoincore-rpc", - "brk_vecs", "fjall", "jiff", "minreq", "serde_json", + "vecdb", "zerocopy", ] @@ -590,10 +590,10 @@ dependencies = [ "brk_parser", "brk_store", "brk_structs", - "brk_vecs", "fjall", "log", "rayon", + "vecdb", ] [[package]] @@ -604,13 +604,13 @@ dependencies = [ "brk_error", "brk_indexer", "brk_structs", - "brk_vecs", "derive_deref", "schemars 1.0.4", "serde", "serde_json", "serde_with", "tabled", + "vecdb", ] [[package]] @@ -650,9 +650,9 @@ dependencies = [ [[package]] name = "brk_rmcp" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0faa9da00a4fc50c5a4f331850c37bf69529e7a88890e08962c1e9a32ca11dbd" +checksum = "34e99c51043db05e5d77c381124c1705c9a360f9a88bef0af44397134929d730" dependencies = [ "base64 0.22.1", "brk_rmcp-macros", @@ -680,9 +680,9 @@ dependencies = [ [[package]] name = "brk_rmcp-macros" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6078c726658d7abc2ec1b55196791fcfb58f20f1dd9f78401c749c5138a79562" +checksum = "c19a9e21ea5789ad190ce5c572bdbba50e589f9dd01acd19600b86442b56f02d" dependencies = [ "darling 0.21.1", "proc-macro2", @@ -1025,7 +1025,6 @@ dependencies = [ "brk_logger", "brk_mcp", "brk_parser", - "brk_vecs", "jiff", "log", "quick_cache", @@ -1033,6 +1032,7 @@ dependencies = [ "tokio", "tower-http", "tracing", + "vecdb", ] [[package]] @@ -1065,44 +1065,17 @@ dependencies = [ "bitcoin", "bitcoincore-rpc", "brk_error", - "brk_vecs", "byteview", "derive_deref", "jiff", "rapidhash", "serde", "serde_bytes", + "vecdb", "zerocopy", "zerocopy-derive", ] -[[package]] -name = "brk_vecs" -version = "0.0.85" -dependencies = [ - "brk_vecs_macros", - "ctrlc", - "libc", - "log", - "memmap2", - "parking_lot", - "pco", - "rayon", - "serde", - "serde_derive", - "serde_json", - "zerocopy", - "zerocopy-derive", -] - -[[package]] -name = "brk_vecs_macros" -version = "0.0.85" -dependencies = [ - "quote", - "syn 2.0.104", -] - [[package]] name = "brotli" version = "8.0.1" @@ -1168,9 +1141,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.31" +version = "1.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2" +checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" dependencies = [ "jobserver", "libc", @@ -1990,9 +1963,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", @@ -2268,7 +2241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "serde", ] @@ -2812,7 +2785,7 @@ checksum = "5c67f2b817263a72b6cc5b46e32467bc4077496f88315ce08c1796647fed84d4" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "oxc_data_structures", "oxc_estree", "rustc-hash", @@ -3314,7 +3287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "indexmap 2.10.0", "serde", ] @@ -3498,7 +3471,7 @@ checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786" dependencies = [ "ahash", "equivalent", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "parking_lot", ] @@ -3597,9 +3570,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "3.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef3d82b018f786967b1a5d34a08ebc3c7a9ab35b5bcbe3e2e057a0a453f26c8" +checksum = "efee4b7317469c6c6e7fdeee3d094313af846a97678d6ed971d83a852d730083" [[package]] name = "rayon" @@ -3685,7 +3658,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010" dependencies = [ - "hashbrown 0.15.4", + "hashbrown 0.15.5", "memchr", ] @@ -3772,9 +3745,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3888,6 +3861,19 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" +[[package]] +name = "seqdb" +version = "0.0.2" +dependencies = [ + "libc", + "log", + "memmap2", + "parking_lot", + "rayon", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "serde" version = "1.0.219" @@ -4050,9 +4036,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" @@ -4700,6 +4686,32 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" +[[package]] +name = "vecdb" +version = "0.0.2" +dependencies = [ + "ctrlc", + "log", + "parking_lot", + "pco", + "rayon", + "seqdb", + "serde", + "serde_derive", + "serde_json", + "vecdb_derive", + "zerocopy", + "zerocopy-derive", +] + +[[package]] +name = "vecdb_derive" +version = "0.0.2" +dependencies = [ + "quote", + "syn 2.0.104", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index ff3c66335..e7725b283 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,8 +38,8 @@ brk_mcp = { version = "0.0.85", path = "crates/brk_mcp" } brk_parser = { version = "0.0.85", path = "crates/brk_parser" } brk_server = { version = "0.0.85", path = "crates/brk_server" } brk_store = { version = "0.0.85", path = "crates/brk_store" } -brk_vecs = { version = "0.0.85", path = "crates/brk_vecs" } -brk_vecs_macros = { version = "0.0.85", path = "crates/brk_vecs_macros" } +# vecdb = { version = "0.0.2", features = ["derive"]} +vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} byteview = "=0.6.1" derive_deref = "1.1.1" fjall = "2.11.2" diff --git a/README.md b/README.md index 6be20ce3f..74c4e7ab1 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,8 @@ In contrast, existing alternatives tend to be either [very costly](https://studi - [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) - [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK -- [`brk_vecs`](https://crates.io/crates/brk_vecs): A KISS index/value store -- [`brk_vecs_macros`](https://crates.io/crates/brk_vecs_macros): Macros for [`brk_vecs`](https://crates.io/crates/brk_vecs) +- [`vecdb`](https://crates.io/crates/vecdb): A KISS index-value storage engine +- [`vecdb_macros`](https://crates.io/crates/vecdb_macros): Macros for [`vecdb`](https://crates.io/crates/vecdb) ## Hosting as a service diff --git a/TODO.md b/TODO.md index f84c34f1d..2f81e4134 100644 --- a/TODO.md +++ b/TODO.md @@ -60,11 +60,6 @@ - add extensions support (.json .csv …) - if format instead of extension then don't download file - add support for https (rustls) - - lru cache - - _vec_ - - add native lock file support (once it's available in stable rust) - - improve compressed mode (slow reads) - - add ema support - __docs__ - _README_ - add a comparison table with alternatives diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 0128c4d53..c9766a324 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -37,7 +37,7 @@ parser = ["brk_parser"] server = ["brk_server"] store = ["brk_store"] structs = ["brk_structs"] -vecs = ["brk_vecs"] +vecs = ["vecdb"] [dependencies] brk_bundler = { workspace = true, optional = true } @@ -53,7 +53,7 @@ brk_parser = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } brk_store = { workspace = true, optional = true } brk_structs = { workspace = true, optional = true } -brk_vecs = { workspace = true, optional = true } +vecdb = { workspace = true, optional = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index b117391f8..62f78b219 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -53,4 +53,4 @@ pub use brk_store as store; #[cfg(feature = "vecs")] #[doc(inline)] -pub use brk_vecs as vecs; +pub use vecdb as vecs; diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index c875ad738..16aaf75ef 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -18,7 +18,7 @@ brk_interface = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_server = { workspace = true } -brk_vecs = { workspace = true } +vecdb = { workspace = true } clap = { version = "4.5.43", features = ["string"] } clap_derive = "4.5.41" color-eyre = "0.6.5" diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index 63b27e454..17aa89fe5 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -32,11 +32,16 @@ pub struct Config { #[arg(long, value_name = "PATH")] brkdir: Option, - /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved + /// Activate fetching prices from BRK's API and the computation of all price related datasets, default: true, saved #[serde(default, deserialize_with = "default_on_error")] #[arg(short = 'F', long, value_name = "BOOL")] fetch: Option, + /// Activate fetching prices from exchanges APIs if `fetch` is also set to `true`, default: true, saved + #[serde(default, deserialize_with = "default_on_error")] + #[arg(long, value_name = "BOOL")] + exchanges: Option, + /// Website served by the server (if active), default: default, saved #[serde(default, deserialize_with = "default_on_error")] #[arg(short, long)] @@ -117,6 +122,10 @@ impl Config { config_saved.fetch = Some(fetch); } + if let Some(exchanges) = config_args.exchanges.take() { + config_saved.exchanges = Some(exchanges); + } + if let Some(website) = config_args.website.take() { config_saved.website = Some(website); } @@ -196,7 +205,9 @@ impl Config { if self.rpc_auth().is_err() { println!( - "No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help." + "Unsuccessful authentication with the RPC client. +First make sure that `bitcoind` is running. If it is then please either set --rpccookiefile or --rpcuser and --rpcpassword as the default values seemed to have failed. +Finally, you can run the program with '-h' for help." ); std::process::exit(1); } @@ -311,9 +322,13 @@ impl Config { self.fetch.is_none_or(|b| b) } + pub fn exchanges(&self) -> bool { + self.exchanges.is_none_or(|b| b) + } + pub fn fetcher(&self) -> Option { self.fetch() - .then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap()) + .then(|| Fetcher::import(self.exchanges(), Some(self.harsdir().as_path())).unwrap()) } pub fn check_collisions(&self) -> bool { diff --git a/crates/brk_cli/src/lib.rs b/crates/brk_cli/src/lib.rs index dd48bd680..0b419b3b0 100644 --- a/crates/brk_cli/src/lib.rs +++ b/crates/brk_cli/src/lib.rs @@ -14,8 +14,8 @@ use brk_computer::Computer; use brk_indexer::Indexer; use brk_interface::Interface; use brk_server::{Server, VERSION}; -use brk_vecs::Exit; use log::info; +use vecdb::Exit; mod bridge; mod config; @@ -136,9 +136,9 @@ pub fn run() -> color_eyre::Result<()> { info!("{} blocks found.", block_count + 1); let starting_indexes = - indexer.index(&parser, rpc, &exit, config.check_collisions())?; + indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap(); - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, starting_indexes, &exit).unwrap(); if let Some(delay) = config.delay() { sleep(Duration::from_secs(delay)) diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 902afc3da..eb9d2f168 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -17,7 +17,7 @@ brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } -brk_vecs = { workspace = true } +vecdb = { workspace = true } derive_deref = { workspace = true } log = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 2086db3b0..240acdbf8 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -9,7 +9,7 @@ use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_vecs::Exit; +use vecdb::Exit; pub fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); @@ -38,7 +38,7 @@ pub fn main() -> Result<()> { let mut indexer = Indexer::forced_import(&outputs_dir)?; - let fetcher = Fetcher::import(None)?; + let fetcher = Fetcher::import(true, None)?; let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?; diff --git a/crates/brk_computer/src/blocks.rs b/crates/brk_computer/src/blocks.rs index 5bb4c8273..9e6ae77f3 100644 --- a/crates/brk_computer/src/blocks.rs +++ b/crates/brk_computer/src/blocks.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; @@ -6,8 +6,8 @@ use brk_structs::{ CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version, Weight, }; -use brk_vecs::{ - AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, VecIterator, +use vecdb::{ + AnyCollectableVec, Computation, Database, EagerVec, Exit, Format, PAGE_SIZE, VecIterator, }; use crate::grouped::Source; @@ -22,7 +22,7 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub height_to_interval: EagerVec, pub height_to_vbytes: EagerVec, @@ -44,18 +44,18 @@ impl Vecs { format: Format, indexes: &indexes::Vecs, ) -> Result { - let file = Arc::new(File::open(&parent.join("blocks"))?); - file.set_min_len(PAGE_SIZE * 1_000_000)?; + let db = Database::open(&parent.join("blocks"))?; + db.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { height_to_interval: EagerVec::forced_import( - &file, + &db, "interval", version + VERSION + Version::ZERO, format, )?, timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "timestamp", Source::Compute, version + VERSION + Version::ZERO, @@ -65,7 +65,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, indexes_to_block_interval: ComputedVecsFromHeight::forced_import( - &file, + &db, "block_interval", Source::None, version + VERSION + Version::ZERO, @@ -78,7 +78,7 @@ impl Vecs { .add_average(), )?, indexes_to_block_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "block_count", Source::Compute, version + VERSION + Version::ZERO, @@ -88,7 +88,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_weight: ComputedVecsFromHeight::forced_import( - &file, + &db, "block_weight", Source::None, version + VERSION + Version::ZERO, @@ -98,7 +98,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_block_size: ComputedVecsFromHeight::forced_import( - &file, + &db, "block_size", Source::None, version + VERSION + Version::ZERO, @@ -108,13 +108,13 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, height_to_vbytes: EagerVec::forced_import( - &file, + &db, "vbytes", version + VERSION + Version::ZERO, format, )?, indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( - &file, + &db, "block_vbytes", Source::None, version + VERSION + Version::ZERO, @@ -124,19 +124,19 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, difficultyepoch_to_timestamp: EagerVec::forced_import( - &file, + &db, "timestamp", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_timestamp: EagerVec::forced_import( - &file, + &db, "timestamp", version + VERSION + Version::ZERO, format, )?, - file, + db, }) } @@ -148,7 +148,7 @@ impl Vecs { exit: &Exit, ) -> Result<()> { self.compute_(indexer, indexes, starting_indexes, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index e7bc7cbb7..4c191cba6 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -1,9 +1,9 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version}; -use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator}; +use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator}; use super::{ Indexes, @@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub indexes_to_coinblocks_created: ComputedVecsFromHeight, pub indexes_to_coinblocks_stored: ComputedVecsFromHeight, @@ -55,14 +55,14 @@ impl Vecs { indexes: &indexes::Vecs, price: Option<&price::Vecs>, ) -> Result { - let file = Arc::new(File::open(&parent.join("cointime"))?); - file.set_min_len(PAGE_SIZE * 1_000_000)?; + let db = Database::open(&parent.join("cointime"))?; + db.set_min_len(PAGE_SIZE * 1_000_000)?; let compute_dollars = price.is_some(); Ok(Self { indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import( - &file, + &db, "coinblocks_created", Source::Compute, version + VERSION + Version::ZERO, @@ -72,7 +72,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import( - &file, + &db, "coinblocks_stored", Source::Compute, version + VERSION + Version::ZERO, @@ -82,7 +82,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_liveliness: ComputedVecsFromHeight::forced_import( - &file, + &db, "liveliness", Source::Compute, version + VERSION + Version::ZERO, @@ -92,7 +92,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaultedness: ComputedVecsFromHeight::forced_import( - &file, + &db, "vaultedness", Source::Compute, version + VERSION + Version::ZERO, @@ -102,7 +102,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import( - &file, + &db, "activity_to_vaultedness_ratio", Source::Compute, version + VERSION + Version::ZERO, @@ -112,7 +112,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "vaulted_supply", Source::Compute, version + VERSION + Version::ONE, @@ -123,7 +123,7 @@ impl Vecs { indexes, )?, indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "active_supply", Source::Compute, version + VERSION + Version::ONE, @@ -134,7 +134,7 @@ impl Vecs { indexes, )?, indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import( - &file, + &db, "thermo_cap", Source::Compute, version + VERSION + Version::ONE, @@ -144,7 +144,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_investor_cap: ComputedVecsFromHeight::forced_import( - &file, + &db, "investor_cap", Source::Compute, version + VERSION + Version::ONE, @@ -154,7 +154,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import( - &file, + &db, "vaulted_cap", Source::Compute, version + VERSION + Version::ONE, @@ -164,7 +164,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_active_cap: ComputedVecsFromHeight::forced_import( - &file, + &db, "active_cap", Source::Compute, version + VERSION + Version::ONE, @@ -174,7 +174,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import( - &file, + &db, "vaulted_price", Source::Compute, version + VERSION + Version::ZERO, @@ -184,7 +184,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "vaulted_price", Source::None, version + VERSION + Version::ZERO, @@ -194,7 +194,7 @@ impl Vecs { true, )?, indexes_to_active_price: ComputedVecsFromHeight::forced_import( - &file, + &db, "active_price", Source::Compute, version + VERSION + Version::ZERO, @@ -204,7 +204,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "active_price", Source::None, version + VERSION + Version::ZERO, @@ -214,7 +214,7 @@ impl Vecs { true, )?, indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import( - &file, + &db, "true_market_mean", Source::Compute, version + VERSION + Version::ZERO, @@ -224,7 +224,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "true_market_mean", Source::None, version + VERSION + Version::ZERO, @@ -234,7 +234,7 @@ impl Vecs { true, )?, indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import( - &file, + &db, "cointime_value_destroyed", Source::Compute, version + VERSION + Version::ZERO, @@ -244,7 +244,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import( - &file, + &db, "cointime_value_created", Source::Compute, version + VERSION + Version::ZERO, @@ -254,7 +254,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import( - &file, + &db, "cointime_value_stored", Source::Compute, version + VERSION + Version::ZERO, @@ -264,7 +264,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_cointime_price: ComputedVecsFromHeight::forced_import( - &file, + &db, "cointime_price", Source::Compute, version + VERSION + Version::ZERO, @@ -274,7 +274,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import( - &file, + &db, "cointime_cap", Source::Compute, version + VERSION + Version::ZERO, @@ -284,7 +284,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "cointime_price", Source::None, version + VERSION + Version::ZERO, @@ -294,7 +294,7 @@ impl Vecs { true, )?, - file, + db, }) } @@ -318,7 +318,7 @@ impl Vecs { stateful, exit, )?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index 18311be2c..bb358b264 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -1,9 +1,9 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{StoredU16, Version}; -use brk_vecs::{AnyCollectableVec, AnyVec, Computation, Exit, File, Format}; +use vecdb::{AnyCollectableVec, AnyVec, Computation, Database, Exit, Format}; use crate::grouped::Source; @@ -17,7 +17,7 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub constant_0: ComputedVecsFromHeight, pub constant_1: ComputedVecsFromHeight, @@ -33,11 +33,11 @@ impl Vecs { format: Format, indexes: &indexes::Vecs, ) -> Result { - let file = Arc::new(File::open(&parent.join("constants"))?); + let db = Database::open(&parent.join("constants"))?; Ok(Self { constant_0: ComputedVecsFromHeight::forced_import( - &file, + &db, "constant_0", Source::Compute, version + VERSION + Version::ZERO, @@ -47,7 +47,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_1: ComputedVecsFromHeight::forced_import( - &file, + &db, "constant_1", Source::Compute, version + VERSION + Version::ZERO, @@ -57,7 +57,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_50: ComputedVecsFromHeight::forced_import( - &file, + &db, "constant_50", Source::Compute, version + VERSION + Version::ZERO, @@ -67,7 +67,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, constant_100: ComputedVecsFromHeight::forced_import( - &file, + &db, "constant_100", Source::Compute, version + VERSION + Version::ZERO, @@ -77,7 +77,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, - file, + db, }) } @@ -89,7 +89,7 @@ impl Vecs { exit: &Exit, ) -> Result<()> { self.compute_(indexer, indexes, starting_indexes, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index ec8a0208a..b20314f2f 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -1,19 +1,19 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_structs::{DateIndex, Height, OHLCCents, Version}; -use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Exit, File, GenericStoredVec, RawVec, - StoredIndex, VecIterator, +use vecdb::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, + RawVec, StoredIndex, VecIterator, }; use super::{Indexes, indexes}; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, fetcher: Fetcher, pub dateindex_to_ohlc_in_cents: RawVec, @@ -22,23 +22,23 @@ pub struct Vecs { impl Vecs { pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result { - let file = Arc::new(File::open(&parent.join("fetched"))?); + let db = Database::open(&parent.join("fetched"))?; Ok(Self { fetcher, dateindex_to_ohlc_in_cents: RawVec::forced_import( - &file, + &db, "ohlc_in_cents", version + Version::ZERO, )?, height_to_ohlc_in_cents: RawVec::forced_import( - &file, + &db, "ohlc_in_cents", version + Version::ZERO, )?, - file, + db, }) } @@ -50,7 +50,7 @@ impl Vecs { exit: &Exit, ) -> Result<()> { self.compute_(indexer, indexes, starting_indexes, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/grouped/builder_computed.rs b/crates/brk_computer/src/grouped/builder_computed.rs index 6b492f9d1..068fd405f 100644 --- a/crates/brk_computer/src/grouped/builder_computed.rs +++ b/crates/brk_computer/src/grouped/builder_computed.rs @@ -1,11 +1,9 @@ -use std::sync::Arc; - use brk_error::Result; use brk_structs::Version; -use brk_vecs::{ +use vecdb::{ AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, - ComputedVec, ComputedVecFrom2, Exit, File, Format, FromCoarserIndex, StoredIndex, + ComputedVec, ComputedVecFrom2, Database, Exit, Format, FromCoarserIndex, StoredIndex, }; use crate::grouped::{EagerVecBuilder, VecBuilderOptions}; @@ -40,7 +38,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, version: Version, format: Format, @@ -67,7 +65,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, &maybe_suffix("first"), version + VERSION + Version::ZERO, format, @@ -92,7 +90,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, name, version + VERSION + Version::ZERO, format, @@ -101,7 +99,7 @@ where source .as_ref() .unwrap_or_else(|| { - dbg!(file, name, I::to_string()); + dbg!(db, name, I::to_string()); panic!() }) .clone() @@ -125,7 +123,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, &maybe_suffix("min"), version + VERSION + Version::ZERO, format, @@ -150,7 +148,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, &maybe_suffix("max"), version + VERSION + Version::ZERO, format, @@ -175,7 +173,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, &maybe_suffix("average"), version + VERSION + Version::ZERO, format, @@ -207,7 +205,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, &(if !options.last && !options.average && !options.min && !options.max { name.to_string() } else { @@ -242,7 +240,7 @@ where Box::new( ComputedVec::forced_import_or_init_from_2( computation, - file, + db, &suffix("cumulative"), version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index 3ff073125..b1de1dc85 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -1,9 +1,7 @@ -use std::sync::Arc; - use brk_error::{Error, Result}; use brk_structs::{CheckedSub, StoredU64, Version}; -use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, File, Format, +use vecdb::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, GenericStoredVec, StoredIndex, StoredRaw, }; @@ -39,7 +37,7 @@ where T: ComputedType, { pub fn forced_import( - file: &Arc, + db: &Database, name: &str, version: Version, format: Format, @@ -61,7 +59,7 @@ where first: options.first.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("first"), version + VERSION + Version::ZERO, format, @@ -71,13 +69,13 @@ where }), last: options.last.then(|| { Box::new( - EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(), + EagerVec::forced_import(db, name, version + Version::ZERO, format).unwrap(), ) }), min: options.min.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("min"), version + VERSION + Version::ZERO, format, @@ -88,7 +86,7 @@ where max: options.max.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("max"), version + VERSION + Version::ZERO, format, @@ -99,7 +97,7 @@ where median: options.median.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("median"), version + VERSION + Version::ZERO, format, @@ -110,7 +108,7 @@ where average: options.average.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("average"), version + VERSION + Version::ZERO, format, @@ -121,7 +119,7 @@ where sum: options.sum.then(|| { Box::new( EagerVec::forced_import( - file, + db, &(if !options.last && !options.average && !options.min && !options.max { name.to_string() } else { @@ -136,7 +134,7 @@ where cumulative: options.cumulative.then(|| { Box::new( EagerVec::forced_import( - file, + db, &suffix("cumulative"), version + VERSION + Version::ZERO, format, @@ -147,7 +145,7 @@ where _90p: options._90p.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("90p"), version + VERSION + Version::ZERO, format, @@ -158,7 +156,7 @@ where _75p: options._75p.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("75p"), version + VERSION + Version::ZERO, format, @@ -169,7 +167,7 @@ where _25p: options._25p.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("25p"), version + VERSION + Version::ZERO, format, @@ -180,7 +178,7 @@ where _10p: options._10p.then(|| { Box::new( EagerVec::forced_import( - file, + db, &maybe_suffix("10p"), version + VERSION + Version::ZERO, format, @@ -203,7 +201,7 @@ where return Ok(()); }; - self.validate_computed_version_or_reset_file(source.version())?; + self.validate_computed_version_or_reset(source.version())?; let index = self.starting_index(max_from); @@ -234,7 +232,7 @@ where where I2: StoredIndex + StoredRaw + CheckedSub, { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( source.version() + first_indexes.version() + count_indexes.version(), )?; @@ -253,10 +251,10 @@ where first_indexes .iter_at(index) - .try_for_each(|(i, first_index)| -> Result<()> { + .try_for_each(|(index, first_index)| -> Result<()> { let first_index = first_index.into_owned(); - let count_index = count_indexes_iter.unwrap_get_inner(i); + let count_index = count_indexes_iter.unwrap_get_inner(index); if let Some(first) = self.first.as_mut() { let f = source_iter @@ -305,7 +303,7 @@ where if let Some(max) = self.max.as_mut() { max.forced_push_at( - i, + index, *values .last() .ok_or(Error::Str("expect some")) @@ -327,27 +325,27 @@ where } if let Some(_90p) = self._90p.as_mut() { - _90p.forced_push_at(i, get_percentile(&values, 0.90), exit)?; + _90p.forced_push_at(index, get_percentile(&values, 0.90), exit)?; } if let Some(_75p) = self._75p.as_mut() { - _75p.forced_push_at(i, get_percentile(&values, 0.75), exit)?; + _75p.forced_push_at(index, get_percentile(&values, 0.75), exit)?; } if let Some(median) = self.median.as_mut() { - median.forced_push_at(i, get_percentile(&values, 0.50), exit)?; + median.forced_push_at(index, get_percentile(&values, 0.50), exit)?; } if let Some(_25p) = self._25p.as_mut() { - _25p.forced_push_at(i, get_percentile(&values, 0.25), exit)?; + _25p.forced_push_at(index, get_percentile(&values, 0.25), exit)?; } if let Some(_10p) = self._10p.as_mut() { - _10p.forced_push_at(i, get_percentile(&values, 0.10), exit)?; + _10p.forced_push_at(index, get_percentile(&values, 0.10), exit)?; } if let Some(min) = self.min.as_mut() { - min.forced_push_at(i, *values.first().unwrap(), exit)?; + min.forced_push_at(index, *values.first().unwrap(), exit)?; } } @@ -357,18 +355,18 @@ where if let Some(average) = self.average.as_mut() { let avg = sum / len; - average.forced_push_at(i, avg, exit)?; + average.forced_push_at(index, avg, exit)?; } if needs_sum_or_cumulative { if let Some(sum_vec) = self.sum.as_mut() { - sum_vec.forced_push_at(i, sum, exit)?; + sum_vec.forced_push_at(index, sum, exit)?; } if let Some(cumulative_vec) = self.cumulative.as_mut() { let t = cumulative.unwrap() + sum; cumulative.replace(t); - cumulative_vec.forced_push_at(i, t, exit)?; + cumulative_vec.forced_push_at(index, t, exit)?; } } } @@ -403,7 +401,7 @@ where panic!("unsupported"); } - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( VERSION + first_indexes.version() + count_indexes.version(), )?; @@ -426,10 +424,10 @@ where first_indexes .iter_at(index) - .try_for_each(|(i, first_index, ..)| -> Result<()> { + .try_for_each(|(index, first_index, ..)| -> Result<()> { let first_index = first_index.into_owned(); - let count_index = count_indexes_iter.unwrap_get_inner(i); + let count_index = count_indexes_iter.unwrap_get_inner(index); if let Some(first) = self.first.as_mut() { let v = source_first_iter @@ -468,7 +466,7 @@ where .map(|(_, v)| v.into_owned()) .collect::>(); values.sort_unstable(); - max.forced_push_at(i, *values.last().unwrap(), exit)?; + max.forced_push_at(index, *values.last().unwrap(), exit)?; } if let Some(min) = self.min.as_mut() { @@ -479,7 +477,7 @@ where .map(|(_, v)| v.into_owned()) .collect::>(); values.sort_unstable(); - min.forced_push_at(i, *values.first().unwrap(), exit)?; + min.forced_push_at(index, *values.first().unwrap(), exit)?; } } @@ -497,7 +495,7 @@ where // TODO: Multiply by count then divide by cumulative // Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example) let avg = cumulative / len; - average.forced_push_at(i, avg, exit)?; + average.forced_push_at(index, avg, exit)?; } if needs_sum_or_cumulative { @@ -511,13 +509,13 @@ where let sum = values.into_iter().fold(T::from(0), |a, b| a + b); if let Some(sum_vec) = self.sum.as_mut() { - sum_vec.forced_push_at(i, sum, exit)?; + sum_vec.forced_push_at(index, sum, exit)?; } if let Some(cumulative_vec) = self.cumulative.as_mut() { let t = cumulative.unwrap() + sum; cumulative.replace(t); - cumulative_vec.forced_push_at(i, t, exit)?; + cumulative_vec.forced_push_at(index, t, exit)?; } } } @@ -665,42 +663,42 @@ where Ok(()) } - pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { + pub fn validate_computed_version_or_reset(&mut self, version: Version) -> Result<()> { if let Some(first) = self.first.as_mut() { - first.validate_computed_version_or_reset_file(Version::ZERO + version)?; + first.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(last) = self.last.as_mut() { - last.validate_computed_version_or_reset_file(Version::ZERO + version)?; + last.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(min) = self.min.as_mut() { - min.validate_computed_version_or_reset_file(Version::ZERO + version)?; + min.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(max) = self.max.as_mut() { - max.validate_computed_version_or_reset_file(Version::ZERO + version)?; + max.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(median) = self.median.as_mut() { - median.validate_computed_version_or_reset_file(Version::ZERO + version)?; + median.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(average) = self.average.as_mut() { - average.validate_computed_version_or_reset_file(Version::ZERO + version)?; + average.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(sum) = self.sum.as_mut() { - sum.validate_computed_version_or_reset_file(Version::ZERO + version)?; + sum.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(cumulative) = self.cumulative.as_mut() { - cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?; + cumulative.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(_90p) = self._90p.as_mut() { - _90p.validate_computed_version_or_reset_file(Version::ZERO + version)?; + _90p.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(_75p) = self._75p.as_mut() { - _75p.validate_computed_version_or_reset_file(Version::ZERO + version)?; + _75p.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(_25p) = self._25p.as_mut() { - _25p.validate_computed_version_or_reset_file(Version::ZERO + version)?; + _25p.validate_computed_version_or_reset(Version::ZERO + version)?; } if let Some(_10p) = self._10p.as_mut() { - _10p.validate_computed_version_or_reset_file(Version::ZERO + version)?; + _10p.validate_computed_version_or_reset(Version::ZERO + version)?; } Ok(()) diff --git a/crates/brk_computer/src/grouped/computed.rs b/crates/brk_computer/src/grouped/computed.rs index 849e61b59..7b7e3bad8 100644 --- a/crates/brk_computer/src/grouped/computed.rs +++ b/crates/brk_computer/src/grouped/computed.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::StoredCompressed; +use vecdb::StoredCompressed; pub trait ComputedType where diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index 654c6ca6a..4449a5c53 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -1,14 +1,12 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{ DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use brk_vecs::{ - AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File, - Format, +use vecdb::{ + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec, + Exit, Format, }; use crate::{Indexes, grouped::ComputedVecBuilder, indexes}; @@ -38,7 +36,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -48,11 +46,11 @@ where options: VecBuilderOptions, ) -> Result { let dateindex = source.is_compute().then(|| { - EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap() + EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap() }); let dateindex_extra = EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -65,7 +63,7 @@ where Ok(Self { weekindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -76,7 +74,7 @@ where options.into(), )?, monthindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -87,7 +85,7 @@ where options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -98,7 +96,7 @@ where options.into(), )?, semesterindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -109,7 +107,7 @@ where options.into(), )?, yearindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -120,7 +118,7 @@ where options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 17be32533..c58054e82 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; @@ -7,9 +5,9 @@ use brk_structs::{ DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use brk_vecs::{ - AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File, - Format, +use vecdb::{ + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec, + Exit, Format, }; use crate::{ @@ -47,7 +45,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -57,11 +55,11 @@ where options: VecBuilderOptions, ) -> Result { let height = source.is_compute().then(|| { - EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap() + EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap() }); let height_extra = EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -69,7 +67,7 @@ where )?; let dateindex = EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -80,7 +78,7 @@ where Ok(Self { weekindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -91,7 +89,7 @@ where options.into(), )?, monthindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -102,7 +100,7 @@ where options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -113,7 +111,7 @@ where options.into(), )?, semesterindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -124,7 +122,7 @@ where options.into(), )?, yearindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -135,7 +133,7 @@ where options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -145,12 +143,12 @@ where indexes.decadeindex_to_decadeindex.boxed_clone(), options.into(), )?, - // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?, + // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, height, height_extra, dateindex, difficultyepoch: EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index 4fe6819df..206f64ed9 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -1,10 +1,8 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{DifficultyEpoch, Height, Version}; -use brk_vecs::{AnyCollectableVec, EagerVec, Exit, File, Format}; +use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format}; use crate::{Indexes, indexes}; @@ -29,17 +27,16 @@ where f64: From, { pub fn forced_import( - file: &Arc, + db: &Database, name: &str, version: Version, format: Format, options: VecBuilderOptions, ) -> Result { - let height = - EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?; + let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)?; let height_extra = EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -52,13 +49,13 @@ where height, height_extra, difficultyepoch: EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, options, )?, - // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?, + // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, }) } diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index 3495a643c..c58f97a4c 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -1,14 +1,12 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{ Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex, }; -use brk_vecs::{ - AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, - Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator, +use vecdb::{ + AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, Database, + EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator, }; use crate::{ @@ -46,7 +44,7 @@ where { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -57,13 +55,13 @@ where ) -> Result { let txindex = source.is_compute().then(|| { Box::new( - EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format) + EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format) .unwrap(), ) }); let height = EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -73,7 +71,7 @@ where let options = options.remove_percentiles(); let dateindex = EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -82,7 +80,7 @@ where Ok(Self { weekindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -93,7 +91,7 @@ where options.into(), )?, monthindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -104,7 +102,7 @@ where options.into(), )?, quarterindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -115,7 +113,7 @@ where options.into(), )?, semesterindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -126,7 +124,7 @@ where options.into(), )?, yearindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -137,7 +135,7 @@ where options.into(), )?, decadeindex: ComputedVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, @@ -152,13 +150,13 @@ where height, dateindex, difficultyepoch: EagerVecBuilder::forced_import( - file, + db, name, version + VERSION + Version::ZERO, format, options, )?, - // halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?, + // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, }) } @@ -325,7 +323,7 @@ impl ComputedVecsFromTxindex { }; self.height - .validate_computed_version_or_reset_file(txindex_version)?; + .validate_computed_version_or_reset(txindex_version)?; let starting_index = self.height.starting_index(starting_indexes.height); @@ -504,7 +502,7 @@ impl ComputedVecsFromTxindex { }; self.height - .validate_computed_version_or_reset_file(txindex_version)?; + .validate_computed_version_or_reset(txindex_version)?; let starting_index = self.height.starting_index(starting_indexes.height); diff --git a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs index bfd12e32a..ecbf3db20 100644 --- a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -1,11 +1,9 @@ -use std::{f32, sync::Arc}; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version}; -use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, - Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator, +use vecdb::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database, + EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator, }; use crate::{Indexes, grouped::source::Source, indexes, price, utils::get_percentile}; @@ -60,7 +58,7 @@ const VERSION: Version = Version::ZERO; impl ComputedRatioVecsFromDateIndex { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -74,7 +72,7 @@ impl ComputedRatioVecsFromDateIndex { Ok(Self { price: source.is_compute().then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, name, Source::Compute, version + VERSION, @@ -86,7 +84,7 @@ impl ComputedRatioVecsFromDateIndex { .unwrap() }), ratio: ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio"), Source::Compute, version + VERSION + Version::ZERO, @@ -97,7 +95,7 @@ impl ComputedRatioVecsFromDateIndex { )?, ratio_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -110,7 +108,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1w_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_1w_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -123,7 +121,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1m_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_1m_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -136,7 +134,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_1y_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -149,7 +147,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_4y_sma: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_4y_sma"), Source::Compute, version + VERSION + Version::ZERO, @@ -162,7 +160,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_sma_momentum_oscillator: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_1y_sma_momentum_oscillator"), Source::Compute, version + VERSION + Version::ZERO, @@ -175,7 +173,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -188,7 +186,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_4y_sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_4y_sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -201,7 +199,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_1y_sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -214,7 +212,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_9: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p99_9"), Source::Compute, version + VERSION + Version::ZERO, @@ -227,7 +225,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_5: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p99_5"), Source::Compute, version + VERSION + Version::ZERO, @@ -240,7 +238,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p99"), Source::Compute, version + VERSION + Version::ZERO, @@ -253,7 +251,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p1"), Source::Compute, version + VERSION + Version::ZERO, @@ -266,7 +264,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_5: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p0_5"), Source::Compute, version + VERSION + Version::ZERO, @@ -279,7 +277,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_1: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p0_1"), Source::Compute, version + VERSION + Version::ZERO, @@ -292,7 +290,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p1sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -305,7 +303,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p2sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p2sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -318,7 +316,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p3sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p3sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -331,7 +329,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m1sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_m1sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -344,7 +342,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m2sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_m2sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -357,7 +355,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m3sd: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_m3sd"), Source::Compute, version + VERSION + Version::ZERO, @@ -370,7 +368,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_9_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p99_9_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -383,7 +381,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_5_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p99_5_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -396,7 +394,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p99_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p99_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -409,7 +407,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p1_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -422,7 +420,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_5_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p0_5_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -435,7 +433,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p0_1_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p0_1_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -448,7 +446,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p1sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p1sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -461,7 +459,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p2sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p2sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -474,7 +472,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_p3sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_p3sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -487,7 +485,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m1sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_m1sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -500,7 +498,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m2sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_m2sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -513,7 +511,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_m3sd_as_price: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_m3sd_as_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -526,7 +524,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_zscore: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_zscore"), Source::Compute, version + VERSION + Version::ZERO, @@ -539,7 +537,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_4y_zscore: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_4y_zscore"), Source::Compute, version + VERSION + Version::ZERO, @@ -552,7 +550,7 @@ impl ComputedRatioVecsFromDateIndex { }), ratio_1y_zscore: extended.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_ratio_1y_zscore"), Source::Compute, version + VERSION + Version::ZERO, @@ -771,7 +769,7 @@ impl ComputedRatioVecsFromDateIndex { self.mut_ratio_vecs() .iter_mut() .try_for_each(|v| -> Result<()> { - v.validate_computed_version_or_reset_file( + v.validate_computed_version_or_reset( Version::ZERO + v.inner_version() + ratio_version, )?; Ok(()) diff --git a/crates/brk_computer/src/grouped/source.rs b/crates/brk_computer/src/grouped/source.rs index b1a621aa0..5a04d2882 100644 --- a/crates/brk_computer/src/grouped/source.rs +++ b/crates/brk_computer/src/grouped/source.rs @@ -1,4 +1,4 @@ -use brk_vecs::AnyBoxedIterableVec; +use vecdb::AnyBoxedIterableVec; #[derive(Clone)] pub enum Source { diff --git a/crates/brk_computer/src/grouped/value_from_dateindex.rs b/crates/brk_computer/src/grouped/value_from_dateindex.rs index 88ab458c9..ff3aa7ace 100644 --- a/crates/brk_computer/src/grouped/value_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/value_from_dateindex.rs @@ -1,10 +1,8 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version}; -use brk_vecs::{ - AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec, +use vecdb::{ + AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec, }; use crate::{ @@ -28,7 +26,7 @@ const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromDateIndex { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -40,7 +38,7 @@ impl ComputedValueVecsFromDateIndex { ) -> Result { Ok(Self { sats: ComputedVecsFromDateIndex::forced_import( - file, + db, name, source, version + VERSION, @@ -50,7 +48,7 @@ impl ComputedValueVecsFromDateIndex { options, )?, bitcoin: ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_in_btc"), Source::Compute, version + VERSION, @@ -61,7 +59,7 @@ impl ComputedValueVecsFromDateIndex { )?, dollars: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &format!("{name}_in_usd"), Source::Compute, version + VERSION, diff --git a/crates/brk_computer/src/grouped/value_from_height.rs b/crates/brk_computer/src/grouped/value_from_height.rs index 7fbd8a47b..ced39a6bb 100644 --- a/crates/brk_computer/src/grouped/value_from_height.rs +++ b/crates/brk_computer/src/grouped/value_from_height.rs @@ -1,10 +1,8 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, Dollars, Height, Sats, Version}; -use brk_vecs::{ - AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec, +use vecdb::{ + AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec, }; use crate::{ @@ -28,7 +26,7 @@ const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromHeight { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -40,7 +38,7 @@ impl ComputedValueVecsFromHeight { ) -> Result { Ok(Self { sats: ComputedVecsFromHeight::forced_import( - file, + db, name, source, version + VERSION, @@ -50,7 +48,7 @@ impl ComputedValueVecsFromHeight { options, )?, bitcoin: ComputedVecsFromHeight::forced_import( - file, + db, &format!("{name}_in_btc"), Source::Compute, version + VERSION, @@ -61,7 +59,7 @@ impl ComputedValueVecsFromHeight { )?, dollars: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &format!("{name}_in_usd"), Source::Compute, version + VERSION, diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index 1a28f8763..77196ce72 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -1,11 +1,9 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; -use brk_vecs::{ +use vecdb::{ AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Computation, ComputedVecFrom3, - Exit, File, Format, LazyVecFrom1, StoredIndex, StoredVec, + Database, Exit, Format, LazyVecFrom1, StoredIndex, StoredVec, }; use crate::{Indexes, grouped::Source, indexes, price}; @@ -38,7 +36,7 @@ const VERSION: Version = Version::ZERO; impl ComputedValueVecsFromTxindex { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, name: &str, indexes: &indexes::Vecs, source: Source, @@ -54,7 +52,7 @@ impl ComputedValueVecsFromTxindex { let name_in_usd = format!("{name}_in_usd"); let sats = ComputedVecsFromTxindex::forced_import( - file, + db, name, source.clone(), version + VERSION, @@ -79,7 +77,7 @@ impl ComputedValueVecsFromTxindex { ); let bitcoin = ComputedVecsFromTxindex::forced_import( - file, + db, &name_in_btc, Source::None, version + VERSION, @@ -92,7 +90,7 @@ impl ComputedValueVecsFromTxindex { let dollars_txindex = price.map(|price| { ComputedVecFrom3::forced_import_or_init_from_3( computation, - file, + db, &name_in_usd, version + VERSION, format, @@ -127,7 +125,7 @@ impl ComputedValueVecsFromTxindex { dollars_txindex, dollars: compute_dollars.then(|| { ComputedVecsFromTxindex::forced_import( - file, + db, &name_in_usd, Source::None, version + VERSION, diff --git a/crates/brk_computer/src/grouped/value_height.rs b/crates/brk_computer/src/grouped/value_height.rs index 4fc0f920b..ff6281d54 100644 --- a/crates/brk_computer/src/grouped/value_height.rs +++ b/crates/brk_computer/src/grouped/value_height.rs @@ -1,9 +1,7 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, Dollars, Height, Sats, Version}; -use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, Exit, File, Format, StoredVec}; +use vecdb::{AnyCollectableVec, CollectableVec, Database, EagerVec, Exit, Format, StoredVec}; use crate::{ Indexes, @@ -23,7 +21,7 @@ const VERSION: Version = Version::ZERO; impl ComputedHeightValueVecs { pub fn forced_import( - file: &Arc, + db: &Database, name: &str, source: Source, version: Version, @@ -32,18 +30,18 @@ impl ComputedHeightValueVecs { ) -> Result { Ok(Self { sats: source.is_compute().then(|| { - EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format) + EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format) .unwrap() }), bitcoin: EagerVec::forced_import( - file, + db, &format!("{name}_in_btc"), version + VERSION + Version::ZERO, format, )?, dollars: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &format!("{name}_in_usd"), version + VERSION + Version::ZERO, format, diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index a57784a0c..39ac93a8b 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -1,4 +1,4 @@ -use std::{ops::Deref, path::Path, sync::Arc}; +use std::{ops::Deref, path::Path}; use brk_error::Result; use brk_indexer::Indexer; @@ -10,16 +10,16 @@ use brk_structs::{ P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64, Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex, }; -use brk_vecs::{ +use vecdb::{ AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1, - ComputedVecFrom2, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, VecIterator, + ComputedVecFrom2, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator, }; const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub dateindex_to_date: EagerVec, pub dateindex_to_dateindex: EagerVec, @@ -105,12 +105,12 @@ impl Vecs { computation: Computation, format: Format, ) -> Result { - let file = Arc::new(File::open(&parent.join("indexes"))?); - file.set_min_len(PAGE_SIZE * 10_000_000)?; + let db = Database::open(&parent.join("indexes"))?; + db.set_min_len(PAGE_SIZE * 10_000_000)?; let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "outputindex", version + VERSION + Version::ZERO, format, @@ -120,7 +120,7 @@ impl Vecs { let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "inputindex", version + VERSION + Version::ZERO, format, @@ -130,7 +130,7 @@ impl Vecs { let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "txindex", version + VERSION + Version::ZERO, format, @@ -140,7 +140,7 @@ impl Vecs { let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2( computation, - &file, + &db, "input_count", version + VERSION + Version::ZERO, format, @@ -163,7 +163,7 @@ impl Vecs { let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2( computation, - &file, + &db, "output_count", version + VERSION + Version::ZERO, format, @@ -186,7 +186,7 @@ impl Vecs { let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2pk33addressindex", version + VERSION + Version::ZERO, format, @@ -195,7 +195,7 @@ impl Vecs { )?; let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2pk65addressindex", version + VERSION + Version::ZERO, format, @@ -204,7 +204,7 @@ impl Vecs { )?; let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2pkhaddressindex", version + VERSION + Version::ZERO, format, @@ -213,7 +213,7 @@ impl Vecs { )?; let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2shaddressindex", version + VERSION + Version::ZERO, format, @@ -222,7 +222,7 @@ impl Vecs { )?; let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2traddressindex", version + VERSION + Version::ZERO, format, @@ -231,7 +231,7 @@ impl Vecs { )?; let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2wpkhaddressindex", version + VERSION + Version::ZERO, format, @@ -240,7 +240,7 @@ impl Vecs { )?; let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2wshaddressindex", version + VERSION + Version::ZERO, format, @@ -249,7 +249,7 @@ impl Vecs { )?; let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2aaddressindex", version + VERSION + Version::ZERO, format, @@ -258,7 +258,7 @@ impl Vecs { )?; let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "p2msoutputindex", version + VERSION + Version::ZERO, format, @@ -267,7 +267,7 @@ impl Vecs { )?; let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "emptyoutputindex", version + VERSION + Version::ZERO, format, @@ -276,7 +276,7 @@ impl Vecs { )?; let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "unknownoutputindex", version + VERSION + Version::ZERO, format, @@ -285,7 +285,7 @@ impl Vecs { )?; let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "opreturnindex", version + VERSION + Version::ZERO, format, @@ -313,265 +313,265 @@ impl Vecs { unknownoutputindex_to_unknownoutputindex, dateindex_to_date: EagerVec::forced_import( - &file, + &db, "date", version + VERSION + Version::ZERO, format, )?, dateindex_to_dateindex: EagerVec::forced_import( - &file, + &db, "dateindex", version + VERSION + Version::ZERO, format, )?, dateindex_to_first_height: EagerVec::forced_import( - &file, + &db, "first_height", version + VERSION + Version::ZERO, format, )?, dateindex_to_monthindex: EagerVec::forced_import( - &file, + &db, "monthindex", version + VERSION + Version::ZERO, format, )?, dateindex_to_weekindex: EagerVec::forced_import( - &file, + &db, "weekindex", version + VERSION + Version::ZERO, format, )?, decadeindex_to_decadeindex: EagerVec::forced_import( - &file, + &db, "decadeindex", version + VERSION + Version::ZERO, format, )?, decadeindex_to_first_yearindex: EagerVec::forced_import( - &file, + &db, "first_yearindex", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_difficultyepoch: EagerVec::forced_import( - &file, + &db, "difficultyepoch", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_first_height: EagerVec::forced_import( - &file, + &db, "first_height", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_first_height: EagerVec::forced_import( - &file, + &db, "first_height", version + VERSION + Version::ZERO, format, )?, halvingepoch_to_halvingepoch: EagerVec::forced_import( - &file, + &db, "halvingepoch", version + VERSION + Version::ZERO, format, )?, height_to_date: EagerVec::forced_import( - &file, + &db, "date", version + VERSION + Version::ZERO, format, )?, height_to_difficultyepoch: EagerVec::forced_import( - &file, + &db, "difficultyepoch", version + VERSION + Version::ZERO, format, )?, height_to_halvingepoch: EagerVec::forced_import( - &file, + &db, "halvingepoch", version + VERSION + Version::ZERO, format, )?, height_to_height: EagerVec::forced_import( - &file, + &db, "height", version + VERSION + Version::ZERO, format, )?, monthindex_to_first_dateindex: EagerVec::forced_import( - &file, + &db, "first_dateindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_monthindex: EagerVec::forced_import( - &file, + &db, "monthindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_quarterindex: EagerVec::forced_import( - &file, + &db, "quarterindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_semesterindex: EagerVec::forced_import( - &file, + &db, "semesterindex", version + VERSION + Version::ZERO, format, )?, monthindex_to_yearindex: EagerVec::forced_import( - &file, + &db, "yearindex", version + VERSION + Version::ZERO, format, )?, quarterindex_to_first_monthindex: EagerVec::forced_import( - &file, + &db, "first_monthindex", version + VERSION + Version::ZERO, format, )?, semesterindex_to_first_monthindex: EagerVec::forced_import( - &file, + &db, "first_monthindex", version + VERSION + Version::ZERO, format, )?, weekindex_to_first_dateindex: EagerVec::forced_import( - &file, + &db, "first_dateindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_first_monthindex: EagerVec::forced_import( - &file, + &db, "first_monthindex", version + VERSION + Version::ZERO, format, )?, quarterindex_to_quarterindex: EagerVec::forced_import( - &file, + &db, "quarterindex", version + VERSION + Version::ZERO, format, )?, semesterindex_to_semesterindex: EagerVec::forced_import( - &file, + &db, "semesterindex", version + VERSION + Version::ZERO, format, )?, weekindex_to_weekindex: EagerVec::forced_import( - &file, + &db, "weekindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_decadeindex: EagerVec::forced_import( - &file, + &db, "decadeindex", version + VERSION + Version::ZERO, format, )?, yearindex_to_yearindex: EagerVec::forced_import( - &file, + &db, "yearindex", version + VERSION + Version::ZERO, format, )?, height_to_date_fixed: EagerVec::forced_import( - &file, + &db, "date_fixed", version + VERSION + Version::ZERO, format, )?, height_to_dateindex: EagerVec::forced_import( - &file, + &db, "dateindex", version + VERSION + Version::ZERO, format, )?, txindex_to_height: EagerVec::forced_import( - &file, + &db, "height", version + VERSION + Version::ZERO, format, )?, height_to_timestamp_fixed: EagerVec::forced_import( - &file, + &db, "timestamp_fixed", version + VERSION + Version::ZERO, format, )?, height_to_txindex_count: EagerVec::forced_import( - &file, + &db, "txindex_count", version + VERSION + Version::ZERO, format, )?, dateindex_to_height_count: EagerVec::forced_import( - &file, + &db, "height_count", version + VERSION + Version::ZERO, format, )?, weekindex_to_dateindex_count: EagerVec::forced_import( - &file, + &db, "dateindex_count", version + VERSION + Version::ZERO, format, )?, difficultyepoch_to_height_count: EagerVec::forced_import( - &file, + &db, "height_count", version + VERSION + Version::ZERO, format, )?, monthindex_to_dateindex_count: EagerVec::forced_import( - &file, + &db, "dateindex_count", version + VERSION + Version::ZERO, format, )?, quarterindex_to_monthindex_count: EagerVec::forced_import( - &file, + &db, "monthindex_count", version + VERSION + Version::ZERO, format, )?, semesterindex_to_monthindex_count: EagerVec::forced_import( - &file, + &db, "monthindex_count", version + VERSION + Version::ZERO, format, )?, yearindex_to_monthindex_count: EagerVec::forced_import( - &file, + &db, "monthindex_count", version + VERSION + Version::ZERO, format, )?, decadeindex_to_yearindex_count: EagerVec::forced_import( - &file, + &db, "yearindex_count", version + VERSION + Version::ZERO, format, )?, outputindex_to_txindex: EagerVec::forced_import( - &file, + &db, "txindex", version + VERSION + Version::ZERO, format, )?, - file, + db, }) } @@ -582,7 +582,7 @@ impl Vecs { exit: &Exit, ) -> Result { let idxs = self.compute_(indexer, starting_indexes, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(idxs) } diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index d8c67e3a5..0710681c2 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -9,8 +9,8 @@ use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_structs::Version; -use brk_vecs::{AnyCollectableVec, Computation, Exit, Format}; use log::info; +use vecdb::{AnyCollectableVec, Computation, Exit, Format}; mod blocks; mod cointime; diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index aa3565a04..6e41ef1b5 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -1,10 +1,10 @@ -use std::{path::Path, sync::Arc, thread}; +use std::{path::Path, thread}; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version}; -use brk_vecs::{ - AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, +use vecdb::{ + AnyCollectableVec, Computation, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator, }; @@ -24,7 +24,7 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub height_to_marketcap: EagerVec, pub height_to_ath: EagerVec, @@ -178,30 +178,30 @@ impl Vecs { format: Format, indexes: &indexes::Vecs, ) -> Result { - let file = Arc::new(File::open(&parent.join("market"))?); - file.set_min_len(PAGE_SIZE * 1_000_000)?; + let db = Database::open(&parent.join("market"))?; + db.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { height_to_marketcap: EagerVec::forced_import( - &file, + &db, "marketcap", version + VERSION + Version::ZERO, format, )?, height_to_ath: EagerVec::forced_import( - &file, + &db, "ath", version + VERSION + Version::ZERO, format, )?, height_to_drawdown: EagerVec::forced_import( - &file, + &db, "drawdown", version + VERSION + Version::ZERO, format, )?, indexes_to_marketcap: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "marketcap", Source::Compute, version + VERSION + Version::ZERO, @@ -211,7 +211,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_ath: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "ath", Source::Compute, version + VERSION + Version::ZERO, @@ -221,7 +221,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_drawdown: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "drawdown", Source::Compute, version + VERSION + Version::ZERO, @@ -231,7 +231,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_days_since_ath: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "days_since_ath", Source::Compute, version + VERSION + Version::ZERO, @@ -241,7 +241,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_max_days_between_aths: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "max_days_between_aths", Source::Compute, version + VERSION + Version::ZERO, @@ -251,7 +251,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_max_years_between_aths: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "max_years_between_aths", Source::Compute, version + VERSION + Version::ZERO, @@ -262,7 +262,7 @@ impl Vecs { )?, indexes_to_1w_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "1w_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -272,7 +272,7 @@ impl Vecs { true, )?, indexes_to_8d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "8d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -282,7 +282,7 @@ impl Vecs { true, )?, indexes_to_13d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "13d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -292,7 +292,7 @@ impl Vecs { true, )?, indexes_to_21d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "21d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -302,7 +302,7 @@ impl Vecs { true, )?, indexes_to_1m_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "1m_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -312,7 +312,7 @@ impl Vecs { true, )?, indexes_to_34d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "34d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -322,7 +322,7 @@ impl Vecs { true, )?, indexes_to_55d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "55d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -332,7 +332,7 @@ impl Vecs { true, )?, indexes_to_89d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "89d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -342,7 +342,7 @@ impl Vecs { true, )?, indexes_to_144d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "144d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -352,7 +352,7 @@ impl Vecs { true, )?, indexes_to_200d_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "200d_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -362,7 +362,7 @@ impl Vecs { true, )?, indexes_to_1y_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "1y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -372,7 +372,7 @@ impl Vecs { true, )?, indexes_to_2y_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "2y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -382,7 +382,7 @@ impl Vecs { true, )?, indexes_to_200w_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "200w_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -392,7 +392,7 @@ impl Vecs { true, )?, indexes_to_4y_sma: ComputedRatioVecsFromDateIndex::forced_import( - &file, + &db, "4y_sma", Source::Compute, version + VERSION + Version::ZERO, @@ -403,7 +403,7 @@ impl Vecs { )?, _1d_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1d_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -413,7 +413,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1w_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -423,7 +423,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -433,7 +433,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -443,7 +443,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6m_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -453,7 +453,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -463,7 +463,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "2y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -473,7 +473,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -483,7 +483,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "4y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -493,7 +493,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "5y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -503,7 +503,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -513,7 +513,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "8y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -523,7 +523,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "10y_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -533,7 +533,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "2y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -543,7 +543,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -553,7 +553,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "4y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -563,7 +563,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "5y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -573,7 +573,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -583,7 +583,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "8y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -593,7 +593,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "10y_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -604,7 +604,7 @@ impl Vecs { )?, _1w_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1w_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -614,7 +614,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -624,7 +624,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -634,7 +634,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6m_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -644,7 +644,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -654,7 +654,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "2y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -664,7 +664,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -674,7 +674,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "4y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -684,7 +684,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "5y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -694,7 +694,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -704,7 +704,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "8y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -714,7 +714,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "10y_dca_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -724,7 +724,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "2y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -734,7 +734,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -744,7 +744,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "4y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -754,7 +754,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "5y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -764,7 +764,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -774,7 +774,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "8y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -784,7 +784,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_cagr: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "10y_dca_cagr", Source::Compute, version + VERSION + Version::ZERO, @@ -794,7 +794,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1w_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -804,7 +804,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -814,7 +814,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -824,7 +824,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6m_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -834,7 +834,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -844,7 +844,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "2y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -854,7 +854,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -864,7 +864,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "4y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -874,7 +874,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "5y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -884,7 +884,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -894,7 +894,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "8y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -904,7 +904,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "10y_dca_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -914,7 +914,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1d_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_1d_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -924,7 +924,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1w_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_1w_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -934,7 +934,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1m_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_1m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -944,7 +944,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_3m_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_3m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -954,7 +954,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_6m_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_6m_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -964,7 +964,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_1y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_1y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -974,7 +974,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_2y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_2y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -984,7 +984,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_3y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_3y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -994,7 +994,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_4y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_4y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1004,7 +1004,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_5y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_5y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1014,7 +1014,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_6y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_6y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1024,7 +1024,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_8y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_8y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1034,7 +1034,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, price_10y_ago: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "price_10y_ago", Source::Compute, version + VERSION + Version::ZERO, @@ -1044,7 +1044,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1w_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1w_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1054,7 +1054,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1m_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1064,7 +1064,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3m_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1074,7 +1074,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6m_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6m_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1084,7 +1084,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _1y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "1y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1094,7 +1094,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _2y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "2y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1104,7 +1104,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _3y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "3y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1114,7 +1114,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _4y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "4y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1124,7 +1124,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _5y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "5y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1134,7 +1134,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _6y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "6y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1144,7 +1144,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _8y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "8y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1154,7 +1154,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, _10y_dca_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "10y_dca_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1165,7 +1165,7 @@ impl Vecs { )?, dca_class_2025_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2025_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1175,7 +1175,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2024_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1185,7 +1185,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2023_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1195,7 +1195,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2022_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1205,7 +1205,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2021_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1215,7 +1215,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2020_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1225,7 +1225,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2019_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1235,7 +1235,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2018_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1245,7 +1245,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2017_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1255,7 +1255,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2016_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1265,7 +1265,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_stack: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2015_stack", Source::Compute, version + VERSION + Version::ZERO, @@ -1276,7 +1276,7 @@ impl Vecs { )?, dca_class_2025_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2025_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1286,7 +1286,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2024_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1296,7 +1296,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2023_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1306,7 +1306,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2022_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1316,7 +1316,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2021_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1326,7 +1326,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2020_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1336,7 +1336,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2019_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1346,7 +1346,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2018_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1356,7 +1356,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2017_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1366,7 +1366,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2016_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1376,7 +1376,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_avg_price: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2015_avg_price", Source::Compute, version + VERSION + Version::ZERO, @@ -1387,7 +1387,7 @@ impl Vecs { )?, dca_class_2025_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2025_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1397,7 +1397,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2024_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2024_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1407,7 +1407,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2023_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2023_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1417,7 +1417,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2022_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2022_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1427,7 +1427,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2021_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2021_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1437,7 +1437,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2020_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2020_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1447,7 +1447,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2019_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2019_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1457,7 +1457,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2018_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2018_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1467,7 +1467,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2017_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2017_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1477,7 +1477,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2016_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2016_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1487,7 +1487,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, dca_class_2015_returns: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "dca_class_2015_returns", Source::Compute, version + VERSION + Version::ZERO, @@ -1498,7 +1498,7 @@ impl Vecs { )?, indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "200d_sma_x2_4", Source::Compute, version + VERSION + Version::ZERO, @@ -1508,7 +1508,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "200d_sma_x0_8", Source::Compute, version + VERSION + Version::ZERO, @@ -1518,7 +1518,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, - file, + db, }) } @@ -1539,7 +1539,7 @@ impl Vecs { starting_indexes, exit, )?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/mining.rs b/crates/brk_computer/src/mining.rs index 64b30fe9a..3a2647ff9 100644 --- a/crates/brk_computer/src/mining.rs +++ b/crates/brk_computer/src/mining.rs @@ -1,9 +1,9 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version}; -use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator}; +use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator}; use crate::grouped::Source; @@ -17,7 +17,7 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub indexes_to_difficulty: ComputedVecsFromHeight, pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex, @@ -32,12 +32,12 @@ impl Vecs { format: Format, indexes: &indexes::Vecs, ) -> Result { - let file = Arc::new(File::open(&parent.join("mining"))?); - file.set_min_len(PAGE_SIZE * 1_000_000)?; + let db = Database::open(&parent.join("mining"))?; + db.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { indexes_to_difficulty: ComputedVecsFromHeight::forced_import( - &file, + &db, "difficulty", Source::None, version + VERSION + Version::ZERO, @@ -47,7 +47,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "difficultyepoch", Source::Compute, version + VERSION + Version::ZERO, @@ -57,7 +57,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "halvingepoch", Source::Compute, version + VERSION + Version::ZERO, @@ -67,7 +67,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, - file, + db, }) } @@ -79,7 +79,7 @@ impl Vecs { exit: &Exit, ) -> Result<()> { self.compute_(indexer, indexes, starting_indexes, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs index d1ac21802..9ec6e6bb7 100644 --- a/crates/brk_computer/src/price.rs +++ b/crates/brk_computer/src/price.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; @@ -6,8 +6,8 @@ use brk_structs::{ Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex, }; -use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File, +use vecdb::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit, Format, GenericStoredVec, PAGE_SIZE, RawVec, }; @@ -21,7 +21,7 @@ use super::{ #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub dateindex_to_close_in_cents: EagerVec>, pub dateindex_to_high_in_cents: EagerVec>, @@ -80,80 +80,76 @@ impl Vecs { format: Format, indexes: &indexes::Vecs, ) -> Result { - let file = Arc::new(File::open(&parent.join("price"))?); - file.set_min_len(PAGE_SIZE * 1_000_000)?; + let db = Database::open(&parent.join("price"))?; + db.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { dateindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, dateindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, dateindex_to_close_in_cents: EagerVec::forced_import( - &file, + &db, "close_in_cents", version + VERSION + Version::ZERO, format, )?, dateindex_to_high_in_cents: EagerVec::forced_import( - &file, + &db, "high_in_cents", version + VERSION + Version::ZERO, format, )?, dateindex_to_low_in_cents: EagerVec::forced_import( - &file, + &db, "low_in_cents", version + VERSION + Version::ZERO, format, )?, dateindex_to_open_in_cents: EagerVec::forced_import( - &file, + &db, "open_in_cents", version + VERSION + Version::ZERO, format, )?, - height_to_ohlc: RawVec::forced_import( - &file, - "ohlc", - version + VERSION + Version::ZERO, - )?, + height_to_ohlc: RawVec::forced_import(&db, "ohlc", version + VERSION + Version::ZERO)?, height_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, height_to_close_in_cents: EagerVec::forced_import( - &file, + &db, "close_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_high_in_cents: EagerVec::forced_import( - &file, + &db, "high_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_low_in_cents: EagerVec::forced_import( - &file, + &db, "low_in_cents", version + VERSION + Version::ZERO, format, )?, height_to_open_in_cents: EagerVec::forced_import( - &file, + &db, "open_in_cents", version + VERSION + Version::ZERO, format, )?, timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "open", Source::Compute, version + VERSION + Version::ZERO, @@ -163,7 +159,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, timeindexes_to_high: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "high", Source::Compute, version + VERSION + Version::ZERO, @@ -173,7 +169,7 @@ impl Vecs { VecBuilderOptions::default().add_max(), )?, timeindexes_to_low: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "low", Source::Compute, version + VERSION + Version::ZERO, @@ -183,7 +179,7 @@ impl Vecs { VecBuilderOptions::default().add_min(), )?, timeindexes_to_close: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "close", Source::Compute, version + VERSION + Version::ZERO, @@ -193,7 +189,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "open_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -203,7 +199,7 @@ impl Vecs { VecBuilderOptions::default().add_first(), )?, timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "high_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -213,7 +209,7 @@ impl Vecs { VecBuilderOptions::default().add_max(), )?, timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "low_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -223,7 +219,7 @@ impl Vecs { VecBuilderOptions::default().add_min(), )?, timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import( - &file, + &db, "close_in_sats", Source::Compute, version + VERSION + VERSION_IN_SATS + Version::ZERO, @@ -233,135 +229,135 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "open", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_first(), )?, chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "high", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_max(), )?, chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "low", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_min(), )?, chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "close", version + VERSION + Version::ZERO, format, VecBuilderOptions::default().add_last(), )?, chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "open_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_first(), )?, chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "high_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_max(), )?, chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "low_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_min(), )?, chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import( - &file, + &db, "close_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, format, VecBuilderOptions::default().add_last(), )?, weekindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, weekindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, difficultyepoch_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, difficultyepoch_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, monthindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, monthindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, quarterindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, quarterindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, semesterindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, semesterindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, yearindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, yearindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - // halvingepoch_to_ohlc: StorableVec::forced_import(file, + // halvingepoch_to_ohlc: StorableVec::forced_import(db, // "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?, decadeindex_to_ohlc: RawVec::forced_import( - &file, + &db, "ohlc", version + VERSION + Version::ZERO, )?, decadeindex_to_ohlc_in_sats: RawVec::forced_import( - &file, + &db, "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - file, + db, }) } @@ -374,7 +370,7 @@ impl Vecs { exit: &Exit, ) -> Result<()> { self.compute_(indexer, indexes, starting_indexes, fetched, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_computer/src/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs index 843784e2b..41cc4f44b 100644 --- a/crates/brk_computer/src/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -1,10 +1,10 @@ -use std::{ops::Deref, path::Path, sync::Arc}; +use std::{ops::Deref, path::Path}; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version}; -use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File, +use vecdb::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit, Format, GenericStoredVec, VecIterator, }; @@ -25,7 +25,7 @@ const VERSION: Version = Version::ZERO; pub struct Vecs { starting_height: Height, - pub state: AddressCohortState, + pub state: Option, pub inner: common::Vecs, @@ -36,14 +36,14 @@ pub struct Vecs { impl Vecs { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, cohort_name: Option<&str>, computation: Computation, format: Format, version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, - states_path: &Path, + states_path: Option<&Path>, compute_relative_to_all: bool, ) -> Result { let compute_dollars = price.is_some(); @@ -52,19 +52,22 @@ impl Vecs { Ok(Self { starting_height: Height::ZERO, - state: AddressCohortState::default_and_import( - states_path, - cohort_name.unwrap_or_default(), - compute_dollars, - )?, + state: states_path.map(|states_path| { + AddressCohortState::default_and_import( + states_path, + cohort_name.unwrap_or_default(), + compute_dollars, + ) + .unwrap() + }), height_to_address_count: EagerVec::forced_import( - file, + db, &suffix("address_count"), version + VERSION + Version::ZERO, format, )?, indexes_to_address_count: ComputedVecsFromHeight::forced_import( - file, + db, &suffix("address_count"), Source::None, version + VERSION + Version::ZERO, @@ -74,7 +77,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, inner: common::Vecs::forced_import( - file, + db, cohort_name, computation, format, @@ -83,6 +86,7 @@ impl Vecs { price, compute_relative_to_all, false, + false, )?, }) } @@ -91,7 +95,9 @@ impl Vecs { impl DynCohortVecs for Vecs { fn starting_height(&self) -> Height { [ - self.state.height().map_or(Height::MAX, |h| h.incremented()), + self.state.as_ref().map_or(Height::MAX, |state| { + state.height().map_or(Height::MAX, |h| h.incremented()) + }), self.height_to_address_count.len().into(), self.inner.starting_height(), ] @@ -108,19 +114,21 @@ impl DynCohortVecs for Vecs { self.starting_height = starting_height; if let Some(prev_height) = starting_height.decremented() { - self.state.address_count = *self + self.state.as_mut().unwrap().address_count = *self .height_to_address_count .into_iter() .unwrap_get_inner(prev_height); } - self.inner - .init(&mut self.starting_height, &mut self.state.inner); + self.inner.init( + &mut self.starting_height, + &mut self.state.as_mut().unwrap().inner, + ); } fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { self.height_to_address_count - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + self.height_to_address_count.inner_version(), )?; @@ -134,11 +142,12 @@ impl DynCohortVecs for Vecs { self.height_to_address_count.forced_push_at( height, - self.state.address_count.into(), + self.state.as_ref().unwrap().address_count.into(), exit, )?; - self.inner.forced_pushed_at(height, exit, &self.state.inner) + self.inner + .forced_pushed_at(height, exit, &self.state.as_ref().unwrap().inner) } fn compute_then_force_push_unrealized_states( @@ -155,7 +164,7 @@ impl DynCohortVecs for Vecs { dateindex, date_price, exit, - &self.state.inner, + &self.state.as_ref().unwrap().inner, ) } @@ -163,7 +172,7 @@ impl DynCohortVecs for Vecs { self.height_to_address_count.safe_flush(exit)?; self.inner - .safe_flush_stateful_vecs(height, exit, &mut self.state.inner) + .safe_flush_stateful_vecs(height, exit, &mut self.state.as_mut().unwrap().inner) } #[allow(clippy::too_many_arguments)] diff --git a/crates/brk_computer/src/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs index 2350326b4..561cec027 100644 --- a/crates/brk_computer/src/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; @@ -6,8 +6,8 @@ use brk_structs::{ AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex, Dollars, GroupFilter, Height, Version, }; -use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format}; use derive_deref::{Deref, DerefMut}; +use vecdb::{AnyIterableVec, Computation, Database, Exit, Format}; use crate::{ Indexes, indexes, market, price, @@ -24,7 +24,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>); impl Vecs { pub fn forced_import( - file: &Arc, + db: &Database, version: Version, _computation: Computation, format: Format, @@ -36,458 +36,458 @@ impl Vecs { AddressGroups { amount_range: ByAmountRange { _0sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_with_0sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _1sat_to_10sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1sat_under_10sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _10sats_to_100sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10sats_under_100sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _100sats_to_1k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100sats_under_1k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _1k_sats_to_10k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1k_sats_under_10k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _10k_sats_to_100k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10k_sats_under_100k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _100k_sats_to_1m_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100k_sats_under_1m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _1m_sats_to_10m_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1m_sats_under_10m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _10m_sats_to_1btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10m_sats_under_1btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _1btc_to_10btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1btc_under_10btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _10btc_to_100btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10btc_under_100btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _100btc_to_1k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100btc_under_1k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _1k_btc_to_10k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1k_btc_under_10k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _10k_btc_to_100k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10k_btc_under_100k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, _100k_btc_or_more: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, )?, }, lt_amount: ByLowerThanAmount { _10sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_10sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_100sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_1k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_10k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_100k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1m_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_1m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10m_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_10m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_1btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_10btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_100btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_1k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_10k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_under_100k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, }, ge_amount: ByGreatEqualAmount { _1sat: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1sat"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100k_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1m_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10m_sats: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _100btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_100btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _1k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_1k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, _10k_btc: address_cohort::Vecs::forced_import( - file, + db, Some("addrs_above_10k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, )?, }, diff --git a/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs index 54a6bf313..53212810c 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs @@ -1,6 +1,6 @@ use brk_structs::{ByAddressType, Height}; -use brk_vecs::VecIterator; use derive_deref::{Deref, DerefMut}; +use vecdb::VecIterator; use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; diff --git a/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs index 8273634aa..f5950c58e 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_structs::{ByAddressType, Height, StoredU64}; -use brk_vecs::{EagerVec, Exit, GenericStoredVec}; +use vecdb::{EagerVec, Exit, GenericStoredVec}; use derive_deref::{Deref, DerefMut}; use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; diff --git a/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs index e4eb38c73..df833a8e1 100644 --- a/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_structs::{ByAddressType, StoredU64}; -use brk_vecs::{AnyCollectableVec, Exit}; use derive_deref::{Deref, DerefMut}; +use vecdb::{AnyCollectableVec, Exit}; use crate::{ Indexes, grouped::ComputedVecsFromHeight, indexes, diff --git a/crates/brk_computer/src/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs index f9b43db91..e0e70ce46 100644 --- a/crates/brk_computer/src/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -1,13 +1,11 @@ -use std::sync::Arc; - use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{ Bitcoin, DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64, Version, }; -use brk_vecs::{ +use vecdb::{ AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, - EagerVec, Exit, File, Format, GenericStoredVec, VecIterator, + Database, EagerVec, Exit, Format, GenericStoredVec, VecIterator, }; use crate::{ @@ -128,7 +126,7 @@ pub struct Vecs { impl Vecs { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, cohort_name: Option<&str>, computation: Computation, format: Format, @@ -137,6 +135,7 @@ impl Vecs { price: Option<&price::Vecs>, compute_relative_to_all: bool, ratio_extended: bool, + compute_adjusted: bool, ) -> Result { let compute_dollars = price.is_some(); @@ -146,7 +145,7 @@ impl Vecs { let dateindex_to_supply_in_profit = compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_profit"), version + VERSION + Version::ZERO, format, @@ -156,7 +155,7 @@ impl Vecs { let dateindex_to_supply_even = compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_even"), version + VERSION + Version::ZERO, format, @@ -166,7 +165,7 @@ impl Vecs { let dateindex_to_supply_in_loss = compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_loss"), version + VERSION + Version::ZERO, format, @@ -176,7 +175,7 @@ impl Vecs { let dateindex_to_unrealized_profit = compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("unrealized_profit"), version + VERSION + Version::ZERO, format, @@ -186,7 +185,7 @@ impl Vecs { let dateindex_to_unrealized_loss = compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("unrealized_loss"), version + VERSION + Version::ZERO, format, @@ -197,7 +196,7 @@ impl Vecs { Ok(Self { height_to_supply_in_profit: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_profit"), version + VERSION + Version::ZERO, format, @@ -206,7 +205,7 @@ impl Vecs { }), indexes_to_supply_in_profit: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_in_profit"), dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -221,7 +220,7 @@ impl Vecs { dateindex_to_supply_in_profit, height_to_supply_even: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_even"), version + VERSION + Version::ZERO, format, @@ -230,7 +229,7 @@ impl Vecs { }), indexes_to_supply_even: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_even"), dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -245,7 +244,7 @@ impl Vecs { dateindex_to_supply_even, height_to_supply_in_loss: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_loss"), version + VERSION + Version::ZERO, format, @@ -254,7 +253,7 @@ impl Vecs { }), indexes_to_supply_in_loss: compute_dollars.then(|| { ComputedValueVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_in_loss"), dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -269,7 +268,7 @@ impl Vecs { dateindex_to_supply_in_loss, height_to_unrealized_profit: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("unrealized_profit"), version + VERSION + Version::ZERO, format, @@ -278,7 +277,7 @@ impl Vecs { }), indexes_to_unrealized_profit: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("unrealized_profit"), dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -292,7 +291,7 @@ impl Vecs { dateindex_to_unrealized_profit, height_to_unrealized_loss: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("unrealized_loss"), version + VERSION + Version::ZERO, format, @@ -301,7 +300,7 @@ impl Vecs { }), height_to_min_price_paid: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("min_price_paid"), version + VERSION + Version::ZERO, format, @@ -310,7 +309,7 @@ impl Vecs { }), height_to_max_price_paid: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("max_price_paid"), version + VERSION + Version::ZERO, format, @@ -319,7 +318,7 @@ impl Vecs { }), indexes_to_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("unrealized_loss"), dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(), version + VERSION + Version::ZERO, @@ -333,7 +332,7 @@ impl Vecs { dateindex_to_unrealized_loss, height_to_realized_cap: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("realized_cap"), version + VERSION + Version::ZERO, format, @@ -342,7 +341,7 @@ impl Vecs { }), indexes_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_cap"), Source::None, version + VERSION + Version::ZERO, @@ -355,7 +354,7 @@ impl Vecs { }), indexes_to_min_price_paid: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("min_price_paid"), Source::None, version + VERSION + Version::ZERO, @@ -368,7 +367,7 @@ impl Vecs { }), indexes_to_max_price_paid: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("max_price_paid"), Source::None, version + VERSION + Version::ZERO, @@ -380,13 +379,13 @@ impl Vecs { .unwrap() }), height_to_supply: EagerVec::forced_import( - file, + db, &suffix("supply"), version + VERSION + Version::ZERO, format, )?, height_to_supply_value: ComputedHeightValueVecs::forced_import( - file, + db, &suffix("supply"), Source::None, version + VERSION + Version::ZERO, @@ -394,7 +393,7 @@ impl Vecs { compute_dollars, )?, indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( - file, + db, &suffix("supply"), Source::Compute, version + VERSION + Version::ONE, @@ -405,13 +404,13 @@ impl Vecs { indexes, )?, height_to_utxo_count: EagerVec::forced_import( - file, + db, &suffix("utxo_count"), version + VERSION + Version::ZERO, format, )?, indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( - file, + db, &suffix("utxo_count"), Source::None, version + VERSION + Version::ZERO, @@ -422,7 +421,7 @@ impl Vecs { )?, indexes_to_realized_price: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_price"), Source::Compute, version + VERSION + Version::ZERO, @@ -435,7 +434,7 @@ impl Vecs { }), indexes_to_realized_price_extra: compute_dollars.then(|| { ComputedRatioVecsFromDateIndex::forced_import( - file, + db, &suffix("realized_price"), Source::None, version + VERSION + Version::ZERO, @@ -448,7 +447,7 @@ impl Vecs { }), height_to_realized_profit: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("realized_profit"), version + VERSION + Version::ZERO, format, @@ -457,7 +456,7 @@ impl Vecs { }), indexes_to_realized_profit: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_profit"), Source::None, version + VERSION + Version::ZERO, @@ -472,7 +471,7 @@ impl Vecs { }), height_to_realized_loss: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("realized_loss"), version + VERSION + Version::ZERO, format, @@ -481,7 +480,7 @@ impl Vecs { }), indexes_to_realized_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_loss"), Source::None, version + VERSION + Version::ZERO, @@ -496,7 +495,7 @@ impl Vecs { }), indexes_to_negative_realized_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("negative_realized_loss"), Source::Compute, version + VERSION + Version::ONE, @@ -509,7 +508,7 @@ impl Vecs { }), height_to_value_created: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("value_created"), version + VERSION + Version::ZERO, format, @@ -518,7 +517,7 @@ impl Vecs { }), indexes_to_value_created: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("value_created"), Source::None, version + VERSION + Version::ZERO, @@ -531,7 +530,7 @@ impl Vecs { }), indexes_to_realized_value: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_value"), Source::Compute, version + VERSION + Version::ZERO, @@ -542,18 +541,18 @@ impl Vecs { ) .unwrap() }), - height_to_adjusted_value_created: compute_dollars.then(|| { + height_to_adjusted_value_created: (compute_dollars && compute_adjusted).then(|| { EagerVec::forced_import( - file, + db, &suffix("adjusted_value_created"), version + VERSION + Version::ZERO, format, ) .unwrap() }), - indexes_to_adjusted_value_created: compute_dollars.then(|| { + indexes_to_adjusted_value_created: (compute_dollars && compute_adjusted).then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("adjusted_value_created"), Source::None, version + VERSION + Version::ZERO, @@ -566,7 +565,7 @@ impl Vecs { }), height_to_value_destroyed: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("value_destroyed"), version + VERSION + Version::ZERO, format, @@ -575,7 +574,7 @@ impl Vecs { }), indexes_to_value_destroyed: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("value_destroyed"), Source::None, version + VERSION + Version::ZERO, @@ -586,18 +585,18 @@ impl Vecs { ) .unwrap() }), - height_to_adjusted_value_destroyed: compute_dollars.then(|| { + height_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted).then(|| { EagerVec::forced_import( - file, + db, &suffix("adjusted_value_destroyed"), version + VERSION + Version::ZERO, format, ) .unwrap() }), - indexes_to_adjusted_value_destroyed: compute_dollars.then(|| { + indexes_to_adjusted_value_destroyed: (compute_dollars && compute_adjusted).then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("adjusted_value_destroyed"), Source::None, version + VERSION + Version::ZERO, @@ -610,7 +609,7 @@ impl Vecs { }), indexes_to_realized_cap_30d_change: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("realized_cap_30d_change"), Source::Compute, version + VERSION + Version::ZERO, @@ -623,7 +622,7 @@ impl Vecs { }), indexes_to_net_realized_profit_and_loss: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("net_realized_profit_and_loss"), Source::Compute, version + VERSION + Version::ZERO, @@ -638,7 +637,7 @@ impl Vecs { }), dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("sell_side_risk_ratio"), version + VERSION + Version::ONE, format, @@ -647,16 +646,16 @@ impl Vecs { }), dateindex_to_spent_output_profit_ratio: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("spent_output_profit_ratio"), version + VERSION + Version::ZERO, format, ) .unwrap() }), - dateindex_to_adjusted_spent_output_profit_ratio: compute_dollars.then(|| { + dateindex_to_adjusted_spent_output_profit_ratio: (compute_dollars && compute_adjusted).then(|| { EagerVec::forced_import( - file, + db, &suffix("adjusted_spent_output_profit_ratio"), version + VERSION + Version::ZERO, format, @@ -664,7 +663,7 @@ impl Vecs { .unwrap() }), height_to_halved_supply_value: ComputedHeightValueVecs::forced_import( - file, + db, &suffix("halved_supply"), Source::Compute, version + VERSION + Version::ZERO, @@ -672,7 +671,7 @@ impl Vecs { compute_dollars, )?, indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import( - file, + db, &suffix("halved_supply"), Source::Compute, version + VERSION + Version::ZERO, @@ -684,7 +683,7 @@ impl Vecs { )?, height_to_negative_unrealized_loss: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("negative_unrealized_loss"), version + VERSION + Version::ZERO, format, @@ -693,7 +692,7 @@ impl Vecs { }), indexes_to_negative_unrealized_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("negative_unrealized_loss"), Source::Compute, version + VERSION + Version::ZERO, @@ -706,7 +705,7 @@ impl Vecs { }), height_to_net_unrealized_profit_and_loss: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("net_unrealized_profit_and_loss"), version + VERSION + Version::ZERO, format, @@ -715,7 +714,7 @@ impl Vecs { }), indexes_to_net_unrealized_profit_and_loss: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("net_unrealized_profit_and_loss"), Source::Compute, version + VERSION + Version::ZERO, @@ -729,7 +728,7 @@ impl Vecs { height_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then( || { EagerVec::forced_import( - file, + db, &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), version + VERSION + Version::ONE, format, @@ -740,7 +739,7 @@ impl Vecs { indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then( || { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("net_unrealized_profit_and_loss_relative_to_market_cap"), Source::Compute, version + VERSION + Version::ONE, @@ -754,7 +753,7 @@ impl Vecs { ), indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_profit_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::ZERO, @@ -767,7 +766,7 @@ impl Vecs { }), indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("realized_loss_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::ZERO, @@ -781,7 +780,7 @@ impl Vecs { indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then( || { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("net_realized_profit_and_loss_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::ONE, @@ -795,7 +794,7 @@ impl Vecs { ), height_to_supply_even_value: compute_dollars.then(|| { ComputedHeightValueVecs::forced_import( - file, + db, &suffix("supply_even"), Source::None, version + VERSION + Version::ZERO, @@ -806,7 +805,7 @@ impl Vecs { }), height_to_supply_in_loss_value: compute_dollars.then(|| { ComputedHeightValueVecs::forced_import( - file, + db, &suffix("supply_in_loss"), Source::None, version + VERSION + Version::ZERO, @@ -817,7 +816,7 @@ impl Vecs { }), height_to_supply_in_profit_value: compute_dollars.then(|| { ComputedHeightValueVecs::forced_import( - file, + db, &suffix("supply_in_profit"), Source::None, version + VERSION + Version::ZERO, @@ -828,7 +827,7 @@ impl Vecs { }), height_to_supply_even_relative_to_own_supply: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_even_relative_to_own_supply"), version + VERSION + Version::ONE, format, @@ -837,7 +836,7 @@ impl Vecs { }), height_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_loss_relative_to_own_supply"), version + VERSION + Version::ONE, format, @@ -846,7 +845,7 @@ impl Vecs { }), height_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_profit_relative_to_own_supply"), version + VERSION + Version::ONE, format, @@ -855,7 +854,7 @@ impl Vecs { }), indexes_to_supply_even_relative_to_own_supply: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_even_relative_to_own_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -868,7 +867,7 @@ impl Vecs { }), indexes_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_in_loss_relative_to_own_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -881,7 +880,7 @@ impl Vecs { }), indexes_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_in_profit_relative_to_own_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -894,7 +893,7 @@ impl Vecs { }), indexes_to_supply_relative_to_circulating_supply: compute_relative_to_all.then(|| { ComputedVecsFromHeight::forced_import( - file, + db, &suffix("supply_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -909,7 +908,7 @@ impl Vecs { && compute_dollars) .then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_even_relative_to_circulating_supply"), version + VERSION + Version::ONE, format, @@ -920,7 +919,7 @@ impl Vecs { && compute_dollars) .then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_loss_relative_to_circulating_supply"), version + VERSION + Version::ONE, format, @@ -931,7 +930,7 @@ impl Vecs { && compute_dollars) .then(|| { EagerVec::forced_import( - file, + db, &suffix("supply_in_profit_relative_to_circulating_supply"), version + VERSION + Version::ONE, format, @@ -942,7 +941,7 @@ impl Vecs { && compute_dollars) .then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_even_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -957,7 +956,7 @@ impl Vecs { && compute_dollars) .then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_in_loss_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -972,7 +971,7 @@ impl Vecs { && compute_dollars) .then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("supply_in_profit_relative_to_circulating_supply"), Source::Compute, version + VERSION + Version::ONE, @@ -984,19 +983,19 @@ impl Vecs { .unwrap() }), height_to_satblocks_destroyed: EagerVec::forced_import( - file, + db, &suffix("satblocks_destroyed"), version + VERSION + Version::ZERO, format, )?, height_to_satdays_destroyed: EagerVec::forced_import( - file, + db, &suffix("satdays_destroyed"), version + VERSION + Version::ZERO, format, )?, indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import( - file, + db, &suffix("coinblocks_destroyed"), Source::Compute, version + VERSION + Version::TWO, @@ -1006,7 +1005,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import( - file, + db, &suffix("coindays_destroyed"), Source::Compute, version + VERSION + Version::TWO, @@ -1017,7 +1016,7 @@ impl Vecs { )?, indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("net_realized_profit_and_loss_cumulative_30d_change"), Source::Compute, version + VERSION + Version::new(3), @@ -1030,7 +1029,7 @@ impl Vecs { }), indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"), Source::Compute, version + VERSION + Version::new(3), @@ -1043,7 +1042,7 @@ impl Vecs { }), indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| { ComputedVecsFromDateIndex::forced_import( - file, + db, &suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"), Source::Compute, version + VERSION + Version::new(3), @@ -1132,28 +1131,27 @@ impl Vecs { } pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { - self.height_to_supply - .validate_computed_version_or_reset_file( - base_version + self.height_to_supply.inner_version(), - )?; + self.height_to_supply.validate_computed_version_or_reset( + base_version + self.height_to_supply.inner_version(), + )?; self.height_to_utxo_count - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + self.height_to_utxo_count.inner_version(), )?; self.height_to_satblocks_destroyed - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + self.height_to_satblocks_destroyed.inner_version(), )?; self.height_to_satdays_destroyed - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + self.height_to_satdays_destroyed.inner_version(), )?; if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() { - height_to_realized_cap.validate_computed_version_or_reset_file( + height_to_realized_cap.validate_computed_version_or_reset( base_version + height_to_realized_cap.inner_version(), )?; @@ -1165,7 +1163,7 @@ impl Vecs { self.height_to_realized_profit .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_realized_profit_inner_version, )?; let height_to_realized_loss_inner_version = self @@ -1176,7 +1174,7 @@ impl Vecs { self.height_to_realized_loss .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_realized_loss_inner_version, )?; let height_to_value_created_inner_version = self @@ -1187,20 +1185,9 @@ impl Vecs { self.height_to_value_created .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_value_created_inner_version, )?; - let height_to_adjusted_value_created_inner_version = self - .height_to_adjusted_value_created - .as_ref() - .unwrap() - .inner_version(); - self.height_to_adjusted_value_created - .as_mut() - .unwrap() - .validate_computed_version_or_reset_file( - base_version + height_to_adjusted_value_created_inner_version, - )?; let height_to_value_destroyed_inner_version = self .height_to_value_destroyed .as_ref() @@ -1209,20 +1196,9 @@ impl Vecs { self.height_to_value_destroyed .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_value_destroyed_inner_version, )?; - let height_to_adjusted_value_destroyed_inner_version = self - .height_to_adjusted_value_destroyed - .as_ref() - .unwrap() - .inner_version(); - self.height_to_adjusted_value_destroyed - .as_mut() - .unwrap() - .validate_computed_version_or_reset_file( - base_version + height_to_adjusted_value_destroyed_inner_version, - )?; let height_to_supply_in_profit_inner_version = self .height_to_supply_in_profit .as_ref() @@ -1231,7 +1207,7 @@ impl Vecs { self.height_to_supply_in_profit .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_supply_in_profit_inner_version, )?; let height_to_supply_in_loss_inner_version = self @@ -1242,7 +1218,7 @@ impl Vecs { self.height_to_supply_in_loss .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_supply_in_loss_inner_version, )?; let height_to_supply_even_inner_version = @@ -1250,7 +1226,7 @@ impl Vecs { self.height_to_supply_even .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_supply_even_inner_version, )?; let height_to_unrealized_profit_inner_version = self @@ -1261,7 +1237,7 @@ impl Vecs { self.height_to_unrealized_profit .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_unrealized_profit_inner_version, )?; let height_to_unrealized_loss_inner_version = self @@ -1272,7 +1248,7 @@ impl Vecs { self.height_to_unrealized_loss .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_unrealized_loss_inner_version, )?; let dateindex_to_supply_in_profit_inner_version = self @@ -1283,7 +1259,7 @@ impl Vecs { self.dateindex_to_supply_in_profit .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + dateindex_to_supply_in_profit_inner_version, )?; let dateindex_to_supply_in_loss_inner_version = self @@ -1294,7 +1270,7 @@ impl Vecs { self.dateindex_to_supply_in_loss .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + dateindex_to_supply_in_loss_inner_version, )?; let dateindex_to_supply_even_inner_version = self @@ -1305,7 +1281,7 @@ impl Vecs { self.dateindex_to_supply_even .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + dateindex_to_supply_even_inner_version, )?; let dateindex_to_unrealized_profit_inner_version = self @@ -1316,7 +1292,7 @@ impl Vecs { self.dateindex_to_unrealized_profit .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + dateindex_to_unrealized_profit_inner_version, )?; let dateindex_to_unrealized_loss_inner_version = self @@ -1327,7 +1303,7 @@ impl Vecs { self.dateindex_to_unrealized_loss .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + dateindex_to_unrealized_loss_inner_version, )?; let height_to_min_price_paid_inner_version = self @@ -1338,7 +1314,7 @@ impl Vecs { self.height_to_min_price_paid .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_min_price_paid_inner_version, )?; let height_to_max_price_paid_inner_version = self @@ -1349,9 +1325,34 @@ impl Vecs { self.height_to_max_price_paid .as_mut() .unwrap() - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + height_to_max_price_paid_inner_version, )?; + + if self.height_to_adjusted_value_created.is_some() { + let height_to_adjusted_value_created_inner_version = self + .height_to_adjusted_value_created + .as_ref() + .unwrap() + .inner_version(); + self.height_to_adjusted_value_created + .as_mut() + .unwrap() + .validate_computed_version_or_reset( + base_version + height_to_adjusted_value_created_inner_version, + )?; + let height_to_adjusted_value_destroyed_inner_version = self + .height_to_adjusted_value_destroyed + .as_ref() + .unwrap() + .inner_version(); + self.height_to_adjusted_value_destroyed + .as_mut() + .unwrap() + .validate_computed_version_or_reset( + base_version + height_to_adjusted_value_destroyed_inner_version, + )?; + } } Ok(()) @@ -1401,18 +1402,21 @@ impl Vecs { .as_mut() .unwrap() .forced_push_at(height, realized.value_created, exit)?; - self.height_to_adjusted_value_created - .as_mut() - .unwrap() - .forced_push_at(height, realized.adj_value_created, exit)?; self.height_to_value_destroyed .as_mut() .unwrap() .forced_push_at(height, realized.value_destroyed, exit)?; - self.height_to_adjusted_value_destroyed - .as_mut() - .unwrap() - .forced_push_at(height, realized.adj_value_destroyed, exit)?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created + .as_mut() + .unwrap() + .forced_push_at(height, realized.adj_value_created, exit)?; + self.height_to_adjusted_value_destroyed + .as_mut() + .unwrap() + .forced_push_at(height, realized.adj_value_destroyed, exit)?; + } } Ok(()) } @@ -1528,18 +1532,10 @@ impl Vecs { .as_mut() .unwrap() .safe_flush(exit)?; - self.height_to_adjusted_value_created - .as_mut() - .unwrap() - .safe_flush(exit)?; self.height_to_value_destroyed .as_mut() .unwrap() .safe_flush(exit)?; - self.height_to_adjusted_value_destroyed - .as_mut() - .unwrap() - .safe_flush(exit)?; self.height_to_supply_in_profit .as_mut() .unwrap() @@ -1588,6 +1584,17 @@ impl Vecs { .as_mut() .unwrap() .safe_flush(exit)?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created + .as_mut() + .unwrap() + .safe_flush(exit)?; + self.height_to_adjusted_value_destroyed + .as_mut() + .unwrap() + .safe_flush(exit)?; + } } state.commit(height)?; @@ -1709,18 +1716,6 @@ impl Vecs { .as_slice(), exit, )?; - self.height_to_adjusted_value_created - .as_mut() - .unwrap() - .compute_sum_of_others( - starting_indexes.height, - others - .iter() - .map(|v| v.height_to_adjusted_value_created.as_ref().unwrap()) - .collect::>() - .as_slice(), - exit, - )?; self.height_to_value_destroyed .as_mut() .unwrap() @@ -1733,18 +1728,6 @@ impl Vecs { .as_slice(), exit, )?; - self.height_to_adjusted_value_destroyed - .as_mut() - .unwrap() - .compute_sum_of_others( - starting_indexes.height, - others - .iter() - .map(|v| v.height_to_adjusted_value_destroyed.as_ref().unwrap()) - .collect::>() - .as_slice(), - exit, - )?; self.height_to_supply_in_profit .as_mut() .unwrap() @@ -1889,6 +1872,41 @@ impl Vecs { .as_slice(), exit, )?; + + if self.height_to_adjusted_value_created.is_some() { + self.height_to_adjusted_value_created + .as_mut() + .unwrap() + .compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| { + v.height_to_adjusted_value_created + .as_ref() + .unwrap_or(v.height_to_value_created.as_ref().unwrap()) + }) + .collect::>() + .as_slice(), + exit, + )?; + self.height_to_adjusted_value_destroyed + .as_mut() + .unwrap() + .compute_sum_of_others( + starting_indexes.height, + others + .iter() + .map(|v| { + v.height_to_adjusted_value_destroyed + .as_ref() + .unwrap_or(v.height_to_value_destroyed.as_ref().unwrap()) + }) + .collect::>() + .as_slice(), + exit, + )?; + } } Ok(()) @@ -2142,16 +2160,6 @@ impl Vecs { Some(self.height_to_value_created.as_ref().unwrap()), )?; - self.indexes_to_adjusted_value_created - .as_mut() - .unwrap() - .compute_rest( - indexes, - starting_indexes, - exit, - Some(self.height_to_adjusted_value_created.as_ref().unwrap()), - )?; - self.indexes_to_value_destroyed .as_mut() .unwrap() @@ -2162,16 +2170,6 @@ impl Vecs { Some(self.height_to_value_destroyed.as_ref().unwrap()), )?; - self.indexes_to_adjusted_value_destroyed - .as_mut() - .unwrap() - .compute_rest( - indexes, - starting_indexes, - exit, - Some(self.height_to_adjusted_value_destroyed.as_ref().unwrap()), - )?; - self.indexes_to_realized_cap_30d_change .as_mut() .unwrap() @@ -2251,24 +2249,6 @@ impl Vecs { exit, )?; - self.dateindex_to_adjusted_spent_output_profit_ratio - .as_mut() - .unwrap() - .compute_divide( - starting_indexes.dateindex, - self.indexes_to_adjusted_value_created - .as_ref() - .unwrap() - .dateindex - .unwrap_sum(), - self.indexes_to_adjusted_value_destroyed - .as_ref() - .unwrap() - .dateindex - .unwrap_sum(), - exit, - )?; - self.dateindex_to_sell_side_risk_ratio .as_mut() .unwrap() @@ -2812,6 +2792,46 @@ impl Vecs { }, )?; } + + if self.indexes_to_adjusted_value_created.is_some() { + self.indexes_to_adjusted_value_created + .as_mut() + .unwrap() + .compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_adjusted_value_created.as_ref().unwrap()), + )?; + + self.indexes_to_adjusted_value_destroyed + .as_mut() + .unwrap() + .compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_adjusted_value_destroyed.as_ref().unwrap()), + )?; + + self.dateindex_to_adjusted_spent_output_profit_ratio + .as_mut() + .unwrap() + .compute_divide( + starting_indexes.dateindex, + self.indexes_to_adjusted_value_created + .as_ref() + .unwrap() + .dateindex + .unwrap_sum(), + self.indexes_to_adjusted_value_destroyed + .as_ref() + .unwrap() + .dateindex + .unwrap_sum(), + exit, + )?; + } } Ok(()) diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 9dc1e6afd..31ad15ef1 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1,4 +1,4 @@ -use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread}; +use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; use brk_error::Result; use brk_indexer::Indexer; @@ -9,12 +9,12 @@ use brk_structs::{ P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, Sats, StoredU64, Timestamp, TypeIndex, Version, }; -use brk_vecs::{ - AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File, - Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator, -}; use log::info; use rayon::prelude::*; +use vecdb::{ + AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database, EagerVec, Exit, + Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator, +}; use crate::{ BlockState, Indexes, SupplyState, Transacted, @@ -52,7 +52,7 @@ const VERSION: Version = Version::new(21); #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, pub chain_state: RawVec, @@ -92,29 +92,29 @@ impl Vecs { price: Option<&price::Vecs>, states_path: &Path, ) -> Result { - let file = Arc::new(File::open(&parent.join("stateful"))?); - file.set_min_len(PAGE_SIZE * 20_000_000)?; - file.set_min_regions(50_000)?; + let db = Database::open(&parent.join("stateful"))?; + db.set_min_len(PAGE_SIZE * 20_000_000)?; + db.set_min_regions(50_000)?; let compute_dollars = price.is_some(); - let chain_file = Arc::new(File::open(&parent.join("chain"))?); + let chain_db = Database::open(&parent.join("chain"))?; Ok(Self { chain_state: RawVec::forced_import( - &chain_file, + &chain_db, "chain", version + VERSION + Version::ZERO, )?, height_to_unspendable_supply: EagerVec::forced_import( - &file, + &db, "unspendable_supply", version + VERSION + Version::ZERO, format, )?, indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "unspendable_supply", Source::None, version + VERSION + Version::ZERO, @@ -125,13 +125,13 @@ impl Vecs { indexes, )?, height_to_opreturn_supply: EagerVec::forced_import( - &file, + &db, "opreturn_supply", version + VERSION + Version::ZERO, format, )?, indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "opreturn_supply", Source::None, version + VERSION + Version::ZERO, @@ -142,7 +142,7 @@ impl Vecs { indexes, )?, indexes_to_address_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "address_count", Source::Compute, version + VERSION + Version::ZERO, @@ -152,7 +152,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "empty_address_count", Source::Compute, version + VERSION + Version::ZERO, @@ -164,49 +164,49 @@ impl Vecs { addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { p2pk65: EagerVec::forced_import( - &file, + &db, "p2pk65_address_count", version + VERSION + Version::ZERO, format, )?, p2pk33: EagerVec::forced_import( - &file, + &db, "p2pk33_address_count", version + VERSION + Version::ZERO, format, )?, p2pkh: EagerVec::forced_import( - &file, + &db, "p2pkh_address_count", version + VERSION + Version::ZERO, format, )?, p2sh: EagerVec::forced_import( - &file, + &db, "p2sh_address_count", version + VERSION + Version::ZERO, format, )?, p2wpkh: EagerVec::forced_import( - &file, + &db, "p2wpkh_address_count", version + VERSION + Version::ZERO, format, )?, p2wsh: EagerVec::forced_import( - &file, + &db, "p2wsh_address_count", version + VERSION + Version::ZERO, format, )?, p2tr: EagerVec::forced_import( - &file, + &db, "p2tr_address_count", version + VERSION + Version::ZERO, format, )?, p2a: EagerVec::forced_import( - &file, + &db, "p2a_address_count", version + VERSION + Version::ZERO, format, @@ -216,49 +216,49 @@ impl Vecs { addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { p2pk65: EagerVec::forced_import( - &file, + &db, "p2pk65_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2pk33: EagerVec::forced_import( - &file, + &db, "p2pk33_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2pkh: EagerVec::forced_import( - &file, + &db, "p2pkh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2sh: EagerVec::forced_import( - &file, + &db, "p2sh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2wpkh: EagerVec::forced_import( - &file, + &db, "p2wpkh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2wsh: EagerVec::forced_import( - &file, + &db, "p2wsh_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2tr: EagerVec::forced_import( - &file, + &db, "p2tr_empty_address_count", version + VERSION + Version::ZERO, format, )?, p2a: EagerVec::forced_import( - &file, + &db, "p2a_empty_address_count", version + VERSION + Version::ZERO, format, @@ -268,7 +268,7 @@ impl Vecs { addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from( ByAddressType { p2pk65: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pk65_address_count", Source::None, version + VERSION + Version::ZERO, @@ -278,7 +278,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pk33: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pk33_address_count", Source::None, version + VERSION + Version::ZERO, @@ -288,7 +288,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pkh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pkh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -298,7 +298,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2sh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2sh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -308,7 +308,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wpkh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2wpkh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -318,7 +318,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wsh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2wsh_address_count", Source::None, version + VERSION + Version::ZERO, @@ -328,7 +328,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2tr: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2tr_address_count", Source::None, version + VERSION + Version::ZERO, @@ -338,7 +338,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2a: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2a_address_count", Source::None, version + VERSION + Version::ZERO, @@ -352,7 +352,7 @@ impl Vecs { addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from( ByAddressType { p2pk65: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pk65_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -362,7 +362,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pk33: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pk33_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -372,7 +372,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2pkh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pkh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -382,7 +382,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2sh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2sh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -392,7 +392,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wpkh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2wpkh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -402,7 +402,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2wsh: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2wsh_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -412,7 +412,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2tr: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2tr_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -422,7 +422,7 @@ impl Vecs { VecBuilderOptions::default().add_last(), )?, p2a: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2a_empty_address_count", Source::None, version + VERSION + Version::ZERO, @@ -434,7 +434,7 @@ impl Vecs { }, ), utxo_cohorts: utxo_cohorts::Vecs::forced_import( - &file, + &db, version, computation, format, @@ -443,7 +443,7 @@ impl Vecs { states_path, )?, address_cohorts: address_cohorts::Vecs::forced_import( - &file, + &db, version, computation, format, @@ -453,58 +453,58 @@ impl Vecs { )?, p2aaddressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2pk33addressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2pk65addressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2pkhaddressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2shaddressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2traddressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2wpkhaddressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, p2wshaddressindex_to_anyaddressindex: RawVec::forced_import( - &file, + &db, "anyaddressindex", version + VERSION + Version::ZERO, )?, loadedaddressindex_to_loadedaddressdata: RawVec::forced_import( - &file, + &db, "loadedaddressdata", version + VERSION + Version::ZERO, )?, emptyaddressindex_to_emptyaddressdata: RawVec::forced_import( - &file, + &db, "emptyaddressdata", version + VERSION + Version::ZERO, )?, - file, + db, }) } @@ -529,7 +529,7 @@ impl Vecs { starting_indexes, exit, )?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } @@ -624,11 +624,11 @@ impl Vecs { .par_iter_mut() .try_for_each(|(_, v)| v.validate_computed_versions(base_version))?; self.height_to_unspendable_supply - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + self.height_to_unspendable_supply.inner_version(), )?; self.height_to_opreturn_supply - .validate_computed_version_or_reset_file( + .validate_computed_version_or_reset( base_version + self.height_to_opreturn_supply.inner_version(), )?; @@ -712,13 +712,14 @@ impl Vecs { separate_utxo_vecs .par_iter_mut() - .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; + .flat_map(|(_, v)| v.state.as_mut()) + .try_for_each(|state| state.reset_price_to_amount())?; info!("Resetting address price maps..."); separate_address_vecs .par_iter_mut() - .try_for_each(|(_, v)| v.state.reset_price_to_amount())?; + .try_for_each(|(_, v)| v.state.as_mut().unwrap().reset_price_to_amount())?; }; let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp()); @@ -820,12 +821,12 @@ impl Vecs { self.utxo_cohorts .as_mut_separate_vecs() .iter_mut() - .for_each(|(_, v)| v.state.reset_single_iteration_values()); + .for_each(|(_, v)| v.state.as_mut().unwrap().reset_single_iteration_values()); self.address_cohorts .as_mut_separate_vecs() .iter_mut() - .for_each(|(_, v)| v.state.reset_single_iteration_values()); + .for_each(|(_, v)| v.state.as_mut().unwrap().reset_single_iteration_values()); let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height); let price = height_to_close_iter @@ -1874,17 +1875,27 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { .get_mut(prev_amount) .1 .state + .as_mut() + .unwrap() .subtract(addressdata); } addressdata.receive(value, price); - vecs.amount_range.get_mut(amount).1.state.add(addressdata); + vecs.amount_range + .get_mut(amount) + .1 + .state + .as_mut() + .unwrap() + .add(addressdata); } else { vecs.amount_range .get_mut(amount) .1 .state + .as_mut() + .unwrap() .receive(addressdata, value, price); } }); @@ -1965,6 +1976,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { .get_mut(prev_amount) .1 .state + .as_mut() + .unwrap() .subtract(addressdata); addressdata.send(value, prev_price)?; @@ -1985,18 +1998,30 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { .unwrap() .insert(type_index, addressdata.into()); } else { - vecs.amount_range.get_mut(amount).1.state.add(addressdata); + vecs.amount_range + .get_mut(amount) + .1 + .state + .as_mut() + .unwrap() + .add(addressdata); } } else { - vecs.amount_range.get_mut(amount).1.state.send( - addressdata, - value, - price, - prev_price, - blocks_old, - days_old, - older_than_hour, - )?; + vecs.amount_range + .get_mut(amount) + .1 + .state + .as_mut() + .unwrap() + .send( + addressdata, + value, + price, + prev_price, + blocks_old, + days_old, + older_than_hour, + )?; } Ok(()) diff --git a/crates/brk_computer/src/stateful/range_map.rs b/crates/brk_computer/src/stateful/range_map.rs index 70eac6e1a..99ddb67e8 100644 --- a/crates/brk_computer/src/stateful/range_map.rs +++ b/crates/brk_computer/src/stateful/range_map.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use brk_vecs::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw}; +use vecdb::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw}; #[derive(Debug)] pub struct RangeMap(BTreeMap); diff --git a/crates/brk_computer/src/stateful/trait.rs b/crates/brk_computer/src/stateful/trait.rs index 6a44974dd..9018d0098 100644 --- a/crates/brk_computer/src/stateful/trait.rs +++ b/crates/brk_computer/src/stateful/trait.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version}; -use brk_vecs::{AnyCollectableVec, AnyIterableVec, Exit}; +use vecdb::{AnyCollectableVec, AnyIterableVec, Exit}; use crate::{Indexes, indexes, market, price}; diff --git a/crates/brk_computer/src/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs index a0555a92d..d0b01576f 100644 --- a/crates/brk_computer/src/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -1,9 +1,9 @@ -use std::{ops::Deref, path::Path, sync::Arc}; +use std::{ops::Deref, path::Path}; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version}; -use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, Exit, File, Format}; +use vecdb::{AnyCollectableVec, AnyIterableVec, Computation, Database, Exit, Format}; use crate::{ Indexes, UTXOCohortState, indexes, market, price, @@ -17,7 +17,7 @@ use crate::{ pub struct Vecs { starting_height: Height, - pub state: UTXOCohortState, + pub state: Option, inner: common::Vecs, } @@ -25,30 +25,34 @@ pub struct Vecs { impl Vecs { #[allow(clippy::too_many_arguments)] pub fn forced_import( - file: &Arc, + db: &Database, cohort_name: Option<&str>, computation: Computation, format: Format, version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, - states_path: &Path, + states_path: Option<&Path>, compute_relative_to_all: bool, ratio_extended: bool, + compute_adjusted: bool, ) -> Result { let compute_dollars = price.is_some(); Ok(Self { starting_height: Height::ZERO, - state: UTXOCohortState::default_and_import( - states_path, - cohort_name.unwrap_or_default(), - compute_dollars, - )?, + state: states_path.map(|states_path| { + UTXOCohortState::default_and_import( + states_path, + cohort_name.unwrap_or_default(), + compute_dollars, + ) + .unwrap() + }), inner: common::Vecs::forced_import( - file, + db, cohort_name, computation, format, @@ -57,6 +61,7 @@ impl Vecs { price, compute_relative_to_all, ratio_extended, + compute_adjusted, )?, }) } @@ -65,7 +70,9 @@ impl Vecs { impl DynCohortVecs for Vecs { fn starting_height(&self) -> Height { [ - self.state.height().map_or(Height::MAX, |h| h.incremented()), + self.state.as_ref().map_or(Height::MAX, |state| { + state.height().map_or(Height::MAX, |h| h.incremented()) + }), self.inner.starting_height(), ] .into_iter() @@ -80,7 +87,8 @@ impl DynCohortVecs for Vecs { self.starting_height = starting_height; - self.inner.init(&mut self.starting_height, &mut self.state); + self.inner + .init(&mut self.starting_height, self.state.as_mut().unwrap()); } fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { @@ -92,7 +100,8 @@ impl DynCohortVecs for Vecs { return Ok(()); } - self.inner.forced_pushed_at(height, exit, &self.state) + self.inner + .forced_pushed_at(height, exit, self.state.as_ref().unwrap()) } fn compute_then_force_push_unrealized_states( @@ -109,13 +118,13 @@ impl DynCohortVecs for Vecs { dateindex, date_price, exit, - &self.state, + self.state.as_mut().unwrap(), ) } fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { self.inner - .safe_flush_stateful_vecs(height, exit, &mut self.state) + .safe_flush_stateful_vecs(height, exit, self.state.as_mut().unwrap()) } #[allow(clippy::too_many_arguments)] diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index 3ace57a9f..b32890410 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, ops::ControlFlow, path::Path, sync::Arc}; +use std::{collections::BTreeMap, ops::ControlFlow, path::Path}; use brk_error::Result; use brk_indexer::Indexer; @@ -7,8 +7,8 @@ use brk_structs::{ ByMinAge, BySpendableType, ByTerm, CheckedSub, DateIndex, Dollars, GroupFilter, HalvingEpoch, Height, Timestamp, UTXOGroups, Version, }; -use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format, StoredIndex}; use derive_deref::{Deref, DerefMut}; +use vecdb::{AnyIterableVec, Computation, Database, Exit, Format, StoredIndex}; use crate::{ Indexes, indexes, market, price, @@ -25,7 +25,7 @@ pub struct Vecs(UTXOGroups<(GroupFilter, utxo_cohort::Vecs)>); impl Vecs { pub fn forced_import( - file: &Arc, + db: &Database, version: Version, _computation: Computation, format: Format, @@ -36,1413 +36,1529 @@ impl Vecs { Ok(Self( UTXOGroups { all: utxo_cohort::Vecs::forced_import( - file, + db, None, _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, false, true, + true, )?, term: ByTerm { short: utxo_cohort::Vecs::forced_import( - file, + db, Some("short_term_holders"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, long: utxo_cohort::Vecs::forced_import( - file, + db, Some("long_term_holders"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, }, epoch: ByEpoch { _0: utxo_cohort::Vecs::forced_import( - file, + db, Some("epoch_0"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _1: utxo_cohort::Vecs::forced_import( - file, + db, Some("epoch_1"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _2: utxo_cohort::Vecs::forced_import( - file, + db, Some("epoch_2"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _3: utxo_cohort::Vecs::forced_import( - file, + db, Some("epoch_3"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _4: utxo_cohort::Vecs::forced_import( - file, + db, Some("epoch_4"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, }, _type: BySpendableType { p2pk65: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2pk65"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2pk33: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2pk33"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2pkh: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2pkh"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2sh: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2sh"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2wpkh: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2wpkh"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2wsh: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2wsh"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2tr: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2tr"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2a: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2a"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, p2ms: utxo_cohort::Vecs::forced_import( - file, + db, Some("p2ms_outputs"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, empty: utxo_cohort::Vecs::forced_import( - file, + db, Some("empty_outputs"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, unknown: utxo_cohort::Vecs::forced_import( - file, + db, Some("unknown_outputs"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, }, max_age: ByMaxAge { _1w: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_1w_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _1m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_1m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _2m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_2m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _3m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_3m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _4m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_4m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _5m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_5m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _6m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_6m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _1y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_1y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _2y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_2y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _3y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_3y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _4y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_4y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _5y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_5y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _6y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_6y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _7y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_7y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _8y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_8y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _10y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_10y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _12y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_12y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, _15y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_15y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, + true, true, true, )?, }, min_age: ByMinAge { _1d: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1d_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _1w: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1w_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _1m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _2m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_2m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _3m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_3m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _4m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_4m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _5m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_5m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _6m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_6m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _1y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _2y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_2y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _3y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_3y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _4y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_4y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _5y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_5y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _6y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_6y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _7y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_7y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _8y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_8y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _10y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_10y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, _12y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_12y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, true, + false, )?, }, age_range: ByAgeRange { up_to_1d: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_up_to_1d_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), + true, true, true, )?, _1d_to_1w: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1d_up_to_1w_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _1w_to_1m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1w_up_to_1m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _1m_to_2m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1m_up_to_2m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _2m_to_3m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_2m_up_to_3m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _3m_to_4m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_3m_up_to_4m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _4m_to_5m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_4m_up_to_5m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _5m_to_6m: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_5m_up_to_6m_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _6m_to_1y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_6m_up_to_1y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _1y_to_2y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_1y_up_to_2y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _2y_to_3y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_2y_up_to_3y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _3y_to_4y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_3y_up_to_4y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _4y_to_5y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_4y_up_to_5y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _5y_to_6y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_5y_up_to_6y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _6y_to_7y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_6y_up_to_7y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _7y_to_8y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_7y_up_to_8y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _8y_to_10y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_8y_up_to_10y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _10y_to_12y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_10y_up_to_12y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, _12y_to_15y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_12y_up_to_15y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, from_15y: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_at_least_15y_old"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, true, + false, )?, }, amount_range: ByAmountRange { _0sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_with_0sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _1sat_to_10sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1sat_under_10sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _10sats_to_100sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10sats_under_100sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _100sats_to_1k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100sats_under_1k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _1k_sats_to_10k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1k_sats_under_10k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _10k_sats_to_100k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10k_sats_under_100k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _100k_sats_to_1m_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100k_sats_under_1m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _1m_sats_to_10m_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1m_sats_under_10m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _10m_sats_to_1btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10m_sats_under_1btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _1btc_to_10btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1btc_under_10btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _10btc_to_100btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10btc_under_100btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _100btc_to_1k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100btc_under_1k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _1k_btc_to_10k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1k_btc_under_10k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _10k_btc_to_100k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10k_btc_under_100k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, _100k_btc_or_more: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + Some(states_path), true, false, + false, )?, }, lt_amount: ByLowerThanAmount { _10sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_10sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_100sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_1k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_10k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_100k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1m_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_1m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10m_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_10m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_1btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_10btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_100btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_1k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_10k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_under_100k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, }, ge_amount: ByGreatEqualAmount { _1sat: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1sat"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100k_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100k_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1m_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10m_sats: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10m_sats"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _100btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_100btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _1k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_1k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, _10k_btc: utxo_cohort::Vecs::forced_import( - file, + db, Some("utxos_above_10k_btc"), _computation, format, version + VERSION + Version::ZERO, indexes, price, - states_path, + None, true, false, + false, )?, }, } @@ -1480,9 +1596,15 @@ impl Vecs { let was = filter.contains(prev_days_old); if is && !was { - state.increment(&block_state.supply, block_state.price); + state + .as_mut() + .unwrap() + .increment(&block_state.supply, block_state.price); } else if was && !is { - state.decrement(&block_state.supply, block_state.price); + state + .as_mut() + .unwrap() + .decrement(&block_state.supply, block_state.price); } }); @@ -1536,7 +1658,7 @@ impl Vecs { _ => unreachable!(), }) .for_each(|(_, vecs)| { - vecs.state.send( + vecs.state.as_mut().unwrap().send( &sent.spendable_supply, current_price, prev_price, @@ -1548,14 +1670,21 @@ impl Vecs { sent.by_type.spendable.as_typed_vec().into_iter().for_each( |(output_type, supply_state)| { - self.0._type.get_mut(output_type).1.state.send( - supply_state, - current_price, - prev_price, - blocks_old, - days_old_float, - older_than_hour, - ) + self.0 + ._type + .get_mut(output_type) + .1 + .state + .as_mut() + .unwrap() + .send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ) }, ); @@ -1563,14 +1692,21 @@ impl Vecs { .as_typed_vec() .into_iter() .for_each(|(group, supply_state)| { - self.0.amount_range.get_mut(group).1.state.send( - supply_state, - current_price, - prev_price, - blocks_old, - days_old_float, - older_than_hour, - ); + self.0 + .amount_range + .get_mut(group) + .1 + .state + .as_mut() + .unwrap() + .send( + supply_state, + current_price, + prev_price, + blocks_old, + days_old_float, + older_than_hour, + ); }); }); } @@ -1584,7 +1720,7 @@ impl Vecs { ] .into_iter() .for_each(|v| { - v.state.receive(&supply_state, price); + v.state.as_mut().unwrap().receive(&supply_state, price); }); self._type @@ -1595,7 +1731,10 @@ impl Vecs { GroupFilter::Type(output_type) => *output_type, _ => unreachable!(), }; - vecs.state.receive(received.by_type.get(output_type), price) + vecs.state + .as_mut() + .unwrap() + .receive(received.by_type.get(output_type), price) }); received @@ -1607,6 +1746,8 @@ impl Vecs { .get_mut(group) .1 .state + .as_mut() + .unwrap() .receive(supply_state, price); }); } diff --git a/crates/brk_computer/src/traits.rs b/crates/brk_computer/src/traits.rs index 5bff19d31..3c7da2162 100644 --- a/crates/brk_computer/src/traits.rs +++ b/crates/brk_computer/src/traits.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32}; -use brk_vecs::{ +use vecdb::{ AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex, VecIterator, Version, }; @@ -32,7 +32,7 @@ impl ComputeDCAStackViaLen for EagerVec { len: usize, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ZERO + self.inner_version() + closes.version(), )?; @@ -83,7 +83,7 @@ impl ComputeDCAStackViaLen for EagerVec { from: DateIndex, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ZERO + self.inner_version() + closes.version(), )?; @@ -143,7 +143,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec { len: usize, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ONE + self.inner_version() + stacks.version(), )?; @@ -176,7 +176,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec { from: DateIndex, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ZERO + self.inner_version() + stacks.version(), )?; @@ -218,7 +218,7 @@ where sats: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ZERO + self.inner_version() + sats.version(), )?; @@ -254,7 +254,7 @@ where price: &impl AnyIterableVec>, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ZERO + self.inner_version() + bitcoin.version(), )?; @@ -292,7 +292,7 @@ where ath: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> { - self.validate_computed_version_or_reset_file( + self.validate_computed_version_or_reset( Version::ZERO + self.inner_version() + ath.version() + close.version(), )?; diff --git a/crates/brk_computer/src/transactions.rs b/crates/brk_computer/src/transactions.rs index c583dfd23..848beb567 100644 --- a/crates/brk_computer/src/transactions.rs +++ b/crates/brk_computer/src/transactions.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; @@ -6,9 +6,9 @@ use brk_structs::{ CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredBool, StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight, }; -use brk_vecs::{ +use vecdb::{ AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec, - ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, PAGE_SIZE, + ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Database, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator, }; @@ -23,7 +23,7 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - file: Arc, + db: Database, // pub txindex_to_is_v1: LazyVec, // pub txindex_to_is_v2: LazyVec, @@ -90,14 +90,14 @@ impl Vecs { format: Format, price: Option<&price::Vecs>, ) -> Result { - let file = Arc::new(File::open(&parent.join("transactions"))?); - file.set_min_len(PAGE_SIZE * 10_000_000)?; + let db = Database::open(&parent.join("transactions"))?; + db.set_min_len(PAGE_SIZE * 10_000_000)?; let compute_dollars = price.is_some(); let inputindex_to_value = ComputedVec::forced_import_or_init_from_2( computation, - &file, + &db, "value", version + VERSION + Version::ZERO, format, @@ -124,7 +124,7 @@ impl Vecs { let txindex_to_weight = ComputedVec::forced_import_or_init_from_2( computation, - &file, + &db, "weight", version + VERSION + Version::ZERO, format, @@ -152,7 +152,7 @@ impl Vecs { let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1( computation, - &file, + &db, "vsize", version + VERSION + Version::ZERO, format, @@ -169,7 +169,7 @@ impl Vecs { let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2( computation, - &file, + &db, "is_coinbase", version + VERSION + Version::ZERO, format, @@ -192,7 +192,7 @@ impl Vecs { let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3( computation, - &file, + &db, "input_value", version + VERSION + Version::ZERO, format, @@ -228,7 +228,7 @@ impl Vecs { // let indexes_to_input_value: ComputedVecsFromTxindex = // ComputedVecsFromTxindex::forced_import( - // file, + // db, // "input_value", // true, // version + VERSION + Version::ZERO, @@ -242,7 +242,7 @@ impl Vecs { let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3( computation, - &file, + &db, "output_value", version + VERSION + Version::ZERO, format, @@ -278,7 +278,7 @@ impl Vecs { // let indexes_to_output_value: ComputedVecsFromTxindex = // ComputedVecsFromTxindex::forced_import( - // file, + // db, // "output_value", // true, // version + VERSION + Version::ZERO, @@ -292,7 +292,7 @@ impl Vecs { let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2( Computation::Eager, - &file, + &db, "fee", version + VERSION + Version::ZERO, format, @@ -315,7 +315,7 @@ impl Vecs { let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2( Computation::Eager, - &file, + &db, "feerate", version + VERSION + Version::ZERO, format, @@ -335,7 +335,7 @@ impl Vecs { Ok(Self { indexes_to_tx_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "tx_count", Source::Compute, version + VERSION + Version::ZERO, @@ -350,7 +350,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_input_count: ComputedVecsFromTxindex::forced_import( - &file, + &db, "input_count", Source::None, version + VERSION + Version::ZERO, @@ -365,7 +365,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_output_count: ComputedVecsFromTxindex::forced_import( - &file, + &db, "output_count", Source::None, version + VERSION + Version::ZERO, @@ -380,7 +380,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( - &file, + &db, "tx_v1", Source::Compute, version + VERSION + Version::ZERO, @@ -390,7 +390,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v2: ComputedVecsFromHeight::forced_import( - &file, + &db, "tx_v2", Source::Compute, version + VERSION + Version::ZERO, @@ -400,7 +400,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_tx_v3: ComputedVecsFromHeight::forced_import( - &file, + &db, "tx_v3", Source::Compute, version + VERSION + Version::ZERO, @@ -410,7 +410,7 @@ impl Vecs { VecBuilderOptions::default().add_sum().add_cumulative(), )?, indexes_to_fee: ComputedValueVecsFromTxindex::forced_import( - &file, + &db, "fee", indexes, Source::Vec(txindex_to_fee.boxed_clone()), @@ -426,7 +426,7 @@ impl Vecs { .add_average(), )?, indexes_to_feerate: ComputedVecsFromTxindex::forced_import( - &file, + &db, "feerate", Source::None, version + VERSION + Version::ZERO, @@ -439,7 +439,7 @@ impl Vecs { .add_average(), )?, indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import( - &file, + &db, "tx_vsize", Source::None, version + VERSION + Version::ZERO, @@ -452,7 +452,7 @@ impl Vecs { .add_average(), )?, indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import( - &file, + &db, "tx_weight", Source::None, version + VERSION + Version::ZERO, @@ -465,7 +465,7 @@ impl Vecs { .add_average(), )?, indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "subsidy", Source::Compute, version + VERSION + Version::ZERO, @@ -481,7 +481,7 @@ impl Vecs { indexes, )?, indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "coinbase", Source::Compute, version + VERSION + Version::ZERO, @@ -497,7 +497,7 @@ impl Vecs { indexes, )?, indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import( - &file, + &db, "unclaimed_rewards", Source::Compute, version + VERSION + Version::ZERO, @@ -508,7 +508,7 @@ impl Vecs { indexes, )?, indexes_to_p2a_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2a_count", Source::Compute, version + VERSION + Version::ZERO, @@ -523,7 +523,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2ms_count", Source::Compute, version + VERSION + Version::ZERO, @@ -538,7 +538,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pk33_count", Source::Compute, version + VERSION + Version::ZERO, @@ -553,7 +553,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pk65_count", Source::Compute, version + VERSION + Version::ZERO, @@ -568,7 +568,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2pkh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -583,7 +583,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2sh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -598,7 +598,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2tr_count", Source::Compute, version + VERSION + Version::ZERO, @@ -613,7 +613,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2wpkh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -628,7 +628,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "p2wsh_count", Source::Compute, version + VERSION + Version::ZERO, @@ -643,7 +643,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "opreturn_count", Source::Compute, version + VERSION + Version::ZERO, @@ -658,7 +658,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "unknownoutput_count", Source::Compute, version + VERSION + Version::ZERO, @@ -673,7 +673,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "emptyoutput_count", Source::Compute, version + VERSION + Version::ZERO, @@ -688,7 +688,7 @@ impl Vecs { .add_cumulative(), )?, indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import( - &file, + &db, "exact_utxo_count", Source::Compute, version + VERSION + Version::ZERO, @@ -708,7 +708,7 @@ impl Vecs { txindex_to_vsize, txindex_to_weight, - file, + db, }) } @@ -721,7 +721,7 @@ impl Vecs { exit: &Exit, ) -> Result<()> { self.compute_(indexer, indexes, starting_indexes, price, exit)?; - self.file.flush_then_punch()?; + self.db.flush_then_punch()?; Ok(()) } diff --git a/crates/brk_error/Cargo.toml b/crates/brk_error/Cargo.toml index 5a0d2e491..5e0b4d690 100644 --- a/crates/brk_error/Cargo.toml +++ b/crates/brk_error/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true build = "build.rs" [dependencies] -brk_vecs = { workspace = true } +vecdb = { workspace = true } bitcoincore-rpc = { workspace = true } fjall = { workspace = true } jiff = { workspace = true } diff --git a/crates/brk_error/src/lib.rs b/crates/brk_error/src/lib.rs index 7eb13ae19..89f48632e 100644 --- a/crates/brk_error/src/lib.rs +++ b/crates/brk_error/src/lib.rs @@ -11,11 +11,13 @@ pub enum Error { BitcoinRPC(bitcoincore_rpc::Error), Jiff(jiff::Error), Fjall(fjall::Error), + VecDB(vecdb::Error), + SeqDB(vecdb::SeqDBError), Minreq(minreq::Error), SystemTimeError(time::SystemTimeError), SerdeJson(serde_json::Error), ZeroCopyError, - Vecs(brk_vecs::Error), + Vecs(vecdb::Error), WrongLength, WrongAddressType, @@ -43,6 +45,18 @@ impl From for Error { } } +impl From for Error { + fn from(value: vecdb::Error) -> Self { + Self::VecDB(value) + } +} + +impl From for Error { + fn from(value: vecdb::SeqDBError) -> Self { + Self::SeqDB(value) + } +} + impl From for Error { fn from(value: bitcoincore_rpc::Error) -> Self { Self::BitcoinRPC(value) @@ -55,12 +69,6 @@ impl From for Error { } } -impl From for Error { - fn from(value: brk_vecs::Error) -> Self { - Self::Vecs(value) - } -} - impl From for Error { fn from(value: jiff::Error) -> Self { Self::Jiff(value) @@ -91,6 +99,8 @@ impl fmt::Display for Error { Error::IO(error) => Display::fmt(&error, f), Error::Minreq(error) => Display::fmt(&error, f), Error::SerdeJson(error) => Display::fmt(&error, f), + Error::VecDB(error) => Display::fmt(&error, f), + Error::SeqDB(error) => Display::fmt(&error, f), Error::Vecs(error) => Display::fmt(&error, f), Error::BitcoinRPC(error) => Display::fmt(&error, f), Error::SystemTimeError(error) => Display::fmt(&error, f), diff --git a/crates/brk_fetcher/examples/main.rs b/crates/brk_fetcher/examples/main.rs index f1eb5ef26..cddddc748 100644 --- a/crates/brk_fetcher/examples/main.rs +++ b/crates/brk_fetcher/examples/main.rs @@ -9,7 +9,7 @@ fn main() -> Result<()> { dbg!(brk.get_from_height(Height::new(900_000))?); dbg!(brk.get_from_date(Date::new(2025, 6, 7))?); - let mut fetcher = Fetcher::import(None)?; + let mut fetcher = Fetcher::import(true, None)?; Binance::fetch_1d().map(|b| { dbg!(b.last_key_value()); diff --git a/crates/brk_fetcher/src/binance.rs b/crates/brk_fetcher/src/binance.rs index e7a0df0ed..44bc813f5 100644 --- a/crates/brk_fetcher/src/binance.rs +++ b/crates/brk_fetcher/src/binance.rs @@ -11,7 +11,7 @@ use brk_structs::{Cents, OHLCCents, Timestamp}; use log::info; use serde_json::Value; -use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, retry}; +use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, default_retry}; #[derive(Clone)] pub struct Binance { @@ -68,17 +68,13 @@ impl Binance { pub fn fetch_1mn() -> Result> { info!("Fetching 1mn prices from Binance..."); - retry( - |_| { - Self::json_to_timestamp_to_ohlc( - &minreq::get(Self::url("interval=1m&limit=1000")) - .send()? - .json()?, - ) - }, - 30, - 10, - ) + default_retry(|_| { + Self::json_to_timestamp_to_ohlc( + &minreq::get(Self::url("interval=1m&limit=1000")) + .send()? + .json()?, + ) + }) } pub fn get_from_1d(&mut self, date: &Date) -> Result { @@ -97,11 +93,9 @@ impl Binance { pub fn fetch_1d() -> Result> { info!("Fetching daily prices from Binance..."); - retry( - |_| Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?), - 30, - 10, - ) + default_retry(|_| { + Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?) + }) } fn read_har(&self) -> Result> { diff --git a/crates/brk_fetcher/src/brk.rs b/crates/brk_fetcher/src/brk.rs index 30f573730..0e0ff69c3 100644 --- a/crates/brk_fetcher/src/brk.rs +++ b/crates/brk_fetcher/src/brk.rs @@ -5,7 +5,7 @@ use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents}; use log::info; use serde_json::Value; -use crate::{Close, Dollars, High, Low, Open, retry}; +use crate::{Close, Dollars, High, Low, Open, default_retry}; #[derive(Default, Clone)] #[allow(clippy::upper_case_acronyms)] @@ -15,7 +15,6 @@ pub struct BRK { } const API_URL: &str = "https://bitcoinresearchkit.org/api/vecs"; -const RETRIES: usize = 10; const CHUNK_SIZE: usize = 10_000; impl BRK { @@ -45,25 +44,21 @@ impl BRK { fn fetch_height_prices(height: Height) -> Result> { info!("Fetching BRK height {height} prices..."); - retry( - |_| { - let url = format!( - "{API_URL}/height-to-ohlc?from={}&to={}", - height, - height + CHUNK_SIZE - ); + default_retry(|_| { + let url = format!( + "{API_URL}/height-to-ohlc?from={}&to={}", + height, + height + CHUNK_SIZE + ); - let body: Value = minreq::get(url).send()?.json()?; + let body: Value = minreq::get(url).send()?.json()?; - body.as_array() - .ok_or(Error::Str("Expect to be an array"))? - .iter() - .map(Self::value_to_ohlc) - .collect::, _>>() - }, - 30, - RETRIES, - ) + body.as_array() + .ok_or(Error::Str("Expect to be an array"))? + .iter() + .map(Self::value_to_ohlc) + .collect::, _>>() + }) } pub fn get_from_date(&mut self, date: Date) -> Result { @@ -94,25 +89,21 @@ impl BRK { fn fetch_date_prices(dateindex: DateIndex) -> Result> { info!("Fetching BRK dateindex {dateindex} prices..."); - retry( - |_| { - let url = format!( - "{API_URL}/dateindex-to-ohlc?from={}&to={}", - dateindex, - dateindex + CHUNK_SIZE - ); + default_retry(|_| { + let url = format!( + "{API_URL}/dateindex-to-ohlc?from={}&to={}", + dateindex, + dateindex + CHUNK_SIZE + ); - let body: Value = minreq::get(url).send()?.json()?; + let body: Value = minreq::get(url).send()?.json()?; - body.as_array() - .ok_or(Error::Str("Expect to be an array"))? - .iter() - .map(Self::value_to_ohlc) - .collect::, _>>() - }, - 30, - RETRIES, - ) + body.as_array() + .ok_or(Error::Str("Expect to be an array"))? + .iter() + .map(Self::value_to_ohlc) + .collect::, _>>() + }) } fn value_to_ohlc(value: &Value) -> Result { diff --git a/crates/brk_fetcher/src/kraken.rs b/crates/brk_fetcher/src/kraken.rs index 99461e970..4a79afd96 100644 --- a/crates/brk_fetcher/src/kraken.rs +++ b/crates/brk_fetcher/src/kraken.rs @@ -5,7 +5,7 @@ use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Times use log::info; use serde_json::Value; -use crate::{Fetcher, retry}; +use crate::{Fetcher, default_retry}; #[derive(Default, Clone)] pub struct Kraken { @@ -35,11 +35,9 @@ impl Kraken { pub fn fetch_1mn() -> Result> { info!("Fetching 1mn prices from Kraken..."); - retry( - |_| Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url(1)).send()?.json()?), - 30, - 10, - ) + default_retry(|_| { + Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url(1)).send()?.json()?) + }) } pub fn get_from_1d(&mut self, date: &Date) -> Result { @@ -57,11 +55,7 @@ impl Kraken { pub fn fetch_1d() -> Result> { info!("Fetching daily prices from Kraken..."); - retry( - |_| Self::json_to_date_to_ohlc(&minreq::get(Self::url(1440)).send()?.json()?), - 30, - 10, - ) + default_retry(|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url(1440)).send()?.json()?)) } fn json_to_timestamp_to_ohlc(json: &Value) -> Result> { diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index c818008b2..4fddbed87 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -23,16 +23,16 @@ const TRIES: usize = 12 * 60; #[derive(Clone)] pub struct Fetcher { - binance: Binance, - kraken: Kraken, + binance: Option, + kraken: Option, brk: BRK, } impl Fetcher { - pub fn import(hars_path: Option<&Path>) -> Result { + pub fn import(exchanges: bool, hars_path: Option<&Path>) -> Result { Ok(Self { - binance: Binance::init(hars_path), - kraken: Kraken::default(), + binance: exchanges.then(|| Binance::init(hars_path)), + kraken: exchanges.then(Kraken::default), brk: BRK::default(), }) } @@ -43,10 +43,17 @@ impl Fetcher { fn get_date_(&mut self, date: Date, tries: usize) -> Result { self.kraken - .get_from_1d(&date) + .as_mut() + .map_or(Err(Error::Str("Kraken off")), |kraken| { + kraken.get_from_1d(&date) + }) .or_else(|_| { // eprintln!("{e}"); - self.binance.get_from_1d(&date) + self.binance + .as_mut() + .map_or(Err(Error::Str("Binance off")), |binance| { + binance.get_from_1d(&date) + }) }) .or_else(|_| { // eprintln!("{e}"); @@ -93,11 +100,17 @@ impl Fetcher { let ohlc = self .kraken - .get_from_1mn(timestamp, previous_timestamp) + .as_mut() + .map_or(Err(Error::Str("Kraken off")), |kraken| { + kraken.get_from_1mn(timestamp, previous_timestamp) + }) .unwrap_or_else(|_report| { // eprintln!("{_report}"); self.binance - .get_from_1mn(timestamp, previous_timestamp) + .as_mut() + .map_or(Err(Error::Str("Binance off")), |binance| { + binance.get_from_1mn(timestamp, previous_timestamp) + }) .unwrap_or_else(|_report| { // // eprintln!("{_report}"); self.brk.get_from_height(height).unwrap_or_else(|_report| { @@ -187,8 +200,12 @@ How to fix this: } pub fn clear(&mut self) { - self.kraken.clear(); - self.binance.clear(); + if let Some(kraken) = self.kraken.as_mut() { + kraken.clear() + } + if let Some(binance) = self.binance.as_mut() { + binance.clear() + } self.brk.clear(); } } diff --git a/crates/brk_fetcher/src/retry.rs b/crates/brk_fetcher/src/retry.rs index df8ccaabe..d6eb4cc78 100644 --- a/crates/brk_fetcher/src/retry.rs +++ b/crates/brk_fetcher/src/retry.rs @@ -3,7 +3,14 @@ use std::{fmt::Debug, thread::sleep, time::Duration}; use brk_error::Result; use log::info; -pub fn retry(function: impl Fn(usize) -> Result, sleep_in_s: u64, retries: usize) -> Result +pub fn default_retry(function: impl Fn(usize) -> Result) -> Result +where + T: Debug, +{ + retry(function, 5, 6) +} + +fn retry(function: impl Fn(usize) -> Result, sleep_in_s: u64, retries: usize) -> Result where T: Debug, { diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index b6bc09d0a..ef0c020e4 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -16,7 +16,7 @@ brk_error = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_store = { workspace = true } -brk_vecs = { workspace = true } +vecdb = { workspace = true } fjall = { workspace = true } log = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 0626da6e3..4ce571403 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -8,7 +8,7 @@ use std::{ use brk_error::Result; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_vecs::Exit; +use vecdb::Exit; fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index a55334e18..9bc8c96d3 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -7,7 +7,7 @@ use brk_structs::{ P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, TxIndex, TypeIndex, UnknownOutputIndex, }; -use brk_vecs::{ +use vecdb::{ AnyIterableVec, AnyStoredIterableVec, GenericStoredVec, StoredIndex, StoredRaw, VecIterator, }; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 7889ecd4b..c51774db4 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -3,7 +3,7 @@ #![doc = include_str!("../examples/indexer.rs")] #![doc = "```"] -use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant}; +use std::{collections::BTreeMap, path::Path, str::FromStr, thread, time::Instant}; use bitcoin::{Transaction, TxIn, TxOut}; use brk_error::{Error, Result}; @@ -15,9 +15,9 @@ use brk_structs::{ OutputType, Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, Vin, Vout, }; -use brk_vecs::{AnyVec, Exit, File, GenericStoredVec, PAGE_SIZE, Reader, VecIterator}; use log::{error, info}; use rayon::prelude::*; +use vecdb::{AnyVec, Database, Exit, GenericStoredVec, PAGE_SIZE, Reader, VecIterator}; mod indexes; mod stores; mod vecs; @@ -27,23 +27,23 @@ pub use stores::*; pub use vecs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1_000; -const COLLISIONS_CHECKED_UP_TO: Height = Height::new(908_700); +const COLLISIONS_CHECKED_UP_TO: Height = Height::new(909_150); const VERSION: Version = Version::ONE; #[derive(Clone)] pub struct Indexer { - pub file: Arc, + pub db: Database, pub vecs: Vecs, pub stores: Stores, } impl Indexer { pub fn forced_import(outputs_dir: &Path) -> Result { - let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?); + let db = Database::open(&outputs_dir.join("indexed/vecs"))?; - let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?; + let vecs = Vecs::forced_import(&db, VERSION + Version::ZERO)?; - file.set_min_len(PAGE_SIZE * 50_000_000)?; + db.set_min_len(PAGE_SIZE * 50_000_000)?; Ok(Self { vecs, @@ -51,7 +51,7 @@ impl Indexer { &outputs_dir.join("indexed/stores"), VERSION + Version::ZERO, )?, - file, + db, }) } @@ -62,10 +62,10 @@ impl Indexer { exit: &Exit, check_collisions: bool, ) -> Result { - let file = self.file.clone(); + let db = self.db.clone(); - // dbg!(self.file.regions().id_to_index()); - // dbg!(self.file.layout()); + // dbg!(self.db.regions().id_to_index()); + // dbg!(self.db.layout()); let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)) .unwrap_or_else(|_report| Indexes::default()); @@ -111,8 +111,8 @@ impl Indexer { vecs.flush(height)?; info!("Flushed vecs in {}s", i.elapsed().as_secs()); let i = Instant::now(); - file.flush()?; - info!("Flushed file in {}s", i.elapsed().as_secs()); + db.flush()?; + info!("Flushed db in {}s", i.elapsed().as_secs()); Ok(()) }; @@ -799,8 +799,8 @@ impl Indexer { } let i = Instant::now(); - file.punch_holes()?; - info!("Punched holes in file in {}s", i.elapsed().as_secs()); + db.punch_holes()?; + info!("Punched holes in db in {}s", i.elapsed().as_secs()); Ok(starting_indexes) } diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index 550c2c13a..63af451cc 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -6,9 +6,9 @@ use brk_structs::{ AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex, OutputType, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, }; -use brk_vecs::VecIterator; use fjall::{PersistMode, TransactionalKeyspace}; use rayon::prelude::*; +use vecdb::VecIterator; use crate::Indexes; diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index edd571b73..636d4887a 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use brk_error::Result; use brk_structs::{ AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex, @@ -9,10 +7,10 @@ use brk_structs::{ RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion, Txid, TypeIndex, UnknownOutputIndex, Version, Weight, }; -use brk_vecs::{ - AnyCollectableVec, AnyStoredVec, CompressedVec, File, GenericStoredVec, RawVec, Stamp, -}; use rayon::prelude::*; +use vecdb::{ + AnyCollectableVec, AnyStoredVec, CompressedVec, Database, GenericStoredVec, RawVec, Stamp, +}; use crate::Indexes; @@ -69,225 +67,221 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import(file: &Arc, version: Version) -> Result { + pub fn forced_import(db: &Database, version: Version) -> Result { Ok(Self { emptyoutputindex_to_txindex: CompressedVec::forced_import( - file, + db, "txindex", version + VERSION + Version::ZERO, )?, height_to_blockhash: RawVec::forced_import( - file, + db, "blockhash", version + VERSION + Version::ZERO, )?, height_to_difficulty: CompressedVec::forced_import( - file, + db, "difficulty", version + VERSION + Version::ZERO, )?, height_to_first_emptyoutputindex: CompressedVec::forced_import( - file, + db, "first_emptyoutputindex", version + VERSION + Version::ZERO, )?, height_to_first_inputindex: CompressedVec::forced_import( - file, + db, "first_inputindex", version + VERSION + Version::ZERO, )?, height_to_first_opreturnindex: CompressedVec::forced_import( - file, + db, "first_opreturnindex", version + VERSION + Version::ZERO, )?, height_to_first_outputindex: CompressedVec::forced_import( - file, + db, "first_outputindex", version + VERSION + Version::ZERO, )?, height_to_first_p2aaddressindex: CompressedVec::forced_import( - file, + db, "first_p2aaddressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2msoutputindex: CompressedVec::forced_import( - file, + db, "first_p2msoutputindex", version + VERSION + Version::ZERO, )?, height_to_first_p2pk33addressindex: CompressedVec::forced_import( - file, + db, "first_p2pk33addressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2pk65addressindex: CompressedVec::forced_import( - file, + db, "first_p2pk65addressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2pkhaddressindex: CompressedVec::forced_import( - file, + db, "first_p2pkhaddressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2shaddressindex: CompressedVec::forced_import( - file, + db, "first_p2shaddressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2traddressindex: CompressedVec::forced_import( - file, + db, "first_p2traddressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2wpkhaddressindex: CompressedVec::forced_import( - file, + db, "first_p2wpkhaddressindex", version + VERSION + Version::ZERO, )?, height_to_first_p2wshaddressindex: CompressedVec::forced_import( - file, + db, "first_p2wshaddressindex", version + VERSION + Version::ZERO, )?, height_to_first_txindex: CompressedVec::forced_import( - file, + db, "first_txindex", version + VERSION + Version::ZERO, )?, height_to_first_unknownoutputindex: CompressedVec::forced_import( - file, + db, "first_unknownoutputindex", version + VERSION + Version::ZERO, )?, height_to_timestamp: CompressedVec::forced_import( - file, + db, "timestamp", version + VERSION + Version::ZERO, )?, height_to_total_size: CompressedVec::forced_import( - file, + db, "total_size", version + VERSION + Version::ZERO, )?, height_to_weight: CompressedVec::forced_import( - file, + db, "weight", version + VERSION + Version::ZERO, )?, inputindex_to_outputindex: RawVec::forced_import( - file, + db, "outputindex", version + VERSION + Version::ZERO, )?, opreturnindex_to_txindex: CompressedVec::forced_import( - file, + db, "txindex", version + VERSION + Version::ZERO, )?, outputindex_to_outputtype: RawVec::forced_import( - file, + db, "outputtype", version + VERSION + Version::ZERO, )?, outputindex_to_typeindex: RawVec::forced_import( - file, + db, "typeindex", version + VERSION + Version::ZERO, )?, outputindex_to_value: RawVec::forced_import( - file, + db, "value", version + VERSION + Version::ZERO, )?, p2aaddressindex_to_p2abytes: RawVec::forced_import( - file, + db, "p2abytes", version + VERSION + Version::ZERO, )?, p2msoutputindex_to_txindex: CompressedVec::forced_import( - file, + db, "txindex", version + VERSION + Version::ZERO, )?, p2pk33addressindex_to_p2pk33bytes: RawVec::forced_import( - file, + db, "p2pk33bytes", version + VERSION + Version::ZERO, )?, p2pk65addressindex_to_p2pk65bytes: RawVec::forced_import( - file, + db, "p2pk65bytes", version + VERSION + Version::ZERO, )?, p2pkhaddressindex_to_p2pkhbytes: RawVec::forced_import( - file, + db, "p2pkhbytes", version + VERSION + Version::ZERO, )?, p2shaddressindex_to_p2shbytes: RawVec::forced_import( - file, + db, "p2shbytes", version + VERSION + Version::ZERO, )?, p2traddressindex_to_p2trbytes: RawVec::forced_import( - file, + db, "p2trbytes", version + VERSION + Version::ZERO, )?, p2wpkhaddressindex_to_p2wpkhbytes: RawVec::forced_import( - file, + db, "p2wpkhbytes", version + VERSION + Version::ZERO, )?, p2wshaddressindex_to_p2wshbytes: RawVec::forced_import( - file, + db, "p2wshbytes", version + VERSION + Version::ZERO, )?, txindex_to_base_size: CompressedVec::forced_import( - file, + db, "base_size", version + VERSION + Version::ZERO, )?, txindex_to_first_inputindex: CompressedVec::forced_import( - file, + db, "first_inputindex", version + VERSION + Version::ZERO, )?, txindex_to_first_outputindex: CompressedVec::forced_import( - file, + db, "first_outputindex", version + VERSION + Version::ZERO, )?, txindex_to_is_explicitly_rbf: CompressedVec::forced_import( - file, + db, "is_explicitly_rbf", version + VERSION + Version::ZERO, )?, txindex_to_rawlocktime: CompressedVec::forced_import( - file, + db, "rawlocktime", version + VERSION + Version::ZERO, )?, txindex_to_total_size: CompressedVec::forced_import( - file, + db, "total_size", version + VERSION + Version::ZERO, )?, - txindex_to_txid: RawVec::forced_import( - file, - "txid", - version + VERSION + Version::ZERO, - )?, + txindex_to_txid: RawVec::forced_import(db, "txid", version + VERSION + Version::ZERO)?, txindex_to_txversion: CompressedVec::forced_import( - file, + db, "txversion", version + VERSION + Version::ZERO, )?, unknownoutputindex_to_txindex: CompressedVec::forced_import( - file, + db, "txindex", version + VERSION + Version::ZERO, )?, diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index 22000258d..3ad8d4dc2 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -13,7 +13,7 @@ brk_computer = { workspace = true } brk_error = { workspace = true } brk_indexer = { workspace = true } brk_structs = { workspace = true } -brk_vecs = { workspace = true } +vecdb = { workspace = true } derive_deref = { workspace = true } schemars = "1.0.4" serde = { workspace = true } diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 9216520ff..bdd69e2bf 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -9,8 +9,8 @@ use brk_computer::Computer; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::Height; -use brk_vecs::{AnyCollectableVec, AnyStoredVec}; use tabled::settings::Style; +use vecdb::{AnyCollectableVec, AnyStoredVec}; mod deser; mod format; diff --git a/crates/brk_interface/src/vecs.rs b/crates/brk_interface/src/vecs.rs index 4054d2d88..d42810a39 100644 --- a/crates/brk_interface/src/vecs.rs +++ b/crates/brk_interface/src/vecs.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use brk_computer::Computer; use brk_indexer::Indexer; -use brk_vecs::AnyCollectableVec; +use vecdb::AnyCollectableVec; use derive_deref::{Deref, DerefMut}; use crate::pagination::{PaginatedIndexParam, PaginationParam}; diff --git a/crates/brk_mcp/Cargo.toml b/crates/brk_mcp/Cargo.toml index 96bae40e7..49d7e13cb 100644 --- a/crates/brk_mcp/Cargo.toml +++ b/crates/brk_mcp/Cargo.toml @@ -12,7 +12,7 @@ build = "build.rs" axum = { workspace = true } brk_interface = { workspace = true } log = { workspace = true } -brk_rmcp = { version = "0.4.1", features = [ +brk_rmcp = { version = "0.5.0", features = [ "transport-worker", "transport-streamable-http-server", ] } diff --git a/crates/brk_parser/Cargo.lock b/crates/brk_parser/Cargo.lock deleted file mode 100644 index c3335d7d9..000000000 --- a/crates/brk_parser/Cargo.lock +++ /dev/null @@ -1,475 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "base58ck" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f" -dependencies = [ - "bitcoin-internals", - "bitcoin_hashes", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bech32" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" - -[[package]] -name = "bitcoin" -version = "0.32.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026" -dependencies = [ - "base58ck", - "bech32", - "bitcoin-internals", - "bitcoin-io", - "bitcoin-units", - "bitcoin_hashes", - "hex-conservative", - "hex_lit", - "secp256k1", - "serde", -] - -[[package]] -name = "bitcoin-internals" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2" -dependencies = [ - "serde", -] - -[[package]] -name = "bitcoin-io" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56" - -[[package]] -name = "bitcoin-units" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" -dependencies = [ - "bitcoin-internals", - "serde", -] - -[[package]] -name = "bitcoin_hashes" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" -dependencies = [ - "bitcoin-io", - "hex-conservative", - "serde", -] - -[[package]] -name = "bitcoincore-rpc" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" -dependencies = [ - "bitcoincore-rpc-json", - "jsonrpc", - "log", - "serde", - "serde_json", -] - -[[package]] -name = "bitcoincore-rpc-json" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" -dependencies = [ - "bitcoin", - "serde", - "serde_json", -] - -[[package]] -name = "biter" -version = "0.2.2" -dependencies = [ - "bitcoin", - "bitcoincore-rpc", - "crossbeam", - "derived-deref", - "rayon", - "serde", - "serde_json", -] - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "cc" -version = "1.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - -[[package]] -name = "derived-deref" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "hex-conservative" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" -dependencies = [ - "arrayvec", -] - -[[package]] -name = "hex_lit" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" - -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "jsonrpc" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" -dependencies = [ - "base64", - "minreq", - "serde", - "serde_json", -] - -[[package]] -name = "libc" -version = "0.2.161" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "minreq" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0" -dependencies = [ - "log", - "serde", - "serde_json", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "secp256k1" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" -dependencies = [ - "bitcoin_hashes", - "rand", - "secp256k1-sys", - "serde", -] - -[[package]] -name = "secp256k1-sys" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" -dependencies = [ - "cc", -] - -[[package]] -name = "serde" -version = "1.0.217" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.217" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.135" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "syn" -version = "2.0.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 6e8bfa648..0102d80d3 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -19,7 +19,7 @@ brk_interface = { workspace = true } brk_logger = { workspace = true } brk_mcp = { workspace = true } brk_parser = { workspace = true } -brk_vecs = { workspace = true } +vecdb = { workspace = true } jiff = { workspace = true } log = { workspace = true } quick_cache = "0.6.16" diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index a902271da..87f0ed4cb 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -9,7 +9,7 @@ use brk_indexer::Indexer; use brk_interface::Interface; use brk_parser::Parser; use brk_server::Server; -use brk_vecs::Exit; +use vecdb::Exit; pub fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); @@ -32,7 +32,7 @@ pub fn main() -> Result<()> { let mut indexer = Indexer::forced_import(outputs_dir)?; - let fetcher = Some(Fetcher::import(None)?); + let fetcher = Some(Fetcher::import(true, None)?); let mut computer = Computer::forced_import(outputs_dir, &indexer, fetcher)?; diff --git a/crates/brk_server/src/api/interface.rs b/crates/brk_server/src/api/interface.rs index 1ea28589f..bcf1101d0 100644 --- a/crates/brk_server/src/api/interface.rs +++ b/crates/brk_server/src/api/interface.rs @@ -9,8 +9,8 @@ use axum::{ }; use brk_error::{Error, Result}; use brk_interface::{Format, Output, Params}; -use brk_vecs::Stamp; use quick_cache::sync::GuardResult; +use vecdb::Stamp; use crate::{HeaderMapExtended, ResponseExtended}; @@ -112,6 +112,7 @@ fn req_to_response_res( headers.insert_cors(); headers.insert_etag(&etag); + headers.insert_cache_control_must_revalidate(); match format { Some(format) => { diff --git a/crates/brk_server/src/extended/header_map.rs b/crates/brk_server/src/extended/header_map.rs index c2c614619..d899a209b 100644 --- a/crates/brk_server/src/extended/header_map.rs +++ b/crates/brk_server/src/extended/header_map.rs @@ -59,7 +59,7 @@ impl HeaderMapExtended for HeaderMap { fn insert_cache_control_must_revalidate(&mut self) { self.insert( header::CACHE_CONTROL, - "public, max-age=0, must-revalidate".parse().unwrap(), + "public, max-age=1, must-revalidate".parse().unwrap(), ); } diff --git a/crates/brk_structs/Cargo.toml b/crates/brk_structs/Cargo.toml index 7794b195e..4437e42e7 100644 --- a/crates/brk_structs/Cargo.toml +++ b/crates/brk_structs/Cargo.toml @@ -12,11 +12,11 @@ build = "build.rs" bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_error = {workspace = true} -brk_vecs = {workspace = true} +vecdb = {workspace = true} byteview = { workspace = true } derive_deref = { workspace = true } jiff = { workspace = true } -rapidhash = "3.0.0" +rapidhash = "3.1.0" serde = { workspace = true } serde_bytes = { workspace = true } zerocopy = { workspace = true } diff --git a/crates/brk_structs/src/lib.rs b/crates/brk_structs/src/lib.rs index 43e5af20c..5bae5e3cb 100644 --- a/crates/brk_structs/src/lib.rs +++ b/crates/brk_structs/src/lib.rs @@ -6,4 +6,4 @@ mod structs; pub use groups::*; pub use structs::*; -pub use brk_vecs::{CheckedSub, Exit, Printable, Version}; +pub use vecdb::{CheckedSub, Exit, Printable, Version}; diff --git a/crates/brk_structs/src/structs/bitcoin.rs b/crates/brk_structs/src/structs/bitcoin.rs index 348b7498c..8b02f50c1 100644 --- a/crates/brk_structs/src/structs/bitcoin.rs +++ b/crates/brk_structs/src/structs/bitcoin.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div, Mul}, }; -use brk_vecs::{CheckedSub, StoredCompressed}; use serde::Serialize; +use vecdb::{CheckedSub, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Sats, StoredF64}; diff --git a/crates/brk_structs/src/structs/cents.rs b/crates/brk_structs/src/structs/cents.rs index ec391e8b3..74892a6f3 100644 --- a/crates/brk_structs/src/structs/cents.rs +++ b/crates/brk_structs/src/structs/cents.rs @@ -1,7 +1,7 @@ use std::ops::{Add, Div, Mul}; -use brk_vecs::{CheckedSub, StoredCompressed}; use serde::Serialize; +use vecdb::{CheckedSub, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Dollars; diff --git a/crates/brk_structs/src/structs/date.rs b/crates/brk_structs/src/structs/date.rs index 1c335734c..2fb3fca47 100644 --- a/crates/brk_structs/src/structs/date.rs +++ b/crates/brk_structs/src/structs/date.rs @@ -1,6 +1,6 @@ -use brk_vecs::StoredCompressed; use jiff::{Span, civil::Date as Date_, tz::TimeZone}; use serde::{Serialize, Serializer}; +use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{DateIndex, Timestamp}; diff --git a/crates/brk_structs/src/structs/dateindex.rs b/crates/brk_structs/src/structs/dateindex.rs index 8b1a66d52..cf236fcdc 100644 --- a/crates/brk_structs/src/structs/dateindex.rs +++ b/crates/brk_structs/src/structs/dateindex.rs @@ -4,7 +4,7 @@ use std::{ }; use brk_error::Error; -use brk_vecs::{CheckedSub, FromCoarserIndex, Printable, StoredCompressed}; +use vecdb::{CheckedSub, FromCoarserIndex, Printable, StoredCompressed}; use jiff::Span; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/decadeindex.rs b/crates/brk_structs/src/structs/decadeindex.rs index 59ec54c00..bc377ecef 100644 --- a/crates/brk_structs/src/structs/decadeindex.rs +++ b/crates/brk_structs/src/structs/decadeindex.rs @@ -3,7 +3,7 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/difficultyepoch.rs b/crates/brk_structs/src/structs/difficultyepoch.rs index 339f6bdb4..e10e06f5e 100644 --- a/crates/brk_structs/src/structs/difficultyepoch.rs +++ b/crates/brk_structs/src/structs/difficultyepoch.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Height; diff --git a/crates/brk_structs/src/structs/dollars.rs b/crates/brk_structs/src/structs/dollars.rs index 52c8b1f05..5e0719c90 100644 --- a/crates/brk_structs/src/structs/dollars.rs +++ b/crates/brk_structs/src/structs/dollars.rs @@ -4,9 +4,9 @@ use std::{ ops::{Add, AddAssign, Div, Mul}, }; -use brk_vecs::{CheckedSub, StoredCompressed}; use derive_deref::Deref; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64}; diff --git a/crates/brk_structs/src/structs/emptyaddressindex.rs b/crates/brk_structs/src/structs/emptyaddressindex.rs index b0d38b6b9..41fcd7fe9 100644 --- a/crates/brk_structs/src/structs/emptyaddressindex.rs +++ b/crates/brk_structs/src/structs/emptyaddressindex.rs @@ -1,6 +1,6 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable}; +use vecdb::{CheckedSub, Printable}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/emptyoutputindex.rs b/crates/brk_structs/src/structs/emptyoutputindex.rs index 2ae1e3c2e..f1c154fac 100644 --- a/crates/brk_structs/src/structs/emptyoutputindex.rs +++ b/crates/brk_structs/src/structs/emptyoutputindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/feerate.rs b/crates/brk_structs/src/structs/feerate.rs index 9ffc48c29..39e084fc3 100644 --- a/crates/brk_structs/src/structs/feerate.rs +++ b/crates/brk_structs/src/structs/feerate.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::StoredCompressed; use serde::Serialize; +use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Sats, StoredU64}; diff --git a/crates/brk_structs/src/structs/halvingepoch.rs b/crates/brk_structs/src/structs/halvingepoch.rs index 529c18f58..39ae6aaae 100644 --- a/crates/brk_structs/src/structs/halvingepoch.rs +++ b/crates/brk_structs/src/structs/halvingepoch.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Height; diff --git a/crates/brk_structs/src/structs/height.rs b/crates/brk_structs/src/structs/height.rs index 034e6dc1d..33aa58028 100644 --- a/crates/brk_structs/src/structs/height.rs +++ b/crates/brk_structs/src/structs/height.rs @@ -4,10 +4,10 @@ use std::{ }; use bitcoincore_rpc::{Client, RpcApi}; -use brk_vecs::{CheckedSub, Printable, Stamp, StoredCompressed}; use byteview::ByteView; use derive_deref::Deref; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, Stamp, StoredCompressed}; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/inputindex.rs b/crates/brk_structs/src/structs/inputindex.rs index 5e2e78e19..c08acefa9 100644 --- a/crates/brk_structs/src/structs/inputindex.rs +++ b/crates/brk_structs/src/structs/inputindex.rs @@ -1,8 +1,8 @@ use std::ops::{Add, AddAssign}; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Vin; diff --git a/crates/brk_structs/src/structs/loadedaddressdata.rs b/crates/brk_structs/src/structs/loadedaddressdata.rs index 5ea97bce7..70803a275 100644 --- a/crates/brk_structs/src/structs/loadedaddressdata.rs +++ b/crates/brk_structs/src/structs/loadedaddressdata.rs @@ -1,6 +1,6 @@ use brk_error::{Error, Result}; -use brk_vecs::CheckedSub; use serde::Serialize; +use vecdb::CheckedSub; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Bitcoin, Dollars, EmptyAddressData, Sats}; diff --git a/crates/brk_structs/src/structs/loadedaddressindex.rs b/crates/brk_structs/src/structs/loadedaddressindex.rs index 5b3b398fe..15b028ffa 100644 --- a/crates/brk_structs/src/structs/loadedaddressindex.rs +++ b/crates/brk_structs/src/structs/loadedaddressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable}; use derive_deref::Deref; use serde::Serialize; +use vecdb::{CheckedSub, Printable}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/monthindex.rs b/crates/brk_structs/src/structs/monthindex.rs index 4ba14246b..7c071abf7 100644 --- a/crates/brk_structs/src/structs/monthindex.rs +++ b/crates/brk_structs/src/structs/monthindex.rs @@ -3,7 +3,7 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/ohlc.rs b/crates/brk_structs/src/structs/ohlc.rs index 5dce4b5d0..671b199e4 100644 --- a/crates/brk_structs/src/structs/ohlc.rs +++ b/crates/brk_structs/src/structs/ohlc.rs @@ -1,8 +1,8 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::StoredCompressed; use derive_deref::{Deref, DerefMut}; use serde::{Serialize, Serializer, ser::SerializeTuple}; +use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Cents, Dollars, Sats}; diff --git a/crates/brk_structs/src/structs/opreturnindex.rs b/crates/brk_structs/src/structs/opreturnindex.rs index 6f8c75b86..38a06e11b 100644 --- a/crates/brk_structs/src/structs/opreturnindex.rs +++ b/crates/brk_structs/src/structs/opreturnindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/outputindex.rs b/crates/brk_structs/src/structs/outputindex.rs index be9d686e4..a7f9eedaa 100644 --- a/crates/brk_structs/src/structs/outputindex.rs +++ b/crates/brk_structs/src/structs/outputindex.rs @@ -1,8 +1,8 @@ use std::ops::{Add, AddAssign}; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::copy_first_8bytes; diff --git a/crates/brk_structs/src/structs/p2aaddressindex.rs b/crates/brk_structs/src/structs/p2aaddressindex.rs index 2842af509..2bcab2a76 100644 --- a/crates/brk_structs/src/structs/p2aaddressindex.rs +++ b/crates/brk_structs/src/structs/p2aaddressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2msoutputindex.rs b/crates/brk_structs/src/structs/p2msoutputindex.rs index ad5643a14..c6e633618 100644 --- a/crates/brk_structs/src/structs/p2msoutputindex.rs +++ b/crates/brk_structs/src/structs/p2msoutputindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2pk33addressindex.rs b/crates/brk_structs/src/structs/p2pk33addressindex.rs index 7a146b395..c1f0ab2c0 100644 --- a/crates/brk_structs/src/structs/p2pk33addressindex.rs +++ b/crates/brk_structs/src/structs/p2pk33addressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2pk65addressindex.rs b/crates/brk_structs/src/structs/p2pk65addressindex.rs index fd1b1c4d5..40657278e 100644 --- a/crates/brk_structs/src/structs/p2pk65addressindex.rs +++ b/crates/brk_structs/src/structs/p2pk65addressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2pkhaddressindex.rs b/crates/brk_structs/src/structs/p2pkhaddressindex.rs index 2d9e9d0a1..ee6a029f9 100644 --- a/crates/brk_structs/src/structs/p2pkhaddressindex.rs +++ b/crates/brk_structs/src/structs/p2pkhaddressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2shaddressindex.rs b/crates/brk_structs/src/structs/p2shaddressindex.rs index d29776d1d..5ea0259d1 100644 --- a/crates/brk_structs/src/structs/p2shaddressindex.rs +++ b/crates/brk_structs/src/structs/p2shaddressindex.rs @@ -1,6 +1,6 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/p2traddressindex.rs b/crates/brk_structs/src/structs/p2traddressindex.rs index f96951483..fc9bf3419 100644 --- a/crates/brk_structs/src/structs/p2traddressindex.rs +++ b/crates/brk_structs/src/structs/p2traddressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2wpkhaddressindex.rs b/crates/brk_structs/src/structs/p2wpkhaddressindex.rs index c1a8b91e3..277041dfc 100644 --- a/crates/brk_structs/src/structs/p2wpkhaddressindex.rs +++ b/crates/brk_structs/src/structs/p2wpkhaddressindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/p2wshaddressindex.rs b/crates/brk_structs/src/structs/p2wshaddressindex.rs index e1586929d..c9e1bd3b0 100644 --- a/crates/brk_structs/src/structs/p2wshaddressindex.rs +++ b/crates/brk_structs/src/structs/p2wshaddressindex.rs @@ -1,6 +1,6 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/quarterindex.rs b/crates/brk_structs/src/structs/quarterindex.rs index f2c665f43..e4c438bb6 100644 --- a/crates/brk_structs/src/structs/quarterindex.rs +++ b/crates/brk_structs/src/structs/quarterindex.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::MonthIndex; diff --git a/crates/brk_structs/src/structs/rawlocktime.rs b/crates/brk_structs/src/structs/rawlocktime.rs index 2e0b13a37..9681de287 100644 --- a/crates/brk_structs/src/structs/rawlocktime.rs +++ b/crates/brk_structs/src/structs/rawlocktime.rs @@ -1,6 +1,6 @@ use bitcoin::absolute::LockTime; -use brk_vecs::StoredCompressed; use serde::Serialize; +use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive( diff --git a/crates/brk_structs/src/structs/sats.rs b/crates/brk_structs/src/structs/sats.rs index 8568ea668..63006bdd2 100644 --- a/crates/brk_structs/src/structs/sats.rs +++ b/crates/brk_structs/src/structs/sats.rs @@ -4,9 +4,9 @@ use std::{ }; use bitcoin::Amount; -use brk_vecs::{CheckedSub, StoredCompressed}; use derive_deref::Deref; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::StoredF64; diff --git a/crates/brk_structs/src/structs/semesterindex.rs b/crates/brk_structs/src/structs/semesterindex.rs index bba339b2c..4c17beeee 100644 --- a/crates/brk_structs/src/structs/semesterindex.rs +++ b/crates/brk_structs/src/structs/semesterindex.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::MonthIndex; diff --git a/crates/brk_structs/src/structs/stored_bool.rs b/crates/brk_structs/src/structs/stored_bool.rs index 7d0328a33..02df2ded5 100644 --- a/crates/brk_structs/src/structs/stored_bool.rs +++ b/crates/brk_structs/src/structs/stored_bool.rs @@ -1,4 +1,4 @@ -use brk_vecs::{Printable, StoredCompressed}; +use vecdb::{Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/stored_f32.rs b/crates/brk_structs/src/structs/stored_f32.rs index aba7820fd..7dcead77e 100644 --- a/crates/brk_structs/src/structs/stored_f32.rs +++ b/crates/brk_structs/src/structs/stored_f32.rs @@ -4,9 +4,9 @@ use std::{ ops::{Add, AddAssign, Div, Mul, Sub}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Dollars, StoredF64}; diff --git a/crates/brk_structs/src/structs/stored_f64.rs b/crates/brk_structs/src/structs/stored_f64.rs index 5ab306141..23bb4be09 100644 --- a/crates/brk_structs/src/structs/stored_f64.rs +++ b/crates/brk_structs/src/structs/stored_f64.rs @@ -4,9 +4,9 @@ use std::{ ops::{Add, AddAssign, Div, Mul}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Bitcoin, Dollars}; diff --git a/crates/brk_structs/src/structs/stored_u16.rs b/crates/brk_structs/src/structs/stored_u16.rs index 7bec1148d..a51fc823e 100644 --- a/crates/brk_structs/src/structs/stored_u16.rs +++ b/crates/brk_structs/src/structs/stored_u16.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/stored_u32.rs b/crates/brk_structs/src/structs/stored_u32.rs index bdcd99b7b..ada3e7654 100644 --- a/crates/brk_structs/src/structs/stored_u32.rs +++ b/crates/brk_structs/src/structs/stored_u32.rs @@ -1,8 +1,8 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{ diff --git a/crates/brk_structs/src/structs/stored_u64.rs b/crates/brk_structs/src/structs/stored_u64.rs index 8fdecb419..7b66e4406 100644 --- a/crates/brk_structs/src/structs/stored_u64.rs +++ b/crates/brk_structs/src/structs/stored_u64.rs @@ -1,8 +1,8 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::Deref; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{ diff --git a/crates/brk_structs/src/structs/stored_u8.rs b/crates/brk_structs/src/structs/stored_u8.rs index 554be7d26..88c2c8353 100644 --- a/crates/brk_structs/src/structs/stored_u8.rs +++ b/crates/brk_structs/src/structs/stored_u8.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::{CheckedSub, Printable}; +use vecdb::{CheckedSub, Printable}; use derive_deref::Deref; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/timestamp.rs b/crates/brk_structs/src/structs/timestamp.rs index 44b1806a0..d0a436f44 100644 --- a/crates/brk_structs/src/structs/timestamp.rs +++ b/crates/brk_structs/src/structs/timestamp.rs @@ -1,9 +1,9 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::{CheckedSub, StoredCompressed}; use derive_deref::Deref; use jiff::{civil::date, tz::TimeZone}; use serde::Serialize; +use vecdb::{CheckedSub, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Date; diff --git a/crates/brk_structs/src/structs/txindex.rs b/crates/brk_structs/src/structs/txindex.rs index f6a9a84d8..014f76c6e 100644 --- a/crates/brk_structs/src/structs/txindex.rs +++ b/crates/brk_structs/src/structs/txindex.rs @@ -1,9 +1,9 @@ use std::ops::{Add, AddAssign}; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use byteview::ByteView; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::copy_first_4bytes; diff --git a/crates/brk_structs/src/structs/txversion.rs b/crates/brk_structs/src/structs/txversion.rs index 8b078f273..91717fb06 100644 --- a/crates/brk_structs/src/structs/txversion.rs +++ b/crates/brk_structs/src/structs/txversion.rs @@ -1,6 +1,6 @@ -use brk_vecs::StoredCompressed; use derive_deref::Deref; use serde::Serialize; +use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::StoredU16; diff --git a/crates/brk_structs/src/structs/typeindex.rs b/crates/brk_structs/src/structs/typeindex.rs index b1be0e57b..071dcc420 100644 --- a/crates/brk_structs/src/structs/typeindex.rs +++ b/crates/brk_structs/src/structs/typeindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, StoredCompressed}; use byteview::ByteView; use serde::Serialize; +use vecdb::{CheckedSub, StoredCompressed}; use zerocopy::IntoBytes; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/brk_structs/src/structs/unknownoutputindex.rs b/crates/brk_structs/src/structs/unknownoutputindex.rs index 6926a54c2..496321a30 100644 --- a/crates/brk_structs/src/structs/unknownoutputindex.rs +++ b/crates/brk_structs/src/structs/unknownoutputindex.rs @@ -1,8 +1,8 @@ use std::ops::Add; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use derive_deref::{Deref, DerefMut}; use serde::Serialize; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::TypeIndex; diff --git a/crates/brk_structs/src/structs/weekindex.rs b/crates/brk_structs/src/structs/weekindex.rs index e603d781d..6eb6a9659 100644 --- a/crates/brk_structs/src/structs/weekindex.rs +++ b/crates/brk_structs/src/structs/weekindex.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Date, DateIndex}; diff --git a/crates/brk_structs/src/structs/weight.rs b/crates/brk_structs/src/structs/weight.rs index d8230ba4b..10cf78eab 100644 --- a/crates/brk_structs/src/structs/weight.rs +++ b/crates/brk_structs/src/structs/weight.rs @@ -1,8 +1,8 @@ use std::ops::{Add, AddAssign, Div}; -use brk_vecs::StoredCompressed; use derive_deref::Deref; use serde::Serialize; +use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive( diff --git a/crates/brk_structs/src/structs/yearindex.rs b/crates/brk_structs/src/structs/yearindex.rs index bea3b6e28..9b65e9af3 100644 --- a/crates/brk_structs/src/structs/yearindex.rs +++ b/crates/brk_structs/src/structs/yearindex.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Div}, }; -use brk_vecs::{CheckedSub, Printable, StoredCompressed}; use serde::{Deserialize, Serialize}; +use vecdb::{CheckedSub, Printable, StoredCompressed}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Date, DateIndex, MonthIndex}; diff --git a/crates/brk_vecs/.gitignore b/crates/brk_vecs/.gitignore deleted file mode 100644 index bb90405e8..000000000 --- a/crates/brk_vecs/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/vecs -/raw -/compressed diff --git a/crates/brk_vecs/Cargo.toml b/crates/brk_vecs/Cargo.toml deleted file mode 100644 index 147e4902b..000000000 --- a/crates/brk_vecs/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "brk_vecs" -description = "A KISS index/value store" -keywords = ["vec", "disk", "data"] -categories = ["database"] -version.workspace = true -edition.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -build = "build.rs" - -[dependencies] -brk_vecs_macros = { workspace = true } -ctrlc = { version = "3.4.7", features = ["termination"] } -libc = "0.2.174" -log = { workspace = true } -memmap2 = "0.9.7" -parking_lot = { workspace = true } -rayon = { workspace = true } -serde = { workspace = true } -serde_derive = { workspace = true } -serde_json = { workspace = true } -zerocopy = { workspace = true } -zerocopy-derive = { workspace = true } -pco = "0.4.6" diff --git a/crates/brk_vecs/README.md b/crates/brk_vecs/README.md deleted file mode 100644 index f6e7e869c..000000000 --- a/crates/brk_vecs/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Vecs - -- Columnar -- Embedded -- Index as keys, thus free -- Only values are stored -- Raw or Compressed diff --git a/crates/brk_vecs/build.rs b/crates/brk_vecs/build.rs deleted file mode 100644 index f765747eb..000000000 --- a/crates/brk_vecs/build.rs +++ /dev/null @@ -1,14 +0,0 @@ -fn main() { - let profile = std::env::var("PROFILE").unwrap_or_default(); - - if profile == "release" { - println!("cargo:rustc-flag=-C"); - println!("cargo:rustc-flag=target-cpu=native"); - - #[cfg(target_arch = "x86_64")] - { - println!("cargo:rustc-flag=-C"); - println!("cargo:rustc-flag=target-feature=+bmi1,+bmi2,+avx2"); - } - } -} diff --git a/crates/brk_vecs/examples/compressed.rs b/crates/brk_vecs/examples/compressed.rs deleted file mode 100644 index d5f17fa45..000000000 --- a/crates/brk_vecs/examples/compressed.rs +++ /dev/null @@ -1,185 +0,0 @@ -use std::{borrow::Cow, collections::BTreeSet, fs, path::Path, sync::Arc}; - -use brk_vecs::{ - AnyStoredVec, AnyVec, CollectableVec, CompressedVec, File, GenericStoredVec, Stamp, - VecIterator, Version, -}; - -#[allow(clippy::upper_case_acronyms)] -type VEC = CompressedVec; - -fn main() -> Result<(), Box> { - let _ = fs::remove_dir_all("compressed"); - - let version = Version::TWO; - - let file = Arc::new(File::open(Path::new("compressed"))?); - - { - let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - - (0..21_u32).for_each(|v| { - vec.push(v); - }); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(1) == Some(Cow::Borrowed(&1))); - assert!(iter.get(2) == Some(Cow::Borrowed(&2))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21).is_none()); - drop(iter); - - vec.flush()?; - - assert!(vec.header().stamp() == Stamp::new(0)); - } - - { - let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - - vec.mut_header().update_stamp(Stamp::new(100)); - - assert!(vec.header().stamp() == Stamp::new(100)); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(1) == Some(Cow::Borrowed(&1))); - assert!(iter.get(2) == Some(Cow::Borrowed(&2))); - assert!(iter.get(3) == Some(Cow::Borrowed(&3))); - assert!(iter.get(4) == Some(Cow::Borrowed(&4))); - assert!(iter.get(5) == Some(Cow::Borrowed(&5))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - drop(iter); - - vec.push(21); - vec.push(22); - - assert!(vec.stored_len() == 21); - assert!(vec.pushed_len() == 2); - assert!(vec.len() == 23); - - let mut iter = vec.into_iter(); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21) == Some(Cow::Borrowed(&21))); - assert!(iter.get(22) == Some(Cow::Borrowed(&22))); - assert!(iter.get(23).is_none()); - drop(iter); - - vec.flush()?; - } - - { - let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - - assert!(vec.header().stamp() == Stamp::new(100)); - - assert!(vec.stored_len() == 23); - assert!(vec.pushed_len() == 0); - assert!(vec.len() == 23); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21) == Some(Cow::Borrowed(&21))); - assert!(iter.get(22) == Some(Cow::Borrowed(&22))); - drop(iter); - - vec.truncate_if_needed(14)?; - - assert!(vec.stored_len() == 14); - assert!(vec.pushed_len() == 0); - assert!(vec.len() == 14); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(5) == Some(Cow::Borrowed(&5))); - assert!(iter.get(20).is_none()); - drop(iter); - - assert!(vec.collect_signed_range(Some(-5), None)? == vec![9, 10, 11, 12, 13]); - - vec.push(vec.len() as u32); - assert!(VecIterator::last(vec.into_iter()) == Some((14, Cow::Borrowed(&14)))); - - assert!( - vec.into_iter() - .map(|(_, v)| v.into_owned()) - .collect::>() - == vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] - ); - } - - { - let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - - vec.reset()?; - - // dbg!(vec.header()); - // assert len - - assert!(vec.pushed_len() == 0); - assert!(vec.stored_len() == 0); - assert!(vec.len() == 0); - - (0..21_u32).for_each(|v| { - vec.push(v); - }); - - assert!(vec.pushed_len() == 21); - assert!(vec.stored_len() == 0); - assert!(vec.len() == 21); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21).is_none()); - drop(iter); - - // let reader = vec.create_static_reader(); - // assert!(vec.take(10, &reader)? == Some(10)); - // assert!(vec.holes() == &BTreeSet::from([10])); - // assert!(vec.get_or_read(10, &reader)?.is_none()); - // drop(reader); - - vec.flush()?; - - // assert!(vec.holes() == &BTreeSet::from([10])); - } - - { - let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - - // assert!(vec.holes() == &BTreeSet::from([10])); - - // let reader = vec.create_static_reader(); - // assert!(vec.get_or_read(10, &reader)?.is_none()); - // drop(reader); - - // vec.update(10, 10)?; - // vec.update(0, 10)?; - - let reader = vec.create_static_reader(); - assert!(vec.holes() == &BTreeSet::new()); - assert!(vec.get_or_read(0, &reader)? == Some(Cow::Borrowed(&0))); - assert!(vec.get_or_read(10, &reader)? == Some(Cow::Borrowed(&10))); - drop(reader); - - vec.flush()?; - } - - { - let vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - - assert!( - vec.collect()? - == vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - ] - ); - } - - Ok(()) -} diff --git a/crates/brk_vecs/examples/file.rs b/crates/brk_vecs/examples/file.rs deleted file mode 100644 index b3c67f5f8..000000000 --- a/crates/brk_vecs/examples/file.rs +++ /dev/null @@ -1,425 +0,0 @@ -use std::{fs, path::Path}; - -use brk_vecs::{File, PAGE_SIZE, Result}; - -fn main() -> Result<()> { - let _ = fs::remove_dir_all("vecs"); - - let file = File::open(Path::new("vecs"))?; - - // let file_min_len = PAGE_SIZE * 1_000_000; - // let min_regions = 20_000; - - // file.set_min_len(file_min_len)?; - // file.set_min_regions(min_regions)?; - - let (region1_i, _) = file.create_region_if_needed("region1")?; - - { - let layout = file.layout(); - assert!(layout.start_to_index().len() == 1); - assert!(layout.start_to_index().first_key_value() == Some((&0, &0))); - assert!(layout.start_to_hole().is_empty()); - - let regions = file.regions(); - assert!( - regions - .get_region_index_from_id("region1") - .is_some_and(|i| i == region1_i) - ); - - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 0); - assert!(region.reserved() == PAGE_SIZE); - } - - file.write_all_to_region(region1_i.into(), &[0, 1, 2, 3, 4])?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 5); - assert!(region.reserved() == PAGE_SIZE); - - assert!(file.mmap()[0..10] == [0, 1, 2, 3, 4, 0, 0, 0, 0, 0]); - } - - file.write_all_to_region(region1_i.into(), &[5, 6, 7, 8, 9])?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 10); - assert!(region.reserved() == PAGE_SIZE); - - assert!(file.mmap()[0..10] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - } - - file.write_all_to_region_at(region1_i.into(), &[1, 2], 0)?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 10); - assert!(region.reserved() == PAGE_SIZE); - - assert!(file.mmap()[0..10] == [1, 2, 2, 3, 4, 5, 6, 7, 8, 9]); - } - - file.write_all_to_region_at(region1_i.into(), &[10, 11, 12, 13, 14, 15, 16, 17, 18], 4)?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 13); - assert!(region.reserved() == PAGE_SIZE); - - assert!( - file.mmap()[0..20] - == [ - 1, 2, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 0, 0, 0, 0, 0 - ] - ); - } - - file.write_all_to_region_at(region1_i.into(), &[0, 0, 0, 0, 0, 1], 13)?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 19); - assert!(region.reserved() == PAGE_SIZE); - - assert!( - file.mmap()[0..20] - == [ - 1, 2, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 0, 0, 0, 1, 0 - ] - ); - } - - dbg!(1); - - file.write_all_to_region_at(region1_i.into(), &[1; 8000], 0)?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 8000); - assert!(region.reserved() == PAGE_SIZE * 2); - - assert!(file.mmap()[0..8000] == [1; 8000]); - assert!(file.mmap()[8000..8001] == [0]); - } - - println!("Disk usage - pre sync: {}", file.disk_usage()); - file.flush()?; - println!("Disk usage - post sync: {}", file.disk_usage()); - - file.truncate_region(region1_i.into(), 10)?; - file.punch_holes()?; - - { - let region = file.get_region(region1_i.into())?; - assert!(region.start() == 0); - assert!(region.len() == 10); - assert!(region.reserved() == PAGE_SIZE * 2); - // We only punch a hole in whole pages (4096 bytes) - // Thus the last byte of the page where the is still data wasn't overwritten when truncating - // And the first byte of the punched page was set to 0 - assert!(file.mmap()[4095..=4096] == [1, 0]); - } - - file.flush()?; - println!("Disk usage - post trunc: {}", file.disk_usage()); - - file.remove_region(region1_i.into())?; - - file.flush()?; - - println!("Disk usage - post remove: {}", file.disk_usage()); - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 1); - assert!(index_to_region[0].is_none()); - assert!(regions.id_to_index().is_empty()); - - let layout = file.layout(); - assert!(layout.start_to_index().is_empty()); - assert!(layout.start_to_hole().len() == 1); - } - - let (region1_i, _) = file.create_region_if_needed("region1")?; - let (region2_i, _) = file.create_region_if_needed("region2")?; - let (region3_i, _) = file.create_region_if_needed("region3")?; - - // dbg!(file.layout()); - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == 0); - assert!(region1.len() == 0); - assert!(region1.reserved() == PAGE_SIZE); - let region2 = file.get_region(region2_i.into())?; - assert!(region2.start() == PAGE_SIZE); - assert!(region2.len() == 0); - assert!(region2.reserved() == PAGE_SIZE); - let region3 = file.get_region(region3_i.into())?; - assert!(region3.start() == PAGE_SIZE * 2); - assert!(region3.len() == 0); - assert!(region3.reserved() == PAGE_SIZE); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 3); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2") == Some(&1)); - assert!(id_to_index.get("region3") == Some(&2)); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 3); - assert!(start_to_index.get(&0) == Some(&0)); - assert!(start_to_index.get(&PAGE_SIZE) == Some(&1)); - assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(&2)); - assert!(layout.start_to_hole().is_empty()); - } - - file.remove_region(region2_i.into())?; - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == 0); - assert!(region1.len() == 0); - assert!(region1.reserved() == PAGE_SIZE); - assert!(file.get_region(region2_i.into()).is_err()); - assert!( - index_to_region - .get(region2_i) - .is_some_and(|opt| opt.is_none()) - ); - let region3 = file.get_region(region3_i.into())?; - assert!(region3.start() == PAGE_SIZE * 2); - assert!(region3.len() == 0); - assert!(region3.reserved() == PAGE_SIZE); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 2); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2").is_none()); - assert!(id_to_index.get("region3") == Some(&2)); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 2); - assert!(start_to_index.get(&0) == Some(®ion1_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i)); - let start_to_hole = layout.start_to_hole(); - assert!(start_to_hole.len() == 1); - assert!(start_to_hole.get(&PAGE_SIZE) == Some(&PAGE_SIZE)); - - drop(regions); - drop(layout); - assert!( - file.remove_region(region2_i.into()) - .is_ok_and(|o| o.is_none()) - ); - } - - let (region2_i, _) = file.create_region_if_needed("region2")?; - - { - assert!(region2_i == 1) - } - - file.remove_region(region2_i.into())?; - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == 0); - assert!(region1.len() == 0); - assert!(region1.reserved() == PAGE_SIZE); - assert!(file.get_region(region2_i.into()).is_err()); - assert!( - index_to_region - .get(region2_i) - .is_some_and(|opt| opt.is_none()) - ); - let region3 = file.get_region(region3_i.into())?; - assert!(region3.start() == PAGE_SIZE * 2); - assert!(region3.len() == 0); - assert!(region3.reserved() == PAGE_SIZE); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 2); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2").is_none()); - assert!(id_to_index.get("region3") == Some(&2)); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 2); - assert!(start_to_index.get(&0) == Some(®ion1_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i)); - let start_to_hole = layout.start_to_hole(); - assert!(start_to_hole.len() == 1); - assert!(start_to_hole.get(&PAGE_SIZE) == Some(&PAGE_SIZE)); - - drop(regions); - drop(layout); - assert!( - file.remove_region(region2_i.into()) - .is_ok_and(|o| o.is_none()) - ); - } - - file.write_all_to_region_at(region1_i.into(), &[1; 8000], 0)?; - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == 0); - assert!(region1.len() == 8000); - assert!(region1.reserved() == 2 * PAGE_SIZE); - assert!(file.get_region(region2_i.into()).is_err()); - assert!( - index_to_region - .get(region2_i) - .is_some_and(|opt| opt.is_none()) - ); - let region3 = file.get_region(region3_i.into())?; - assert!(region3.start() == PAGE_SIZE * 2); - assert!(region3.len() == 0); - assert!(region3.reserved() == PAGE_SIZE); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 2); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2").is_none()); - assert!(id_to_index.get("region3") == Some(&2)); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 2); - assert!(start_to_index.get(&0) == Some(®ion1_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i)); - let start_to_hole = layout.start_to_hole(); - assert!(start_to_hole.is_empty()); - } - - let (region2_i, _) = file.create_region_if_needed("region2")?; - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == 0); - assert!(region1.len() == 8000); - assert!(region1.reserved() == 2 * PAGE_SIZE); - let region2 = file.get_region(region2_i.into())?; - assert!(region2.start() == PAGE_SIZE * 3); - assert!(region2.len() == 0); - assert!(region2.reserved() == PAGE_SIZE); - let region3 = file.get_region(region3_i.into())?; - assert!(region3.start() == PAGE_SIZE * 2); - assert!(region3.len() == 0); - assert!(region3.reserved() == PAGE_SIZE); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 3); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2") == Some(&1)); - assert!(id_to_index.get("region3") == Some(&2)); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 3); - assert!(start_to_index.get(&0) == Some(®ion1_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 3)) == Some(®ion2_i)); - let start_to_hole = layout.start_to_hole(); - assert!(start_to_hole.is_empty()); - } - - file.remove_region(region3_i.into())?; - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == 0); - assert!(region1.len() == 8000); - assert!(region1.reserved() == 2 * PAGE_SIZE); - let region2 = file.get_region(region2_i.into())?; - assert!(region2.start() == PAGE_SIZE * 3); - assert!(region2.len() == 0); - assert!(region2.reserved() == PAGE_SIZE); - assert!(file.get_region(region3_i.into()).is_err()); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 2); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2") == Some(&1)); - assert!(id_to_index.get("region3").is_none()); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 2); - assert!(start_to_index.get(&0) == Some(®ion1_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 3)) == Some(®ion2_i)); - let start_to_hole = layout.start_to_hole(); - assert!(start_to_hole.get(&(PAGE_SIZE * 2)) == Some(&PAGE_SIZE)); - } - - file.write_all_to_region(region1_i.into(), &[1; 8000])?; - - { - let regions = file.regions(); - let index_to_region = regions.index_to_region(); - assert!(index_to_region.len() == 3); - let region1 = file.get_region(region1_i.into())?; - assert!(region1.start() == PAGE_SIZE * 4); - assert!(region1.len() == 16_000); - assert!(region1.reserved() == 4 * PAGE_SIZE); - let region2 = file.get_region(region2_i.into())?; - assert!(region2.start() == PAGE_SIZE * 3); - assert!(region2.len() == 0); - assert!(region2.reserved() == PAGE_SIZE); - assert!(file.get_region(region3_i.into()).is_err()); - let id_to_index = regions.id_to_index(); - assert!(id_to_index.len() == 2); - assert!(id_to_index.get("region1") == Some(&0)); - assert!(id_to_index.get("region2") == Some(&1)); - assert!(id_to_index.get("region3").is_none()); - - let layout = file.layout(); - let start_to_index = layout.start_to_index(); - assert!(start_to_index.len() == 2); - assert!(start_to_index.get(&(PAGE_SIZE * 4)) == Some(®ion1_i)); - assert!(start_to_index.get(&(PAGE_SIZE * 3)) == Some(®ion2_i)); - let start_to_hole = layout.start_to_hole(); - assert!(start_to_hole.get(&0) == Some(&(PAGE_SIZE * 3))); - } - - file.write_all_to_region(region2_i.into(), &[1; 6000])?; - - let (region4_i, _) = file.create_region_if_needed("region4")?; - file.remove_region(region2_i.into())?; - file.remove_region(region4_i.into())?; - - let regions = file.regions(); - dbg!(®ions); - let layout = file.layout(); - dbg!(&layout); - - Ok(()) -} diff --git a/crates/brk_vecs/examples/raw.rs b/crates/brk_vecs/examples/raw.rs deleted file mode 100644 index 83ea0aff6..000000000 --- a/crates/brk_vecs/examples/raw.rs +++ /dev/null @@ -1,185 +0,0 @@ -use std::{borrow::Cow, collections::BTreeSet, fs, path::Path, sync::Arc}; - -use brk_vecs::{ - AnyStoredVec, AnyVec, CollectableVec, File, GenericStoredVec, RawVec, Stamp, VecIterator, - Version, -}; - -#[allow(clippy::upper_case_acronyms)] -type VEC = RawVec; - -fn main() -> Result<(), Box> { - let _ = fs::remove_dir_all("raw"); - - let version = Version::TWO; - - let file = Arc::new(File::open(Path::new("raw"))?); - - { - let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; - - (0..21_u32).for_each(|v| { - vec.push(v); - }); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(1) == Some(Cow::Borrowed(&1))); - assert!(iter.get(2) == Some(Cow::Borrowed(&2))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21).is_none()); - drop(iter); - - vec.flush()?; - - assert!(vec.header().stamp() == Stamp::new(0)); - } - - { - let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; - - vec.mut_header().update_stamp(Stamp::new(100)); - - assert!(vec.header().stamp() == Stamp::new(100)); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(1) == Some(Cow::Borrowed(&1))); - assert!(iter.get(2) == Some(Cow::Borrowed(&2))); - assert!(iter.get(3) == Some(Cow::Borrowed(&3))); - assert!(iter.get(4) == Some(Cow::Borrowed(&4))); - assert!(iter.get(5) == Some(Cow::Borrowed(&5))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - drop(iter); - - vec.push(21); - vec.push(22); - - assert!(vec.stored_len() == 21); - assert!(vec.pushed_len() == 2); - assert!(vec.len() == 23); - - let mut iter = vec.into_iter(); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21) == Some(Cow::Borrowed(&21))); - assert!(iter.get(22) == Some(Cow::Borrowed(&22))); - assert!(iter.get(23).is_none()); - drop(iter); - - vec.flush()?; - } - - { - let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; - - assert!(vec.header().stamp() == Stamp::new(100)); - - assert!(vec.stored_len() == 23); - assert!(vec.pushed_len() == 0); - assert!(vec.len() == 23); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21) == Some(Cow::Borrowed(&21))); - assert!(iter.get(22) == Some(Cow::Borrowed(&22))); - drop(iter); - - vec.truncate_if_needed(14)?; - - assert!(vec.stored_len() == 14); - assert!(vec.pushed_len() == 0); - assert!(vec.len() == 14); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(5) == Some(Cow::Borrowed(&5))); - assert!(iter.get(20).is_none()); - drop(iter); - - assert!(vec.collect_signed_range(Some(-5), None)? == vec![9, 10, 11, 12, 13]); - - vec.push(vec.len() as u32); - assert!(VecIterator::last(vec.into_iter()) == Some((14, Cow::Borrowed(&14)))); - - assert!( - vec.into_iter() - .map(|(_, v)| v.into_owned()) - .collect::>() - == vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] - ); - } - - { - let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; - - vec.reset()?; - - // dbg!(vec.header()); - // assert len - - assert!(vec.pushed_len() == 0); - assert!(vec.stored_len() == 0); - assert!(vec.len() == 0); - - (0..21_u32).for_each(|v| { - vec.push(v); - }); - - assert!(vec.pushed_len() == 21); - assert!(vec.stored_len() == 0); - assert!(vec.len() == 21); - - let mut iter = vec.into_iter(); - assert!(iter.get(0) == Some(Cow::Borrowed(&0))); - assert!(iter.get(20) == Some(Cow::Borrowed(&20))); - assert!(iter.get(21).is_none()); - drop(iter); - - let reader = vec.create_static_reader(); - assert!(vec.take(10, &reader)? == Some(10)); - assert!(vec.holes() == &BTreeSet::from([10])); - assert!(vec.get_or_read(10, &reader)?.is_none()); - drop(reader); - - vec.flush()?; - - assert!(vec.holes() == &BTreeSet::from([10])); - } - - { - let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; - - assert!(vec.holes() == &BTreeSet::from([10])); - - let reader = vec.create_static_reader(); - assert!(vec.get_or_read(10, &reader)?.is_none()); - drop(reader); - - vec.update(10, 10)?; - vec.update(0, 10)?; - - let reader = vec.create_static_reader(); - assert!(vec.holes() == &BTreeSet::new()); - assert!(vec.get_or_read(0, &reader)? == Some(Cow::Borrowed(&10))); - assert!(vec.get_or_read(10, &reader)? == Some(Cow::Borrowed(&10))); - drop(reader); - - vec.flush()?; - } - - { - let vec: VEC = RawVec::forced_import(&file, "vec", version)?; - - assert!( - vec.collect()? - == vec![ - 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - ] - ); - } - - Ok(()) -} diff --git a/crates/brk_vecs/src/error.rs b/crates/brk_vecs/src/error.rs deleted file mode 100644 index a10b3ea30..000000000 --- a/crates/brk_vecs/src/error.rs +++ /dev/null @@ -1,103 +0,0 @@ -use std::{ - fmt::{self, Debug, Display}, - fs, io, result, time, -}; - -use crate::Version; - -pub type Result = result::Result; - -#[derive(Debug)] -pub enum Error { - IO(io::Error), - TryLockError(fs::TryLockError), - SerdeJson(serde_json::Error), - SystemTimeError(time::SystemTimeError), - PCO(pco::errors::PcoError), - ZeroCopyError, - - WrongEndian, - DifferentVersion { found: Version, expected: Version }, - IndexTooHigh, - ExpectVecToHaveIndex, - FailedKeyTryIntoUsize, - DifferentCompressionMode, - WrongLength, - Str(&'static str), - String(String), -} - -impl From for Error { - fn from(value: time::SystemTimeError) -> Self { - Self::SystemTimeError(value) - } -} - -impl From for Error { - fn from(value: io::Error) -> Self { - Self::IO(value) - } -} - -impl From for Error { - fn from(value: fs::TryLockError) -> Self { - Self::TryLockError(value) - } -} - -impl From for Error { - fn from(value: pco::errors::PcoError) -> Self { - Self::PCO(value) - } -} - -impl From> for Error { - fn from(_: zerocopy::error::ConvertError) -> Self { - Self::ZeroCopyError - } -} - -impl From> for Error { - fn from(_: zerocopy::error::SizeError) -> Self { - Self::ZeroCopyError - } -} - -impl From for Error { - fn from(error: serde_json::Error) -> Self { - Self::SerdeJson(error) - } -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::IO(error) => Display::fmt(&error, f), - Error::TryLockError(_) => write!( - f, - "Couldn't lock file. It must be already opened by another process." - ), - Error::PCO(error) => Display::fmt(&error, f), - Error::SystemTimeError(error) => Display::fmt(&error, f), - Error::SerdeJson(error) => Display::fmt(&error, f), - Error::ZeroCopyError => write!(f, "ZeroCopy error"), - - Error::WrongEndian => write!(f, "Wrong endian"), - Error::DifferentVersion { found, expected } => { - write!( - f, - "Different version found: {found:?}, expected: {expected:?}" - ) - } - Error::IndexTooHigh => write!(f, "Index too high"), - Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"), - Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"), - Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"), - Error::WrongLength => write!(f, "Wrong length"), - Error::Str(s) => write!(f, "{s}"), - Error::String(s) => write!(f, "{s}"), - } - } -} - -impl std::error::Error for Error {} diff --git a/crates/brk_vecs/src/exit.rs b/crates/brk_vecs/src/exit.rs deleted file mode 100644 index 96420c4fa..000000000 --- a/crates/brk_vecs/src/exit.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::{process::exit, sync::Arc}; - -use log::info; -use parking_lot::{RwLock, RwLockReadGuard}; - -#[derive(Default, Clone)] -pub struct Exit(Arc>); - -impl Exit { - pub fn new() -> Self { - Self(Arc::new(RwLock::new(()))) - } - - pub fn set_ctrlc_handler(&self) { - let copy = self.0.clone(); - - ctrlc::set_handler(move || { - if copy.is_locked() { - info!("Waiting to exit safely..."); - } - let _lock = copy.write(); - info!("Exiting..."); - exit(0); - }) - .expect("Error setting Ctrl-C handler"); - } - - pub fn lock(&self) -> RwLockReadGuard<'_, ()> { - self.0.read() - } -} diff --git a/crates/brk_vecs/src/file/identifier.rs b/crates/brk_vecs/src/file/identifier.rs deleted file mode 100644 index d081fb81a..000000000 --- a/crates/brk_vecs/src/file/identifier.rs +++ /dev/null @@ -1,23 +0,0 @@ -#[derive(Debug, Clone)] -pub enum Identifier { - Number(usize), - String(String), -} - -impl<'a> From<&'a str> for Identifier { - fn from(value: &'a str) -> Self { - Self::String(value.to_owned()) - } -} - -impl From for Identifier { - fn from(value: String) -> Self { - Self::String(value) - } -} - -impl From for Identifier { - fn from(value: usize) -> Self { - Self::Number(value) - } -} diff --git a/crates/brk_vecs/src/file/layout.rs b/crates/brk_vecs/src/file/layout.rs deleted file mode 100644 index d12154513..000000000 --- a/crates/brk_vecs/src/file/layout.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::collections::BTreeMap; - -use crate::{Error, Result}; - -use super::{Region, Regions}; - -#[derive(Debug)] -pub struct Layout { - start_to_index: BTreeMap, - start_to_hole: BTreeMap, - start_to_reserved: BTreeMap, -} - -impl From<&Regions> for Layout { - fn from(value: &Regions) -> Self { - let mut start_to_index = BTreeMap::new(); - - let index_to_region = value.index_to_region(); - - value - .index_to_region() - .iter() - .enumerate() - .flat_map(|(index, opt)| opt.as_ref().map(|region| (index, region))) - .for_each(|(index, region)| { - let region = region.read(); - let start = region.start(); - start_to_index.insert(start, index); - }); - - let mut start_to_hole = BTreeMap::new(); - - let mut prev_end = 0; - - start_to_index.iter().for_each(|(&start, &index)| { - if prev_end != start { - start_to_hole.insert(prev_end, start - prev_end); - } - let reserved = index_to_region[index].as_ref().unwrap().read().reserved(); - prev_end = start + reserved; - }); - - Self { - start_to_index, - start_to_hole, - start_to_reserved: BTreeMap::default(), - } - } -} - -impl Layout { - pub fn start_to_index(&self) -> &BTreeMap { - &self.start_to_index - } - - pub fn start_to_hole(&self) -> &BTreeMap { - &self.start_to_hole - } - - pub fn len(&self, regions: &Regions) -> u64 { - let mut len = 0; - let mut start = 0; - if let Some((start_reserved, reserved)) = self.get_last_reserved() { - start = start_reserved; - len = start + reserved; - } - if let Some((hole_start, gap)) = self.get_last_hole() - && hole_start > start - { - start = hole_start; - len = start + gap; - } - if let Some((region_start, region_index)) = self.get_last_region() - && region_start > start - { - len = region_start - + regions - .get_region_from_index(region_index) - .unwrap() - .read() - .reserved(); - } - len - } - - pub fn get_last_region_index(&self) -> Option { - self.get_last_region().map(|(_, index)| index) - } - - pub fn get_last_region(&self) -> Option<(u64, usize)> { - self.start_to_index - .last_key_value() - .map(|(start, index)| (*start, *index)) - } - - fn get_last_hole(&self) -> Option<(u64, u64)> { - self.start_to_hole - .last_key_value() - .map(|(start, gap)| (*start, *gap)) - } - - fn get_last_reserved(&self) -> Option<(u64, u64)> { - self.start_to_reserved - .last_key_value() - .map(|(start, reserved)| (*start, *reserved)) - } - - pub fn is_last_anything(&self, index: usize) -> bool { - if let Some((last_start, last_index)) = self.get_last_region() - && last_index == index - && self - .get_last_hole() - .is_none_or(|(hole_start, _)| last_start > hole_start) - && self - .get_last_reserved() - .is_none_or(|(reserved_start, _)| last_start > reserved_start) - { - true - } else { - false - } - } - - pub fn insert_region(&mut self, start: u64, index: usize) { - assert!(self.start_to_index.insert(start, index).is_none()) - // TODO: Other checks related to holes and reserved ? - } - - pub fn move_region(&mut self, new_start: u64, index: usize, region: &Region) -> Result<()> { - self.remove_region(index, region)?; - self.insert_region(new_start, index); - Ok(()) - } - - pub fn remove_region(&mut self, index: usize, region: &Region) -> Result<()> { - // info!("Remove region {index}"); - - let start = region.start(); - let mut reserved = region.reserved(); - - let removed = self.start_to_index.remove(&start); - - if removed.is_none_or(|index_| index != index_) { - dbg!((index, removed)); - return Err(Error::Str( - "Something went wrong, indexes of removed region should be the same", - )); - } - - reserved += self - .start_to_hole - .remove(&(start + reserved)) - .unwrap_or_default(); - - if let Some((&hole_start, gap)) = self.start_to_hole.range_mut(..start).next_back() - && hole_start + *gap == start - { - *gap += reserved; - } else { - self.start_to_hole.insert(start, reserved); - } - - Ok(()) - } - - pub fn get_hole(&self, start: u64) -> Option { - self.start_to_hole.get(&start).copied() - } - - pub fn find_smallest_adequate_hole(&self, reserved: u64) -> Option { - self.start_to_hole - .iter() - .filter(|(_, gap)| **gap >= reserved) - .map(|(start, gap)| (gap, start)) - .collect::>() - .pop_first() - .map(|(_, s)| *s) - } - - pub fn remove_or_compress_hole(&mut self, start: u64, compress_by: u64) { - if let Some(gap) = self.start_to_hole.remove(&start) - && gap != compress_by - { - if gap > compress_by { - self.start_to_hole - .insert(start + compress_by, gap - compress_by); - } else { - panic!("Hole too small"); - } - } - } - - pub fn reserve(&mut self, start: u64, reserved: u64) { - if self.start_to_reserved.insert(start, reserved).is_some() { - unreachable!(); - } - } - - pub fn reserved(&mut self, start: u64) -> Option { - self.start_to_reserved.remove(&start) - } -} diff --git a/crates/brk_vecs/src/file/mod.rs b/crates/brk_vecs/src/file/mod.rs deleted file mode 100644 index b5d08cf5f..000000000 --- a/crates/brk_vecs/src/file/mod.rs +++ /dev/null @@ -1,616 +0,0 @@ -use std::{ - fs::{self, OpenOptions}, - os::unix::io::AsRawFd, - path::{Path, PathBuf}, - sync::Arc, -}; - -use libc::off_t; -use memmap2::{MmapMut, MmapOptions}; -use parking_lot::{RwLock, RwLockReadGuard}; - -mod identifier; -mod layout; -mod reader; -mod region; -mod regions; - -pub use identifier::*; -use layout::*; -use rayon::prelude::*; -pub use reader::*; -pub use region::*; -use regions::*; - -use crate::{Error, Result}; - -pub const PAGE_SIZE: u64 = 4096; -pub const PAGE_SIZE_MINUS_1: u64 = PAGE_SIZE - 1; -const GB: u64 = 1024 * 1024 * 1024; - -#[derive(Debug)] -pub struct File { - path: PathBuf, - regions: RwLock, - layout: RwLock, - file: RwLock, - mmap: RwLock, -} - -impl File { - pub fn open(path: &Path) -> Result { - fs::create_dir_all(path)?; - - let regions = Regions::open(path)?; - - let layout = Layout::from(®ions); - - let file = OpenOptions::new() - .read(true) - .create(true) - .write(true) - .truncate(false) - .open(Self::data_path_(path))?; - file.try_lock()?; - - let mmap = Self::create_mmap(&file)?; - - Ok(Self { - path: path.to_owned(), - file: RwLock::new(file), - mmap: RwLock::new(mmap), - regions: RwLock::new(regions), - layout: RwLock::new(layout), - }) - } - - pub fn file_len(&self) -> Result { - Ok(self.file.read().metadata()?.len()) - } - - pub fn set_min_len(&self, len: u64) -> Result<()> { - let len = Self::ceil_number_to_page_size_multiple(len); - - let file_len = self.file_len()?; - if file_len < len { - let mut mmap = self.mmap.write(); - let file = self.file.write(); - file.set_len(len)?; - *mmap = Self::create_mmap(&file)?; - Ok(()) - } else { - Ok(()) - } - } - - pub fn set_min_regions(&self, regions: usize) -> Result<()> { - self.regions - .write() - .set_min_len((regions * SIZE_OF_REGION) as u64)?; - self.set_min_len(regions as u64 * PAGE_SIZE) - } - - pub fn create_region_if_needed(&self, id: &str) -> Result<(usize, Arc>)> { - let regions = self.regions.read(); - if let Some(index) = regions.get_region_index_from_id(id) { - return Ok((index, regions.get_region_from_index(index).unwrap())); - } - drop(regions); - - let mut regions = self.regions.write(); - let mut layout = self.layout.write(); - - let start = if let Some(start) = layout.find_smallest_adequate_hole(PAGE_SIZE) { - layout.remove_or_compress_hole(start, PAGE_SIZE); - start - } else { - let start = layout - .get_last_region_index() - .map(|index| { - let region_opt = regions.get_region_from_index(index); - let region = region_opt.as_ref().unwrap().read(); - region.start() + region.reserved() - }) - .unwrap_or_default(); - - let len = start + PAGE_SIZE; - - self.set_min_len(len)?; - - start - }; - - let (index, region) = regions.create_region(id.to_owned(), start)?; - - layout.insert_region(start, index); - - Ok((index, region)) - } - - pub fn get_region(&self, identifier: Identifier) -> Result> { - let regions = self.regions.read(); - let region_opt = regions.get_region(identifier); - let region_arc = region_opt.ok_or(Error::Str("Unknown region"))?; - let region = region_arc.read(); - let region: RwLockReadGuard<'static, Region> = unsafe { std::mem::transmute(region) }; - Ok(region) - } - - pub fn create_region_reader<'a>(&'a self, identifier: Identifier) -> Result> { - let mmap: RwLockReadGuard<'a, MmapMut> = self.mmap.read(); - let region = self.get_region(identifier)?; - Ok(Reader::new(mmap, region)) - } - - #[inline] - pub fn write_all_to_region(&self, identifier: Identifier, data: &[u8]) -> Result<()> { - self.write_all_to_region_at_(identifier, data, None, false) - } - - #[inline] - pub fn write_all_to_region_at( - &self, - identifier: Identifier, - data: &[u8], - at: u64, - ) -> Result<()> { - self.write_all_to_region_at_(identifier, data, Some(at), false) - } - - #[inline] - pub fn truncate_write_all_to_region( - &self, - identifier: Identifier, - at: u64, - data: &[u8], - ) -> Result<()> { - self.write_all_to_region_at_(identifier, data, Some(at), true) - } - - fn write_all_to_region_at_( - &self, - identifier: Identifier, - data: &[u8], - at: Option, - truncate: bool, - ) -> Result<()> { - let regions = self.regions.read(); - let Some(region_lock) = regions.get_region(identifier.clone()) else { - return Err(Error::Str("Unknown region")); - }; - - let region_index = regions.identifier_to_index(identifier).unwrap(); - - let region = region_lock.read(); - let start = region.start(); - let reserved = region.reserved(); - let len = region.len(); - drop(region); - - let data_len = data.len() as u64; - let new_len = at.map_or(len + data_len, |at| { - assert!(at <= len); - let new_len = at + data_len; - if truncate { new_len } else { new_len.max(len) } - }); - let write_start = start + at.unwrap_or(len); - - if at.is_some_and(|at| at > reserved) { - return Err(Error::Str("Invalid at parameter")); - } - - // Write to reserved space if possible - if new_len <= reserved { - // info!( - // "Write {data_len} bytes to {region_index} reserved space at {write_start} (start = {start}, at = {at:?}, len = {len})" - // ); - - if at.is_none() { - self.write(write_start, data); - } - - let mut region = region_lock.write(); - - if at.is_some() { - self.write(write_start, data); - } - - region.set_len(new_len); - regions.write_to_mmap(®ion, region_index); - - return Ok(()); - } - - assert!(new_len > reserved); - let mut new_reserved = reserved; - while new_len > new_reserved { - new_reserved *= 2; - } - assert!(new_len <= new_reserved); - let added_reserve = new_reserved - reserved; - - let mut layout = self.layout.write(); - - // If is last continue writing - if layout.is_last_anything(region_index) { - // info!("{region_index} Append to file at {write_start}"); - - self.set_min_len(start + new_reserved)?; - let mut region = region_lock.write(); - region.set_reserved(new_reserved); - drop(region); - drop(layout); - - self.write(write_start, data); - - let mut region = region_lock.write(); - region.set_len(new_len); - regions.write_to_mmap(®ion, region_index); - - return Ok(()); - } - - // Expand region to the right if gap is wide enough - let hole_start = start + reserved; - if layout - .get_hole(hole_start) - .is_some_and(|gap| gap >= added_reserve) - { - // info!("Expand {region_index} to hole"); - - layout.remove_or_compress_hole(hole_start, added_reserve); - let mut region = region_lock.write(); - region.set_reserved(new_reserved); - drop(region); - drop(layout); - - self.write(write_start, data); - - let mut region = region_lock.write(); - region.set_len(new_len); - regions.write_to_mmap(®ion, region_index); - - return Ok(()); - } - - // Find hole big enough to move the region - if let Some(hole_start) = layout.find_smallest_adequate_hole(new_reserved) { - // info!("Move {region_index} to hole at {hole_start}"); - - layout.remove_or_compress_hole(hole_start, new_reserved); - drop(layout); - - self.write( - hole_start, - &self.mmap.read()[start as usize..write_start as usize], - ); - self.write(hole_start + at.unwrap_or(len), data); - - let mut region = region_lock.write(); - let mut layout = self.layout.write(); - layout.move_region(hole_start, region_index, ®ion)?; - drop(layout); - - region.set_start(hole_start); - region.set_reserved(new_reserved); - region.set_len(new_len); - regions.write_to_mmap(®ion, region_index); - - return Ok(()); - } - - let new_start = layout.len(®ions); - // Write at the end - // info!( - // "Move {region_index} to the end, from {start}..{} to {new_start}..{}", - // start + reserved, - // new_start + new_reserved - // ); - self.set_min_len(new_start + new_reserved)?; - layout.reserve(new_start, new_reserved); - drop(layout); - - self.write( - new_start, - &self.mmap.read()[start as usize..write_start as usize], - ); - self.write(new_start + at.unwrap_or(len), data); - - let mut region = region_lock.write(); - let mut layout = self.layout.write(); - layout.move_region(new_start, region_index, ®ion)?; - assert!(layout.reserved(new_start) == Some(new_reserved)); - drop(layout); - - region.set_start(new_start); - region.set_reserved(new_reserved); - region.set_len(new_len); - regions.write_to_mmap(®ion, region_index); - - Ok(()) - } - - fn write(&self, at: u64, data: &[u8]) { - let mmap = self.mmap.read(); - - let data_len = data.len(); - let start = at as usize; - let end = start + data_len; - - if end > mmap.len() { - unreachable!("Trying to write beyond mmap") - } - - let slice = unsafe { std::slice::from_raw_parts_mut(mmap.as_ptr() as *mut u8, mmap.len()) }; - - slice[start..end].copy_from_slice(data); - } - - /// - /// From relative to start - /// - /// Non destructive - /// - pub fn truncate_region(&self, identifier: Identifier, from: u64) -> Result<()> { - let Some(region) = self.regions.read().get_region(identifier.clone()) else { - return Err(Error::Str("Unknown region")); - }; - let mut region_ = region.write(); - let len = region_.len(); - if from == len { - return Ok(()); - } else if from > len { - return Err(Error::Str("Truncating further than length")); - } - region_.set_len(from); - Ok(()) - } - - pub fn remove_region(&self, identifier: Identifier) -> Result>>> { - let mut regions = self.regions.write(); - - let mut layout = self.layout.write(); - - let index_opt = regions.identifier_to_index(identifier.clone()); - - let Some(region) = regions.remove_region(identifier)? else { - return Ok(None); - }; - - let index = index_opt.unwrap(); - - let region_ = region.write(); - - layout.remove_region(index, ®ion_)?; - - drop(region_); - - Ok(Some(region)) - } - - fn create_mmap(file: &fs::File) -> Result { - Ok(unsafe { MmapOptions::new().map_mut(file)? }) - } - - pub fn regions(&self) -> RwLockReadGuard<'_, Regions> { - self.regions.read() - } - - pub fn layout(&self) -> RwLockReadGuard<'_, Layout> { - self.layout.read() - } - - pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> { - self.mmap.read() - } - - fn ceil_number_to_page_size_multiple(num: u64) -> u64 { - (num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1 - } - - fn data_path(&self) -> PathBuf { - Self::data_path_(&self.path) - } - fn data_path_(path: &Path) -> PathBuf { - path.join("data") - } - - pub fn disk_usage(&self) -> String { - let path = self.data_path(); - - let output = std::process::Command::new("du") - .arg("-h") - .arg(&path) - .output() - .expect("Failed to run du"); - - String::from_utf8_lossy(&output.stdout) - .replace(path.to_str().unwrap(), " ") - .trim() - .to_string() - } - - pub fn flush(&self) -> Result<()> { - let mmap = self.mmap.read(); - let regions = self.regions.read(); - mmap.flush()?; - regions.flush() - } - - pub fn flush_then_punch(&self) -> Result<()> { - self.flush()?; - self.punch_holes() - } - - pub fn punch_holes(&self) -> Result<()> { - let file = self.file.write(); - let mut mmap = self.mmap.write(); - let regions = self.regions.read(); - let layout = self.layout.read(); - - let mut punched = regions - .index_to_region() - .par_iter() - .flatten() - .map(|region_lock| -> Result { - let region = region_lock.read(); - let rstart = region.start(); - let len = region.len(); - let reserved = region.reserved(); - let ceil_len = Self::ceil_number_to_page_size_multiple(len); - assert!(len <= ceil_len); - if ceil_len > reserved { - panic!() - } else if ceil_len < reserved { - let start = rstart + ceil_len; - let hole = reserved - ceil_len; - if Self::approx_has_punchable_data(&mmap, start, hole) { - // info!( - // "dbg: {:?}", - // (region, rstart, len, ceil_len, reserved, start, hole) - // ); - // info!("Punching a hole of {hole} bytes at {start}..."); - Self::punch_hole(&file, start, hole)?; - return Ok(1); - } - } - Ok(0) - }) - .sum::>()?; - - punched += layout - .start_to_hole() - .par_iter() - .map(|(&start, &hole)| -> Result { - if Self::approx_has_punchable_data(&mmap, start, hole) { - // info!("dbg: {:?}", (start, hole)); - // info!("Punching a hole of {hole} bytes at {start}..."); - Self::punch_hole(&file, start, hole)?; - Ok(1) - } else { - Ok(0) - } - }) - .sum::>()?; - - if punched > 0 { - // info!("Remaping post hole punching..."); - unsafe { - libc::fsync(file.as_raw_fd()); - } - *mmap = Self::create_mmap(&file)?; - } - - Ok(()) - } - - fn approx_has_punchable_data(mmap: &MmapMut, start: u64, len: u64) -> bool { - assert!(start % PAGE_SIZE == 0); - assert!(len % PAGE_SIZE == 0); - - let min = start as usize; - let max = (start + len) as usize; - let check = |start, end| { - assert!(start >= min); - assert!(end < max); - let start_is_some = mmap[start] != 0; - // if start_is_some { - // info!("mmap[start = {}] = {}", start, mmap[start]) - // } - let end_is_some = mmap[end] != 0; - // if end_is_some { - // info!("mmap[end = {}] = {}", end, mmap[end]) - // } - start_is_some || end_is_some - }; - - // Check first page (first and last byte) - let first_page_start = start as usize; - let first_page_end = (start + PAGE_SIZE - 1) as usize; - if check(first_page_start, first_page_end) { - return true; - } - - // Check last page (first and last byte) - let last_page_start = (start + len - PAGE_SIZE) as usize; - let last_page_end = (start + len - 1) as usize; - if check(last_page_start, last_page_end) { - return true; - } - - // For large lengths, check at 1GB intervals - if len > GB { - let num_gb_checks = (len / GB) as usize; - for i in 1..num_gb_checks { - let gb_boundary = start + (i as u64 * GB); - let page_start = gb_boundary as usize; - let page_end = (gb_boundary + PAGE_SIZE - 1) as usize; - - if check(page_start, page_end) { - return true; - } - } - } - - false - } - - #[cfg(target_os = "macos")] - fn punch_hole(file: &fs::File, start: u64, length: u64) -> Result<()> { - let fpunchhole = FPunchhole { - fp_flags: 0, - reserved: 0, - fp_offset: start as libc::off_t, - fp_length: length as libc::off_t, - }; - - let result = unsafe { - libc::fcntl( - file.as_raw_fd(), - libc::F_PUNCHHOLE, - &fpunchhole as *const FPunchhole, - ) - }; - - if result == -1 { - let err = std::io::Error::last_os_error(); - return Err(Error::String(format!("Failed to punch hole: {err}"))); - } - - Ok(()) - } - - #[cfg(target_os = "linux")] - fn punch_hole(file: &fs::File, start: u64, length: u64) -> Result<()> { - let result = unsafe { - libc::fallocate( - file.as_raw_fd(), - libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE, - start as libc::off_t, - length as libc::off_t, - ) - }; - - if result == -1 { - let err = std::io::Error::last_os_error(); - return Err(Error::String(format!("Failed to punch hole: {err}"))); - } - - Ok(()) - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - fn punch_hole(_file: &fs::File, _start: u64, _length: u64) -> Result<()> { - Err(Error::String( - "Hole punching not supported on this platform".to_string(), - )) - } -} - -#[repr(C)] -struct FPunchhole { - fp_flags: u32, - reserved: u32, - fp_offset: off_t, - fp_length: off_t, -} diff --git a/crates/brk_vecs/src/file/reader.rs b/crates/brk_vecs/src/file/reader.rs deleted file mode 100644 index 62be55c54..000000000 --- a/crates/brk_vecs/src/file/reader.rs +++ /dev/null @@ -1,39 +0,0 @@ -use memmap2::MmapMut; -use parking_lot::RwLockReadGuard; - -use super::Region; - -#[derive(Debug)] -pub struct Reader<'a> { - mmap: RwLockReadGuard<'a, MmapMut>, - region: RwLockReadGuard<'static, Region>, -} - -impl<'a> Reader<'a> { - pub fn new( - mmap: RwLockReadGuard<'a, MmapMut>, - region: RwLockReadGuard<'static, Region>, - ) -> Self { - Self { mmap, region } - } - - pub fn read(&self, offset: u64, len: u64) -> &[u8] { - assert!(offset + len <= self.region.len()); - let start = self.region.start() + offset; - let end = start + len; - &self.mmap[start as usize..end as usize] - } - - pub fn read_all(&self) -> &[u8] { - self.read(0, self.region().len()) - } - - pub fn region(&self) -> &Region { - &self.region - } - - pub fn prefixed(&self, offset: u64) -> &[u8] { - let start = self.region.start() + offset; - &self.mmap[start as usize..] - } -} diff --git a/crates/brk_vecs/src/file/region.rs b/crates/brk_vecs/src/file/region.rs deleted file mode 100644 index 552003b79..000000000 --- a/crates/brk_vecs/src/file/region.rs +++ /dev/null @@ -1,81 +0,0 @@ -use memmap2::MmapMut; -use parking_lot::RwLockReadGuard; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use super::{File, PAGE_SIZE, Reader}; - -#[derive(Debug, Clone, FromBytes, IntoBytes, Immutable, KnownLayout)] -#[repr(C)] -pub struct Region { - /// Must be multiple of 4096 - start: u64, - len: u64, - /// Must be multiple of 4096, greater or equal to len - reserved: u64, - padding: u64, -} - -pub const SIZE_OF_REGION: usize = size_of::(); - -impl Region { - pub fn new(start: u64, len: u64, reserved: u64) -> Self { - assert!(start % PAGE_SIZE == 0); - assert!(reserved >= PAGE_SIZE); - assert!(reserved % PAGE_SIZE == 0); - assert!(len <= reserved); - - Self { - start, - len, - reserved, - padding: 0, - } - } - - pub fn start(&self) -> u64 { - self.start - } - - pub fn set_start(&mut self, start: u64) { - assert!(start % PAGE_SIZE == 0); - self.start = start - } - - pub fn len(&self) -> u64 { - self.len - } - - pub fn set_len(&mut self, len: u64) { - assert!(len <= self.reserved()); - self.len = len - } - - pub fn reserved(&self) -> u64 { - self.reserved - } - - pub fn set_reserved(&mut self, reserved: u64) { - assert!(self.len() <= reserved); - assert!(reserved >= PAGE_SIZE); - assert!(reserved % PAGE_SIZE == 0); - - self.reserved = reserved; - } - - pub fn left(&self) -> u64 { - self.reserved - self.len - } -} - -pub trait RegionReader { - fn create_reader(self, file: &'_ File) -> Reader<'_>; -} - -impl<'a> RegionReader for RwLockReadGuard<'a, Region> { - fn create_reader(self, file: &File) -> Reader<'static> { - let region: RwLockReadGuard<'static, Region> = unsafe { std::mem::transmute(self) }; - let mmap: RwLockReadGuard<'static, MmapMut> = - unsafe { std::mem::transmute(file.mmap.read()) }; - Reader::new(mmap, region) - } -} diff --git a/crates/brk_vecs/src/file/regions.rs b/crates/brk_vecs/src/file/regions.rs deleted file mode 100644 index 3dd5edddb..000000000 --- a/crates/brk_vecs/src/file/regions.rs +++ /dev/null @@ -1,277 +0,0 @@ -use std::{ - collections::HashMap, - fs::{self, OpenOptions}, - io::{Cursor, Read}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use memmap2::MmapMut; -use parking_lot::{RwLock, RwLockWriteGuard}; -use zerocopy::{FromBytes, IntoBytes}; - -use crate::{Error, Result}; - -use super::{ - Identifier, PAGE_SIZE, - region::{Region, SIZE_OF_REGION}, -}; - -#[derive(Debug)] -pub struct Regions { - id_to_index: HashMap, - id_to_index_path: PathBuf, - index_to_region: Vec>>>, - index_to_region_file: fs::File, - index_to_region_mmap: MmapMut, -} - -impl Regions { - pub fn open(path: &Path) -> Result { - let path = path.join("regions"); - - fs::create_dir_all(&path)?; - - let id_to_index_path = path.join("id_to_index"); - - let id_to_index: HashMap = - Self::deserialize(&fs::read(&id_to_index_path).unwrap_or_default()).unwrap_or_default(); - - let index_to_region_file = OpenOptions::new() - .read(true) - .create(true) - .write(true) - .truncate(false) - .open(path.join("index_to_region"))?; - index_to_region_file.try_lock()?; - - let index_to_region_mmap = unsafe { MmapMut::map_mut(&index_to_region_file)? }; - - let mut index_to_region: Vec>>> = vec![]; - - id_to_index - .iter() - .try_for_each(|(_, &index)| -> Result<()> { - let start = index * SIZE_OF_REGION; - let end = start + SIZE_OF_REGION; - let region = Region::read_from_bytes(&index_to_region_mmap[start..end])?; - if index_to_region.len() < index + 1 { - index_to_region.resize_with(index + 1, Default::default); - } - index_to_region - .get_mut(index) - .unwrap() - .replace(Arc::new(RwLock::new(region))); - Ok(()) - })?; - - // TODO: Removes Nones from vec if needed, update map accordingly and save them - - Ok(Self { - id_to_index, - id_to_index_path, - index_to_region, - index_to_region_file, - index_to_region_mmap, - }) - } - - pub fn set_min_len(&mut self, len: u64) -> Result<()> { - if self.index_to_region_mmap.len() < len as usize { - self.index_to_region_file.set_len(len)?; - self.index_to_region_mmap = unsafe { MmapMut::map_mut(&self.index_to_region_file)? }; - } - Ok(()) - } - - pub fn create_region( - &mut self, - id: String, - start: u64, - ) -> Result<(usize, Arc>)> { - let index = self - .index_to_region - .iter() - .enumerate() - .find(|(_, opt)| opt.is_none()) - .map(|(index, _)| index) - .unwrap_or_else(|| self.index_to_region.len()); - - let region = Region::new(start, 0, PAGE_SIZE); - - self.set_min_len(((index + 1) * SIZE_OF_REGION) as u64)?; - - let region_lock = RwLock::new(region); - - self.write_to_mmap(®ion_lock.write(), index); - - let region_arc = Arc::new(region_lock); - - let region_opt = Some(region_arc.clone()); - if index < self.index_to_region.len() { - self.index_to_region[index] = region_opt - } else { - self.index_to_region.push(region_opt); - self.index_to_region_mmap.flush()?; - } - - if self.id_to_index.insert(id, index).is_some() { - return Err(Error::Str("Already exists")); - } - self.flush_id_to_index()?; - - Ok((index, region_arc)) - } - - fn flush_id_to_index(&mut self) -> Result<()> { - fs::write(&self.id_to_index_path, Self::serialize(&self.id_to_index))?; - Ok(()) - } - - #[inline] - pub fn get_region(&self, identifier: Identifier) -> Option>> { - match identifier { - Identifier::Number(index) => self.get_region_from_index(index), - Identifier::String(id) => self.get_region_from_id(&id), - } - } - - #[inline] - pub fn get_region_from_index(&self, index: usize) -> Option>> { - self.index_to_region.get(index).cloned().flatten() - } - - #[inline] - pub fn get_region_from_id(&self, id: &str) -> Option>> { - self.get_region_index_from_id(id) - .and_then(|index| self.get_region_from_index(index)) - } - - #[inline] - pub fn get_region_index_from_id(&self, id: &str) -> Option { - self.id_to_index.get(id).copied() - } - - fn find_id_from_index(&self, index: usize) -> Option<&String> { - Some( - self.id_to_index - .iter() - .find(|(_, v)| **v == index) - .unwrap() - .0, - ) - } - - #[inline] - pub fn index_to_region(&self) -> &[Option>>] { - &self.index_to_region - } - - #[inline] - pub fn id_to_index(&self) -> &HashMap { - &self.id_to_index - } - - #[inline] - pub fn identifier_to_index(&self, identifier: Identifier) -> Option { - match identifier { - Identifier::Number(index) => Some(index), - Identifier::String(id) => self.get_region_index_from_id(&id), - } - } - - pub fn remove_region(&mut self, identifier: Identifier) -> Result>>> { - match identifier { - Identifier::Number(index) => self.remove_region_from_index(index), - Identifier::String(id) => self.remove_region_from_id(&id), - } - } - - pub fn remove_region_from_id(&mut self, id: &str) -> Result>>> { - let Some(index) = self.get_region_index_from_id(id) else { - return Ok(None); - }; - self.remove_region_from_index(index) - } - - pub fn remove_region_from_index( - &mut self, - index: usize, - ) -> Result>>> { - let Some(region) = self.index_to_region.get_mut(index).and_then(Option::take) else { - return Ok(None); - }; - - self.index_to_region_mmap.flush()?; - - self.id_to_index - .remove(&self.find_id_from_index(index).unwrap().to_owned()); - - self.flush_id_to_index()?; - - Ok(Some(region)) - } - - pub fn write_to_mmap(&self, region: &RwLockWriteGuard, index: usize) { - let mmap = &self.index_to_region_mmap; - let start = index * SIZE_OF_REGION; - let end = start + SIZE_OF_REGION; - - if end > mmap.len() { - unreachable!("Trying to write beyond mmap") - } - - let slice = unsafe { std::slice::from_raw_parts_mut(mmap.as_ptr() as *mut u8, mmap.len()) }; - - slice[start..end].copy_from_slice(region.as_bytes()); - } - - pub fn flush(&self) -> Result<()> { - self.index_to_region_mmap.flush().map_err(|e| e.into()) - } - - fn serialize(map: &HashMap) -> Vec { - let mut buffer = Vec::new(); - - buffer.extend_from_slice(&map.len().to_ne_bytes()); - - for (key, value) in map { - buffer.extend_from_slice(key.len().as_bytes()); - buffer.extend_from_slice(key.as_bytes()); - buffer.extend_from_slice(value.as_bytes()); - } - - buffer - } - - fn deserialize(data: &[u8]) -> Result> { - let mut cursor = Cursor::new(data); - let mut buffer = [0u8; 8]; - - cursor - .read_exact(&mut buffer) - .map_err(|_| Error::Str("Failed to read entry count"))?; - let entry_count = usize::read_from_bytes(&buffer)?; - - let mut map = HashMap::with_capacity(entry_count); - - for _ in 0..entry_count { - cursor - .read_exact(&mut buffer) - .map_err(|_| Error::Str("Failed to read key length"))?; - let key_len = usize::read_from_bytes(&buffer)?; - - let mut key_bytes = vec![0u8; key_len]; - cursor.read_exact(&mut key_bytes)?; - let key = - String::from_utf8(key_bytes).map_err(|_| Error::Str("Invalid UTF-8 in key"))?; - - cursor.read_exact(&mut buffer)?; - let value = usize::read_from_bytes(&buffer)?; - - map.insert(key, value); - } - - Ok(map) - } -} diff --git a/crates/brk_vecs/src/lib.rs b/crates/brk_vecs/src/lib.rs deleted file mode 100644 index a23f34d6c..000000000 --- a/crates/brk_vecs/src/lib.rs +++ /dev/null @@ -1,32 +0,0 @@ -#![doc = include_str!("../README.md")] -#![doc = "\n## Examples\n"] -#![doc = "\n### File\n\n```rust"] -#![doc = include_str!("../examples/file.rs")] -#![doc = "```\n"] -#![doc = "\n### Raw\n\n```rust"] -#![doc = include_str!("../examples/raw.rs")] -#![doc = "```"] - -mod error; -mod exit; -mod file; -mod stamp; -mod traits; -mod variants; -mod version; - -pub use brk_vecs_macros::StoredCompressed; -pub use pco::data_types::LatentType; - -use variants::*; - -pub use error::*; -pub use exit::*; -pub use file::{File, PAGE_SIZE, Reader}; -pub use stamp::*; -pub use traits::*; -pub use variants::{ - CompressedVec, Computation, ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, - EagerVec, Format, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, StoredVec, -}; -pub use version::*; diff --git a/crates/brk_vecs/src/stamp.rs b/crates/brk_vecs/src/stamp.rs deleted file mode 100644 index e31fa6a5c..000000000 --- a/crates/brk_vecs/src/stamp.rs +++ /dev/null @@ -1,24 +0,0 @@ -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -#[derive( - Debug, Default, Clone, Copy, PartialEq, Eq, FromBytes, IntoBytes, Immutable, KnownLayout, -)] -pub struct Stamp(u64); - -impl Stamp { - pub fn new(stamp: u64) -> Self { - Self(stamp) - } -} - -impl From for Stamp { - fn from(value: u64) -> Self { - Self(value) - } -} - -impl From for u64 { - fn from(value: Stamp) -> Self { - value.0 - } -} diff --git a/crates/brk_vecs/src/traits/any.rs b/crates/brk_vecs/src/traits/any.rs deleted file mode 100644 index 7c8498c64..000000000 --- a/crates/brk_vecs/src/traits/any.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::{Stamp, Version}; - -pub fn i64_to_usize(i: i64, len: usize) -> usize { - if i >= 0 { - (i as usize).min(len) - } else { - let v = len as i64 + i; - if v < 0 { 0 } else { v as usize } - } -} - -pub trait AnyVec: Send + Sync { - fn version(&self) -> Version; - fn name(&self) -> &str; - fn len(&self) -> usize; - fn is_empty(&self) -> bool { - self.len() == 0 - } - fn index_type_to_string(&self) -> &'static str; - fn value_type_to_size_of(&self) -> usize; - fn etag(&self, stamp: Stamp, to: Option) -> String { - let len = self.len(); - format!( - "{}-{}-{}", - to.map_or(len, |to| { - if to.is_negative() { - len.checked_sub(to.unsigned_abs() as usize) - .unwrap_or_default() - } else { - to as usize - } - }), - u64::from(self.version()), - u64::from(stamp), - ) - } - - #[inline] - fn i64_to_usize(&self, i: i64) -> usize { - let len = self.len(); - i64_to_usize(i, len) - } -} diff --git a/crates/brk_vecs/src/traits/checked_sub.rs b/crates/brk_vecs/src/traits/checked_sub.rs deleted file mode 100644 index 857d82881..000000000 --- a/crates/brk_vecs/src/traits/checked_sub.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub trait CheckedSub: Sized { - fn checked_sub(self, rhs: Rhs) -> Option; -} diff --git a/crates/brk_vecs/src/traits/collectable.rs b/crates/brk_vecs/src/traits/collectable.rs deleted file mode 100644 index bdde9acdf..000000000 --- a/crates/brk_vecs/src/traits/collectable.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::{Error, Result, i64_to_usize}; - -use super::{AnyIterableVec, AnyVec, StoredIndex, StoredRaw}; - -pub trait CollectableVec: AnyVec + AnyIterableVec -where - Self: Clone, - I: StoredIndex, - T: StoredRaw, -{ - fn collect(&self) -> Result> { - self.collect_range(None, None) - } - - fn collect_range(&self, from: Option, to: Option) -> Result> { - let len = self.len(); - let from = from.unwrap_or_default(); - let to = to.map_or(len, |to| to.min(len)); - - if from >= len || from >= to { - return Ok(vec![]); - } - - Ok(self - .iter_at_(from) - .take(to - from) - .map(|(_, v)| v.into_owned()) - .collect::>()) - } - - #[inline] - fn i64_to_usize_(i: i64, len: usize) -> usize { - if i >= 0 { - (i as usize).min(len) - } else { - let v = len as i64 + i; - if v < 0 { 0 } else { v as usize } - } - } - - fn collect_signed_range(&self, from: Option, to: Option) -> Result> { - let from = from.map(|i| self.i64_to_usize(i)); - let to = to.map(|i| self.i64_to_usize(i)); - self.collect_range(from, to) - } - - #[inline] - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - self.collect_range(from, to)? - .into_iter() - .map(|v| serde_json::to_value(v).map_err(Error::from)) - .collect::>>() - } -} - -impl CollectableVec for V -where - V: AnyVec + AnyIterableVec + Clone, - I: StoredIndex, - T: StoredRaw, -{ -} - -pub trait AnyCollectableVec: AnyVec { - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result>; - - fn range_count(&self, from: Option, to: Option) -> usize { - let len = self.len(); - let from = from.map(|i| i64_to_usize(i, len)); - let to = to.map(|i| i64_to_usize(i, len)); - (from.unwrap_or_default()..to.unwrap_or(len)).count() - } - - fn range_weight(&self, from: Option, to: Option) -> usize { - self.range_count(from, to) * self.value_type_to_size_of() - } -} diff --git a/crates/brk_vecs/src/traits/compressed.rs b/crates/brk_vecs/src/traits/compressed.rs deleted file mode 100644 index 4993c3a9f..000000000 --- a/crates/brk_vecs/src/traits/compressed.rs +++ /dev/null @@ -1,98 +0,0 @@ -use pco::data_types::Number; - -use super::StoredRaw; - -pub trait TransparentStoredCompressed {} - -pub trait StoredCompressed -where - Self: StoredRaw + Copy + 'static + TransparentStoredCompressed, -{ - type NumberType: pco::data_types::Number; -} - -pub trait AsInnerSlice -where - T: Number, -{ - fn as_inner_slice(&self) -> &[T]; -} - -impl AsInnerSlice for [T] -where - T: StoredCompressed, -{ - fn as_inner_slice(&self) -> &[T::NumberType] { - assert_eq!( - std::mem::size_of::(), - std::mem::size_of::() - ); - assert_eq!( - std::mem::align_of::(), - std::mem::align_of::() - ); - unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T::NumberType, self.len()) } - } -} - -pub trait FromInnerSlice { - fn from_inner_slice(slice: Vec) -> Vec - where - Self: Sized; -} - -impl FromInnerSlice for T -where - T: StoredCompressed, -{ - fn from_inner_slice(vec: Vec) -> Vec { - assert_eq!( - std::mem::size_of::(), - std::mem::size_of::() - ); - assert_eq!( - std::mem::align_of::(), - std::mem::align_of::() - ); - - let mut vec = std::mem::ManuallyDrop::new(vec); - unsafe { Vec::from_raw_parts(vec.as_mut_ptr() as *mut T, vec.len(), vec.capacity()) } - } -} - -impl TransparentStoredCompressed for u16 {} -impl StoredCompressed for u16 { - type NumberType = u16; -} -impl TransparentStoredCompressed for u32 {} -impl StoredCompressed for u32 { - type NumberType = u32; -} -impl TransparentStoredCompressed for u64 {} -impl StoredCompressed for u64 { - type NumberType = u64; -} -impl TransparentStoredCompressed for i16 {} -impl StoredCompressed for i16 { - type NumberType = i16; -} -impl TransparentStoredCompressed for i32 {} -impl StoredCompressed for i32 { - type NumberType = i32; -} -impl TransparentStoredCompressed for i64 {} -impl StoredCompressed for i64 { - type NumberType = i64; -} -impl TransparentStoredCompressed for f32 {} -impl StoredCompressed for f32 { - type NumberType = f32; -} -impl TransparentStoredCompressed for f64 {} -impl StoredCompressed for f64 { - type NumberType = f64; -} -impl TransparentStoredCompressed for () {} -impl StoredCompressed for () { - type NumberType = u16; -} diff --git a/crates/brk_vecs/src/traits/from_coarser.rs b/crates/brk_vecs/src/traits/from_coarser.rs deleted file mode 100644 index 93c1f697e..000000000 --- a/crates/brk_vecs/src/traits/from_coarser.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::ops::RangeInclusive; - -pub trait FromCoarserIndex -where - T: Ord + From, -{ - fn min_from(coarser: T) -> usize; - fn max_from_(coarser: T) -> usize; - fn max_from(coarser: T, len: usize) -> usize { - Self::max_from_(coarser).min(len - 1) - } - fn inclusive_range_from(coarser: T, len: usize) -> RangeInclusive - where - T: Clone, - { - Self::min_from(coarser.clone())..=Self::max_from(coarser, len) - } -} diff --git a/crates/brk_vecs/src/traits/generic.rs b/crates/brk_vecs/src/traits/generic.rs deleted file mode 100644 index 05f9e2976..000000000 --- a/crates/brk_vecs/src/traits/generic.rs +++ /dev/null @@ -1,290 +0,0 @@ -use std::{ - borrow::Cow, - cmp::Ordering, - collections::{BTreeMap, BTreeSet}, -}; - -use crate::{AnyStoredVec, Error, Exit, HEADER_OFFSET, Result, Stamp, file::Reader}; - -const ONE_KIB: usize = 1024; -const ONE_MIB: usize = ONE_KIB * ONE_KIB; -const MAX_CACHE_SIZE: usize = 256 * ONE_MIB; - -use super::{StoredIndex, StoredRaw}; - -pub trait GenericStoredVec: Send + Sync -where - Self: AnyStoredVec, - I: StoredIndex, - T: StoredRaw, -{ - const SIZE_OF_T: usize = size_of::(); - - /// - /// Be careful with deadlocks - /// - /// You'll want to drop the reader before mutable ops - /// - fn create_reader(&'_ self) -> Reader<'_> { - self.create_static_reader() - } - - /// - /// Be careful with deadlocks - /// - /// You'll want to drop the reader before mutable ops - /// - fn create_static_reader(&self) -> Reader<'static> { - unsafe { - std::mem::transmute( - self.file() - .create_region_reader(self.region_index().into()) - .unwrap(), - ) - } - } - - #[inline] - fn unwrap_read(&self, index: I, reader: &Reader) -> T { - self.read(index, reader).unwrap() - } - #[inline] - fn read(&self, index: I, reader: &Reader) -> Result { - self.read_(index.to_usize()?, reader) - } - fn read_(&self, index: usize, reader: &Reader) -> Result; - - #[inline] - fn get_or_read(&'_ self, index: I, reader: &Reader) -> Result>> { - self.get_or_read_(index.to_usize()?, reader) - } - #[inline] - fn get_or_read_(&'_ self, index: usize, reader: &Reader) -> Result>> { - let stored_len = self.stored_len(); - - let holes = self.holes(); - if !holes.is_empty() && holes.contains(&index) { - return Ok(None); - } - - if index >= stored_len { - let pushed = self.pushed(); - let j = index - stored_len; - if j >= pushed.len() { - return Ok(None); - } - return Ok(pushed.get(j).map(Cow::Borrowed)); - } - - let updated = self.updated(); - if !updated.is_empty() - && let Some(updated) = updated.get(&index) - { - return Ok(Some(Cow::Borrowed(updated))); - } - - Ok(Some(Cow::Owned(self.read_(index, reader)?))) - } - - #[inline] - fn len_(&self) -> usize { - self.stored_len() + self.pushed_len() - } - - fn pushed(&self) -> &[T]; - #[inline] - fn pushed_len(&self) -> usize { - self.pushed().len() - } - fn mut_pushed(&mut self) -> &mut Vec; - #[inline] - fn push(&mut self, value: T) { - self.mut_pushed().push(value) - } - - #[inline] - fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { - let len = self.len(); - match len.cmp(&index.to_usize()?) { - Ordering::Greater => { - // dbg!(len, index, &self.pathbuf); - // panic!(); - Ok(()) - } - Ordering::Equal => { - self.push(value); - Ok(()) - } - Ordering::Less => { - dbg!(index, value, len, self.header(), self.region_index()); - Err(Error::IndexTooHigh) - } - } - } - - #[inline] - fn forced_push_at(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { - match self.len().cmp(&index.to_usize()?) { - Ordering::Less => { - return Err(Error::IndexTooHigh); - } - ord => { - if ord == Ordering::Greater { - self.safe_truncate_if_needed(index, exit)?; - } - self.push(value); - } - } - - let pushed_bytes = self.pushed_len() * Self::SIZE_OF_T; - if pushed_bytes >= MAX_CACHE_SIZE { - // info!("pushed_bytes ({pushed_bytes}) >= MAX_CACHE_SIZE ({MAX_CACHE_SIZE})"); - self.safe_flush(exit)?; - } - - Ok(()) - } - - #[inline] - fn update_or_push(&mut self, index: I, value: T) -> Result<()> { - let len = self.len(); - match len.cmp(&index.to_usize()?) { - Ordering::Less => { - dbg!(index, value, len, self.header()); - Err(Error::IndexTooHigh) - } - Ordering::Equal => { - self.push(value); - Ok(()) - } - Ordering::Greater => self.update(index, value), - } - } - - fn get_first_empty_index(&self) -> I { - self.holes() - .first() - .cloned() - .unwrap_or_else(|| self.len_()) - .into() - } - - #[inline] - fn fill_first_hole_or_push(&mut self, value: T) -> Result { - Ok( - if let Some(hole) = self.mut_holes().pop_first().map(I::from) { - self.update(hole, value)?; - hole - } else { - self.push(value); - I::from(self.len() - 1) - }, - ) - } - - fn holes(&self) -> &BTreeSet; - fn mut_holes(&mut self) -> &mut BTreeSet; - fn take(&mut self, index: I, reader: &Reader) -> Result> { - let opt = self.get_or_read(index, reader)?.map(|v| v.into_owned()); - if opt.is_some() { - self.unchecked_delete(index); - } - Ok(opt) - } - #[inline] - fn delete(&mut self, index: I) { - if index.unwrap_to_usize() < self.len() { - self.unchecked_delete(index); - } - } - #[inline] - #[doc(hidden)] - fn unchecked_delete(&mut self, index: I) { - let uindex = index.unwrap_to_usize(); - let updated = self.mut_updated(); - if !updated.is_empty() { - updated.remove(&uindex); - } - self.mut_holes().insert(uindex); - } - - fn updated(&self) -> &BTreeMap; - fn mut_updated(&mut self) -> &mut BTreeMap; - #[inline] - fn update(&mut self, index: I, value: T) -> Result<()> { - let uindex = index.unwrap_to_usize(); - let stored_len = self.stored_len(); - - if uindex >= stored_len { - if let Some(prev) = self.mut_pushed().get_mut(uindex - stored_len) { - *prev = value; - return Ok(()); - } else { - return Err(Error::IndexTooHigh); - } - } - - let holes = self.mut_holes(); - if !holes.is_empty() { - holes.remove(&index.unwrap_to_usize()); - } - - self.mut_updated().insert(index.unwrap_to_usize(), value); - - Ok(()) - } - - fn reset(&mut self) -> Result<()>; - - #[inline] - fn reset_(&mut self) -> Result<()> { - self.file().remove_region(self.holes_region_name().into())?; - self.file() - .truncate_region(self.region_index().into(), HEADER_OFFSET as u64) - } - - #[inline] - fn is_pushed_empty(&self) -> bool { - self.pushed_len() == 0 - } - - #[inline] - fn has(&self, index: I) -> Result { - Ok(self.has_(index.to_usize()?)) - } - #[inline] - fn has_(&self, index: usize) -> bool { - index < self.len_() - } - - fn truncate_if_needed(&mut self, index: I) -> Result<()>; - - fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> { - let _lock = exit.lock(); - self.truncate_if_needed(index) - } - - #[inline] - fn truncate_if_needed_with_stamp(&mut self, index: I, stamp: Stamp) -> Result<()> { - self.update_stamp(stamp); - self.truncate_if_needed(index) - } - - fn index_to_name(&self) -> String { - format!("{}_to_{}", I::to_string(), self.name()) - } - - fn vec_region_name(&self) -> String { - Self::vec_region_name_(self.name()) - } - fn vec_region_name_(name: &str) -> String { - format!("{}_to_{name}", I::to_string()) - } - - fn holes_region_name(&self) -> String { - Self::holes_region_name_(self.name()) - } - fn holes_region_name_(name: &str) -> String { - format!("{}_holes", Self::vec_region_name_(name)) - } -} diff --git a/crates/brk_vecs/src/traits/index.rs b/crates/brk_vecs/src/traits/index.rs deleted file mode 100644 index 7d6c00955..000000000 --- a/crates/brk_vecs/src/traits/index.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::{fmt::Debug, ops::Add}; - -use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; - -use crate::{Error, Printable, Result}; - -pub trait StoredIndex -where - Self: Debug - + Default - + Copy - + Clone - + PartialEq - + Eq - + PartialOrd - + Ord - + From - + TryInto - + Add - + TryFromBytes - + IntoBytes - + Immutable - + KnownLayout - + Send - + Sync - + Printable, -{ - fn unwrap_to_usize(self) -> usize; - fn to_usize(self) -> Result; - fn decremented(self) -> Option; -} - -impl StoredIndex for I -where - I: Debug - + Default - + Copy - + Clone - + PartialEq - + Eq - + PartialOrd - + Ord - + From - + TryInto - + Add - + TryFromBytes - + IntoBytes - + Immutable - + KnownLayout - + Send - + Sync - + Printable, -{ - #[inline] - fn unwrap_to_usize(self) -> usize { - self.to_usize().unwrap() - } - - #[inline] - fn to_usize(self) -> Result { - self.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) - } - - #[inline] - fn decremented(self) -> Option { - self.unwrap_to_usize().checked_sub(1).map(Self::from) - } -} diff --git a/crates/brk_vecs/src/traits/iterable.rs b/crates/brk_vecs/src/traits/iterable.rs deleted file mode 100644 index b04555b03..000000000 --- a/crates/brk_vecs/src/traits/iterable.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::{AnyStoredVec, AnyVec, BoxedVecIterator, StoredIndex, StoredRaw}; - -pub trait AnyIterableVec: AnyVec { - #[allow(clippy::wrong_self_convention)] - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredRaw + 'a; - - fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredRaw + 'a, - { - self.boxed_iter() - } - - fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredRaw + 'a, - { - let mut iter = self.boxed_iter(); - iter.set(i); - iter - } - - fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredRaw + 'a, - { - let mut iter = self.boxed_iter(); - iter.set_(i); - iter - } -} - -pub trait AnyStoredIterableVec: AnyIterableVec + AnyStoredVec {} - -impl AnyStoredIterableVec for U where U: 'static + AnyIterableVec + AnyStoredVec -{} - -pub trait AnyCloneableIterableVec: AnyIterableVec { - fn boxed_clone(&self) -> Box>; -} - -impl AnyCloneableIterableVec for U -where - U: 'static + AnyIterableVec + Clone, -{ - fn boxed_clone(&self) -> Box> { - Box::new(self.clone()) - } -} - -impl Clone for Box> { - fn clone(&self) -> Self { - self.boxed_clone() - } -} - -pub type AnyBoxedIterableVec = Box>; diff --git a/crates/brk_vecs/src/traits/iterator.rs b/crates/brk_vecs/src/traits/iterator.rs deleted file mode 100644 index a33ae2845..000000000 --- a/crates/brk_vecs/src/traits/iterator.rs +++ /dev/null @@ -1,103 +0,0 @@ -use std::{borrow::Cow, iter::Skip}; - -use super::{Printable, StoredIndex, StoredRaw}; - -pub trait BaseVecIterator: Iterator { - fn mut_index(&mut self) -> &mut usize; - - #[inline] - fn set_(&mut self, i: usize) { - *self.mut_index() = i; - } - - #[inline] - fn next_at(&mut self, i: usize) -> Option { - self.set_(i); - self.next() - } - - fn len(&self) -> usize; - - fn name(&self) -> &str; - - fn is_empty(&self) -> bool { - self.len() == 0 - } - - fn skip(self, _: usize) -> Skip - where - Self: Sized, - { - todo!("") - } -} - -pub trait VecIterator<'a>: BaseVecIterator)> { - type I: StoredIndex; - type T: StoredRaw + 'a; - - #[inline] - fn set(&mut self, i: Self::I) { - self.set_(i.unwrap_to_usize()) - } - - #[inline] - fn get_(&mut self, i: usize) -> Option> { - self.next_at(i).map(|(_, v)| v) - } - - #[inline] - fn get(&mut self, i: Self::I) -> Option> { - self.get_(i.unwrap_to_usize()) - } - - #[inline] - fn unwrap_get_inner(&mut self, i: Self::I) -> Self::T { - self.unwrap_get_inner_(i.unwrap_to_usize()) - } - - #[inline] - fn unwrap_get_inner_(&mut self, i: usize) -> Self::T { - self.get_(i) - .unwrap_or_else(|| { - dbg!(self.name(), i, self.len(), std::any::type_name::()); - panic!("unwrap_get_inner_") - }) - .into_owned() - } - - #[inline] - fn get_inner(&mut self, i: Self::I) -> Option { - self.get_(i.unwrap_to_usize()).map(|v| v.into_owned()) - } - - fn last(mut self) -> Option - where - Self: Sized, - { - let len = self.len(); - if len == 0 { - return None; - } - let i = len - 1; - self.set_(i); - self.next() - } - - fn index_type_to_string(&self) -> &'static str { - Self::I::to_string() - } -} - -impl<'a, I, T, Iter> VecIterator<'a> for Iter -where - Iter: BaseVecIterator)>, - I: StoredIndex, - T: StoredRaw + 'a, -{ - type I = I; - type T = T; -} - -pub type BoxedVecIterator<'a, I, T> = - Box)> + 'a>; diff --git a/crates/brk_vecs/src/traits/mod.rs b/crates/brk_vecs/src/traits/mod.rs deleted file mode 100644 index 900aef8bf..000000000 --- a/crates/brk_vecs/src/traits/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod any; -mod checked_sub; -mod collectable; -mod compressed; -mod from_coarser; -mod generic; -mod index; -mod iterable; -mod iterator; -mod printable; -mod raw; -mod stored; - -pub use any::*; -pub use checked_sub::*; -pub use collectable::*; -pub use compressed::*; -pub use from_coarser::*; -pub use generic::*; -pub use index::*; -pub use iterable::*; -pub use iterator::*; -pub use printable::*; -pub use raw::*; -pub use stored::*; diff --git a/crates/brk_vecs/src/traits/printable.rs b/crates/brk_vecs/src/traits/printable.rs deleted file mode 100644 index b2f405821..000000000 --- a/crates/brk_vecs/src/traits/printable.rs +++ /dev/null @@ -1,14 +0,0 @@ -pub trait Printable { - fn to_string() -> &'static str; - fn to_possible_strings() -> &'static [&'static str]; -} - -impl Printable for usize { - fn to_string() -> &'static str { - "usize" - } - - fn to_possible_strings() -> &'static [&'static str] { - &["usize"] - } -} diff --git a/crates/brk_vecs/src/traits/raw.rs b/crates/brk_vecs/src/traits/raw.rs deleted file mode 100644 index f2f9447a4..000000000 --- a/crates/brk_vecs/src/traits/raw.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::fmt::Debug; - -use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -pub trait StoredRaw -where - Self: Sized - + Debug - + Clone - + FromBytes - + IntoBytes - + Immutable - + KnownLayout - + Send - + Sync - + Serialize, -{ -} - -impl StoredRaw for T where - T: Sized - + Debug - + Clone - + FromBytes - + IntoBytes - + Immutable - + KnownLayout - + Send - + Sync - + Serialize -{ -} diff --git a/crates/brk_vecs/src/traits/stored.rs b/crates/brk_vecs/src/traits/stored.rs deleted file mode 100644 index 55004e770..000000000 --- a/crates/brk_vecs/src/traits/stored.rs +++ /dev/null @@ -1,40 +0,0 @@ -use parking_lot::RwLock; - -use crate::{AnyVec, Exit, File, Result, Stamp, file::Region, variants::Header}; - -pub trait AnyStoredVec: AnyVec { - fn file(&self) -> &File; - - fn region_index(&self) -> usize; - - fn region(&self) -> &RwLock; - - fn header(&self) -> &Header; - - fn mut_header(&mut self) -> &mut Header; - - fn flush(&mut self) -> Result<()>; - - #[inline] - fn safe_flush(&mut self, exit: &Exit) -> Result<()> { - // info!("safe flush {}", self.name()); - let _lock = exit.lock(); - self.flush() - } - - fn stored_len(&self) -> usize; - - fn update_stamp(&mut self, stamp: Stamp) { - self.mut_header().update_stamp(stamp); - } - - fn stamp(&self) -> Stamp { - self.header().stamp() - } - - #[inline] - fn stamped_flush(&mut self, stamp: Stamp) -> Result<()> { - self.update_stamp(stamp); - self.flush() - } -} diff --git a/crates/brk_vecs/src/variants/compressed/mod.rs b/crates/brk_vecs/src/variants/compressed/mod.rs deleted file mode 100644 index 082d9200c..000000000 --- a/crates/brk_vecs/src/variants/compressed/mod.rs +++ /dev/null @@ -1,537 +0,0 @@ -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet}, - mem, - sync::Arc, -}; - -use parking_lot::{RwLock, RwLockReadGuard}; -use rayon::prelude::*; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, AsInnerSlice, BaseVecIterator, - BoxedVecIterator, CollectableVec, Error, File, Format, FromInnerSlice, GenericStoredVec, - HEADER_OFFSET, Header, RawVec, Reader, Result, StoredCompressed, StoredIndex, Version, - file::Region, -}; - -mod page; -mod pages; - -use page::*; -use pages::*; - -const ONE_KIB: usize = 1024; -const MAX_PAGE_SIZE: usize = 16 * ONE_KIB; -const PCO_COMPRESSION_LEVEL: usize = 4; - -const VERSION: Version = Version::ONE; - -#[derive(Debug)] -pub struct CompressedVec { - inner: RawVec, - pages: Arc>, -} - -impl CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; - - /// Same as import but will reset the vec under certain errors, so be careful ! - pub fn forced_import(file: &Arc, name: &str, mut version: Version) -> Result { - version = version + VERSION; - let res = Self::import(file, name, version); - match res { - Err(Error::DifferentCompressionMode) - | Err(Error::WrongEndian) - | Err(Error::WrongLength) - | Err(Error::DifferentVersion { .. }) => { - let _ = file.remove_region(Self::vec_region_name_(name).into()); - let _ = file.remove_region(Self::holes_region_name_(name).into()); - let _ = file.remove_region(Self::pages_region_name_(name).into()); - Self::import(file, name, version) - } - _ => res, - } - } - - pub fn import(file: &Arc, name: &str, version: Version) -> Result { - let inner = RawVec::any_import(file, name, version, Format::Compressed)?; - - let pages = Pages::import(file, &Self::pages_region_name_(name))?; - - Ok(Self { - inner, - pages: Arc::new(RwLock::new(pages)), - }) - } - - fn decode_page(&self, page_index: usize, reader: &Reader) -> Result> { - Self::decode_page_(self.stored_len(), page_index, reader, &self.pages.read()) - } - - fn decode_page_( - stored_len: usize, - page_index: usize, - reader: &Reader, - pages: &Pages, - ) -> Result> { - if Self::page_index_to_index(page_index) >= stored_len { - return Err(Error::IndexTooHigh); - } else if page_index >= pages.len() { - return Err(Error::ExpectVecToHaveIndex); - } - - let page = pages.get(page_index).unwrap(); - let len = page.bytes as u64; - let offset = page.start; - - let slice = reader.read(offset, len); - - let vec: Vec = pco::standalone::simple_decompress(slice)?; - let vec = T::from_inner_slice(vec); - - if vec.len() != page.values as usize { - dbg!((offset, len)); - dbg!(vec); - unreachable!() - } - - Ok(vec) - } - - fn compress_page(chunk: &[T]) -> Vec { - if chunk.len() > Self::PER_PAGE { - panic!(); - } - - pco::standalone::simpler_compress(chunk.as_inner_slice(), PCO_COMPRESSION_LEVEL).unwrap() - } - - #[inline] - fn index_to_page_index(index: usize) -> usize { - index / Self::PER_PAGE - } - - #[inline] - fn page_index_to_index(page_index: usize) -> usize { - page_index * Self::PER_PAGE - } - - #[inline] - pub fn iter(&self) -> CompressedVecIterator<'_, I, T> { - self.into_iter() - } - - #[inline] - pub fn iter_at(&self, i: I) -> CompressedVecIterator<'_, I, T> { - self.iter_at_(i.unwrap_to_usize()) - } - - #[inline] - pub fn iter_at_(&self, i: usize) -> CompressedVecIterator<'_, I, T> { - let mut iter = self.into_iter(); - iter.set_(i); - iter - } - - fn pages_region_name_(name: &str) -> String { - format!("{}_pages", Self::vec_region_name_(name)) - } -} - -impl Clone for CompressedVec { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - pages: self.pages.clone(), - } - } -} - -impl AnyVec for CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn version(&self) -> Version { - self.inner.version() - } - - #[inline] - fn name(&self) -> &str { - self.inner.name() - } - - #[inline] - fn len(&self) -> usize { - self.len_() - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyStoredVec for CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn file(&self) -> &File { - self.inner.file() - } - - fn region(&self) -> &RwLock { - self.inner.region() - } - - fn region_index(&self) -> usize { - self.inner.region_index() - } - - fn header(&self) -> &Header { - self.inner.header() - } - - fn mut_header(&mut self) -> &mut Header { - self.inner.mut_header() - } - - #[inline] - fn stored_len(&self) -> usize { - self.pages.read().stored_len(Self::PER_PAGE) - } - - fn flush(&mut self) -> Result<()> { - self.inner.write_header_if_needed()?; - - if self.is_pushed_empty() { - // info!("Nothing to push {}", self.region_index()); - return Ok(()); - } - - let stored_len = self.stored_len(); - - let mut pages = self.pages.write(); - - let mut starting_page_index = pages.len(); - let mut values = vec![]; - let mut truncate_at = None; - - if stored_len % Self::PER_PAGE != 0 { - assert!(!pages.is_empty()); - - let last_page_index = pages.len() - 1; - - let reader = self.create_reader(); - - values = Self::decode_page_(stored_len, last_page_index, &reader, &pages) - .inspect_err(|_| { - dbg!(( - last_page_index, - &pages, - self.region_index(), - &self.region().read() - )); - }) - .unwrap(); - - let start = pages.pop().unwrap().start; - truncate_at.replace(start); - starting_page_index = last_page_index; - } - - let compressed = values - .into_par_iter() - .chain(mem::take(self.inner.mut_pushed()).into_par_iter()) - .chunks(Self::PER_PAGE) - .map(|chunk| (Self::compress_page(chunk.as_slice()), chunk.len())) - .collect::>(); - - compressed.iter().enumerate().for_each(|(i, (bytes, len))| { - let page_index = starting_page_index + i; - - let start = if page_index != 0 { - let prev = pages.get(page_index - 1).unwrap(); - prev.start + prev.bytes as u64 - } else { - HEADER_OFFSET as u64 - }; - - let page = Page::new(start, bytes.len() as u32, *len as u32); - - pages.checked_push(page_index, page); - }); - - let buf = compressed - .into_iter() - .flat_map(|(v, _)| v) - .collect::>(); - - let file = self.file(); - - if let Some(truncate_at) = truncate_at { - // info!("truncate_write_all_to_region {}", self.region_index()); - file.truncate_write_all_to_region(self.region_index().into(), truncate_at, &buf)?; - } else { - // info!("write_all_to_region {}", self.region_index()); - file.write_all_to_region(self.region_index().into(), &buf)?; - } - - pages.flush(file)?; - - Ok(()) - } -} - -impl GenericStoredVec for CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result { - let page_index = Self::index_to_page_index(index); - let decoded_index = index % Self::PER_PAGE; - Ok(unsafe { - *self - .decode_page(page_index, reader)? - .get_unchecked(decoded_index) - }) - } - - #[inline] - fn pushed(&self) -> &[T] { - self.inner.pushed() - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - self.inner.mut_pushed() - } - #[inline] - fn holes(&self) -> &BTreeSet { - self.inner.holes() - } - #[inline] - fn mut_holes(&mut self) -> &mut BTreeSet { - panic!("unsupported") - } - #[inline] - fn updated(&self) -> &BTreeMap { - self.inner.updated() - } - #[inline] - fn mut_updated(&mut self) -> &mut BTreeMap { - panic!("unsupported") - } - - fn reset(&mut self) -> Result<()> { - let file = self.file(); - self.pages.write().reset(file)?; - self.reset_() - } - - fn truncate_if_needed(&mut self, index: I) -> Result<()> { - let index = index.to_usize()?; - - if index >= self.stored_len() { - return Ok(()); - } - - if index == 0 { - self.reset()?; - return Ok(()); - } - - let stored_len = self.stored_len(); - - let mut pages = self.pages.write(); - - let last_page_index = pages.len() - 1; - - let page_index = Self::index_to_page_index(index); - - let values = Self::decode_page_( - stored_len, - last_page_index, - &self.create_static_reader(), - &pages, - )?; - - let mut buf = vec![]; - - let mut page = pages.truncate(page_index).unwrap(); - - let decoded_index = index % Self::PER_PAGE; - - let from = page.start; - - if decoded_index != 0 { - let chunk = &values[..decoded_index]; - - buf = Self::compress_page(chunk); - - page.values = chunk.len() as u32; - page.bytes = buf.len() as u32; - - pages.checked_push(page_index, page); - } - - let file = self.file(); - - pages.flush(file)?; - - file.truncate_write_all_to_region(self.region_index().into(), from, &buf)?; - - Ok(()) - } -} - -#[derive(Debug)] -pub struct CompressedVecIterator<'a, I, T> { - vec: &'a CompressedVec, - reader: Reader<'a>, - decoded: Option<(usize, Vec)>, - pages: RwLockReadGuard<'a, Pages>, - stored_len: usize, - index: usize, -} - -impl CompressedVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredCompressed, -{ - const SIZE_OF_T: usize = size_of::(); - const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; -} - -impl BaseVecIterator for CompressedVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - &mut self.index - } - - #[inline] - fn len(&self) -> usize { - self.vec.len() - } - - #[inline] - fn name(&self) -> &str { - self.vec.name() - } -} - -impl<'a, I, T> Iterator for CompressedVecIterator<'a, I, T> -where - I: StoredIndex, - T: StoredCompressed, -{ - type Item = (I, Cow<'a, T>); - - fn next(&mut self) -> Option { - let i = self.index; - let stored_len = self.stored_len; - - let result = if i >= stored_len { - let j = i - stored_len; - if j >= self.vec.pushed_len() { - return None; - } - self.vec - .pushed() - .get(j) - .map(|v| (I::from(i), Cow::Borrowed(v))) - } else { - let page_index = i / Self::PER_PAGE; - - if self.decoded.as_ref().is_none_or(|b| b.0 != page_index) { - let values = CompressedVec::::decode_page_( - stored_len, - page_index, - &self.reader, - &self.pages, - ) - .unwrap(); - self.decoded.replace((page_index, values)); - } - - self.decoded - .as_ref() - .unwrap() - .1 - .get(i % Self::PER_PAGE) - .map(|v| (I::from(i), Cow::Owned(*v))) - }; - - self.index += 1; - - result - } -} - -impl<'a, I, T> IntoIterator for &'a CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = CompressedVecIterator<'a, I, T>; - - fn into_iter(self) -> Self::IntoIter { - let pages = self.pages.read(); - let stored_len = self.stored_len(); - - CompressedVecIterator { - vec: self, - reader: self.create_static_reader(), - decoded: None, - pages, - index: 0, - stored_len, - } - } -} - -impl AnyIterableVec for CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec for CompressedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/compressed/page.rs b/crates/brk_vecs/src/variants/compressed/page.rs deleted file mode 100644 index 33ea4ebc8..000000000 --- a/crates/brk_vecs/src/variants/compressed/page.rs +++ /dev/null @@ -1,19 +0,0 @@ -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -#[derive(Debug, Clone, IntoBytes, Immutable, FromBytes, KnownLayout)] -#[repr(C)] -pub struct Page { - pub start: u64, - pub bytes: u32, - pub values: u32, -} - -impl Page { - pub fn new(start: u64, bytes: u32, values: u32) -> Self { - Self { - start, - bytes, - values, - } - } -} diff --git a/crates/brk_vecs/src/variants/compressed/pages.rs b/crates/brk_vecs/src/variants/compressed/pages.rs deleted file mode 100644 index e55b03593..000000000 --- a/crates/brk_vecs/src/variants/compressed/pages.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::sync::Arc; - -use parking_lot::RwLock; -use zerocopy::{FromBytes, IntoBytes}; - -use crate::{ - File, Result, - file::{Region, RegionReader}, -}; - -use super::Page; - -#[derive(Debug, Clone)] -pub struct Pages { - _region: Arc>, - region_index: usize, - - vec: Vec, - change_at: Option, -} - -impl Pages { - const SIZE_OF_PAGE: usize = size_of::(); - - pub fn import(file: &File, name: &str) -> Result { - let (region_index, _region) = file.create_region_if_needed(name)?; - - let vec = _region - .read() - .create_reader(file) - .read_all() - .chunks(Self::SIZE_OF_PAGE) - .map(|b| Page::read_from_bytes(b).map_err(|e| e.into())) - .collect::>()?; - - Ok(Self { - _region, - region_index, - vec, - change_at: None, - }) - } - - pub fn flush(&mut self, file: &File) -> Result<()> { - if self.change_at.is_none() { - return Ok(()); - } - - let change_at = self.change_at.take().unwrap(); - let at = (change_at * Self::SIZE_OF_PAGE) as u64; - - file.truncate_write_all_to_region( - self.region_index.into(), - at, - self.vec[change_at..].as_bytes(), - )?; - - Ok(()) - } - - pub fn len(&self) -> usize { - self.vec.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn get(&self, page_index: usize) -> Option<&Page> { - self.vec.get(page_index) - } - - pub fn last(&self) -> Option<&Page> { - self.vec.last() - } - - pub fn pop(&mut self) -> Option { - let popped = self.vec.pop(); - if popped.is_some() { - self.set_changed_at(self.vec.len()); - } - popped - } - - pub fn checked_push(&mut self, page_index: usize, page: Page) { - if page_index != self.vec.len() { - panic!(); - } - - self.set_changed_at(page_index); - - self.vec.push(page); - } - - fn set_changed_at(&mut self, page_index: usize) { - if self.change_at.is_none_or(|pi| pi > page_index) { - self.change_at.replace(page_index); - } - } - - pub fn reset(&mut self, file: &File) -> Result<()> { - self.truncate(0); - self.flush(file) - } - - pub fn truncate(&mut self, page_index: usize) -> Option { - let page = self.get(page_index).cloned(); - self.vec.truncate(page_index); - self.set_changed_at(page_index); - page - } - - pub fn stored_len(&self, per_page: usize) -> usize { - if let Some(last) = self.last() { - (self.len() - 1) * per_page + last.values as usize - } else { - 0 - } - } -} diff --git a/crates/brk_vecs/src/variants/computed/computation.rs b/crates/brk_vecs/src/variants/computed/computation.rs deleted file mode 100644 index eceb25524..000000000 --- a/crates/brk_vecs/src/variants/computed/computation.rs +++ /dev/null @@ -1,18 +0,0 @@ -use serde_derive::{Deserialize, Serialize}; - -#[derive(Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize)] -pub enum Computation { - Eager, - #[default] - Lazy, -} - -impl Computation { - pub fn eager(&self) -> bool { - *self == Self::Eager - } - - pub fn lazy(&self) -> bool { - *self == Self::Lazy - } -} diff --git a/crates/brk_vecs/src/variants/computed/mod.rs b/crates/brk_vecs/src/variants/computed/mod.rs deleted file mode 100644 index 9b35d2665..000000000 --- a/crates/brk_vecs/src/variants/computed/mod.rs +++ /dev/null @@ -1,379 +0,0 @@ -use std::{borrow::Cow, sync::Arc}; - -use crate::{ - AnyBoxedIterableVec, AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, - BoxedVecIterator, CollectableVec, Exit, File, Format, Result, StoredCompressed, StoredIndex, - StoredRaw, Version, -}; - -use super::{ - ComputeFrom1, ComputeFrom2, ComputeFrom3, EagerVec, LazyVecFrom1, LazyVecFrom1Iterator, - LazyVecFrom2, LazyVecFrom2Iterator, LazyVecFrom3, LazyVecFrom3Iterator, StoredVecIterator, -}; - -mod computation; - -pub use computation::*; - -#[derive(Clone)] -pub enum Dependencies -where - S1T: Clone, - S2T: Clone, - S3T: Clone, -{ - From1(AnyBoxedIterableVec, ComputeFrom1), - From2( - (AnyBoxedIterableVec, AnyBoxedIterableVec), - ComputeFrom2, - ), - From3( - ( - AnyBoxedIterableVec, - AnyBoxedIterableVec, - AnyBoxedIterableVec, - ), - ComputeFrom3, - ), -} - -pub type ComputedVecFrom1 = ComputedVec; -pub type ComputedVecFrom2 = - ComputedVec; -pub type ComputedVecFrom3 = - ComputedVec; - -#[derive(Clone)] -pub enum ComputedVec -where - S1T: Clone, - S2T: Clone, - S3T: Clone, -{ - Eager { - vec: EagerVec, - deps: Dependencies, - }, - LazyFrom1(LazyVecFrom1), - LazyFrom2(LazyVecFrom2), - LazyFrom3(LazyVecFrom3), -} - -impl ComputedVec -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - pub fn forced_import_or_init_from_1( - computation: Computation, - file: &Arc, - name: &str, - version: Version, - format: Format, - source: AnyBoxedIterableVec, - compute: ComputeFrom1, - ) -> Result { - Ok(match computation { - Computation::Eager => Self::Eager { - vec: EagerVec::forced_import(file, name, version, format)?, - deps: Dependencies::From1(source, compute), - }, - Computation::Lazy => { - Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute)) - } - }) - } - - #[allow(clippy::too_many_arguments)] - pub fn forced_import_or_init_from_2( - computation: Computation, - file: &Arc, - name: &str, - version: Version, - format: Format, - source1: AnyBoxedIterableVec, - source2: AnyBoxedIterableVec, - compute: ComputeFrom2, - ) -> Result { - Ok(match computation { - Computation::Eager => Self::Eager { - vec: EagerVec::forced_import(file, name, version, format)?, - deps: Dependencies::From2((source1, source2), compute), - }, - Computation::Lazy => { - Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute)) - } - }) - } - - #[allow(clippy::too_many_arguments)] - pub fn forced_import_or_init_from_3( - computation: Computation, - file: &Arc, - name: &str, - version: Version, - format: Format, - source1: AnyBoxedIterableVec, - source2: AnyBoxedIterableVec, - source3: AnyBoxedIterableVec, - compute: ComputeFrom3, - ) -> Result { - Ok(match computation { - Computation::Eager => Self::Eager { - vec: EagerVec::forced_import(file, name, version, format)?, - deps: Dependencies::From3((source1, source2, source3), compute), - }, - Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init( - name, version, source1, source2, source3, compute, - )), - }) - } - - pub fn compute_if_necessary( - &mut self, - max_from: I, - len_source: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> { - let (vec, dependencies) = if let ComputedVec::Eager { - vec, - deps: dependencies, - } = self - { - (vec, dependencies) - } else { - return Ok(()); - }; - - let len = len_source.len(); - - match dependencies { - Dependencies::From1(source, compute) => { - let version = source.version(); - let mut iter = source.iter(); - let t = |i: I| compute(i, &mut *iter).map(|v| (i, v)).unwrap(); - vec.compute_to(max_from, len, version, t, exit) - } - Dependencies::From2((source1, source2), compute) => { - let version = source1.version() + source2.version(); - let mut iter1 = source1.iter(); - let mut iter2 = source2.iter(); - let t = |i: I| { - compute(i, &mut *iter1, &mut *iter2) - .map(|v| (i, v)) - .unwrap() - }; - vec.compute_to(max_from, len, version, t, exit) - } - Dependencies::From3((source1, source2, source3), compute) => { - let version = source1.version() + source2.version() + source3.version(); - let mut iter1 = source1.iter(); - let mut iter2 = source2.iter(); - let mut iter3 = source3.iter(); - let t = |i: I| { - compute(i, &mut *iter1, &mut *iter2, &mut *iter3) - .map(|v| (i, v)) - .unwrap() - }; - vec.compute_to(max_from, len, version, t, exit) - } - } - } -} - -impl AnyVec for ComputedVec -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - fn version(&self) -> Version { - match self { - ComputedVec::Eager { vec, .. } => vec.version(), - ComputedVec::LazyFrom1(v) => v.version(), - ComputedVec::LazyFrom2(v) => v.version(), - ComputedVec::LazyFrom3(v) => v.version(), - } - } - - fn name(&self) -> &str { - match self { - ComputedVec::Eager { vec, .. } => vec.name(), - ComputedVec::LazyFrom1(v) => v.name(), - ComputedVec::LazyFrom2(v) => v.name(), - ComputedVec::LazyFrom3(v) => v.name(), - } - } - - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - fn len(&self) -> usize { - match self { - ComputedVec::Eager { vec, .. } => vec.len(), - ComputedVec::LazyFrom1(v) => v.len(), - ComputedVec::LazyFrom2(v) => v.len(), - ComputedVec::LazyFrom3(v) => v.len(), - } - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> -where - S1T: Clone, - S2T: Clone, - S3T: Clone, -{ - Eager(StoredVecIterator<'a, I, T>), - LazyFrom1(LazyVecFrom1Iterator<'a, I, T, S1I, S1T>), - LazyFrom2(LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>), - LazyFrom3(LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>), -} - -impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator - for ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - fn next(&mut self) -> Option { - match self { - Self::Eager(i) => i.next(), - Self::LazyFrom1(i) => i.next(), - Self::LazyFrom2(i) => i.next(), - Self::LazyFrom3(i) => i.next(), - } - } -} - -impl BaseVecIterator - for ComputedVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - match self { - Self::Eager(i) => i.mut_index(), - Self::LazyFrom1(i) => i.mut_index(), - Self::LazyFrom2(i) => i.mut_index(), - Self::LazyFrom3(i) => i.mut_index(), - } - } - - fn len(&self) -> usize { - match self { - Self::Eager(i) => i.len(), - Self::LazyFrom1(i) => i.len(), - Self::LazyFrom2(i) => i.len(), - Self::LazyFrom3(i) => i.len(), - } - } - - #[inline] - fn name(&self) -> &str { - match self { - Self::Eager(i) => i.name(), - Self::LazyFrom1(i) => i.name(), - Self::LazyFrom2(i) => i.name(), - Self::LazyFrom3(i) => i.name(), - } - } -} - -impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator - for &'a ComputedVec -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; - - fn into_iter(self) -> Self::IntoIter { - match self { - ComputedVec::Eager { vec, .. } => ComputedVecIterator::Eager(vec.into_iter()), - ComputedVec::LazyFrom1(v) => ComputedVecIterator::LazyFrom1(v.into_iter()), - ComputedVec::LazyFrom2(v) => ComputedVecIterator::LazyFrom2(v.into_iter()), - ComputedVec::LazyFrom3(v) => ComputedVecIterator::LazyFrom3(v.into_iter()), - } - } -} - -impl AnyIterableVec - for ComputedVec -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec - for ComputedVec -where - I: StoredIndex, - T: StoredCompressed, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/eager.rs b/crates/brk_vecs/src/variants/eager.rs deleted file mode 100644 index 0a47e7525..000000000 --- a/crates/brk_vecs/src/variants/eager.rs +++ /dev/null @@ -1,1117 +0,0 @@ -use core::error; -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet}, - f32, - fmt::Debug, - ops::{Add, Div, Mul, Sub}, - sync::Arc, -}; - -use log::info; -use parking_lot::RwLock; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, BoxedVecIterator, CheckedSub, - CollectableVec, Exit, File, Format, GenericStoredVec, Reader, Result, StoredCompressed, - StoredIndex, StoredRaw, StoredVec, StoredVecIterator, VecIterator, Version, file::Region, - variants::Header, -}; - -#[derive(Debug, Clone)] -pub struct EagerVec(StoredVec); - -impl EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - pub fn forced_import( - file: &Arc, - value_name: &str, - version: Version, - format: Format, - ) -> Result { - Ok(Self(StoredVec::forced_import( - file, value_name, version, format, - )?)) - } - - #[inline] - pub fn get_or_read<'a, 'b>(&'a self, index: I, reader: &'b Reader) -> Result>> - where - 'a: 'b, - { - self.0.get_or_read(index, reader) - } - - pub fn inner_version(&self) -> Version { - self.0.header().vec_version() - } - - fn update_computed_version(&mut self, computed_version: Version) { - self.mut_header().update_computed_version(computed_version); - } - - pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { - if version != self.header().computed_version() { - self.update_computed_version(version); - if !self.is_empty() { - self.reset()?; - } - } - - if self.is_empty() { - info!( - "Computing {}_to_{}...", - self.index_type_to_string(), - self.name() - ) - } - - Ok(()) - } - - pub fn compute_to( - &mut self, - max_from: I, - to: usize, - version: Version, - mut t: F, - exit: &Exit, - ) -> Result<()> - where - F: FnMut(I) -> (I, T), - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + version, - )?; - - let index = max_from.min(I::from(self.len())); - (index.to_usize()?..to).try_for_each(|i| { - let (i, v) = t(I::from(i)); - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_range( - &mut self, - max_from: I, - other: &impl AnyIterableVec, - t: F, - exit: &Exit, - ) -> Result<()> - where - A: StoredRaw, - F: FnMut(I) -> (I, T), - { - self.compute_to(max_from, other.len(), other.version(), t, exit) - } - - pub fn compute_from_index( - &mut self, - max_from: I, - other: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredRaw, - { - self.compute_to( - max_from, - other.len(), - other.version(), - |i| (i, T::from(i)), - exit, - ) - } - - pub fn compute_transform( - &mut self, - max_from: A, - other: &impl AnyIterableVec, - mut t: F, - exit: &Exit, - ) -> Result<()> - where - A: StoredIndex, - B: StoredRaw, - F: FnMut((A, B, &Self)) -> (I, T), - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + other.version(), - )?; - - let index = max_from.min(A::from(self.len())); - other.iter_at(index).try_for_each(|(a, b)| { - let (i, v) = t((a, b.into_owned(), self)); - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_add( - &mut self, - max_from: I, - added: &impl AnyIterableVec, - adder: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: Add, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + added.version() + adder.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut adder_iter = adder.iter(); - - added.iter_at(index).try_for_each(|(i, v)| { - let v = v.into_owned() + adder_iter.unwrap_get_inner(i); - - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_subtract( - &mut self, - max_from: I, - subtracted: &impl AnyIterableVec, - subtracter: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: CheckedSub, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + subtracted.version() + subtracter.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut subtracter_iter = subtracter.iter(); - - subtracted.iter_at(index).try_for_each(|(i, v)| { - let v = v - .into_owned() - .checked_sub(subtracter_iter.unwrap_get_inner(i)) - .unwrap(); - - self.forced_push_at(i, v, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_max( - &mut self, - max_from: I, - source: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: From + Ord, - T2: StoredRaw, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + source.version(), - )?; - - let index = max_from.min(I::from(self.len())); - - let mut prev = None; - - source.iter_at(index).try_for_each(|(i, v)| { - if prev.is_none() { - let i = i.unwrap_to_usize(); - prev.replace(if i > 0 { - self.into_iter().unwrap_get_inner_(i - 1) - } else { - T::from(source.iter().unwrap_get_inner_(0)) - }); - } - let max = prev.unwrap().max(T::from(v.into_owned())); - prev.replace(max); - - self.forced_push_at(i, max, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_multiply( - &mut self, - max_from: I, - multiplied: &impl AnyIterableVec, - multiplier: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T2: StoredRaw + Mul, - T3: StoredRaw, - T4: StoredRaw, - T: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + multiplied.version() + multiplier.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut multiplier_iter = multiplier.iter(); - - multiplied.iter_at(index).try_for_each(|(i, v)| { - let v = v.into_owned() * multiplier_iter.unwrap_get_inner(i); - - self.forced_push_at(i, v.into(), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_divide( - &mut self, - max_from: I, - divided: &impl AnyIterableVec, - divider: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T2: StoredRaw + Mul, - T3: StoredRaw, - T4: Div + From, - T5: CheckedSub, - T: From, - { - self.compute_divide_(max_from, divided, divider, exit, false, false) - } - - pub fn compute_percentage( - &mut self, - max_from: I, - divided: &impl AnyIterableVec, - divider: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T2: StoredRaw + Mul, - T3: StoredRaw, - T4: Div + From, - T5: CheckedSub, - T: From, - { - self.compute_divide_(max_from, divided, divider, exit, true, false) - } - - pub fn compute_percentage_difference( - &mut self, - max_from: I, - divided: &impl AnyIterableVec, - divider: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T2: StoredRaw + Mul, - T3: StoredRaw, - T4: Div + From, - T5: CheckedSub, - T: From, - { - self.compute_divide_(max_from, divided, divider, exit, true, true) - } - - pub fn compute_divide_( - &mut self, - max_from: I, - divided: &impl AnyIterableVec, - divider: &impl AnyIterableVec, - exit: &Exit, - as_percentage: bool, - as_difference: bool, - ) -> Result<()> - where - T2: StoredRaw + Mul, - T3: StoredRaw, - T4: Div + From, - T5: CheckedSub, - T: From, - { - self.validate_computed_version_or_reset_file( - Version::ONE + self.inner_version() + divided.version() + divider.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let multiplier = if as_percentage { 100 } else { 1 }; - - let mut divider_iter = divider.iter(); - divided.iter_at(index).try_for_each(|(i, divided)| { - let divided = divided.into_owned(); - let divider = divider_iter.unwrap_get_inner(i); - - let v = if as_percentage { - divided * multiplier - } else { - T4::from(divided) - }; - let mut v = v / divider; - if as_difference { - v = v.checked_sub(multiplier).unwrap(); - } - self.forced_push_at(i, T::from(v), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_inverse_more_to_less( - &mut self, - max_from: T, - other: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - I: StoredRaw + StoredIndex, - T: StoredIndex, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + other.version(), - )?; - - let index = max_from.min( - VecIterator::last(self.into_iter()).map_or_else(T::default, |(_, v)| v.into_owned()), - ); - let mut prev_i = None; - other.iter_at(index).try_for_each(|(v, i)| -> Result<()> { - let i = i.into_owned(); - if prev_i.is_some_and(|prev_i| prev_i == i) { - return Ok(()); - } - if self.iter().get_inner(i).is_none_or(|old_v| old_v > v) { - self.forced_push_at(i, v, exit)?; - } - prev_i.replace(i); - Ok(()) - })?; - - self.safe_flush(exit) - } - - pub fn compute_inverse_less_to_more( - &mut self, - max_from: T, - first_indexes: &impl AnyIterableVec, - indexes_count: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - I: StoredRaw, - T: StoredIndex, - T2: StoredRaw, - usize: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO - + self.inner_version() - + first_indexes.version() - + indexes_count.version(), - )?; - - let mut indexes_count_iter = indexes_count.iter(); - - let index = max_from.min(T::from(self.len())); - first_indexes - .iter_at(index) - .try_for_each(|(value, first_index)| { - let first_index = (first_index).to_usize()?; - let count = usize::from(indexes_count_iter.unwrap_get_inner(value)); - (first_index..first_index + count) - .try_for_each(|index| self.forced_push_at(I::from(index), value, exit)) - })?; - - self.safe_flush(exit) - } - - pub fn compute_count_from_indexes( - &mut self, - max_from: I, - first_indexes: &impl AnyIterableVec, - other_to_else: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredRaw - + StoredIndex - + Copy - + Add - + CheckedSub - + TryInto - + Default, - >::Error: error::Error + 'static, - T3: StoredRaw, - { - let opt: Option bool>> = None; - self.compute_filtered_count_from_indexes_(max_from, first_indexes, other_to_else, opt, exit) - } - - pub fn compute_filtered_count_from_indexes( - &mut self, - max_from: I, - first_indexes: &impl AnyIterableVec, - other_to_else: &impl AnyIterableVec, - filter: F, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredRaw - + StoredIndex - + Copy - + Add - + CheckedSub - + TryInto - + Default, - >::Error: error::Error + 'static, - T3: StoredRaw, - F: FnMut(T2) -> bool, - { - self.compute_filtered_count_from_indexes_( - max_from, - first_indexes, - other_to_else, - Some(Box::new(filter)), - exit, - ) - } - - fn compute_filtered_count_from_indexes_( - &mut self, - max_from: I, - first_indexes: &impl AnyIterableVec, - other_to_else: &impl AnyIterableVec, - mut filter: Option bool + '_>>, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredRaw - + StoredIndex - + Copy - + Add - + CheckedSub - + TryInto - + Default, - T3: StoredRaw, - >::Error: error::Error + 'static, - { - self.validate_computed_version_or_reset_file( - Version::ZERO - + self.inner_version() - + first_indexes.version() - + other_to_else.version(), - )?; - - let mut other_iter = first_indexes.iter(); - let index = max_from.min(I::from(self.len())); - first_indexes - .iter_at(index) - .try_for_each(|(i, first_index)| { - let end = other_iter - .get_inner(i + 1) - .map(|v| v.unwrap_to_usize()) - .unwrap_or_else(|| other_to_else.len()); - - let range = first_index.unwrap_to_usize()..end; - let count = if let Some(filter) = filter.as_mut() { - range.into_iter().filter(|i| filter(T2::from(*i))).count() - } else { - range.count() - }; - self.forced_push_at(i, T::from(T2::from(count)), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_is_first_ordered( - &mut self, - max_from: I, - self_to_other: &impl AnyIterableVec, - other_to_self: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - I: StoredRaw, - T: From, - A: StoredIndex + StoredRaw, - { - self.validate_computed_version_or_reset_file( - Version::ZERO - + self.inner_version() - + self_to_other.version() - + other_to_self.version(), - )?; - - let mut other_to_self_iter = other_to_self.iter(); - let index = max_from.min(I::from(self.len())); - self_to_other.iter_at(index).try_for_each(|(i, other)| { - self.forced_push_at( - i, - T::from(other_to_self_iter.unwrap_get_inner(other.into_owned()) == i), - exit, - ) - })?; - - self.safe_flush(exit) - } - - pub fn compute_sum_from_indexes( - &mut self, - max_from: I, - first_indexes: &impl AnyIterableVec, - indexes_count: &impl AnyIterableVec, - source: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: From + Add, - T2: StoredIndex + StoredRaw, - T3: StoredRaw, - usize: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO - + self.inner_version() - + first_indexes.version() - + indexes_count.version(), - )?; - - let mut indexes_count_iter = indexes_count.iter(); - let mut source_iter = source.iter(); - let index = max_from.min(I::from(self.len())); - first_indexes - .iter_at(index) - .try_for_each(|(i, first_index)| { - let count = usize::from(indexes_count_iter.unwrap_get_inner(i)); - let first_index = first_index.unwrap_to_usize(); - let range = first_index..first_index + count; - let mut sum = T::from(0_usize); - range.into_iter().for_each(|i| { - sum = sum + source_iter.unwrap_get_inner(T2::from(i)); - }); - self.forced_push_at(i, sum, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_sum_of_others( - &mut self, - max_from: I, - others: &[&impl AnyIterableVec], - exit: &Exit, - ) -> Result<()> - where - T: From + Add, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + others.iter().map(|v| v.version()).sum(), - )?; - - if others.is_empty() { - unreachable!("others should've length of 1 at least"); - } - - let mut others_iter = others[1..].iter().map(|v| v.iter()).collect::>(); - - let index = max_from.min(I::from(self.len())); - others - .first() - .unwrap() - .iter_at(index) - .try_for_each(|(i, v)| { - let mut sum = v.into_owned(); - others_iter.iter_mut().for_each(|iter| { - sum = sum + iter.unwrap_get_inner(i); - }); - self.forced_push_at(i, sum, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_min_of_others( - &mut self, - max_from: I, - others: &[&impl AnyIterableVec], - exit: &Exit, - ) -> Result<()> - where - T: From + Add + Ord, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + others.iter().map(|v| v.version()).sum(), - )?; - - if others.is_empty() { - unreachable!("others should've length of 1 at least"); - } - - let mut others_iter = others[1..].iter().map(|v| v.iter()).collect::>(); - - let index = max_from.min(I::from(self.len())); - others - .first() - .unwrap() - .iter_at(index) - .try_for_each(|(i, v)| { - let min = v.into_owned(); - let min = others_iter - .iter_mut() - .map(|iter| iter.unwrap_get_inner(i)) - .min() - .map_or(min, |min2| min.min(min2)); - self.forced_push_at(i, min, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_max_of_others( - &mut self, - max_from: I, - others: &[&impl AnyIterableVec], - exit: &Exit, - ) -> Result<()> - where - T: From + Add + Ord, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + others.iter().map(|v| v.version()).sum(), - )?; - - if others.is_empty() { - unreachable!("others should've length of 1 at least"); - } - - let mut others_iter = others[1..].iter().map(|v| v.iter()).collect::>(); - - let index = max_from.min(I::from(self.len())); - others - .first() - .unwrap() - .iter_at(index) - .try_for_each(|(i, v)| { - let max = v.into_owned(); - let max = others_iter - .iter_mut() - .map(|iter| iter.unwrap_get_inner(i)) - .max() - .map_or(max, |max2| max.max(max2)); - self.forced_push_at(i, max, exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_sma( - &mut self, - max_from: I, - source: &impl AnyIterableVec, - sma: usize, - exit: &Exit, - ) -> Result<()> - where - T: Add + From + Div + From, - T2: StoredRaw, - f32: From + From, - { - self.compute_sma_(max_from, source, sma, exit, None) - } - - pub fn compute_sma_( - &mut self, - max_from: I, - source: &impl AnyIterableVec, - sma: usize, - exit: &Exit, - min_i: Option, - ) -> Result<()> - where - T: Add + From + Div + From, - T2: StoredRaw, - f32: From + From, - { - self.validate_computed_version_or_reset_file( - Version::ONE + self.inner_version() + source.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut prev = None; - let min_prev_i = min_i.unwrap_or_default().unwrap_to_usize(); - let mut other_iter = source.iter(); - source.iter_at(index).try_for_each(|(i, value)| { - let value = value.into_owned(); - - if min_i.is_none() || min_i.is_some_and(|min_i| min_i <= i) { - if prev.is_none() { - let i = i.unwrap_to_usize(); - prev.replace(if i > min_prev_i { - self.into_iter().unwrap_get_inner_(i - 1) - } else { - T::from(0.0) - }); - } - - let processed_values_count = i.unwrap_to_usize() - min_prev_i + 1; - let len = (processed_values_count).min(sma); - - let value = f32::from(value); - - let sma = T::from(if processed_values_count > sma { - let prev_sum = f32::from(prev.unwrap()) * len as f32; - let value_to_subtract = f32::from( - other_iter.unwrap_get_inner_(i.unwrap_to_usize().checked_sub(sma).unwrap()), - ); - (prev_sum - value_to_subtract + value) / len as f32 - } else { - (f32::from(prev.unwrap()) * (len - 1) as f32 + value) / len as f32 - }); - - prev.replace(sma); - self.forced_push_at(i, sma, exit) - } else { - self.forced_push_at(i, T::from(f32::NAN), exit) - } - })?; - - self.safe_flush(exit) - } - - pub fn compute_previous_value( - &mut self, - max_from: I, - source: &impl AnyIterableVec, - len: usize, - exit: &Exit, - ) -> Result<()> - where - I: CheckedSub, - T2: StoredRaw + Default, - f32: From, - T: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + source.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut source_iter = source.iter(); - (index.to_usize()?..source.len()).try_for_each(|i| { - let i = I::from(i); - - let previous_value = i - .checked_sub(I::from(len)) - .map(|prev_i| f32::from(source_iter.unwrap_get_inner(prev_i))) - .unwrap_or(f32::NAN); - - self.forced_push_at(i, T::from(previous_value), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_change( - &mut self, - max_from: I, - source: &impl AnyIterableVec, - len: usize, - exit: &Exit, - ) -> Result<()> - where - I: CheckedSub, - T: CheckedSub + Default, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + source.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut source_iter = source.iter(); - source.iter_at(index).try_for_each(|(i, current)| { - let current = current.into_owned(); - - let prev = i - .checked_sub(I::from(len)) - .map(|prev_i| source_iter.unwrap_get_inner(prev_i)) - .unwrap_or_default(); - - self.forced_push_at(i, current.checked_sub(prev).unwrap(), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_percentage_change( - &mut self, - max_from: I, - source: &impl AnyIterableVec, - len: usize, - exit: &Exit, - ) -> Result<()> - where - I: CheckedSub, - T2: StoredRaw + Default, - f32: From, - T: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + source.version(), - )?; - - let index = max_from.min(I::from(self.len())); - let mut source_iter = source.iter(); - source.iter_at(index).try_for_each(|(i, b)| { - let previous_value = f32::from( - i.checked_sub(I::from(len)) - .map(|prev_i| source_iter.unwrap_get_inner(prev_i)) - .unwrap_or_default(), - ); - - let last_value = f32::from(b.into_owned()); - - let percentage_change = ((last_value / previous_value) - 1.0) * 100.0; - - self.forced_push_at(i, T::from(percentage_change), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_cagr( - &mut self, - max_from: I, - percentage_returns: &impl AnyIterableVec, - days: usize, - exit: &Exit, - ) -> Result<()> - where - I: CheckedSub, - T2: StoredRaw + Default, - f32: From, - T: From, - { - self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner_version() + percentage_returns.version(), - )?; - - if days % 365 != 0 { - panic!("bad days"); - } - - let years = days / 365; - let index = max_from.min(I::from(self.len())); - percentage_returns - .iter_at(index) - .try_for_each(|(i, percentage)| { - let percentage = percentage.into_owned(); - - let cagr = (((f32::from(percentage) / 100.0 + 1.0).powf(1.0 / years as f32)) - 1.0) - * 100.0; - - self.forced_push_at(i, T::from(cagr), exit) - })?; - - self.safe_flush(exit) - } - - pub fn compute_zscore( - &mut self, - max_from: I, - ratio: &impl AnyIterableVec, - sma: &impl AnyIterableVec, - sd: &impl AnyIterableVec, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredRaw + Sub + Div, - T3: StoredRaw, - T4: StoredRaw, - T2: StoredRaw, - f32: From + From + From, - { - let mut sma_iter = sma.iter(); - let mut sd_iter = sd.iter(); - - self.compute_transform( - max_from, - ratio, - |(i, ratio, ..)| { - let sma = sma_iter.unwrap_get_inner(i); - let sd = sd_iter.unwrap_get_inner(i); - (i, (ratio - sma) / sd) - }, - exit, - ) - } -} - -impl AnyVec for EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn version(&self) -> Version { - self.0.header().computed_version() - } - - #[inline] - fn name(&self) -> &str { - self.0.name() - } - - #[inline] - fn len(&self) -> usize { - self.0.len() - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyStoredVec for EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn file(&self) -> &File { - self.0.file() - } - - fn region_index(&self) -> usize { - self.0.region_index() - } - - fn region(&self) -> &RwLock { - self.0.region() - } - - fn header(&self) -> &Header { - self.0.header() - } - - fn mut_header(&mut self) -> &mut Header { - self.0.mut_header() - } - - fn flush(&mut self) -> Result<()> { - self.0.flush() - } - - fn stored_len(&self) -> usize { - self.0.stored_len() - } -} - -impl GenericStoredVec for EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result { - self.0.read_(index, reader) - } - - #[inline] - fn pushed(&self) -> &[T] { - self.0.pushed() - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - self.0.mut_pushed() - } - - #[inline] - fn holes(&self) -> &BTreeSet { - self.0.holes() - } - #[inline] - fn mut_holes(&mut self) -> &mut BTreeSet { - self.0.mut_holes() - } - - #[inline] - fn updated(&self) -> &BTreeMap { - self.0.updated() - } - #[inline] - fn mut_updated(&mut self) -> &mut BTreeMap { - self.0.mut_updated() - } - - #[inline] - fn truncate_if_needed(&mut self, index: I) -> Result<()> { - self.0.truncate_if_needed(index) - } - - #[inline] - fn reset(&mut self) -> Result<()> { - self.0.reset() - } -} - -impl<'a, I, T> IntoIterator for &'a EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = StoredVecIterator<'a, I, T>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl AnyIterableVec for EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - I: StoredIndex, - T: StoredRaw + 'a, - { - Box::new(self.0.into_iter()) - } -} - -impl AnyCollectableVec for EagerVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/lazy/lazy1.rs b/crates/brk_vecs/src/variants/lazy/lazy1.rs deleted file mode 100644 index ab22a7461..000000000 --- a/crates/brk_vecs/src/variants/lazy/lazy1.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::borrow::Cow; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, AnyBoxedIterableVec, - BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredRaw, Version, -}; - -pub type ComputeFrom1 = - for<'a> fn(I, &mut dyn BaseVecIterator)>) -> Option; - -#[derive(Clone)] -pub struct LazyVecFrom1 -where - S1T: Clone, -{ - name: String, - version: Version, - source: AnyBoxedIterableVec, - compute: ComputeFrom1, -} - -impl LazyVecFrom1 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, -{ - pub fn init( - name: &str, - version: Version, - source: AnyBoxedIterableVec, - compute: ComputeFrom1, - ) -> Self { - if I::to_string() != S1I::to_string() { - unreachable!() - } - - Self { - name: name.to_string(), - version, - source, - compute, - } - } - - fn version(&self) -> Version { - self.version - } -} - -pub struct LazyVecFrom1Iterator<'a, I, T, S1I, S1T> -where - S1T: Clone, -{ - lazy: &'a LazyVecFrom1, - source: BoxedVecIterator<'a, S1I, S1T>, - index: usize, -} - -impl<'a, I, T, S1I, S1T> Iterator for LazyVecFrom1Iterator<'a, I, T, S1I, S1T> -where - I: StoredIndex, - T: StoredRaw + 'a, - S1I: StoredIndex, - S1T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - - fn next(&mut self) -> Option { - if self.index >= self.len() { - return None; - } - let index = I::from(self.index); - let opt = (self.lazy.compute)(index, &mut *self.source).map(|v| (index, Cow::Owned(v))); - if opt.is_some() { - self.index += 1; - } - opt - } -} - -impl BaseVecIterator for LazyVecFrom1Iterator<'_, I, T, S1I, S1T> -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - &mut self.index - } - - #[inline] - fn len(&self) -> usize { - self.source.len() - } - - #[inline] - fn name(&self) -> &str { - self.source.name() - } -} - -impl<'a, I, T, S1I, S1T> IntoIterator for &'a LazyVecFrom1 -where - I: StoredIndex, - T: StoredRaw + 'a, - S1I: StoredIndex, - S1T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = LazyVecFrom1Iterator<'a, I, T, S1I, S1T>; - - fn into_iter(self) -> Self::IntoIter { - LazyVecFrom1Iterator { - lazy: self, - source: self.source.iter(), - index: 0, - } - } -} - -impl AnyVec for LazyVecFrom1 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, -{ - fn version(&self) -> Version { - self.version() - } - - fn name(&self) -> &str { - self.name.as_str() - } - - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - fn len(&self) -> usize { - self.source.len() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyIterableVec for LazyVecFrom1 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec for LazyVecFrom1 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/lazy/lazy2.rs b/crates/brk_vecs/src/variants/lazy/lazy2.rs deleted file mode 100644 index d8ca303cd..000000000 --- a/crates/brk_vecs/src/variants/lazy/lazy2.rs +++ /dev/null @@ -1,234 +0,0 @@ -use std::borrow::Cow; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, AnyBoxedIterableVec, - BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredRaw, Version, -}; - -pub type ComputeFrom2 = for<'a> fn( - I, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, -) -> Option; - -#[derive(Clone)] -pub struct LazyVecFrom2 -where - S1T: Clone, - S2T: Clone, -{ - name: String, - version: Version, - source1: AnyBoxedIterableVec, - source2: AnyBoxedIterableVec, - compute: ComputeFrom2, -} - -impl LazyVecFrom2 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - pub fn init( - name: &str, - version: Version, - source1: AnyBoxedIterableVec, - source2: AnyBoxedIterableVec, - compute: ComputeFrom2, - ) -> Self { - if ([ - source1.index_type_to_string(), - source2.index_type_to_string(), - ]) - .into_iter() - .filter(|t| *t == I::to_string()) - .count() - == 0 - { - panic!("At least one should have same index"); - } - - Self { - name: name.to_string(), - version, - source1, - source2, - compute, - } - } - - fn version(&self) -> Version { - self.version - } -} - -pub struct LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> -where - S1T: Clone, - S2T: Clone, -{ - lazy: &'a LazyVecFrom2, - source1: BoxedVecIterator<'a, S1I, S1T>, - source2: BoxedVecIterator<'a, S2I, S2T>, - index: usize, -} - -impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> -where - I: StoredIndex, - T: StoredRaw + 'a, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - - fn next(&mut self) -> Option { - let index = I::from(self.index); - let opt = (self.lazy.compute)(index, &mut *self.source1, &mut *self.source2) - .map(|v| (index, Cow::Owned(v))); - if opt.is_some() { - self.index += 1; - } - opt - } -} - -impl BaseVecIterator - for LazyVecFrom2Iterator<'_, I, T, S1I, S1T, S2I, S2T> -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - &mut self.index - } - - #[inline] - fn len(&self) -> usize { - let len1 = if self.source1.index_type_to_string() == I::to_string() { - self.source1.len() - } else { - usize::MAX - }; - let len2 = if self.source2.index_type_to_string() == I::to_string() { - self.source2.len() - } else { - usize::MAX - }; - len1.min(len2) - } - - #[inline] - fn name(&self) -> &str { - self.source1.name() - } -} - -impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVecFrom2 -where - I: StoredIndex, - T: StoredRaw + 'a, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>; - - fn into_iter(self) -> Self::IntoIter { - LazyVecFrom2Iterator { - lazy: self, - source1: self.source1.iter(), - source2: self.source2.iter(), - index: 0, - } - } -} - -impl AnyVec for LazyVecFrom2 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - fn version(&self) -> Version { - self.version() - } - - fn name(&self) -> &str { - self.name.as_str() - } - - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - fn len(&self) -> usize { - let len1 = if self.source1.index_type_to_string() == I::to_string() { - self.source1.len() - } else { - usize::MAX - }; - let len2 = if self.source2.index_type_to_string() == I::to_string() { - self.source2.len() - } else { - usize::MAX - }; - len1.min(len2) - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyIterableVec for LazyVecFrom2 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec for LazyVecFrom2 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/lazy/lazy3.rs b/crates/brk_vecs/src/variants/lazy/lazy3.rs deleted file mode 100644 index 08c326cce..000000000 --- a/crates/brk_vecs/src/variants/lazy/lazy3.rs +++ /dev/null @@ -1,276 +0,0 @@ -use std::borrow::Cow; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, AnyBoxedIterableVec, - BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredRaw, Version, -}; - -pub type ComputeFrom3 = for<'a> fn( - I, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, - &mut dyn BaseVecIterator)>, -) -> Option; - -#[derive(Clone)] -pub struct LazyVecFrom3 -where - S1T: Clone, - S2T: Clone, - S3T: Clone, -{ - name: String, - version: Version, - source1: AnyBoxedIterableVec, - source2: AnyBoxedIterableVec, - source3: AnyBoxedIterableVec, - compute: ComputeFrom3, -} - -impl LazyVecFrom3 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - pub fn init( - name: &str, - version: Version, - source1: AnyBoxedIterableVec, - source2: AnyBoxedIterableVec, - source3: AnyBoxedIterableVec, - compute: ComputeFrom3, - ) -> Self { - if ([ - source1.index_type_to_string(), - source2.index_type_to_string(), - source3.index_type_to_string(), - ]) - .into_iter() - .filter(|t| *t == I::to_string()) - .count() - == 0 - { - panic!("At least one should have same index"); - } - - Self { - name: name.to_string(), - version, - source1, - source2, - source3, - compute, - } - } - - fn version(&self) -> Version { - self.version - } -} - -pub struct LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> -where - S1T: Clone, - S2T: Clone, - S3T: Clone, -{ - lazy: &'a LazyVecFrom3, - source1: BoxedVecIterator<'a, S1I, S1T>, - source2: BoxedVecIterator<'a, S2I, S2T>, - source3: BoxedVecIterator<'a, S3I, S3T>, - index: usize, -} - -impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator - for LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> -where - I: StoredIndex, - T: StoredRaw + 'a, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - - fn next(&mut self) -> Option { - let index = I::from(self.index); - let opt = (self.lazy.compute)( - index, - &mut *self.source1, - &mut *self.source2, - &mut *self.source3, - ) - .map(|v| (index, Cow::Owned(v))); - if opt.is_some() { - self.index += 1; - } - opt - } -} - -impl BaseVecIterator - for LazyVecFrom3Iterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - &mut self.index - } - - #[inline] - fn len(&self) -> usize { - let len1 = if self.source1.index_type_to_string() == I::to_string() { - self.source1.len() - } else { - usize::MAX - }; - let len2 = if self.source2.index_type_to_string() == I::to_string() { - self.source2.len() - } else { - usize::MAX - }; - let len3 = if self.source3.index_type_to_string() == I::to_string() { - self.source3.len() - } else { - usize::MAX - }; - len1.min(len2).min(len3) - } - - #[inline] - fn name(&self) -> &str { - self.source1.name() - } -} - -impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator - for &'a LazyVecFrom3 -where - I: StoredIndex, - T: StoredRaw + 'a, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; - - fn into_iter(self) -> Self::IntoIter { - LazyVecFrom3Iterator { - lazy: self, - source1: self.source1.iter(), - source2: self.source2.iter(), - source3: self.source3.iter(), - index: 0, - } - } -} - -impl AnyVec for LazyVecFrom3 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - fn version(&self) -> Version { - self.version() - } - - fn name(&self) -> &str { - self.name.as_str() - } - - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - fn len(&self) -> usize { - let len1 = if self.source1.index_type_to_string() == I::to_string() { - self.source1.len() - } else { - usize::MAX - }; - let len2 = if self.source2.index_type_to_string() == I::to_string() { - self.source2.len() - } else { - usize::MAX - }; - let len3 = if self.source3.index_type_to_string() == I::to_string() { - self.source3.len() - } else { - usize::MAX - }; - len1.min(len2).min(len3) - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyIterableVec - for LazyVecFrom3 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec - for LazyVecFrom3 -where - I: StoredIndex, - T: StoredRaw, - S1I: StoredIndex, - S1T: StoredRaw, - S2I: StoredIndex, - S2T: StoredRaw, - S3I: StoredIndex, - S3T: StoredRaw, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/lazy/mod.rs b/crates/brk_vecs/src/variants/lazy/mod.rs deleted file mode 100644 index 490b7f251..000000000 --- a/crates/brk_vecs/src/variants/lazy/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod lazy1; -mod lazy2; -mod lazy3; - -pub use lazy1::*; -pub use lazy2::*; -pub use lazy3::*; diff --git a/crates/brk_vecs/src/variants/mod.rs b/crates/brk_vecs/src/variants/mod.rs deleted file mode 100644 index db898658d..000000000 --- a/crates/brk_vecs/src/variants/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod compressed; -mod computed; -mod eager; -mod lazy; -mod raw; -mod stored; - -pub use compressed::*; -pub use computed::*; -pub use eager::*; -pub use lazy::*; -pub use raw::*; -pub use stored::*; diff --git a/crates/brk_vecs/src/variants/raw/header.rs b/crates/brk_vecs/src/variants/raw/header.rs deleted file mode 100644 index 9541682b4..000000000 --- a/crates/brk_vecs/src/variants/raw/header.rs +++ /dev/null @@ -1,204 +0,0 @@ -use std::sync::Arc; - -use parking_lot::RwLock; -use zerocopy::{FromBytes, IntoBytes}; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use crate::{Error, File, Result, Stamp, Version}; - -use super::Format; - -const HEADER_VERSION: Version = Version::ONE; -pub const HEADER_OFFSET: usize = size_of::(); - -#[derive(Debug, Clone)] -pub struct Header { - inner: Arc>, - modified: bool, -} - -impl Header { - pub fn create_and_write( - file: &File, - region_index: usize, - vec_version: Version, - format: Format, - ) -> Result { - let inner = HeaderInner::create_and_write(file, region_index, vec_version, format)?; - Ok(Self { - inner: Arc::new(RwLock::new(inner)), - modified: false, - }) - } - - pub fn import_and_verify( - file: &File, - region_index: usize, - region_len: u64, - vec_version: Version, - format: Format, - ) -> Result { - let inner = - HeaderInner::import_and_verify(file, region_index, region_len, vec_version, format)?; - Ok(Self { - inner: Arc::new(RwLock::new(inner)), - modified: false, - }) - } - - pub fn update_stamp(&mut self, stamp: Stamp) { - self.modified = true; - self.inner.write().stamp = stamp; - } - - pub fn update_format(&mut self, format: Format) { - self.modified = true; - self.inner.write().compressed = ZeroCopyBool::from(format); - } - - pub fn update_computed_version(&mut self, computed_version: Version) { - self.modified = true; - self.inner.write().computed_version = computed_version; - } - - pub fn modified(&self) -> bool { - self.modified - } - - pub fn vec_version(&self) -> Version { - self.inner.read().vec_version - } - - pub fn computed_version(&self) -> Version { - self.inner.read().computed_version - } - - pub fn stamp(&self) -> Stamp { - self.inner.read().stamp - } - - pub fn write(&mut self, file: &File, region_index: usize) -> Result<()> { - self.inner.read().write(file, region_index)?; - self.modified = false; - Ok(()) - } -} - -#[derive(Debug, Clone, FromBytes, IntoBytes, Immutable, KnownLayout)] -#[repr(C)] -struct HeaderInner { - pub header_version: Version, - pub vec_version: Version, - pub computed_version: Version, - pub stamp: Stamp, - pub compressed: ZeroCopyBool, - pub padding: [u8; 31], -} - -impl HeaderInner { - pub fn create_and_write( - file: &File, - region_index: usize, - vec_version: Version, - format: Format, - ) -> Result { - let header = Self { - header_version: HEADER_VERSION, - vec_version, - computed_version: Version::default(), - stamp: Stamp::default(), - compressed: ZeroCopyBool::from(format), - padding: Default::default(), - }; - header.write(file, region_index)?; - Ok(header) - } - - pub fn write(&self, file: &File, region_index: usize) -> Result<()> { - file.write_all_to_region_at(region_index.into(), self.as_bytes(), 0) - } - - pub fn import_and_verify( - file: &File, - region_index: usize, - region_len: u64, - vec_version: Version, - format: Format, - ) -> Result { - let len = region_len; - - if len < HEADER_OFFSET as u64 { - return Err(Error::WrongLength); - } - - let reader = file.create_region_reader(region_index.into())?; - let slice = reader.read(0, HEADER_OFFSET as u64); - let header = HeaderInner::read_from_bytes(slice)?; - - if header.header_version != HEADER_VERSION { - return Err(Error::DifferentVersion { - found: header.header_version, - expected: HEADER_VERSION, - }); - } - if header.vec_version != vec_version { - return Err(Error::DifferentVersion { - found: header.vec_version, - expected: vec_version, - }); - } - if header.compressed.is_broken() { - return Err(Error::WrongEndian); - } - if (header.compressed.is_true() && format.is_raw()) - || (header.compressed.is_false() && format.is_compressed()) - { - return Err(Error::DifferentCompressionMode); - } - - Ok(header) - } -} - -#[derive( - Debug, - Clone, - Copy, - Default, - PartialEq, - Eq, - PartialOrd, - Ord, - FromBytes, - IntoBytes, - Immutable, - KnownLayout, -)] -pub struct ZeroCopyBool(u8); - -impl ZeroCopyBool { - pub const TRUE: Self = Self(1); - pub const FALSE: Self = Self(0); - - pub fn is_true(&self) -> bool { - *self == Self::TRUE - } - - pub fn is_false(&self) -> bool { - *self == Self::FALSE - } - - pub fn is_broken(&self) -> bool { - *self > Self::TRUE - } -} - -impl From for ZeroCopyBool { - fn from(value: Format) -> Self { - if value.is_raw() { - Self::FALSE - } else { - Self::TRUE - } - } -} diff --git a/crates/brk_vecs/src/variants/raw/mod.rs b/crates/brk_vecs/src/variants/raw/mod.rs deleted file mode 100644 index 926d54ea3..000000000 --- a/crates/brk_vecs/src/variants/raw/mod.rs +++ /dev/null @@ -1,432 +0,0 @@ -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet}, - marker::PhantomData, - mem, - sync::Arc, -}; - -use parking_lot::RwLock; -use zerocopy::{FromBytes, IntoBytes}; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, BaseVecIterator, BoxedVecIterator, - CollectableVec, Error, File, GenericStoredVec, Reader, Result, StoredIndex, StoredRaw, Version, - file::{Region, RegionReader}, -}; - -use super::Format; - -mod header; - -pub use header::*; - -const VERSION: Version = Version::ONE; - -#[derive(Debug)] -pub struct RawVec { - file: Arc, - region: Arc>, - region_index: usize, - - header: Header, - name: &'static str, - pushed: Vec, - has_stored_holes: bool, - holes: BTreeSet, - updated: BTreeMap, - phantom: PhantomData, -} - -impl RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - /// Same as import but will reset the vec under certain errors, so be careful ! - pub fn forced_import(file: &Arc, name: &str, mut version: Version) -> Result { - version = version + VERSION; - let res = Self::import(file, name, version); - match res { - Err(Error::DifferentCompressionMode) - | Err(Error::WrongEndian) - | Err(Error::WrongLength) - | Err(Error::DifferentVersion { .. }) => { - let _ = file.remove_region(Self::vec_region_name_(name).into()); - let _ = file.remove_region(Self::holes_region_name_(name).into()); - Self::import(file, name, version) - } - _ => res, - } - } - - pub fn import(file: &Arc, name: &str, version: Version) -> Result { - Self::any_import(file, name, version, Format::Raw) - } - - pub fn any_import( - file: &Arc, - name: &str, - version: Version, - format: Format, - ) -> Result { - let (region_index, region) = file.create_region_if_needed(&Self::vec_region_name_(name))?; - - let region_len = region.read().len() as usize; - if region_len > 0 - && (region_len < HEADER_OFFSET - || (format.is_raw() && (region_len - HEADER_OFFSET) % Self::SIZE_OF_T != 0)) - { - return Err(Error::Str("Region was saved incorrectly")); - } - - let header = if region_len == 0 { - Header::create_and_write(file, region_index, version, format)? - } else { - Header::import_and_verify(file, region_index, region.read().len(), version, format)? - }; - - let holes = if let Ok(holes) = file.get_region(Self::holes_region_name_(name).into()) { - Some( - holes - .create_reader(file) - .read_all() - .chunks(size_of::()) - .map(|b| -> Result { usize::read_from_bytes(b).map_err(|e| e.into()) }) - .collect::>>()?, - ) - } else { - None - }; - - Ok(Self { - file: file.clone(), - region: region.clone(), - region_index, - header, - name: Box::leak(Box::new(name.to_string())), - pushed: vec![], - has_stored_holes: holes.is_some(), - holes: holes.unwrap_or_default(), - updated: BTreeMap::new(), - phantom: PhantomData, - }) - } - - #[inline] - pub fn iter(&self) -> RawVecIterator<'_, I, T> { - self.into_iter() - } - - #[inline] - pub fn iter_at(&self, i: I) -> RawVecIterator<'_, I, T> { - self.iter_at_(i.unwrap_to_usize()) - } - - #[inline] - pub fn iter_at_(&self, i: usize) -> RawVecIterator<'_, I, T> { - let mut iter = self.into_iter(); - iter.set_(i); - iter - } - - pub fn write_header_if_needed(&mut self) -> Result<()> { - if self.header.modified() { - self.header.write(&self.file, self.region_index)?; - } - Ok(()) - } -} - -impl Clone for RawVec { - fn clone(&self) -> Self { - Self { - file: self.file.clone(), - region: self.region.clone(), - region_index: self.region_index, - header: self.header.clone(), - name: self.name, - pushed: vec![], - updated: BTreeMap::new(), - has_stored_holes: false, - holes: BTreeSet::new(), - phantom: PhantomData, - } - } -} - -impl AnyVec for RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - #[inline] - fn version(&self) -> Version { - self.header.vec_version() - } - - #[inline] - fn name(&self) -> &str { - self.name - } - - #[inline] - fn len(&self) -> usize { - self.len_() - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyStoredVec for RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - fn header(&self) -> &Header { - &self.header - } - - fn mut_header(&mut self) -> &mut Header { - &mut self.header - } - - #[inline] - fn stored_len(&self) -> usize { - (self.region.read().len() as usize - HEADER_OFFSET) / Self::SIZE_OF_T - } - - fn flush(&mut self) -> Result<()> { - self.write_header_if_needed()?; - - let has_new_data = !self.is_pushed_empty(); - let has_updated_data = !self.updated.is_empty(); - let has_holes = !self.holes.is_empty(); - let had_holes = self.has_stored_holes && !has_holes; - - if !has_new_data && !has_updated_data && !has_holes && !had_holes { - return Ok(()); - } - - if has_new_data || has_updated_data { - let file = &self.file; - - if has_new_data { - file.write_all_to_region( - self.region_index.into(), - mem::take(&mut self.pushed).as_bytes(), - )?; - } - - if has_updated_data { - mem::take(&mut self.updated) - .into_iter() - .try_for_each(|(i, v)| -> Result<()> { - let bytes = v.as_bytes(); - let at = ((i * Self::SIZE_OF_T) + HEADER_OFFSET) as u64; - file.write_all_to_region_at(self.region_index.into(), bytes, at)?; - Ok(()) - })?; - } - } - - if has_holes || had_holes { - if has_holes { - self.has_stored_holes = true; - let (holes_index, _) = self - .file - .create_region_if_needed(&self.holes_region_name())?; - let bytes = self - .holes - .iter() - .flat_map(|i| i.to_ne_bytes()) - .collect::>(); - self.file - .truncate_write_all_to_region(holes_index.into(), 0, &bytes)?; - } else if had_holes { - self.has_stored_holes = false; - let _ = self.file.remove_region(self.holes_region_name().into()); - } - } - - Ok(()) - } - - fn file(&self) -> &File { - &self.file - } - - fn region_index(&self) -> usize { - self.region_index - } - - fn region(&self) -> &RwLock { - &self.region - } -} - -impl GenericStoredVec for RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result { - T::read_from_prefix(reader.prefixed((index * Self::SIZE_OF_T + HEADER_OFFSET) as u64)) - .map(|(v, _)| v) - .map_err(Error::from) - } - - #[inline] - fn pushed(&self) -> &[T] { - self.pushed.as_slice() - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - &mut self.pushed - } - - #[inline] - fn holes(&self) -> &BTreeSet { - &self.holes - } - #[inline] - fn mut_holes(&mut self) -> &mut BTreeSet { - &mut self.holes - } - - #[inline] - fn updated(&self) -> &BTreeMap { - &self.updated - } - #[inline] - fn mut_updated(&mut self) -> &mut BTreeMap { - &mut self.updated - } - - fn truncate_if_needed(&mut self, index: I) -> Result<()> { - let index = index.to_usize()?; - - if index >= self.stored_len() { - return Ok(()); - } - - if index == 0 { - self.reset()?; - return Ok(()); - } - - let from = index * Self::SIZE_OF_T + HEADER_OFFSET; - - self.file - .truncate_region(self.region_index.into(), from as u64) - } - - fn reset(&mut self) -> Result<()> { - self.reset_() - } -} - -#[derive(Debug)] -pub struct RawVecIterator<'a, I, T> { - vec: &'a RawVec, - reader: Reader<'a>, - index: usize, -} - -impl BaseVecIterator for RawVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredRaw, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - &mut self.index - } - - #[inline] - fn len(&self) -> usize { - self.vec.len() - } - - #[inline] - fn name(&self) -> &str { - self.vec.name() - } -} - -impl<'a, I, T> Iterator for RawVecIterator<'a, I, T> -where - I: StoredIndex, - T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - - fn next(&mut self) -> Option { - let index = self.index; - - let opt = self - .vec - .get_or_read_(index, &self.reader) - .unwrap() - .map(|v| (I::from(index), v)); - - if opt.is_some() { - self.index += 1; - } - - opt - } -} - -impl<'a, I, T> IntoIterator for &'a RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = RawVecIterator<'a, I, T>; - - fn into_iter(self) -> Self::IntoIter { - RawVecIterator { - vec: self, - reader: self.create_static_reader(), - index: 0, - } - } -} - -impl AnyIterableVec for RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec for RawVec -where - I: StoredIndex, - T: StoredRaw, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/stamped/mod.rs b/crates/brk_vecs/src/variants/stamped/mod.rs deleted file mode 100644 index d27faa392..000000000 --- a/crates/brk_vecs/src/variants/stamped/mod.rs +++ /dev/null @@ -1,232 +0,0 @@ -use std::{borrow::Cow, cmp::Ordering, fmt::Debug, sync::Arc}; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Error, File, - Format, GenericStoredVec, Header, Result, StoredCompressed, StoredIndex, StoredVec, Version, - file::Reader, -}; - -use super::StoredVecIterator; - -mod stamp; - -pub use stamp::*; - -#[derive(Debug, Clone)] -pub struct StampedVec(StoredVec); - -impl StampedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - pub fn forced_import( - file: &Arc, - name: &str, - version: Version, - format: Format, - ) -> Result { - Ok(Self(StoredVec::forced_import(file, name, version, format)?)) - } - - #[inline] - pub fn unwrap_read(&self, index: I, reader: &Reader) -> T { - self.0.unwrap_read(index, reader) - } - - #[inline] - pub fn get_or_read<'a, 'b>(&'a self, index: I, reader: &'b Reader) -> Result>> - where - 'a: 'b, - { - self.0.get_or_read(index, reader) - } - - #[inline] - pub fn update_or_push(&mut self, index: I, value: T) -> Result<()> { - self.0.update_or_push(index, value) - } - - #[inline] - pub fn checked_push(&mut self, index: I, value: T) -> Result<()> { - let len = self.0.len(); - match len.cmp(&index.to_usize()?) { - Ordering::Greater => { - dbg!(index, value, len, self.0.header()); - Err(Error::IndexTooLow) - } - Ordering::Equal => { - self.0.push(value); - Ok(()) - } - Ordering::Less => { - dbg!(index, value, len, self.0.header()); - Err(Error::IndexTooHigh) - } - } - } - - #[inline] - pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { - let len = self.0.len(); - match len.cmp(&index.to_usize()?) { - Ordering::Greater => { - // dbg!(len, index, &self.pathbuf); - // panic!(); - Ok(()) - } - Ordering::Equal => { - self.0.push(value); - Ok(()) - } - Ordering::Less => { - dbg!(index, value, len, self.0.header()); - Err(Error::IndexTooHigh) - } - } - } - - #[inline] - pub fn fill_first_hole_or_push(&mut self, value: T) -> Result { - self.0.fill_first_hole_or_push(value) - } - - pub fn update(&mut self, index: I, value: T) -> Result<()> { - self.0.update(index, value) - } - - pub fn take(&mut self, index: I, reader: &Reader) -> Result> { - self.0.take(index, reader) - } - - pub fn delete(&mut self, index: I) { - self.0.delete(index) - } - - fn update_stamp(&mut self, stamp: Stamp) { - self.0.mut_header().update_stamp(stamp); - } - - pub fn reset(&mut self) -> Result<()> { - self.update_stamp(Stamp::default()); - self.0.reset() - } - - pub fn truncate_if_needed(&mut self, index: I, stamp: Stamp) -> Result<()> { - self.update_stamp(stamp); - self.0.truncate_if_needed(index)?; - Ok(()) - } - - pub fn flush(&mut self, stamp: Stamp) -> Result<()> { - self.update_stamp(stamp); - self.0.flush() - } - - pub fn header(&self) -> &Header { - self.0.header() - } - - #[inline] - pub fn hasnt(&self, index: I) -> Result { - self.0.has(index).map(|b| !b) - } - - pub fn create_reader(&self) -> Reader { - self.0.create_reader() - } - - pub fn create_static_reader(&self) -> Reader<'static> { - self.0.create_static_reader() - } -} - -impl AnyVec for StampedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn version(&self) -> Version { - self.0.version() - } - - #[inline] - fn name(&self) -> &str { - self.0.name() - } - - #[inline] - fn len(&self) -> usize { - self.0.len() - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -pub trait AnyStampedVec: AnyVec { - fn stamp(&self) -> Stamp; - fn flush(&mut self, stamp: Stamp) -> Result<()>; -} - -impl AnyStampedVec for StampedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn stamp(&self) -> Stamp { - self.0.header().stamp() - } - - fn flush(&mut self, stamp: Stamp) -> Result<()> { - self.flush(stamp) - } -} - -impl<'a, I, T> IntoIterator for &'a StampedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = StoredVecIterator<'a, I, T>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl AnyIterableVec for StampedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec for StampedVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/variants/stored/format.rs b/crates/brk_vecs/src/variants/stored/format.rs deleted file mode 100644 index 0bd4fcb32..000000000 --- a/crates/brk_vecs/src/variants/stored/format.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::{fs, io, path::Path}; - -use serde_derive::{Deserialize, Serialize}; - -use crate::{Error, Result}; - -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum Format { - Compressed, - #[default] - Raw, -} - -impl Format { - pub fn write(&self, path: &Path) -> Result<(), io::Error> { - fs::write(path, self.as_bytes()) - } - - pub fn is_raw(&self) -> bool { - *self == Self::Raw - } - - pub fn is_compressed(&self) -> bool { - *self == Self::Compressed - } - - fn as_bytes(&self) -> Vec { - if self.is_compressed() { - vec![1] - } else { - vec![0] - } - } - - fn from_bytes(bytes: &[u8]) -> Self { - if bytes.len() != 1 { - panic!(); - } - if bytes[0] == 1 { - Self::Compressed - } else if bytes[0] == 0 { - Self::Raw - } else { - panic!() - } - } - - pub fn validate(&self, path: &Path) -> Result<()> { - if let Ok(prev_compressed) = Format::try_from(path) - && prev_compressed != *self - { - return Err(Error::DifferentCompressionMode); - } - - Ok(()) - } -} - -impl TryFrom<&Path> for Format { - type Error = Error; - fn try_from(value: &Path) -> Result { - Ok(Self::from_bytes(&fs::read(value)?)) - } -} diff --git a/crates/brk_vecs/src/variants/stored/mod.rs b/crates/brk_vecs/src/variants/stored/mod.rs deleted file mode 100644 index 871cd5fa2..000000000 --- a/crates/brk_vecs/src/variants/stored/mod.rs +++ /dev/null @@ -1,314 +0,0 @@ -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; - -use crate::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, BaseVecIterator, BoxedVecIterator, - CollectableVec, File, GenericStoredVec, Header, Result, StoredCompressed, StoredIndex, Version, - file::{Reader, Region}, -}; - -use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator}; - -mod format; - -pub use format::*; -use parking_lot::RwLock; - -#[derive(Debug, Clone)] -pub enum StoredVec { - Raw(RawVec), - Compressed(CompressedVec), -} - -impl StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - pub fn forced_import( - file: &Arc, - name: &str, - version: Version, - format: Format, - ) -> Result { - if version == Version::ZERO { - dbg!(file, name); - panic!("Version must be at least 1, can't verify endianess otherwise"); - } - - if format.is_compressed() { - Ok(Self::Compressed(CompressedVec::forced_import( - file, name, version, - )?)) - } else { - Ok(Self::Raw(RawVec::forced_import(file, name, version)?)) - } - } -} - -impl AnyVec for StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn version(&self) -> Version { - match self { - StoredVec::Raw(v) => v.version(), - StoredVec::Compressed(v) => v.version(), - } - } - - #[inline] - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - #[inline] - fn len(&self) -> usize { - self.pushed_len() + self.stored_len() - } - - fn name(&self) -> &str { - match self { - StoredVec::Raw(v) => v.name(), - StoredVec::Compressed(v) => v.name(), - } - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } -} - -impl AnyStoredVec for StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn file(&self) -> &File { - match self { - StoredVec::Raw(v) => v.file(), - StoredVec::Compressed(v) => v.file(), - } - } - - #[inline] - fn region_index(&self) -> usize { - match self { - StoredVec::Raw(v) => v.region_index(), - StoredVec::Compressed(v) => v.region_index(), - } - } - - #[inline] - fn region(&self) -> &RwLock { - match self { - StoredVec::Raw(v) => v.region(), - StoredVec::Compressed(v) => v.region(), - } - } - - #[inline] - fn header(&self) -> &Header { - match self { - StoredVec::Raw(v) => v.header(), - StoredVec::Compressed(v) => v.header(), - } - } - - #[inline] - fn mut_header(&mut self) -> &mut Header { - match self { - StoredVec::Raw(v) => v.mut_header(), - StoredVec::Compressed(v) => v.mut_header(), - } - } - - #[inline] - fn stored_len(&self) -> usize { - match self { - StoredVec::Raw(v) => v.stored_len(), - StoredVec::Compressed(v) => v.stored_len(), - } - } - - fn flush(&mut self) -> Result<()> { - match self { - StoredVec::Raw(v) => v.flush(), - StoredVec::Compressed(v) => v.flush(), - } - } -} - -impl GenericStoredVec for StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn read_(&self, index: usize, reader: &Reader) -> Result { - match self { - StoredVec::Raw(v) => v.read_(index, reader), - StoredVec::Compressed(v) => v.read_(index, reader), - } - } - - #[inline] - fn pushed(&self) -> &[T] { - match self { - StoredVec::Raw(v) => v.pushed(), - StoredVec::Compressed(v) => v.pushed(), - } - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - match self { - StoredVec::Raw(v) => v.mut_pushed(), - StoredVec::Compressed(v) => v.mut_pushed(), - } - } - - #[inline] - fn holes(&self) -> &BTreeSet { - match self { - StoredVec::Raw(v) => v.holes(), - StoredVec::Compressed(v) => v.holes(), - } - } - #[inline] - fn mut_holes(&mut self) -> &mut BTreeSet { - match self { - StoredVec::Raw(v) => v.mut_holes(), - StoredVec::Compressed(v) => v.mut_holes(), - } - } - - #[inline] - fn updated(&self) -> &BTreeMap { - match self { - StoredVec::Raw(v) => v.updated(), - StoredVec::Compressed(v) => v.updated(), - } - } - #[inline] - fn mut_updated(&mut self) -> &mut BTreeMap { - match self { - StoredVec::Raw(v) => v.mut_updated(), - StoredVec::Compressed(v) => v.mut_updated(), - } - } - - #[inline] - fn truncate_if_needed(&mut self, index: I) -> Result<()> { - match self { - StoredVec::Raw(v) => v.truncate_if_needed(index), - StoredVec::Compressed(v) => v.truncate_if_needed(index), - } - } - - #[inline] - fn reset(&mut self) -> Result<()> { - match self { - StoredVec::Raw(v) => v.reset(), - StoredVec::Compressed(v) => v.reset(), - } - } -} - -#[derive(Debug)] -pub enum StoredVecIterator<'a, I, T> { - Raw(RawVecIterator<'a, I, T>), - Compressed(CompressedVecIterator<'a, I, T>), -} - -impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T> -where - I: StoredIndex, - T: StoredCompressed, -{ - type Item = (I, Cow<'a, T>); - fn next(&mut self) -> Option { - match self { - Self::Compressed(i) => i.next(), - Self::Raw(i) => i.next(), - } - } -} - -impl BaseVecIterator for StoredVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredCompressed, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - match self { - Self::Compressed(iter) => iter.mut_index(), - Self::Raw(iter) => iter.mut_index(), - } - } - - fn len(&self) -> usize { - match self { - Self::Compressed(i) => i.len(), - Self::Raw(i) => i.len(), - } - } - - #[inline] - fn name(&self) -> &str { - match self { - Self::Compressed(i) => i.name(), - Self::Raw(i) => i.name(), - } - } -} - -impl<'a, I, T> IntoIterator for &'a StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - type Item = (I, Cow<'a, T>); - type IntoIter = StoredVecIterator<'a, I, T>; - - fn into_iter(self) -> Self::IntoIter { - match self { - StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()), - StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()), - } - } -} - -impl AnyIterableVec for StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> - where - T: 'a, - { - Box::new(self.into_iter()) - } -} - -impl AnyCollectableVec for StoredVec -where - I: StoredIndex, - T: StoredCompressed, -{ - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - CollectableVec::collect_range_serde_json(self, from, to) - } -} diff --git a/crates/brk_vecs/src/version.rs b/crates/brk_vecs/src/version.rs deleted file mode 100644 index eff79c0c6..000000000 --- a/crates/brk_vecs/src/version.rs +++ /dev/null @@ -1,103 +0,0 @@ -use std::{ - fs, - io::{self, Read}, - iter::Sum, - ops::Add, - path::Path, -}; - -use zerocopy::{FromBytes, IntoBytes}; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use crate::{Error, Result}; - -#[derive( - Default, - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - FromBytes, - IntoBytes, - Immutable, - KnownLayout, -)] -pub struct Version(u64); - -impl Version { - pub const ZERO: Self = Self(0); - pub const ONE: Self = Self(1); - pub const TWO: Self = Self(2); - - pub const fn new(v: u64) -> Self { - Self(v) - } - - pub fn write(&self, path: &Path) -> Result<(), io::Error> { - fs::write(path, self.as_bytes()) - } - - pub fn swap_bytes(self) -> Self { - Self(self.0.swap_bytes()) - } - - /// - /// Ok(true) if existed and is same. - /// - /// Ok(false) if didn't exist. - /// - pub fn validate(&self, path: &Path) -> Result { - if let Ok(prev_version) = Version::try_from(path) { - if prev_version != *self { - if prev_version.swap_bytes() == *self { - return Err(Error::WrongEndian); - } - return Err(Error::DifferentVersion { - found: prev_version, - expected: *self, - }); - } - - Ok(true) - } else { - Ok(false) - } - } -} - -impl From for u64 { - fn from(value: Version) -> u64 { - value.0 - } -} - -impl From for Version { - fn from(value: u64) -> Self { - Self(value) - } -} - -impl TryFrom<&Path> for Version { - type Error = Error; - fn try_from(value: &Path) -> Result { - let mut buf = [0; 8]; - fs::read(value)?.as_slice().read_exact(&mut buf)?; - Ok(*(Self::ref_from_bytes(&buf)?)) - } -} - -impl Add for Version { - type Output = Self; - fn add(self, rhs: Version) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -impl Sum for Version { - fn sum>(iter: I) -> Self { - iter.fold(Self::ZERO, Add::add) - } -} diff --git a/crates/brk_vecs_macros/Cargo.toml b/crates/brk_vecs_macros/Cargo.toml deleted file mode 100644 index 6aed06197..000000000 --- a/crates/brk_vecs_macros/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "brk_vecs_macros" -description = "Macros for brk_vecs" -keywords = ["vec", "disk", "data"] -categories = ["database"] -version.workspace = true -edition.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -build = "build.rs" - -[lib] -proc-macro = true - -[dependencies] -syn = "2.0" -quote = "1.0" diff --git a/crates/brk_vecs_macros/build.rs b/crates/brk_vecs_macros/build.rs deleted file mode 100644 index a4055a31e..000000000 --- a/crates/brk_vecs_macros/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -fn main() { - let profile = std::env::var("PROFILE").unwrap_or_default(); - - if profile == "release" { - println!("cargo:rustc-flag=-C"); - println!("cargo:rustc-flag=target-cpu=native"); - } -} diff --git a/crates/brk_vecs_macros/src/lib.rs b/crates/brk_vecs_macros/src/lib.rs deleted file mode 100644 index 081321103..000000000 --- a/crates/brk_vecs_macros/src/lib.rs +++ /dev/null @@ -1,57 +0,0 @@ -use proc_macro::TokenStream; -use quote::quote; -use syn::{Data, DataStruct, DeriveInput, Fields, parse_macro_input}; - -#[proc_macro_derive(StoredCompressed)] -pub fn derive_stored_compressed(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input as DeriveInput); - let struct_name = &input.ident; - let generics = &input.generics; - let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); - - let inner_type = match &input.data { - Data::Struct(DataStruct { - fields: Fields::Unnamed(fields), - .. - }) if fields.unnamed.len() == 1 => &fields.unnamed[0].ty, - _ => { - return syn::Error::new_spanned( - &input.ident, - "StoredCompressed can only be derived for single-field tuple structs", - ) - .to_compile_error() - .into(); - } - }; - - // Check if we have generic parameters - let has_generics = !generics.params.is_empty(); - - let expanded = if has_generics { - // Generic case: add where clause constraint - let where_clause = if where_clause.is_some() { - quote! { #where_clause #inner_type: StoredCompressed, } - } else { - quote! { where #inner_type: StoredCompressed, } - }; - - quote! { - impl #impl_generics ::brk_vecs::TransparentStoredCompressed<<#inner_type as StoredCompressed>::NumberType> for #struct_name #ty_generics #where_clause {} - - impl #impl_generics StoredCompressed for #struct_name #ty_generics #where_clause { - type NumberType = <#inner_type as StoredCompressed>::NumberType; - } - } - } else { - // Non-generic case: use the original approach - quote! { - impl ::brk_vecs::TransparentStoredCompressed<<#inner_type as StoredCompressed>::NumberType> for #struct_name {} - - impl StoredCompressed for #struct_name { - type NumberType = <#inner_type as StoredCompressed>::NumberType; - } - } - }; - - TokenStream::from(expanded) -}