diff --git a/README.md b/README.md index 66dc9ddbc..32a725b04 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi - [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer` - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) -- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec +- [`brk_vec`](https://crates.io/crates/brk_vec): A storeable vec - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/) ## Hosting as a service diff --git a/TODO.md b/TODO.md index 8ae918195..d5eab02c1 100644 --- a/TODO.md +++ b/TODO.md @@ -56,6 +56,7 @@ - add extensions support (.json .csv …) - if format instead of extension then don't download file - add support for https (rustls) + - lru cache - _vec_ - add native lock file support (once it's available in stable rust) - improve compressed mode (slow reads) diff --git a/crates/brk_computer/src/vecs/mod.rs b/crates/brk_computer/src/all.rs similarity index 95% rename from crates/brk_computer/src/vecs/mod.rs rename to crates/brk_computer/src/all.rs index 3c53ec106..902614870 100644 --- a/crates/brk_computer/src/vecs/mod.rs +++ b/crates/brk_computer/src/all.rs @@ -5,22 +5,11 @@ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, Computation, Format}; - -pub mod blocks; -pub mod cointime; -pub mod constants; -pub mod fetched; -pub mod grouped; -pub mod indexes; -pub mod market; -pub mod mining; -pub mod stateful; -pub mod transactions; - -pub use indexes::Indexes; use log::info; -use crate::stores::Stores; +use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions}; + +use super::stateful; const VERSION: Version = Version::ONE; @@ -130,7 +119,6 @@ impl Vecs { starting_indexes: brk_indexer::Indexes, fetcher: Option<&mut Fetcher>, exit: &Exit, - stores: &mut Stores, ) -> color_eyre::Result<()> { info!("Computing indexes..."); let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; @@ -188,7 +176,6 @@ impl Vecs { &self.market, &mut starting_indexes, exit, - stores, )?; self.cointime.compute( diff --git a/crates/brk_computer/src/vecs/blocks.rs b/crates/brk_computer/src/blocks.rs similarity index 99% rename from crates/brk_computer/src/vecs/blocks.rs rename to crates/brk_computer/src/blocks.rs index 9a3a0da59..26eb86b6d 100644 --- a/crates/brk_computer/src/vecs/blocks.rs +++ b/crates/brk_computer/src/blocks.rs @@ -8,7 +8,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format}; -use crate::vecs::grouped::Source; +use crate::grouped::Source; use super::{ Indexes, diff --git a/crates/brk_computer/src/vecs/cointime.rs b/crates/brk_computer/src/cointime.rs similarity index 99% rename from crates/brk_computer/src/vecs/cointime.rs rename to crates/brk_computer/src/cointime.rs index c98e410e7..78a9d462c 100644 --- a/crates/brk_computer/src/vecs/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -5,16 +5,13 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; -use crate::vecs::{ - fetched, - grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source}, - stateful, transactions, -}; - use super::{ - Indexes, - grouped::{ComputedVecsFromHeight, VecBuilderOptions}, - indexes, + Indexes, fetched, + grouped::{ + ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight, + Source, VecBuilderOptions, + }, + indexes, stateful, transactions, }; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/constants.rs b/crates/brk_computer/src/constants.rs similarity index 99% rename from crates/brk_computer/src/vecs/constants.rs rename to crates/brk_computer/src/constants.rs index 6607df3d5..2b6c0b8fb 100644 --- a/crates/brk_computer/src/vecs/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format}; -use crate::vecs::grouped::Source; +use crate::grouped::Source; use super::{ Indexes, diff --git a/crates/brk_computer/src/vecs/fetched.rs b/crates/brk_computer/src/fetched.rs similarity index 99% rename from crates/brk_computer/src/vecs/fetched.rs rename to crates/brk_computer/src/fetched.rs index 8ae500870..8b7ea7b15 100644 --- a/crates/brk_computer/src/vecs/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -13,7 +13,7 @@ use brk_vec::{ VecIterator, }; -use crate::vecs::grouped::Source; +use crate::grouped::Source; use super::{ Indexes, diff --git a/crates/brk_computer/src/vecs/grouped/builder_computed.rs b/crates/brk_computer/src/grouped/builder_computed.rs similarity index 99% rename from crates/brk_computer/src/vecs/grouped/builder_computed.rs rename to crates/brk_computer/src/grouped/builder_computed.rs index 8115570c9..25b828a06 100644 --- a/crates/brk_computer/src/vecs/grouped/builder_computed.rs +++ b/crates/brk_computer/src/grouped/builder_computed.rs @@ -7,7 +7,7 @@ use brk_vec::{ ComputedVec, ComputedVecFrom2, Format, StoredIndex, }; -use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions}; +use crate::grouped::{EagerVecBuilder, VecBuilderOptions}; use super::ComputedType; diff --git a/crates/brk_computer/src/vecs/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs similarity index 100% rename from crates/brk_computer/src/vecs/grouped/builder_eager.rs rename to crates/brk_computer/src/grouped/builder_eager.rs diff --git a/crates/brk_computer/src/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs similarity index 99% rename from crates/brk_computer/src/vecs/grouped/from_dateindex.rs rename to crates/brk_computer/src/grouped/from_dateindex.rs index 77a366e60..a49c68a2c 100644 --- a/crates/brk_computer/src/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -10,7 +10,7 @@ use brk_vec::{ AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, }; -use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes}; +use crate::{Indexes, grouped::ComputedVecBuilder, indexes}; use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions}; diff --git a/crates/brk_computer/src/vecs/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs similarity index 99% rename from crates/brk_computer/src/vecs/grouped/from_height.rs rename to crates/brk_computer/src/grouped/from_height.rs index 0ed075529..15630aa05 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -10,7 +10,7 @@ use brk_vec::{ AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format, }; -use crate::vecs::{ +use crate::{ Indexes, grouped::{ComputedVecBuilder, Source}, indexes, diff --git a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs similarity index 98% rename from crates/brk_computer/src/vecs/grouped/from_height_strict.rs rename to crates/brk_computer/src/grouped/from_height_strict.rs index 1147a7f54..cefd2df77 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, EagerVec, Format}; -use crate::vecs::{Indexes, indexes}; +use crate::{Indexes, indexes}; use super::{ComputedType, EagerVecBuilder, VecBuilderOptions}; diff --git a/crates/brk_computer/src/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs similarity index 99% rename from crates/brk_computer/src/vecs/grouped/from_txindex.rs rename to crates/brk_computer/src/grouped/from_txindex.rs index a9eb32948..a20e0dbef 100644 --- a/crates/brk_computer/src/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -11,7 +11,7 @@ use brk_vec::{ Format, StoredIndex, VecIterator, }; -use crate::vecs::{ +use crate::{ Indexes, fetched, grouped::{ComputedVecBuilder, Source}, indexes, diff --git a/crates/brk_computer/src/vecs/grouped/mod.rs b/crates/brk_computer/src/grouped/mod.rs similarity index 100% rename from crates/brk_computer/src/vecs/grouped/mod.rs rename to crates/brk_computer/src/grouped/mod.rs diff --git a/crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs similarity index 99% rename from crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs rename to crates/brk_computer/src/grouped/ratio_from_dateindex.rs index 1bc7f6f77..5d144d157 100644 --- a/crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -8,10 +8,7 @@ use brk_vec::{ StoredIndex, VecIterator, }; -use crate::{ - utils::get_percentile, - vecs::{Indexes, fetched, grouped::source::Source, indexes}, -}; +use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile}; use super::{ComputedVecsFromDateIndex, VecBuilderOptions}; diff --git a/crates/brk_computer/src/vecs/grouped/source.rs b/crates/brk_computer/src/grouped/source.rs similarity index 100% rename from crates/brk_computer/src/vecs/grouped/source.rs rename to crates/brk_computer/src/grouped/source.rs diff --git a/crates/brk_computer/src/vecs/grouped/type.rs b/crates/brk_computer/src/grouped/type.rs similarity index 100% rename from crates/brk_computer/src/vecs/grouped/type.rs rename to crates/brk_computer/src/grouped/type.rs diff --git a/crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs b/crates/brk_computer/src/grouped/value_from_dateindex.rs similarity index 98% rename from crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs rename to crates/brk_computer/src/grouped/value_from_dateindex.rs index 124fec3ea..e2ec90ba1 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/value_from_dateindex.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; -use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes}; +use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes}; use super::{Source, VecBuilderOptions}; diff --git a/crates/brk_computer/src/vecs/grouped/value_from_height.rs b/crates/brk_computer/src/grouped/value_from_height.rs similarity index 98% rename from crates/brk_computer/src/vecs/grouped/value_from_height.rs rename to crates/brk_computer/src/grouped/value_from_height.rs index b582475aa..033463328 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_height.rs +++ b/crates/brk_computer/src/grouped/value_from_height.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec}; -use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; +use crate::{Indexes, fetched, grouped::Source, indexes}; use super::{ComputedVecsFromHeight, VecBuilderOptions}; diff --git a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs similarity index 99% rename from crates/brk_computer/src/vecs/grouped/value_from_txindex.rs rename to crates/brk_computer/src/grouped/value_from_txindex.rs index 534ee33b5..4b529bdb2 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -8,7 +8,7 @@ use brk_vec::{ Format, LazyVecFrom1, StoredIndex, StoredVec, }; -use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; +use crate::{Indexes, fetched, grouped::Source, indexes}; use super::{ComputedVecsFromTxindex, VecBuilderOptions}; diff --git a/crates/brk_computer/src/vecs/grouped/value_height.rs b/crates/brk_computer/src/grouped/value_height.rs similarity index 98% rename from crates/brk_computer/src/vecs/grouped/value_height.rs rename to crates/brk_computer/src/grouped/value_height.rs index bd2c34036..24848351d 100644 --- a/crates/brk_computer/src/vecs/grouped/value_height.rs +++ b/crates/brk_computer/src/grouped/value_height.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec}; -use crate::vecs::{Indexes, fetched, grouped::Source, indexes}; +use crate::{Indexes, fetched, grouped::Source, indexes}; #[derive(Clone)] pub struct ComputedHeightValueVecs { diff --git a/crates/brk_computer/src/vecs/indexes.rs b/crates/brk_computer/src/indexes.rs similarity index 99% rename from crates/brk_computer/src/vecs/indexes.rs rename to crates/brk_computer/src/indexes.rs index 9c7ed5cda..9cac4dbe8 100644 --- a/crates/brk_computer/src/vecs/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -15,8 +15,6 @@ use brk_vec::{ ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator, }; -use crate::vecs::indexes; - const VERSION: Version = Version::ZERO; #[derive(Clone)] @@ -1206,7 +1204,7 @@ pub struct Indexes { } impl Indexes { - pub fn update_from_height(&mut self, height: Height, indexes: &indexes::Vecs) { + pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) { self.indexes.height = height; self.dateindex = DateIndex::try_from( indexes diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index c2e5b1cbc..d92820b5b 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -12,20 +12,28 @@ use brk_indexer::Indexer; use brk_vec::{Computation, Format}; use log::info; +mod all; +mod blocks; +mod cointime; +mod constants; +mod fetched; +mod grouped; +mod indexes; +mod market; +mod mining; +mod stateful; mod states; -mod stores; +mod transactions; mod utils; -mod vecs; + +use indexes::Indexes; use states::*; -use stores::Stores; -use vecs::Vecs; #[derive(Clone)] pub struct Computer { fetcher: Option, - pub vecs: Vecs, - pub stores: Stores, + pub vecs: all::Vecs, } const VERSION: Version = Version::ONE; @@ -40,7 +48,7 @@ impl Computer { format: Format, ) -> color_eyre::Result { Ok(Self { - vecs: Vecs::import( + vecs: all::Vecs::import( // TODO: Give self.path, join inside import &outputs_dir.join("vecs/computed"), VERSION + Version::ZERO, @@ -49,12 +57,6 @@ impl Computer { computation, format, )?, - stores: Stores::import( - // TODO: Give self.path, join inside import - &outputs_dir.join("stores"), - VERSION + Version::ZERO, - &indexer.stores.keyspace, - )?, fetcher, }) } @@ -68,12 +70,7 @@ impl Computer { exit: &Exit, ) -> color_eyre::Result<()> { info!("Computing..."); - self.vecs.compute( - indexer, - starting_indexes, - self.fetcher.as_mut(), - exit, - &mut self.stores, - ) + self.vecs + .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit) } } diff --git a/crates/brk_computer/src/vecs/market.rs b/crates/brk_computer/src/market.rs similarity index 99% rename from crates/brk_computer/src/vecs/market.rs rename to crates/brk_computer/src/market.rs index db1fcf27e..7560aa285 100644 --- a/crates/brk_computer/src/vecs/market.rs +++ b/crates/brk_computer/src/market.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, Computation, EagerVec, Format, StoredIndex, VecIterator}; -use crate::vecs::grouped::Source; +use crate::grouped::Source; use super::{ Indexes, fetched, diff --git a/crates/brk_computer/src/vecs/mining.rs b/crates/brk_computer/src/mining.rs similarity index 99% rename from crates/brk_computer/src/vecs/mining.rs rename to crates/brk_computer/src/mining.rs index 0124e61c9..09cea1805 100644 --- a/crates/brk_computer/src/vecs/mining.rs +++ b/crates/brk_computer/src/mining.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator}; -use crate::vecs::grouped::Source; +use crate::grouped::Source; use super::{ Indexes, diff --git a/crates/brk_computer/src/vecs/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs similarity index 96% rename from crates/brk_computer/src/vecs/stateful/address_cohort.rs rename to crates/brk_computer/src/stateful/address_cohort.rs index 68df3badd..ff5ea45fd 100644 --- a/crates/brk_computer/src/vecs/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -8,16 +8,14 @@ use brk_vec::{ }; use crate::{ - states::AddressCohortState, - vecs::{ - Indexes, fetched, - grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, - indexes, market, - stateful::{ - common, - r#trait::{CohortVecs, DynCohortVecs}, - }, + Indexes, fetched, + grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions}, + indexes, market, + stateful::{ + common, + r#trait::{CohortVecs, DynCohortVecs}, }, + states::AddressCohortState, }; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs similarity index 99% rename from crates/brk_computer/src/vecs/stateful/address_cohorts.rs rename to crates/brk_computer/src/stateful/address_cohorts.rs index 5f0e623dc..0bde42eed 100644 --- a/crates/brk_computer/src/vecs/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -9,7 +9,7 @@ use brk_vec::{Computation, Format}; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; -use crate::vecs::{ +use crate::{ Indexes, fetched, indexes, stateful::{ address_cohort, diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs similarity index 94% rename from crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs rename to crates/brk_computer/src/stateful/addresstype_to_addresscount.rs index ee387cff6..334c7207e 100644 --- a/crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_addresscount.rs @@ -2,7 +2,7 @@ use brk_core::{ByAddressType, Height}; use brk_vec::VecIterator; use derive_deref::{Deref, DerefMut}; -use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; +use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount; #[derive(Debug, Default, Deref, DerefMut)] pub struct AddressTypeToAddressCount(ByAddressType); diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_height_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs similarity index 94% rename from crates/brk_computer/src/vecs/stateful/addresstype_to_height_to_addresscount.rs rename to crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs index 98f700654..48c009a5c 100644 --- a/crates/brk_computer/src/vecs/stateful/addresstype_to_height_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs @@ -3,7 +3,7 @@ use brk_exit::Exit; use brk_vec::EagerVec; use derive_deref::{Deref, DerefMut}; -use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; +use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount; #[derive(Debug, Clone, Deref, DerefMut)] pub struct AddressTypeToHeightToAddressCount(ByAddressType>); diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_indexes_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs similarity index 99% rename from crates/brk_computer/src/vecs/stateful/addresstype_to_indexes_to_addresscount.rs rename to crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs index fd384f0cd..73aa42124 100644 --- a/crates/brk_computer/src/vecs/stateful/addresstype_to_indexes_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs @@ -3,7 +3,7 @@ use brk_exit::Exit; use brk_vec::AnyCollectableVec; use derive_deref::{Deref, DerefMut}; -use crate::vecs::{ +use crate::{ Indexes, grouped::ComputedVecsFromHeight, indexes, stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, }; diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_set.rs b/crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs similarity index 100% rename from crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_set.rs rename to crates/brk_computer/src/stateful/addresstype_to_typeindex_set.rs diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_tree.rs b/crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs similarity index 100% rename from crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_tree.rs rename to crates/brk_computer/src/stateful/addresstype_to_typeindex_tree.rs diff --git a/crates/brk_computer/src/vecs/stateful/addresstype_to_vec.rs b/crates/brk_computer/src/stateful/addresstype_to_vec.rs similarity index 100% rename from crates/brk_computer/src/vecs/stateful/addresstype_to_vec.rs rename to crates/brk_computer/src/stateful/addresstype_to_vec.rs diff --git a/crates/brk_computer/src/vecs/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs similarity index 99% rename from crates/brk_computer/src/vecs/stateful/common.rs rename to crates/brk_computer/src/stateful/common.rs index 98b7d15a9..65b043ac7 100644 --- a/crates/brk_computer/src/vecs/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -11,16 +11,13 @@ use brk_vec::{ }; use crate::{ - states::CohortState, - vecs::{ - Indexes, fetched, - grouped::{ - ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, - ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, - Source, VecBuilderOptions, - }, - indexes, market, + Indexes, fetched, + grouped::{ + ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex, + ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions, }, + indexes, market, + states::CohortState, }; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs b/crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs similarity index 83% rename from crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs rename to crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs index 5e74cf426..65449bc63 100644 --- a/crates/brk_computer/src/vecs/stateful/height_to_addresstype_to_vec.rs +++ b/crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use brk_core::Height; use derive_deref::{Deref, DerefMut}; -use crate::vecs::stateful::AddressTypeToVec; +use crate::stateful::AddressTypeToVec; #[derive(Debug, Default, Deref, DerefMut)] pub struct HeightToAddressTypeToVec(pub BTreeMap>); diff --git a/crates/brk_computer/src/vecs/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs similarity index 99% rename from crates/brk_computer/src/vecs/stateful/mod.rs rename to crates/brk_computer/src/stateful/mod.rs index 2e4fb4a50..6f14e4f51 100644 --- a/crates/brk_computer/src/vecs/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -16,16 +16,13 @@ use rayon::prelude::*; use crate::{ BlockState, SupplyState, Transacted, - stores::Stores, - vecs::{ - grouped::{ComputedVecsFromHeight, Source}, - market, - stateful::{ - addresstype_to_addresscount::AddressTypeToAddressCount, - addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, - addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount, - addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs, - }, + grouped::{ComputedVecsFromHeight, Source}, + market, + stateful::{ + addresstype_to_addresscount::AddressTypeToAddressCount, + addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, + addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount, + addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs, }, }; @@ -461,7 +458,6 @@ impl Vecs { // Must take ownership as its indexes will be updated for this specific function starting_indexes: &mut Indexes, exit: &Exit, - stores: &mut Stores, ) -> color_eyre::Result<()> { let height_to_first_outputindex = &indexer.vecs.height_to_first_outputindex; let height_to_first_inputindex = &indexer.vecs.height_to_first_inputindex; diff --git a/crates/brk_computer/src/vecs/stateful/range_map.rs b/crates/brk_computer/src/stateful/range_map.rs similarity index 100% rename from crates/brk_computer/src/vecs/stateful/range_map.rs rename to crates/brk_computer/src/stateful/range_map.rs diff --git a/crates/brk_computer/src/vecs/stateful/trait.rs b/crates/brk_computer/src/stateful/trait.rs similarity index 97% rename from crates/brk_computer/src/vecs/stateful/trait.rs rename to crates/brk_computer/src/stateful/trait.rs index db138ebed..8b059d52f 100644 --- a/crates/brk_computer/src/vecs/stateful/trait.rs +++ b/crates/brk_computer/src/stateful/trait.rs @@ -3,7 +3,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, AnyIterableVec}; -use crate::vecs::{Indexes, fetched, indexes, market}; +use crate::{Indexes, fetched, indexes, market}; pub trait DynCohortVecs: Send + Sync { fn starting_height(&self) -> Height; diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs similarity index 96% rename from crates/brk_computer/src/vecs/stateful/utxo_cohort.rs rename to crates/brk_computer/src/stateful/utxo_cohort.rs index 4058b5906..959edf4b0 100644 --- a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -6,13 +6,10 @@ use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format}; use crate::{ - UTXOCohortState, - vecs::{ - Indexes, fetched, indexes, market, - stateful::{ - common, - r#trait::{CohortVecs, DynCohortVecs}, - }, + Indexes, UTXOCohortState, fetched, indexes, market, + stateful::{ + common, + r#trait::{CohortVecs, DynCohortVecs}, }, }; diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs similarity index 99% rename from crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs rename to crates/brk_computer/src/stateful/utxo_cohorts.rs index e2e816fa5..b8c1e9957 100644 --- a/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -11,8 +11,9 @@ use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; use crate::{ + Indexes, fetched, indexes, + stateful::r#trait::DynCohortVecs, states::{BlockState, Transacted}, - vecs::{Indexes, fetched, indexes, stateful::r#trait::DynCohortVecs}, }; use super::{r#trait::CohortVecs, utxo_cohort}; diff --git a/crates/brk_computer/src/vecs/stateful/withaddressdatasource.rs b/crates/brk_computer/src/stateful/withaddressdatasource.rs similarity index 100% rename from crates/brk_computer/src/vecs/stateful/withaddressdatasource.rs rename to crates/brk_computer/src/stateful/withaddressdatasource.rs diff --git a/crates/brk_computer/src/stores.rs b/crates/brk_computer/src/stores.rs index f52d94ac1..572a8d639 100644 --- a/crates/brk_computer/src/stores.rs +++ b/crates/brk_computer/src/stores.rs @@ -9,7 +9,7 @@ use brk_store::{AnyStore, Store}; use fjall::{PersistMode, TransactionalKeyspace}; use log::info; -use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource}; +use crate::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource}; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/transactions.rs b/crates/brk_computer/src/transactions.rs similarity index 99% rename from crates/brk_computer/src/vecs/transactions.rs rename to crates/brk_computer/src/transactions.rs index d2e008f30..220e544ea 100644 --- a/crates/brk_computer/src/vecs/transactions.rs +++ b/crates/brk_computer/src/transactions.rs @@ -11,17 +11,13 @@ use brk_vec::{ ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator, }; -use crate::vecs::grouped::Source; - -use super::{ - Indexes, fetched, - grouped::{ - ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, - ComputedVecsFromTxindex, VecBuilderOptions, - }, - indexes, +use crate::grouped::{ + ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, + ComputedVecsFromTxindex, Source, VecBuilderOptions, }; +use super::{Indexes, fetched, indexes}; + const VERSION: Version = Version::ZERO; #[derive(Clone)] diff --git a/crates/brk_core/src/structs/anyaddressdataindex.rs b/crates/brk_core/src/structs/anyaddressdataindex.rs new file mode 100644 index 000000000..bc0ac6120 --- /dev/null +++ b/crates/brk_core/src/structs/anyaddressdataindex.rs @@ -0,0 +1,95 @@ +use std::ops::Add; + +use byteview::ByteView; +use derive_deref::{Deref, DerefMut}; +use serde::Serialize; +use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::{CheckedSub, Printable, TypeIndex}; + +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] +pub struct AnyAddressDataIndex(TypeIndex); +// impl From for AddressDataIndex { +// fn from(value: TypeIndex) -> Self { +// Self(value) +// } +// } +// impl From for TypeIndex { +// fn from(value: AddressDataIndex) -> Self { +// value.0 +// } +// } +// impl From for u32 { +// fn from(value: AddressDataIndex) -> Self { +// Self::from(*value) +// } +// } +// impl From for AddressDataIndex { +// fn from(value: u32) -> Self { +// Self(TypeIndex::from(value)) +// } +// } +// impl From for usize { +// fn from(value: AddressDataIndex) -> Self { +// Self::from(*value) +// } +// } +// impl From for AddressDataIndex { +// fn from(value: usize) -> Self { +// Self(TypeIndex::from(value)) +// } +// } +// impl Add for AddressDataIndex { +// type Output = Self; +// fn add(self, rhs: usize) -> Self::Output { +// Self(*self + rhs) +// } +// } +// impl CheckedSub for AddressDataIndex { +// fn checked_sub(self, rhs: Self) -> Option { +// self.0.checked_sub(rhs.0).map(Self) +// } +// } + +// impl Printable for AddressDataIndex { +// fn to_string() -> &'static str { +// "p2pk33addressindex" +// } + +// fn to_possible_strings() -> &'static [&'static str] { +// &["addr", "p2pk33addr", "p2pk33addressindex"] +// } +// } + +// impl From for AddressDataIndex { +// fn from(value: ByteView) -> Self { +// Self::read_from_bytes(&value).unwrap() +// } +// } +// impl From for ByteView { +// fn from(value: AddressDataIndex) -> Self { +// Self::from(&value) +// } +// } +// impl From<&AddressDataIndex> for ByteView { +// fn from(value: &AddressDataIndex) -> Self { +// Self::new(value.as_bytes()) +// } +// } diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_core/src/structs/mod.rs index e21c12922..49628b181 100644 --- a/crates/brk_core/src/structs/mod.rs +++ b/crates/brk_core/src/structs/mod.rs @@ -1,6 +1,7 @@ mod addressbytes; mod addressbyteshash; mod addressdata; +mod anyaddressdataindex; mod bitcoin; mod blockhash; mod blockhashprefix; @@ -59,6 +60,7 @@ mod yearindex; pub use addressbytes::*; pub use addressbyteshash::*; pub use addressdata::*; +pub use anyaddressdataindex::*; pub use bitcoin::*; pub use blockhash::*; pub use blockhashprefix::*; diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index e9bf3df43..5ecac38dc 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -29,7 +29,7 @@ pub use params::{IdParam, Params, ParamsOpt}; pub use table::Tabled; use vecs::Vecs; -use crate::vecs::{IdToVec, IndexToVec}; +use crate::{IdToVec, IndexToVec}; pub struct Interface<'a> { vecs: Vecs<'a>, diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index 677712d17..d8bbfb936 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brk_vec" -description = "A push-only, truncable, compressable, saveable Vec" +description = "A storeable vec" keywords = ["vec", "disk", "data"] categories = ["database"] version.workspace = true diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index fc5500411..dccb39b02 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -9,7 +9,6 @@ type VEC = StoredVec; fn main() -> Result<(), Box> { let _ = fs::remove_dir_all("./vec"); - let _ = fs::remove_file("./vec"); let version = Version::TWO; let format = Format::Raw; @@ -102,6 +101,38 @@ fn main() -> Result<(), Box> { dbg!(iter.get(0.into())); dbg!(iter.get(20.into())); dbg!(iter.get(21.into())); + + let mmap = vec.create_mmap()?; + dbg!(vec.take(10.into(), &mmap)?); + dbg!(vec.get_or_read(10.into(), &mmap)); + dbg!(vec.holes()); + vec.flush()?; + dbg!(vec.holes()); + } + + { + let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?; + + let mmap = vec.create_mmap()?; + + dbg!(vec.holes()); + dbg!(vec.get_or_read(10.into(), &mmap)?); + + vec.update(10.into(), 10); + vec.update(0.into(), 10); + dbg!( + vec.holes(), + vec.get_or_read(0.into(), &mmap), + vec.get_or_read(10.into(), &mmap)? + ); + + vec.flush()?; + } + + { + let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?; + + dbg!(vec.collect()); } Ok(()) diff --git a/crates/brk_vec/src/traits/generic.rs b/crates/brk_vec/src/traits/generic.rs index 4b0ffd2cf..8b2ff9421 100644 --- a/crates/brk_vec/src/traits/generic.rs +++ b/crates/brk_vec/src/traits/generic.rs @@ -1,11 +1,12 @@ use std::{ borrow::Cow, - fs::{File, OpenOptions}, + collections::{BTreeMap, BTreeSet}, + fs::{self, File, OpenOptions}, io::{self, Seek, SeekFrom, Write}, path::{Path, PathBuf}, }; -use brk_core::Result; +use brk_core::{Error, Result}; use memmap2::Mmap; use crate::{AnyVec, HEADER_OFFSET, Header}; @@ -44,10 +45,22 @@ where if j >= pushed.len() { return Ok(None); } - Ok(pushed.get(j).map(Cow::Borrowed)) - } else { - Ok(self.read_(index, mmap)?.map(Cow::Owned)) + return Ok(pushed.get(j).map(Cow::Borrowed)); } + + let updated = self.updated(); + if !updated.is_empty() + && let Some(updated) = updated.get(&index) + { + return Ok(Some(Cow::Borrowed(updated))); + } + + let holes = self.holes(); + if !holes.is_empty() && holes.contains(&index) { + return Ok(None); + } + + Ok(self.read_(index, mmap)?.map(Cow::Owned)) } #[inline] @@ -72,6 +85,47 @@ where self.mut_pushed().push(value) } + fn holes(&self) -> &BTreeSet; + fn mut_holes(&mut self) -> &mut BTreeSet; + fn take(&mut self, index: I, mmap: &Mmap) -> Result> { + let opt = self.get_or_read(index, mmap)?.map(|v| v.into_owned()); + if opt.is_some() { + let uindex = index.unwrap_to_usize(); + let updated = self.mut_updated(); + if !updated.is_empty() { + updated.remove(&uindex); + } + self.mut_holes().insert(uindex); + } + Ok(opt) + } + + fn updated(&self) -> &BTreeMap; + fn mut_updated(&mut self) -> &mut BTreeMap; + #[inline] + fn update(&mut self, index: I, value: T) -> Result<()> { + let uindex = index.unwrap_to_usize(); + let stored_len = self.stored_len(); + + if uindex >= stored_len { + if let Some(prev) = self.mut_pushed().get_mut(uindex - stored_len) { + *prev = value; + return Ok(()); + } else { + return Err(Error::IndexTooHigh); + } + } + + let holes = self.mut_holes(); + if !holes.is_empty() { + holes.remove(&index.unwrap_to_usize()); + } + + self.mut_updated().insert(index.unwrap_to_usize(), value); + + Ok(()) + } + fn header(&self) -> &Header; fn mut_header(&mut self) -> &mut Header; @@ -85,14 +139,24 @@ where parent.join(name) } + #[inline] fn path(&self) -> PathBuf { Self::path_(self.parent(), self.name()) } - + #[inline] fn path_(parent: &Path, name: &str) -> PathBuf { Self::folder_(parent, name).join(I::to_string()) } + #[inline] + fn holes_path(&self) -> PathBuf { + Self::holes_path_(self.parent(), self.name()) + } + #[inline] + fn holes_path_(parent: &Path, name: &str) -> PathBuf { + Self::folder_(parent, name).join(format!("{}_holes", I::to_string())) + } + // --- fn open_file(&self) -> io::Result { @@ -134,6 +198,10 @@ where #[inline] fn reset_(&mut self) -> Result<()> { + let holes_path = self.holes_path(); + if fs::exists(&holes_path)? { + fs::remove_file(&holes_path)?; + } let mut file = self.open_file()?; self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[]) } diff --git a/crates/brk_vec/src/traits/index.rs b/crates/brk_vec/src/traits/index.rs index 1687c7a5d..723d28b45 100644 --- a/crates/brk_vec/src/traits/index.rs +++ b/crates/brk_vec/src/traits/index.rs @@ -1,6 +1,7 @@ use std::{fmt::Debug, ops::Add}; use brk_core::{Error, Printable, Result}; +use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; pub trait StoredIndex where @@ -15,6 +16,10 @@ where + TryInto + From + Add + + TryFromBytes + + IntoBytes + + Immutable + + KnownLayout + Send + Sync + Printable, @@ -37,6 +42,10 @@ where + TryInto + From + Add + + TryFromBytes + + IntoBytes + + Immutable + + KnownLayout + Send + Sync + Printable, diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index d81485a0b..c3bd11a6f 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -1,5 +1,6 @@ use std::{ borrow::Cow, + collections::{BTreeMap, BTreeSet}, fs, mem, path::{Path, PathBuf}, sync::Arc, @@ -57,6 +58,8 @@ where } pub fn import(parent: &Path, name: &str, version: Version) -> Result { + panic!("Compressed vecs are a work in progress right now, please use raw vecs instead"); + let mut inner = RawVec::import(parent, name, version)?; let pages_meta = { @@ -196,6 +199,22 @@ where fn mut_pushed(&mut self) -> &mut Vec { self.inner.mut_pushed() } + #[inline] + fn holes(&self) -> &BTreeSet { + self.inner.holes() + } + #[inline] + fn mut_holes(&mut self) -> &mut BTreeSet { + self.inner.mut_holes() + } + #[inline] + fn updated(&self) -> &BTreeMap { + self.inner.updated() + } + #[inline] + fn mut_updated(&mut self) -> &mut BTreeMap { + self.inner.mut_updated() + } #[inline] fn path(&self) -> PathBuf { diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 5847979a8..8217fdefa 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -1,9 +1,11 @@ use std::{ borrow::Cow, + collections::{BTreeMap, BTreeSet}, fs::{self, File}, io, marker::PhantomData, mem, + os::unix::fs::FileExt, path::{Path, PathBuf}, sync::{ Arc, @@ -24,12 +26,17 @@ const VERSION: Version = Version::ONE; #[derive(Debug)] pub struct RawVec { + // --- Needed for &, TODO: Weak copy ? header: Header, parent: PathBuf, name: &'static str, - pushed: Vec, - local_stored_len: Option, shared_stored_len: Arc, + // --- Needed for &mut + pushed: Vec, + has_stored_holes: bool, + holes: BTreeSet, + updated: BTreeMap, + local_stored_len: Option, phantom: PhantomData, } @@ -90,17 +97,36 @@ where 0 }; + let mut has_stored_holes = false; + let holes_path = Self::holes_path_(parent, name); + let holes = if fs::exists(&holes_path)? { + has_stored_holes = true; + let bytes = fs::read(&holes_path)?; + bytes + .chunks(size_of::()) + .map(|b| -> Result { + Ok(usize::from_ne_bytes(brk_core::copy_first_8bytes(b)?)) + }) + .collect::>>()? + } else { + BTreeSet::new() + }; + Ok(Self { header, name: Box::leak(Box::new(name.to_string())), parent: parent.to_owned(), pushed: vec![], + has_stored_holes, + holes, + updated: BTreeMap::new(), local_stored_len: Some(stored_len), shared_stored_len: Arc::new(AtomicUsize::new(stored_len)), phantom: PhantomData, }) } + #[doc(hidden)] pub fn set_stored_len(&mut self, len: usize) { self.local_stored_len.replace(len); self.shared_stored_len.store(len, Ordering::Relaxed); @@ -171,6 +197,24 @@ where &mut self.pushed } + #[inline] + fn holes(&self) -> &BTreeSet { + &self.holes + } + #[inline] + fn mut_holes(&mut self) -> &mut BTreeSet { + &mut self.holes + } + + #[inline] + fn updated(&self) -> &BTreeMap { + &self.updated + } + #[inline] + fn mut_updated(&mut self) -> &mut BTreeMap { + &mut self.updated + } + #[inline] fn parent(&self) -> &Path { &self.parent @@ -181,33 +225,70 @@ where let pushed_len = self.pushed_len(); - if pushed_len == 0 { + let has_new_data = pushed_len != 0; + let has_updated_data = !self.updated.is_empty(); + let has_holes = !self.holes.is_empty(); + let had_holes = self.has_stored_holes && !has_holes; + + if !has_new_data && !has_updated_data && !has_holes && !had_holes { return Ok(()); } - let bytes = { - let pushed = &mut self.pushed; + if has_new_data || has_updated_data { + let mut file = file_opt.unwrap_or(self.open_file()?); - let mut bytes: Vec = vec![0; pushed.len() * Self::SIZE_OF_T]; + if has_new_data { + let bytes = { + let mut bytes: Vec = vec![0; pushed_len * Self::SIZE_OF_T]; - let unsafe_bytes = UnsafeSlice::new(&mut bytes); + let unsafe_bytes = UnsafeSlice::new(&mut bytes); - mem::take(pushed) - .into_par_iter() - .enumerate() - .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes())); + mem::take(&mut self.pushed) + .into_par_iter() + .enumerate() + .for_each(|(i, v)| { + unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes()) + }); - bytes - }; + bytes + }; - let mut file = file_opt.unwrap_or(self.open_file()?); - self.file_write_all(&mut file, &bytes)?; + self.file_write_all(&mut file, &bytes)?; - if let Some(local_stored_len) = self.local_stored_len.as_mut() { - *local_stored_len += pushed_len; + if let Some(local_stored_len) = self.local_stored_len.as_mut() { + *local_stored_len += pushed_len; + } + self.shared_stored_len + .fetch_add(pushed_len, Ordering::Relaxed); + } + + if has_updated_data { + mem::take(&mut self.updated) + .into_iter() + .try_for_each(|(i, v)| -> Result<()> { + file.write_all_at( + v.as_bytes(), + ((i * Self::SIZE_OF_T) + HEADER_OFFSET) as u64, + )?; + Ok(()) + })?; + } + } + + if has_holes || had_holes { + let holes_path = self.holes_path(); + if has_holes { + fs::write( + &holes_path, + self.holes + .iter() + .flat_map(|i| i.to_ne_bytes()) + .collect::>(), + )?; + } else if had_holes { + let _ = fs::remove_file(&holes_path); + } } - self.shared_stored_len - .fetch_add(pushed_len, Ordering::Relaxed); Ok(()) } @@ -278,9 +359,12 @@ impl Clone for RawVec { parent: self.parent.clone(), name: self.name, pushed: vec![], - phantom: PhantomData, + updated: BTreeMap::new(), + has_stored_holes: false, + holes: BTreeSet::new(), local_stored_len: None, shared_stored_len: self.shared_stored_len.clone(), + phantom: PhantomData, } } } diff --git a/crates/brk_vec/src/variants/stored.rs b/crates/brk_vec/src/variants/stored.rs index 20f81283f..765ca9dcb 100644 --- a/crates/brk_vec/src/variants/stored.rs +++ b/crates/brk_vec/src/variants/stored.rs @@ -1,5 +1,6 @@ use std::{ borrow::Cow, + collections::{BTreeMap, BTreeSet}, path::{Path, PathBuf}, }; @@ -105,6 +106,36 @@ where } } + #[inline] + fn holes(&self) -> &BTreeSet { + match self { + StoredVec::Raw(v) => v.holes(), + StoredVec::Compressed(v) => v.holes(), + } + } + #[inline] + fn mut_holes(&mut self) -> &mut BTreeSet { + match self { + StoredVec::Raw(v) => v.mut_holes(), + StoredVec::Compressed(v) => v.mut_holes(), + } + } + + #[inline] + fn updated(&self) -> &BTreeMap { + match self { + StoredVec::Raw(v) => v.updated(), + StoredVec::Compressed(v) => v.updated(), + } + } + #[inline] + fn mut_updated(&mut self) -> &mut BTreeMap { + match self { + StoredVec::Raw(v) => v.mut_updated(), + StoredVec::Compressed(v) => v.mut_updated(), + } + } + #[inline] fn path(&self) -> PathBuf { match self { diff --git a/docker/DOCKER.md b/docker/README.md similarity index 100% rename from docker/DOCKER.md rename to docker/README.md