computer: convert stores to vecs part 1

crates/brk_computer/src/all.rs

@@ -5,22 +5,11 @@ use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format};
-
-pub mod blocks;
-pub mod cointime;
-pub mod constants;
-pub mod fetched;
-pub mod grouped;
-pub mod indexes;
-pub mod market;
-pub mod mining;
-pub mod stateful;
-pub mod transactions;
-
-pub use indexes::Indexes;
 use log::info;
 
-use crate::stores::Stores;
+use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
+
+use super::stateful;
 
 const VERSION: Version = Version::ONE;
 
@@ -130,7 +119,6 @@ impl Vecs {
         starting_indexes: brk_indexer::Indexes,
         fetcher: Option<&mut Fetcher>,
         exit: &Exit,
-        stores: &mut Stores,
     ) -> color_eyre::Result<()> {
         info!("Computing indexes...");
         let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
@@ -188,7 +176,6 @@ impl Vecs {
             &self.market,
             &mut starting_indexes,
             exit,
-            stores,
         )?;
 
         self.cointime.compute(

crates/brk_computer/src/blocks.rs

@@ -8,7 +8,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
 
-use crate::vecs::grouped::Source;
+use crate::grouped::Source;
 
 use super::{
     Indexes,

crates/brk_computer/src/cointime.rs

@@ -5,16 +5,13 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
 
-use crate::vecs::{
-    fetched,
-    grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source},
-    stateful, transactions,
-};
-
 use super::{
-    Indexes,
-    grouped::{ComputedVecsFromHeight, VecBuilderOptions},
-    indexes,
+    Indexes, fetched,
+    grouped::{
+        ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
+        Source, VecBuilderOptions,
+    },
+    indexes, stateful, transactions,
 };
 
 const VERSION: Version = Version::ZERO;

crates/brk_computer/src/constants.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
 
-use crate::vecs::grouped::Source;
+use crate::grouped::Source;
 
 use super::{
     Indexes,

crates/brk_computer/src/fetched.rs

@@ -13,7 +13,7 @@ use brk_vec::{
     VecIterator,
 };
 
-use crate::vecs::grouped::Source;
+use crate::grouped::Source;
 
 use super::{
     Indexes,

crates/brk_computer/src/grouped/builder_computed.rs

@@ -7,7 +7,7 @@ use brk_vec::{
     ComputedVec, ComputedVecFrom2, Format, StoredIndex,
 };
 
-use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions};
+use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
 
 use super::ComputedType;
 

crates/brk_computer/src/grouped/from_dateindex.rs

@@ -10,7 +10,7 @@ use brk_vec::{
     AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
 };
 
-use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes};
+use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
 
 use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions};
 

crates/brk_computer/src/grouped/from_height.rs

@@ -10,7 +10,7 @@ use brk_vec::{
     AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
 };
 
-use crate::vecs::{
+use crate::{
     Indexes,
     grouped::{ComputedVecBuilder, Source},
     indexes,

crates/brk_computer/src/grouped/from_height_strict.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, EagerVec, Format};
 
-use crate::vecs::{Indexes, indexes};
+use crate::{Indexes, indexes};
 
 use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
 

crates/brk_computer/src/grouped/from_txindex.rs

@@ -11,7 +11,7 @@ use brk_vec::{
     Format, StoredIndex, VecIterator,
 };
 
-use crate::vecs::{
+use crate::{
     Indexes, fetched,
     grouped::{ComputedVecBuilder, Source},
     indexes,

crates/brk_computer/src/grouped/ratio_from_dateindex.rs

@@ -8,10 +8,7 @@ use brk_vec::{
     StoredIndex, VecIterator,
 };
 
-use crate::{
-    utils::get_percentile,
-    vecs::{Indexes, fetched, grouped::source::Source, indexes},
-};
+use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile};
 
 use super::{ComputedVecsFromDateIndex, VecBuilderOptions};
 

crates/brk_computer/src/grouped/value_from_dateindex.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
 
-use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
+use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
 
 use super::{Source, VecBuilderOptions};
 

crates/brk_computer/src/grouped/value_from_height.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
 
-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};
 
 use super::{ComputedVecsFromHeight, VecBuilderOptions};
 

crates/brk_computer/src/grouped/value_from_txindex.rs

@@ -8,7 +8,7 @@ use brk_vec::{
     Format, LazyVecFrom1, StoredIndex, StoredVec,
 };
 
-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};
 
 use super::{ComputedVecsFromTxindex, VecBuilderOptions};
 

crates/brk_computer/src/grouped/value_height.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
 
-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};
 
 #[derive(Clone)]
 pub struct ComputedHeightValueVecs {

crates/brk_computer/src/indexes.rs

@@ -15,8 +15,6 @@ use brk_vec::{
     ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator,
 };
 
-use crate::vecs::indexes;
-
 const VERSION: Version = Version::ZERO;
 
 #[derive(Clone)]
@@ -1206,7 +1204,7 @@ pub struct Indexes {
 }
 
 impl Indexes {
-    pub fn update_from_height(&mut self, height: Height, indexes: &indexes::Vecs) {
+    pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
         self.indexes.height = height;
         self.dateindex = DateIndex::try_from(
             indexes

crates/brk_computer/src/lib.rs

@@ -12,20 +12,28 @@ use brk_indexer::Indexer;
 use brk_vec::{Computation, Format};
 use log::info;
 
+mod all;
+mod blocks;
+mod cointime;
+mod constants;
+mod fetched;
+mod grouped;
+mod indexes;
+mod market;
+mod mining;
+mod stateful;
 mod states;
-mod stores;
+mod transactions;
 mod utils;
-mod vecs;
+
+use indexes::Indexes;
 
 use states::*;
-use stores::Stores;
-use vecs::Vecs;
 
 #[derive(Clone)]
 pub struct Computer {
     fetcher: Option<Fetcher>,
-    pub vecs: Vecs,
-    pub stores: Stores,
+    pub vecs: all::Vecs,
 }
 
 const VERSION: Version = Version::ONE;
@@ -40,7 +48,7 @@ impl Computer {
         format: Format,
     ) -> color_eyre::Result<Self> {
         Ok(Self {
-            vecs: Vecs::import(
+            vecs: all::Vecs::import(
                 // TODO: Give self.path, join inside import
                 &outputs_dir.join("vecs/computed"),
                 VERSION + Version::ZERO,
@@ -49,12 +57,6 @@ impl Computer {
                 computation,
                 format,
             )?,
-            stores: Stores::import(
-                // TODO: Give self.path, join inside import
-                &outputs_dir.join("stores"),
-                VERSION + Version::ZERO,
-                &indexer.stores.keyspace,
-            )?,
             fetcher,
         })
     }
@@ -68,12 +70,7 @@ impl Computer {
         exit: &Exit,
     ) -> color_eyre::Result<()> {
         info!("Computing...");
-        self.vecs.compute(
-            indexer,
-            starting_indexes,
-            self.fetcher.as_mut(),
-            exit,
-            &mut self.stores,
-        )
+        self.vecs
+            .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
     }
 }

crates/brk_computer/src/market.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, EagerVec, Format, StoredIndex, VecIterator};
 
-use crate::vecs::grouped::Source;
+use crate::grouped::Source;
 
 use super::{
     Indexes, fetched,

crates/brk_computer/src/mining.rs

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
 
-use crate::vecs::grouped::Source;
+use crate::grouped::Source;
 
 use super::{
     Indexes,

crates/brk_computer/src/stateful/address_cohort.rs

@@ -8,16 +8,14 @@ use brk_vec::{
 };
 
 use crate::{
-    states::AddressCohortState,
-    vecs::{
-        Indexes, fetched,
-        grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
-        indexes, market,
-        stateful::{
-            common,
-            r#trait::{CohortVecs, DynCohortVecs},
-        },
+    Indexes, fetched,
+    grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
+    indexes, market,
+    stateful::{
+        common,
+        r#trait::{CohortVecs, DynCohortVecs},
     },
+    states::AddressCohortState,
 };
 
 const VERSION: Version = Version::ZERO;

crates/brk_computer/src/stateful/address_cohorts.rs

@@ -9,7 +9,7 @@ use brk_vec::{Computation, Format};
 use derive_deref::{Deref, DerefMut};
 use rayon::prelude::*;
 
-use crate::vecs::{
+use crate::{
     Indexes, fetched, indexes,
     stateful::{
         address_cohort,

crates/brk_computer/src/stateful/addresstype_to_addresscount.rs

@@ -2,7 +2,7 @@ use brk_core::{ByAddressType, Height};
 use brk_vec::VecIterator;
 use derive_deref::{Deref, DerefMut};
 
-use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
+use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
 
 #[derive(Debug, Default, Deref, DerefMut)]
 pub struct AddressTypeToAddressCount(ByAddressType<usize>);

crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs

@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_vec::EagerVec;
 use derive_deref::{Deref, DerefMut};
 
-use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
+use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
 
 #[derive(Debug, Clone, Deref, DerefMut)]
 pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>);

crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs

@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_vec::AnyCollectableVec;
 use derive_deref::{Deref, DerefMut};
 
-use crate::vecs::{
+use crate::{
     Indexes, grouped::ComputedVecsFromHeight, indexes,
     stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
 };

crates/brk_computer/src/stateful/common.rs

@@ -11,16 +11,13 @@ use brk_vec::{
 };
 
 use crate::{
-    states::CohortState,
-    vecs::{
-        Indexes, fetched,
-        grouped::{
-            ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex,
-            ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
-            Source, VecBuilderOptions,
-        },
-        indexes, market,
+    Indexes, fetched,
+    grouped::{
+        ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
+        ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions,
     },
+    indexes, market,
+    states::CohortState,
 };
 
 const VERSION: Version = Version::ZERO;

crates/brk_computer/src/stateful/height_to_addresstype_to_vec.rs

@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
 use brk_core::Height;
 use derive_deref::{Deref, DerefMut};
 
-use crate::vecs::stateful::AddressTypeToVec;
+use crate::stateful::AddressTypeToVec;
 
 #[derive(Debug, Default, Deref, DerefMut)]
 pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);

crates/brk_computer/src/stateful/mod.rs

@@ -16,16 +16,13 @@ use rayon::prelude::*;
 
 use crate::{
     BlockState, SupplyState, Transacted,
-    stores::Stores,
-    vecs::{
-        grouped::{ComputedVecsFromHeight, Source},
-        market,
-        stateful::{
-            addresstype_to_addresscount::AddressTypeToAddressCount,
-            addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
-            addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount,
-            addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs,
-        },
+    grouped::{ComputedVecsFromHeight, Source},
+    market,
+    stateful::{
+        addresstype_to_addresscount::AddressTypeToAddressCount,
+        addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
+        addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount,
+        addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs,
     },
 };
 
@@ -461,7 +458,6 @@ impl Vecs {
         // Must take ownership as its indexes will be updated for this specific function
         starting_indexes: &mut Indexes,
         exit: &Exit,
-        stores: &mut Stores,
     ) -> color_eyre::Result<()> {
         let height_to_first_outputindex = &indexer.vecs.height_to_first_outputindex;
         let height_to_first_inputindex = &indexer.vecs.height_to_first_inputindex;

crates/brk_computer/src/stateful/trait.rs

@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec};
 
-use crate::vecs::{Indexes, fetched, indexes, market};
+use crate::{Indexes, fetched, indexes, market};
 
 pub trait DynCohortVecs: Send + Sync {
     fn starting_height(&self) -> Height;

crates/brk_computer/src/stateful/utxo_cohort.rs

@@ -6,13 +6,10 @@ use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format};
 
 use crate::{
-    UTXOCohortState,
-    vecs::{
-        Indexes, fetched, indexes, market,
-        stateful::{
-            common,
-            r#trait::{CohortVecs, DynCohortVecs},
-        },
+    Indexes, UTXOCohortState, fetched, indexes, market,
+    stateful::{
+        common,
+        r#trait::{CohortVecs, DynCohortVecs},
     },
 };
 

crates/brk_computer/src/stateful/utxo_cohorts.rs

@@ -11,8 +11,9 @@ use derive_deref::{Deref, DerefMut};
 use rayon::prelude::*;
 
 use crate::{
+    Indexes, fetched, indexes,
+    stateful::r#trait::DynCohortVecs,
     states::{BlockState, Transacted},
-    vecs::{Indexes, fetched, indexes, stateful::r#trait::DynCohortVecs},
 };
 
 use super::{r#trait::CohortVecs, utxo_cohort};

crates/brk_computer/src/stores.rs

@@ -9,7 +9,7 @@ use brk_store::{AnyStore, Store};
 use fjall::{PersistMode, TransactionalKeyspace};
 use log::info;
 
-use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
+use crate::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
 
 const VERSION: Version = Version::ZERO;
 

crates/brk_computer/src/transactions.rs

@@ -11,17 +11,13 @@ use brk_vec::{
     ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
 };
 
-use crate::vecs::grouped::Source;
-
-use super::{
-    Indexes, fetched,
-    grouped::{
-        ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
-        ComputedVecsFromTxindex, VecBuilderOptions,
-    },
-    indexes,
+use crate::grouped::{
+    ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
+    ComputedVecsFromTxindex, Source, VecBuilderOptions,
 };
 
+use super::{Indexes, fetched, indexes};
+
 const VERSION: Version = Version::ZERO;
 
 #[derive(Clone)]

crates/brk_core/src/structs/anyaddressdataindex.rs

@@ -0,0 +1,95 @@
+use std::ops::Add;
+
+use byteview::ByteView;
+use derive_deref::{Deref, DerefMut};
+use serde::Serialize;
+use zerocopy::{FromBytes, IntoBytes};
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Clone,
+    Copy,
+    Deref,
+    DerefMut,
+    Default,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct AnyAddressDataIndex(TypeIndex);
+// impl From<TypeIndex> for AddressDataIndex {
+//     fn from(value: TypeIndex) -> Self {
+//         Self(value)
+//     }
+// }
+// impl From<AddressDataIndex> for TypeIndex {
+//     fn from(value: AddressDataIndex) -> Self {
+//         value.0
+//     }
+// }
+// impl From<AddressDataIndex> for u32 {
+//     fn from(value: AddressDataIndex) -> Self {
+//         Self::from(*value)
+//     }
+// }
+// impl From<u32> for AddressDataIndex {
+//     fn from(value: u32) -> Self {
+//         Self(TypeIndex::from(value))
+//     }
+// }
+// impl From<AddressDataIndex> for usize {
+//     fn from(value: AddressDataIndex) -> Self {
+//         Self::from(*value)
+//     }
+// }
+// impl From<usize> for AddressDataIndex {
+//     fn from(value: usize) -> Self {
+//         Self(TypeIndex::from(value))
+//     }
+// }
+// impl Add<usize> for AddressDataIndex {
+//     type Output = Self;
+//     fn add(self, rhs: usize) -> Self::Output {
+//         Self(*self + rhs)
+//     }
+// }
+// impl CheckedSub<AddressDataIndex> for AddressDataIndex {
+//     fn checked_sub(self, rhs: Self) -> Option<Self> {
+//         self.0.checked_sub(rhs.0).map(Self)
+//     }
+// }
+
+// impl Printable for AddressDataIndex {
+//     fn to_string() -> &'static str {
+//         "p2pk33addressindex"
+//     }
+
+//     fn to_possible_strings() -> &'static [&'static str] {
+//         &["addr", "p2pk33addr", "p2pk33addressindex"]
+//     }
+// }
+
+// impl From<ByteView> for AddressDataIndex {
+//     fn from(value: ByteView) -> Self {
+//         Self::read_from_bytes(&value).unwrap()
+//     }
+// }
+// impl From<AddressDataIndex> for ByteView {
+//     fn from(value: AddressDataIndex) -> Self {
+//         Self::from(&value)
+//     }
+// }
+// impl From<&AddressDataIndex> for ByteView {
+//     fn from(value: &AddressDataIndex) -> Self {
+//         Self::new(value.as_bytes())
+//     }
+// }

crates/brk_core/src/structs/mod.rs

@@ -1,6 +1,7 @@
 mod addressbytes;
 mod addressbyteshash;
 mod addressdata;
+mod anyaddressdataindex;
 mod bitcoin;
 mod blockhash;
 mod blockhashprefix;
@@ -59,6 +60,7 @@ mod yearindex;
 pub use addressbytes::*;
 pub use addressbyteshash::*;
 pub use addressdata::*;
+pub use anyaddressdataindex::*;
 pub use bitcoin::*;
 pub use blockhash::*;
 pub use blockhashprefix::*;

crates/brk_interface/src/lib.rs

@@ -29,7 +29,7 @@ pub use params::{IdParam, Params, ParamsOpt};
 pub use table::Tabled;
 use vecs::Vecs;
 
-use crate::vecs::{IdToVec, IndexToVec};
+use crate::{IdToVec, IndexToVec};
 
 pub struct Interface<'a> {
     vecs: Vecs<'a>,

crates/brk_vec/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brk_vec"
-description = "A push-only, truncable, compressable, saveable Vec"
+description = "A storeable vec"
 keywords = ["vec", "disk", "data"]
 categories = ["database"]
 version.workspace = true

crates/brk_vec/examples/main.rs

@@ -9,7 +9,6 @@ type VEC = StoredVec<I, u32>;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let _ = fs::remove_dir_all("./vec");
-    let _ = fs::remove_file("./vec");
 
     let version = Version::TWO;
     let format = Format::Raw;
@@ -102,6 +101,38 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         dbg!(iter.get(0.into()));
         dbg!(iter.get(20.into()));
         dbg!(iter.get(21.into()));
+
+        let mmap = vec.create_mmap()?;
+        dbg!(vec.take(10.into(), &mmap)?);
+        dbg!(vec.get_or_read(10.into(), &mmap));
+        dbg!(vec.holes());
+        vec.flush()?;
+        dbg!(vec.holes());
+    }
+
+    {
+        let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+
+        let mmap = vec.create_mmap()?;
+
+        dbg!(vec.holes());
+        dbg!(vec.get_or_read(10.into(), &mmap)?);
+
+        vec.update(10.into(), 10);
+        vec.update(0.into(), 10);
+        dbg!(
+            vec.holes(),
+            vec.get_or_read(0.into(), &mmap),
+            vec.get_or_read(10.into(), &mmap)?
+        );
+
+        vec.flush()?;
+    }
+
+    {
+        let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+
+        dbg!(vec.collect());
     }
 
     Ok(())

crates/brk_vec/src/traits/generic.rs

@@ -1,11 +1,12 @@
 use std::{
     borrow::Cow,
-    fs::{File, OpenOptions},
+    collections::{BTreeMap, BTreeSet},
+    fs::{self, File, OpenOptions},
     io::{self, Seek, SeekFrom, Write},
     path::{Path, PathBuf},
 };
 
-use brk_core::Result;
+use brk_core::{Error, Result};
 use memmap2::Mmap;
 
 use crate::{AnyVec, HEADER_OFFSET, Header};
@@ -44,10 +45,22 @@ where
             if j >= pushed.len() {
                 return Ok(None);
             }
-            Ok(pushed.get(j).map(Cow::Borrowed))
-        } else {
-            Ok(self.read_(index, mmap)?.map(Cow::Owned))
+            return Ok(pushed.get(j).map(Cow::Borrowed));
         }
+
+        let updated = self.updated();
+        if !updated.is_empty()
+            && let Some(updated) = updated.get(&index)
+        {
+            return Ok(Some(Cow::Borrowed(updated)));
+        }
+
+        let holes = self.holes();
+        if !holes.is_empty() && holes.contains(&index) {
+            return Ok(None);
+        }
+
+        Ok(self.read_(index, mmap)?.map(Cow::Owned))
     }
 
     #[inline]
@@ -72,6 +85,47 @@ where
         self.mut_pushed().push(value)
     }
 
+    fn holes(&self) -> &BTreeSet<usize>;
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize>;
+    fn take(&mut self, index: I, mmap: &Mmap) -> Result<Option<T>> {
+        let opt = self.get_or_read(index, mmap)?.map(|v| v.into_owned());
+        if opt.is_some() {
+            let uindex = index.unwrap_to_usize();
+            let updated = self.mut_updated();
+            if !updated.is_empty() {
+                updated.remove(&uindex);
+            }
+            self.mut_holes().insert(uindex);
+        }
+        Ok(opt)
+    }
+
+    fn updated(&self) -> &BTreeMap<usize, T>;
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T>;
+    #[inline]
+    fn update(&mut self, index: I, value: T) -> Result<()> {
+        let uindex = index.unwrap_to_usize();
+        let stored_len = self.stored_len();
+
+        if uindex >= stored_len {
+            if let Some(prev) = self.mut_pushed().get_mut(uindex - stored_len) {
+                *prev = value;
+                return Ok(());
+            } else {
+                return Err(Error::IndexTooHigh);
+            }
+        }
+
+        let holes = self.mut_holes();
+        if !holes.is_empty() {
+            holes.remove(&index.unwrap_to_usize());
+        }
+
+        self.mut_updated().insert(index.unwrap_to_usize(), value);
+
+        Ok(())
+    }
+
     fn header(&self) -> &Header;
     fn mut_header(&mut self) -> &mut Header;
 
@@ -85,14 +139,24 @@ where
         parent.join(name)
     }
 
+    #[inline]
     fn path(&self) -> PathBuf {
         Self::path_(self.parent(), self.name())
     }
-
+    #[inline]
     fn path_(parent: &Path, name: &str) -> PathBuf {
         Self::folder_(parent, name).join(I::to_string())
     }
 
+    #[inline]
+    fn holes_path(&self) -> PathBuf {
+        Self::holes_path_(self.parent(), self.name())
+    }
+    #[inline]
+    fn holes_path_(parent: &Path, name: &str) -> PathBuf {
+        Self::folder_(parent, name).join(format!("{}_holes", I::to_string()))
+    }
+
     // ---
 
     fn open_file(&self) -> io::Result<File> {
@@ -134,6 +198,10 @@ where
 
     #[inline]
     fn reset_(&mut self) -> Result<()> {
+        let holes_path = self.holes_path();
+        if fs::exists(&holes_path)? {
+            fs::remove_file(&holes_path)?;
+        }
         let mut file = self.open_file()?;
         self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[])
     }

crates/brk_vec/src/traits/index.rs

@@ -1,6 +1,7 @@
 use std::{fmt::Debug, ops::Add};
 
 use brk_core::{Error, Printable, Result};
+use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
 
 pub trait StoredIndex
 where
@@ -15,6 +16,10 @@ where
         + TryInto<usize>
         + From<usize>
         + Add<usize, Output = Self>
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
         + Send
         + Sync
         + Printable,
@@ -37,6 +42,10 @@ where
         + TryInto<usize>
         + From<usize>
         + Add<usize, Output = Self>
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
         + Send
         + Sync
         + Printable,

crates/brk_vec/src/variants/compressed.rs

@@ -1,5 +1,6 @@
 use std::{
     borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
     fs, mem,
     path::{Path, PathBuf},
     sync::Arc,
@@ -57,6 +58,8 @@ where
     }
 
     pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
+        panic!("Compressed vecs are a work in progress right now, please use raw vecs instead");
+
         let mut inner = RawVec::import(parent, name, version)?;
 
         let pages_meta = {
@@ -196,6 +199,22 @@ where
     fn mut_pushed(&mut self) -> &mut Vec<T> {
         self.inner.mut_pushed()
     }
+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        self.inner.holes()
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        self.inner.mut_holes()
+    }
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        self.inner.updated()
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        self.inner.mut_updated()
+    }
 
     #[inline]
     fn path(&self) -> PathBuf {

crates/brk_vec/src/variants/raw.rs

@@ -1,9 +1,11 @@
 use std::{
     borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
     fs::{self, File},
     io,
     marker::PhantomData,
     mem,
+    os::unix::fs::FileExt,
     path::{Path, PathBuf},
     sync::{
         Arc,
@@ -24,12 +26,17 @@ const VERSION: Version = Version::ONE;
 
 #[derive(Debug)]
 pub struct RawVec<I, T> {
+    // --- Needed for &, TODO: Weak copy ?
     header: Header,
     parent: PathBuf,
     name: &'static str,
-    pushed: Vec<T>,
-    local_stored_len: Option<usize>,
     shared_stored_len: Arc<AtomicUsize>,
+    // --- Needed for &mut
+    pushed: Vec<T>,
+    has_stored_holes: bool,
+    holes: BTreeSet<usize>,
+    updated: BTreeMap<usize, T>,
+    local_stored_len: Option<usize>,
     phantom: PhantomData<I>,
 }
 
@@ -90,17 +97,36 @@ where
             0
         };
 
+        let mut has_stored_holes = false;
+        let holes_path = Self::holes_path_(parent, name);
+        let holes = if fs::exists(&holes_path)? {
+            has_stored_holes = true;
+            let bytes = fs::read(&holes_path)?;
+            bytes
+                .chunks(size_of::<usize>())
+                .map(|b| -> Result<usize> {
+                    Ok(usize::from_ne_bytes(brk_core::copy_first_8bytes(b)?))
+                })
+                .collect::<Result<BTreeSet<usize>>>()?
+        } else {
+            BTreeSet::new()
+        };
+
         Ok(Self {
             header,
             name: Box::leak(Box::new(name.to_string())),
             parent: parent.to_owned(),
             pushed: vec![],
+            has_stored_holes,
+            holes,
+            updated: BTreeMap::new(),
             local_stored_len: Some(stored_len),
             shared_stored_len: Arc::new(AtomicUsize::new(stored_len)),
             phantom: PhantomData,
         })
     }
 
+    #[doc(hidden)]
     pub fn set_stored_len(&mut self, len: usize) {
         self.local_stored_len.replace(len);
         self.shared_stored_len.store(len, Ordering::Relaxed);
@@ -171,6 +197,24 @@ where
         &mut self.pushed
     }
 
+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        &self.holes
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        &mut self.holes
+    }
+
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        &self.updated
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        &mut self.updated
+    }
+
     #[inline]
     fn parent(&self) -> &Path {
         &self.parent
@@ -181,33 +225,70 @@ where
 
         let pushed_len = self.pushed_len();
 
-        if pushed_len == 0 {
+        let has_new_data = pushed_len != 0;
+        let has_updated_data = !self.updated.is_empty();
+        let has_holes = !self.holes.is_empty();
+        let had_holes = self.has_stored_holes && !has_holes;
+
+        if !has_new_data && !has_updated_data && !has_holes && !had_holes {
             return Ok(());
         }
 
-        let bytes = {
-            let pushed = &mut self.pushed;
+        if has_new_data || has_updated_data {
+            let mut file = file_opt.unwrap_or(self.open_file()?);
 
-            let mut bytes: Vec<u8> = vec![0; pushed.len() * Self::SIZE_OF_T];
+            if has_new_data {
+                let bytes = {
+                    let mut bytes: Vec<u8> = vec![0; pushed_len * Self::SIZE_OF_T];
 
-            let unsafe_bytes = UnsafeSlice::new(&mut bytes);
+                    let unsafe_bytes = UnsafeSlice::new(&mut bytes);
 
-            mem::take(pushed)
-                .into_par_iter()
-                .enumerate()
-                .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes()));
+                    mem::take(&mut self.pushed)
+                        .into_par_iter()
+                        .enumerate()
+                        .for_each(|(i, v)| {
+                            unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes())
+                        });
 
-            bytes
-        };
+                    bytes
+                };
 
-        let mut file = file_opt.unwrap_or(self.open_file()?);
-        self.file_write_all(&mut file, &bytes)?;
+                self.file_write_all(&mut file, &bytes)?;
 
-        if let Some(local_stored_len) = self.local_stored_len.as_mut() {
-            *local_stored_len += pushed_len;
+                if let Some(local_stored_len) = self.local_stored_len.as_mut() {
+                    *local_stored_len += pushed_len;
+                }
+                self.shared_stored_len
+                    .fetch_add(pushed_len, Ordering::Relaxed);
+            }
+
+            if has_updated_data {
+                mem::take(&mut self.updated)
+                    .into_iter()
+                    .try_for_each(|(i, v)| -> Result<()> {
+                        file.write_all_at(
+                            v.as_bytes(),
+                            ((i * Self::SIZE_OF_T) + HEADER_OFFSET) as u64,
+                        )?;
+                        Ok(())
+                    })?;
+            }
+        }
+
+        if has_holes || had_holes {
+            let holes_path = self.holes_path();
+            if has_holes {
+                fs::write(
+                    &holes_path,
+                    self.holes
+                        .iter()
+                        .flat_map(|i| i.to_ne_bytes())
+                        .collect::<Vec<_>>(),
+                )?;
+            } else if had_holes {
+                let _ = fs::remove_file(&holes_path);
+            }
         }
-        self.shared_stored_len
-            .fetch_add(pushed_len, Ordering::Relaxed);
 
         Ok(())
     }
@@ -278,9 +359,12 @@ impl<I, T> Clone for RawVec<I, T> {
             parent: self.parent.clone(),
             name: self.name,
             pushed: vec![],
-            phantom: PhantomData,
+            updated: BTreeMap::new(),
+            has_stored_holes: false,
+            holes: BTreeSet::new(),
             local_stored_len: None,
             shared_stored_len: self.shared_stored_len.clone(),
+            phantom: PhantomData,
         }
     }
 }

crates/brk_vec/src/variants/stored.rs

@@ -1,5 +1,6 @@
 use std::{
     borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
     path::{Path, PathBuf},
 };
 
@@ -105,6 +106,36 @@ where
         }
     }
 
+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        match self {
+            StoredVec::Raw(v) => v.holes(),
+            StoredVec::Compressed(v) => v.holes(),
+        }
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        match self {
+            StoredVec::Raw(v) => v.mut_holes(),
+            StoredVec::Compressed(v) => v.mut_holes(),
+        }
+    }
+
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        match self {
+            StoredVec::Raw(v) => v.updated(),
+            StoredVec::Compressed(v) => v.updated(),
+        }
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        match self {
+            StoredVec::Raw(v) => v.mut_updated(),
+            StoredVec::Compressed(v) => v.mut_updated(),
+        }
+    }
+
     #[inline]
     fn path(&self) -> PathBuf {
         match self {