global: snapshot

crates/brk_vec/src/lib.rs

@@ -1,4 +1,7 @@
 #![doc = include_str!("../README.md")]
+#![doc = "\n## Example\n\n```rust"]
+#![doc = include_str!("main.rs")]
+#![doc = "```"]
 
 use std::{
     cmp::Ordering,
@@ -26,23 +29,6 @@ pub use enums::*;
 pub use structs::*;
 pub use traits::*;
 
-type Buffer = Vec<u8>;
-
-/// Uses `Mmap` instead of `File`
-///
-/// Used in `/indexer`
-pub const CACHED_GETS: u8 = 0;
-
-/// Will use the same `File` for every read, so not thread safe
-///
-/// Used in `/computer`
-pub const SINGLE_THREAD: u8 = 1;
-
-/// Will spin up a new `File` for every read
-///
-/// Used in `/server`
-pub const STATELESS: u8 = 2;
-
 ///
 /// A very small, fast, efficient and simple storable Vec
 ///
@@ -55,16 +41,15 @@ pub const STATELESS: u8 = 2;
 /// If you don't call `.flush()` it just acts as a normal Vec
 ///
 #[derive(Debug)]
-pub struct StorableVec<I, T, const MODE: u8> {
+pub struct StorableVec<I, T> {
     version: Version,
     pathbuf: PathBuf,
     file: File,
     /// **Number of values NOT number of bytes**
     file_len: usize,
     file_position: u64,
-    buf: Buffer,
-    /// Only for CACHED_GETS
-    cache: Vec<OnceLock<Box<memmap2::Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
+    buf: Vec<u8>,
+    mmaps: Vec<OnceLock<Box<memmap2::Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
     pushed: Vec<T>,
     // updated: BTreeMap<usize, T>,
     // inserted: BTreeMap<usize, T>,
@@ -76,11 +61,12 @@ pub struct StorableVec<I, T, const MODE: u8> {
 
 /// In bytes
 const MAX_PAGE_SIZE: usize = 4 * 4096;
-const ONE_MB: usize = 1000 * 1024;
+const ONE_MB: usize = 1024 * 1024;
 // const MAX_CACHE_SIZE: usize = usize::MAX;
 const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
+const FLUSH_EVERY: usize = 10_000;
 
-impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
+impl<I, T> StorableVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,
@@ -106,11 +92,9 @@ where
     pub fn import(path: &Path, version: Version) -> Result<Self> {
         fs::create_dir_all(path)?;
 
-        if MODE != STATELESS {
-            let path = Self::path_version_(path);
-            version.validate(path.as_ref())?;
-            version.write(path.as_ref())?;
-        }
+        let version_path = Self::path_version_(path);
+        version.validate(version_path.as_ref())?;
+        version.write(version_path.as_ref())?;
 
         let file = Self::open_file_(&Self::path_vec_(path))?;
 
@@ -121,7 +105,7 @@ where
             file_len: Self::read_disk_len_(&file)?,
             file,
             buf: Self::create_buffer(),
-            cache: vec![],
+            mmaps: vec![],
             pushed: vec![],
             // updated: BTreeMap::new(),
             // inserted: BTreeMap::new(),
@@ -131,13 +115,13 @@ where
             // opened_mmaps: AtomicUsize::new(0),
         };
 
-        slf.reset_disk_related_state()?;
+        slf.reset_file_metadata()?;
 
         Ok(slf)
     }
 
     #[inline]
-    fn create_buffer() -> Buffer {
+    fn create_buffer() -> Vec<u8> {
         vec![0; Self::SIZE_OF_T]
     }
 
@@ -153,9 +137,12 @@ where
             .open(path)
     }
 
-    fn open_file_at_then_read(&self, index: usize) -> Result<T> {
+    pub fn open_then_read(&self, index: I) -> Result<T> {
+        self.open_then_read_(Self::i_to_usize(index)?)
+    }
+    fn open_then_read_(&self, index: usize) -> Result<T> {
         let mut file = self.open_file()?;
-        Self::seek(&mut file, Self::index_to_byte_index(index))?;
+        Self::seek_(&mut file, Self::index_to_byte_index(index))?;
         let mut buf = Self::create_buffer();
         Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned())
     }
@@ -167,35 +154,33 @@ where
         Ok(Self::byte_index_to_index(file.metadata()?.len() as usize))
     }
 
-    fn reset_disk_related_state(&mut self) -> io::Result<()> {
+    fn reset_file_metadata(&mut self) -> io::Result<()> {
         self.file_len = self.read_disk_len()?;
-        self.file_position = 0;
-        self.reset_cache()
+        self.file_position = self.file.seek(SeekFrom::Start(0))?;
+        Ok(())
     }
 
-    fn reset_cache(&mut self) -> io::Result<()> {
-        match MODE {
-            CACHED_GETS => {
-                self.cache.par_iter_mut().for_each(|lock| {
-                    lock.take();
-                });
+    pub fn reset_mmaps(&mut self) -> io::Result<()> {
+        self.mmaps.par_iter_mut().for_each(|lock| {
+            lock.take();
+        });
 
-                let len = (self.file_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
-                let len = Self::CACHE_LENGTH.min(len);
+        let len = (self.file_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
+        let len = Self::CACHE_LENGTH.min(len);
 
-                if self.cache.len() != len {
-                    self.cache.resize_with(len, Default::default);
-                    // self.cache.shrink_to_fit();
-                }
-
-                Ok(())
-            }
-            _ => Ok(()),
+        if self.mmaps.len() != len {
+            self.mmaps.resize_with(len, Default::default);
         }
+
+        Ok(())
     }
 
     #[inline]
-    fn seek(file: &mut File, byte_index: u64) -> io::Result<u64> {
+    fn seek(&mut self, byte_index: u64) -> io::Result<u64> {
+        self.file.seek(SeekFrom::Start(byte_index))
+    }
+    #[inline]
+    fn seek_(file: &mut File, byte_index: u64) -> io::Result<u64> {
         file.seek(SeekFrom::Start(byte_index))
     }
 
@@ -206,12 +191,168 @@ where
     }
 
     #[inline]
-    fn push_(&mut self, value: T) {
+    pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
+        self.get_(Self::i_to_usize(index)?)
+    }
+    fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
+        match self.index_to_pushed_index(index) {
+            Ok(index) => {
+                if let Some(index) = index {
+                    return Ok(self.pushed.get(index).map(|v| Value::Ref(v)));
+                }
+            }
+            Err(Error::IndexTooHigh) => return Ok(None),
+            Err(Error::IndexTooLow) => {}
+            Err(error) => return Err(error),
+        }
+
+        // if !self.updated.is_empty() {
+        //     if let Some(v) = self.updated.get(&index) {
+        //         return Ok(Some(v));
+        //     }
+        // }
+
+        let page_index = index / Self::PER_PAGE;
+        let last_index = self.file_len - 1;
+        let max_page_index = last_index / Self::PER_PAGE;
+        let min_page_index = (max_page_index + 1) - self.mmaps.len();
+
+        // let min_open_page = self.min.load(AtomicOrdering::SeqCst);
+
+        // if self.min.load(AtomicOrdering::SeqCst) {
+        //     self.min.set(value)
+        // }
+
+        if !self.mmaps.is_empty() && page_index >= min_page_index {
+            let mmap = &**self
+                .mmaps
+                .get(page_index - min_page_index)
+                .ok_or(Error::MmapsVecIsTooSmall)?
+                .get_or_init(|| {
+                    Box::new(unsafe {
+                        memmap2::MmapOptions::new()
+                            .len(Self::PAGE_SIZE)
+                            .offset((page_index * Self::PAGE_SIZE) as u64)
+                            .map(&self.file)
+                            .unwrap()
+                    })
+                });
+
+            let range = Self::index_to_byte_range(index);
+            let slice = &mmap[range];
+            return Ok(Some(Value::Ref(T::try_ref_from_bytes(slice)?)));
+        }
+
+        Ok(self.open_then_read_(index).map_or(None, |v| Some(Value::Owned(v))))
+    }
+
+    #[inline]
+    pub fn read(&mut self, index: I) -> Result<Option<&T>> {
+        self.read_(Self::i_to_usize(index)?)
+    }
+    #[inline]
+    pub fn read_(&mut self, index: usize) -> Result<Option<&T>> {
+        let byte_index = Self::index_to_byte_index(index);
+        if self.file_position != byte_index {
+            self.file_position = self.seek(Self::index_to_byte_index(index))?;
+        }
+        match Self::read_exact(&mut self.file, &mut self.buf) {
+            Ok(value) => {
+                self.file_position += Self::SIZE_OF_T as u64;
+                Ok(Some(value))
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    fn read_last(&mut self) -> Result<Option<&T>> {
+        let len = self.len();
+        if len == 0 {
+            return Ok(None);
+        }
+        self.read_(len - 1)
+    }
+
+    pub fn iter<F>(&self, f: F) -> Result<()>
+    where
+        F: FnMut((I, &T)) -> Result<()>,
+    {
+        self.iter_from(I::default(), f)
+    }
+
+    pub fn iter_from<F>(&self, mut index: I, mut f: F) -> Result<()>
+    where
+        F: FnMut((I, &T)) -> Result<()>,
+    {
+        let mut file = self.open_file()?;
+
+        let disk_len = I::from(Self::read_disk_len_(&file)?);
+        let pushed_len = I::from(self.pushed_len());
+
+        Self::seek_(&mut file, Self::index_to_byte_index(Self::i_to_usize(index)?))?;
+
+        let mut buf = Self::create_buffer();
+
+        while index < disk_len {
+            f((index, Self::read_exact(&mut file, &mut buf)?))?;
+            index = index + 1;
+        }
+
+        let disk_len = Self::i_to_usize(disk_len)?;
+        let mut i = I::default();
+        while i < pushed_len {
+            f((i + disk_len, self.pushed.get(Self::i_to_usize(i)?).as_ref().unwrap()))?;
+            i = i + 1;
+        }
+
+        Ok(())
+    }
+
+    pub fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
+        if !self.pushed.is_empty() {
+            return Err(Error::UnsupportedUnflushedState);
+        }
+
+        let mut file = self.open_file()?;
+
+        let len = Self::read_disk_len_(&file)?;
+
+        let from = from.map_or(0, |from| {
+            if from >= 0 {
+                from as usize
+            } else {
+                (len as i64 + from) as usize
+            }
+        });
+
+        let to = to.map_or(len, |to| {
+            if to >= 0 {
+                to as usize
+            } else {
+                (len as i64 + to) as usize
+            }
+        });
+
+        if from >= to {
+            return Err(Error::RangeFromAfterTo);
+        }
+
+        Self::seek_(&mut file, Self::index_to_byte_index(from))?;
+
+        let mut buf = Self::create_buffer();
+
+        Ok((from..to)
+            .map(|_| Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned()).unwrap())
+            .collect::<Vec<_>>())
+    }
+
+    #[inline]
+    pub fn push(&mut self, value: T) {
         self.pushed.push(value)
     }
 
     #[inline]
-    fn push_if_needed_(&mut self, index: I, value: T) -> Result<()> {
+    pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
         match self.len().cmp(&Self::i_to_usize(index)?) {
             Ordering::Greater => {
                 // dbg!(len, index, &self.pathbuf);
@@ -229,6 +370,27 @@ where
         }
     }
 
+    #[inline]
+    fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
+        match self.len().cmp(&Self::i_to_usize(index)?) {
+            Ordering::Less => {
+                return Err(Error::IndexTooHigh);
+            }
+            ord => {
+                if ord == Ordering::Greater {
+                    self.safe_truncate_if_needed(index, exit)?;
+                }
+                self.pushed.push(value);
+            }
+        }
+
+        if self.pushed_len() >= FLUSH_EVERY {
+            Ok(self.safe_flush(exit)?)
+        } else {
+            Ok(())
+        }
+    }
+
     #[inline]
     pub fn len(&self) -> usize {
         self.file_len + self.pushed_len()
@@ -258,7 +420,7 @@ where
         self.has(index).map(|b| !b)
     }
 
-    fn _flush(&mut self) -> io::Result<()> {
+    pub fn flush(&mut self) -> io::Result<()> {
         if self.pushed.is_empty() {
             return Ok(());
         }
@@ -274,16 +436,35 @@ where
 
         self.file.write_all(&bytes)?;
 
-        self.reset_disk_related_state()?;
+        self.reset_file_metadata()?;
 
         Ok(())
     }
 
-    fn reset(&mut self) -> Result<()> {
+    pub fn safe_flush(&mut self, exit: &Exit) -> io::Result<()> {
+        if exit.triggered() {
+            return Ok(());
+        }
+        exit.block();
+        self.flush()?;
+        exit.release();
+        Ok(())
+    }
+
+    pub fn reset_file(&mut self) -> Result<()> {
         self.truncate_if_needed(I::from(0))?;
         Ok(())
     }
 
+    fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
+        let path = self.path_computed_version();
+        if version.validate(path.as_ref()).is_err() {
+            self.reset_file()?;
+        }
+        version.write(path.as_ref())?;
+        Ok(())
+    }
+
     pub fn truncate_if_needed(&mut self, index: I) -> Result<Option<T>> {
         let index = Self::i_to_usize(index)?;
 
@@ -291,14 +472,23 @@ where
             return Ok(None);
         }
 
-        let value_at_index = self.open_file_at_then_read(index).ok();
+        let value_at_index = self.open_then_read_(index).ok();
 
         self.file.set_len(Self::index_to_byte_index(index))?;
 
-        self.reset_disk_related_state()?;
+        self.reset_file_metadata()?;
 
         Ok(value_at_index)
     }
+    pub fn safe_truncate_if_needed(&mut self, index: I, exit: &Exit) -> Result<()> {
+        if exit.triggered() {
+            return Ok(());
+        }
+        exit.block();
+        self.truncate_if_needed(index)?;
+        exit.release();
+        Ok(())
+    }
 
     #[inline]
     fn i_to_usize(index: I) -> Result<usize> {
@@ -357,228 +547,19 @@ where
         path.join("version")
     }
 
-    pub fn index_type_to_string(&self) -> &str {
-        std::any::type_name::<I>()
-    }
-}
-
-impl<I, T> StorableVec<I, T, CACHED_GETS>
-where
-    I: StoredIndex,
-    T: StoredType,
-{
-    #[inline]
-    pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
-        self.get_(Self::i_to_usize(index)?)
-    }
-    fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
-        match self.index_to_pushed_index(index) {
-            Ok(index) => {
-                if let Some(index) = index {
-                    return Ok(self.pushed.get(index).map(|v| Value::Ref(v)));
-                }
-            }
-            Err(Error::IndexTooHigh) => return Ok(None),
-            Err(Error::IndexTooLow) => {}
-            Err(error) => return Err(error),
-        }
-
-        // if !self.updated.is_empty() {
-        //     if let Some(v) = self.updated.get(&index) {
-        //         return Ok(Some(v));
-        //     }
-        // }
-
-        let page_index = index / Self::PER_PAGE;
-        let last_index = self.file_len - 1;
-        let max_page_index = last_index / Self::PER_PAGE;
-        let min_page_index = (max_page_index + 1) - self.cache.len();
-
-        // let min_open_page = self.min.load(AtomicOrdering::SeqCst);
-
-        // if self.min.load(AtomicOrdering::SeqCst) {
-        //     self.min.set(value)
-        // }
-
-        if page_index >= min_page_index {
-            let mmap = &**self
-                .cache
-                .get(page_index - min_page_index)
-                .ok_or(Error::MmapsVecIsTooSmall)?
-                .get_or_init(|| {
-                    Box::new(unsafe {
-                        memmap2::MmapOptions::new()
-                            .len(Self::PAGE_SIZE)
-                            .offset((page_index * Self::PAGE_SIZE) as u64)
-                            .map(&self.file)
-                            .unwrap()
-                    })
-                });
-
-            let range = Self::index_to_byte_range(index);
-            let slice = &mmap[range];
-            return Ok(Some(Value::Ref(T::try_ref_from_bytes(slice)?)));
-        }
-
-        Ok(Some(Value::Owned(self.open_file_at_then_read(index)?.to_owned())))
-    }
-
-    pub fn get_or_default(&self, index: I) -> Result<T>
-    where
-        T: Default + Clone,
-    {
-        Ok(self.get(index)?.map(|v| (*v).clone()).unwrap_or(Default::default()))
-    }
-
-    pub fn iter_from<F>(&self, mut index: I, mut f: F) -> Result<()>
-    where
-        F: FnMut((I, Value<T>)) -> Result<()>,
-    {
-        let disk_len = I::from(Self::read_disk_len_(&self.file)?);
-
-        while index < disk_len {
-            f((index, self.get(index)?.unwrap()))?;
-            index = index + 1;
-        }
-
-        let mut index = I::from(0);
-        let pushed_len = I::from(self.pushed_len());
-        let disk_len = Self::i_to_usize(disk_len)?;
-        while index < pushed_len {
-            f(((index + disk_len), self.get(index)?.unwrap()))?;
-            index = index + 1;
-        }
-
-        Ok(())
-    }
-
-    #[inline]
-    pub fn push(&mut self, value: T) {
-        self.push_(value)
-    }
-
-    #[inline]
-    pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
-        self.push_if_needed_(index, value)
-    }
-
-    #[inline]
-    pub fn flush(&mut self) -> io::Result<()> {
-        self._flush()
-    }
-}
-
-const FLUSH_EVERY: usize = 10_000;
-impl<I, T> StorableVec<I, T, SINGLE_THREAD>
-where
-    I: StoredIndex,
-    T: StoredType,
-{
-    pub fn get(&mut self, index: I) -> Result<&T> {
-        self.get_(Self::i_to_usize(index)?)
-    }
-    fn get_(&mut self, index: usize) -> Result<&T> {
-        let byte_index = Self::index_to_byte_index(index);
-        if self.file_position != byte_index {
-            self.file_position = Self::seek(&mut self.file, byte_index)?;
-        }
-        let res = Self::read_exact(&mut self.file, &mut self.buf);
-        if res.is_ok() {
-            self.file_position += Self::SIZE_OF_T as u64;
-        }
-        res
-    }
-
-    fn last(&mut self) -> Result<Option<&T>> {
-        let len = self.len();
-        if len == 0 {
-            return Ok(None);
-        }
-        Ok(self.get_(len - 1).ok())
-    }
-
-    // #[inline]
-    // fn push(&mut self, value: T) {
-    //     self.push_(value)
-    // }
-
-    #[inline]
-    fn blocked_push_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
-        self.push_if_needed_(index, value)?;
-
-        if self.pushed_len() >= FLUSH_EVERY {
-            Ok(self.blocked_flush(exit)?)
-        } else {
-            Ok(())
-        }
-    }
-
-    // pub fn iter<F>(&mut self, f: F) -> Result<()>
-    // where
-    //     F: FnMut((I, &T)) -> Result<()>,
-    // {
-    //     self.iter_from(I::default(), f)
-    // }
-
-    pub fn iter_from<F>(&mut self, mut index: I, mut f: F) -> Result<()>
-    where
-        F: FnMut((I, &T)) -> Result<()>,
-    {
-        // let pushed_len = self.pushed_len();
-
-        // self.seek_if_needed(index)?;
-
-        if !self.pushed.is_empty() {
-            return Err(Error::UnsupportedUnflushedState);
-        }
-
-        let disk_len = I::from(Self::read_disk_len_(&self.file)?);
-
-        while index < disk_len {
-            f((index, self.get(index)?))?;
-            index = index + 1;
-        }
-
-        // i = 0;
-        // while i < pushed_len {
-        //     f((I::from(i + disk_len), self.pushed.get(i).as_ref().unwrap()))?;
-        //     i += 1;
-        // }
-
-        Ok(())
-    }
-
     #[inline]
     fn path_computed_version(&self) -> PathBuf {
         self.path().join("computed_version")
     }
-    // #[inline]
-    // fn path_computed_version_(path: &Path) -> PathBuf {
-    //     path.join("computed_version")
-    // }
 
-    fn validate_or_reset(&mut self, version: Version) -> Result<()> {
-        let path = self.path_computed_version();
-        if version.validate(path.as_ref()).is_err() {
-            self.reset()?;
-        }
-        version.write(path.as_ref())?;
-        Ok(())
-    }
-
-    fn blocked_flush(&mut self, exit: &Exit) -> io::Result<()> {
-        if exit.triggered() {
-            return Ok(());
-        }
-        exit.block();
-        self._flush()?;
-        exit.release();
-        Ok(())
+    pub fn index_type_to_string(&self) -> &str {
+        std::any::type_name::<I>()
     }
 
     pub fn compute_transform<A, F>(
         &mut self,
-        other: &mut StorableVec<I, A, SINGLE_THREAD>,
+        max_from: I,
+        other: &mut StorableVec<I, A>,
         t: F,
         exit: &Exit,
     ) -> Result<()>
@@ -586,78 +567,91 @@ where
         A: StoredType,
         F: Fn(&A) -> T,
     {
-        self.validate_or_reset(Version::from(0) + self.version + other.version)?;
+        self.validate_computed_version_or_reset_file(Version::from(0) + self.version + other.version)?;
 
-        other.iter_from(I::from(self.len()), |(i, a)| self.blocked_push_if_needed(i, t(a), exit))?;
-        Ok(self.blocked_flush(exit)?)
+        let index = max_from.min(I::from(self.len()));
+        other.iter_from(index, |(i, a)| self.push_and_flush_if_needed(i, t(a), exit))?;
+
+        Ok(self.safe_flush(exit)?)
     }
 
     pub fn compute_inverse_more_to_less(
         &mut self,
-        other: &mut StorableVec<T, I, SINGLE_THREAD>,
+        max_from: T,
+        other: &mut StorableVec<T, I>,
         exit: &Exit,
     ) -> Result<()>
     where
-        I: StoredType,
+        I: StoredType + StoredIndex,
         T: StoredIndex,
     {
-        self.validate_or_reset(Version::from(0) + self.version + other.version)?;
+        self.validate_computed_version_or_reset_file(Version::from(0) + self.version + other.version)?;
 
-        let index = self.last()?.cloned().unwrap_or_default();
-        other.iter_from(index, |(v, i)| self.blocked_push_if_needed(*i, v, exit))?;
-        Ok(self.blocked_flush(exit)?)
+        let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
+        other.iter_from(index, |(v, i)| self.push_and_flush_if_needed(*i, v, exit))?;
+
+        Ok(self.safe_flush(exit)?)
     }
 
     pub fn compute_inverse_less_to_more(
         &mut self,
-        first_indexes: &mut StorableVec<T, I, SINGLE_THREAD>,
-        last_indexes: &mut StorableVec<T, I, SINGLE_THREAD>,
+        max_from: T,
+        first_indexes: &mut StorableVec<T, I>,
+        last_indexes: &mut StorableVec<T, I>,
         exit: &Exit,
     ) -> Result<()>
     where
         I: StoredType,
         T: StoredIndex,
     {
-        self.validate_or_reset(Version::from(0) + self.version + first_indexes.version + last_indexes.version)?;
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version + first_indexes.version + last_indexes.version,
+        )?;
 
-        first_indexes.iter_from(T::from(self.len()), |(value, first_index)| {
+        let index = max_from.min(T::from(self.len()));
+        first_indexes.iter_from(index, |(value, first_index)| {
             let first_index = Self::i_to_usize(*first_index)?;
-            let last_index = Self::i_to_usize(*last_indexes.get(value)?)?;
-            (first_index..last_index).try_for_each(|index| self.blocked_push_if_needed(I::from(index), value, exit))
+            let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
+            (first_index..last_index).try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit))
         })?;
-        Ok(self.blocked_flush(exit)?)
+
+        Ok(self.safe_flush(exit)?)
     }
 
     pub fn compute_last_index_from_first(
         &mut self,
-        first_indexes: &mut StorableVec<I, T, SINGLE_THREAD>,
+        max_from: I,
+        first_indexes: &mut StorableVec<I, T>,
         final_len: usize,
         exit: &Exit,
     ) -> Result<()>
     where
         T: Copy + From<usize> + Sub<T, Output = T> + StoredIndex,
     {
-        self.validate_or_reset(Version::from(0) + self.version + first_indexes.version)?;
+        self.validate_computed_version_or_reset_file(Version::from(0) + self.version + first_indexes.version)?;
 
+        let index = max_from.min(I::from(self.len()));
         let one = T::from(1);
         let mut prev_index: Option<I> = None;
-        first_indexes.iter_from(I::from(self.len()), |(i, v)| {
+        first_indexes.iter_from(index, |(i, v)| {
             if let Some(prev_index) = prev_index {
-                self.blocked_push_if_needed(prev_index, *v - one, exit)?;
+                self.push_and_flush_if_needed(prev_index, *v - one, exit)?;
             }
             prev_index.replace(i);
             Ok(())
         })?;
         if let Some(prev_index) = prev_index {
-            self.blocked_push_if_needed(prev_index, T::from(final_len) - one, exit)?;
+            self.push_and_flush_if_needed(prev_index, T::from(final_len) - one, exit)?;
         }
-        Ok(self.blocked_flush(exit)?)
+
+        Ok(self.safe_flush(exit)?)
     }
 
     pub fn compute_count_from_indexes<T2>(
         &mut self,
-        first_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
-        last_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
+        max_from: I,
+        first_indexes: &mut StorableVec<I, T2>,
+        last_indexes: &mut StorableVec<I, T2>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -665,20 +659,25 @@ where
         T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
         <T2 as TryInto<T>>::Error: error::Error + 'static,
     {
-        self.validate_or_reset(Version::from(0) + self.version + first_indexes.version + last_indexes.version)?;
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version + first_indexes.version + last_indexes.version,
+        )?;
 
-        first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
-            let last_index = last_indexes.get(i)?;
+        let index = max_from.min(I::from(self.len()));
+        first_indexes.iter_from(index, |(i, first_index)| {
+            let last_index = last_indexes.read(i)?.unwrap();
             let count = *last_index + 1_usize - *first_index;
-            self.blocked_push_if_needed(i, count.into(), exit)
+            self.push_and_flush_if_needed(i, count.into(), exit)
         })?;
-        Ok(self.blocked_flush(exit)?)
+
+        Ok(self.safe_flush(exit)?)
     }
 
     pub fn compute_is_first_ordered<A>(
         &mut self,
-        self_to_other: &mut StorableVec<I, A, SINGLE_THREAD>,
-        other_to_self: &mut StorableVec<A, I, SINGLE_THREAD>,
+        max_from: I,
+        self_to_other: &mut StorableVec<I, A>,
+        other_to_self: &mut StorableVec<A, I>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -686,18 +685,23 @@ where
         T: From<bool>,
         A: StoredIndex + StoredType,
     {
-        self.validate_or_reset(Version::from(0) + self.version + self_to_other.version + other_to_self.version)?;
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version + self_to_other.version + other_to_self.version,
+        )?;
 
-        self_to_other.iter_from(I::from(self.len()), |(i, other)| {
-            self.blocked_push_if_needed(i, T::from(other_to_self.get(*other)? == &i), exit)
+        let index = max_from.min(I::from(self.len()));
+        self_to_other.iter_from(index, |(i, other)| {
+            self.push_and_flush_if_needed(i, T::from(other_to_self.read(*other)?.unwrap() == &i), exit)
         })?;
-        Ok(self.blocked_flush(exit)?)
+
+        Ok(self.safe_flush(exit)?)
     }
 
     pub fn compute_sum_from_indexes<T2, F>(
         &mut self,
-        first_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
-        last_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
+        max_from: I,
+        first_indexes: &mut StorableVec<I, T2>,
+        last_indexes: &mut StorableVec<I, T2>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -706,70 +710,22 @@ where
         <T2 as TryInto<T>>::Error: error::Error + 'static,
         F: Fn(&T2) -> T,
     {
-        self.validate_or_reset(Version::from(0) + self.version + first_indexes.version + last_indexes.version)?;
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version + first_indexes.version + last_indexes.version,
+        )?;
 
-        first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
-            let last_index = last_indexes.get(i)?;
+        let index = max_from.min(I::from(self.len()));
+        first_indexes.iter_from(index, |(index, first_index)| {
+            let last_index = last_indexes.read(index)?.unwrap();
             let count = *last_index + 1_usize - *first_index;
-            self.blocked_push_if_needed(i, count.into(), exit)
+            self.push_and_flush_if_needed(index, count.into(), exit)
         })?;
-        Ok(self.blocked_flush(exit)?)
+
+        Ok(self.safe_flush(exit)?)
     }
 }
 
-impl<I, T> StorableVec<I, T, STATELESS>
-where
-    I: StoredIndex,
-    T: StoredType,
-{
-    #[inline]
-    pub fn get(&self, index: I) -> Result<Option<T>> {
-        Ok(Some(self.open_file_at_then_read(Self::i_to_usize(index)?)?))
-    }
-
-    pub fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
-        if !self.pushed.is_empty() {
-            return Err(Error::UnsupportedUnflushedState);
-        }
-
-        let mut file = self.open_file()?;
-
-        let len = Self::read_disk_len_(&file)?;
-
-        let from = from.map_or(0, |from| {
-            if from >= 0 {
-                from as usize
-            } else {
-                (len as i64 + from) as usize
-            }
-        });
-
-        let to = to.map_or(len, |to| {
-            if to >= 0 {
-                to as usize
-            } else {
-                (len as i64 + to) as usize
-            }
-        });
-
-        if from >= to {
-            return Err(Error::RangeFromAfterTo);
-        }
-
-        Self::seek(&mut file, Self::index_to_byte_index(from))?;
-
-        let mut buf = Self::create_buffer();
-
-        Ok((from..to)
-            .map(|_| Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned()).unwrap())
-            .collect::<Vec<_>>())
-    }
-
-    // Add iter iter_from iter_range collect..
-    // + add memory cap
-}
-
-impl<I, T> Clone for StorableVec<I, T, STATELESS>
+impl<I, T> Clone for StorableVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,

crates/brk_vec/src/main.rs

@@ -1,11 +1,10 @@
 use std::path::Path;
 
-use brk_vec::{CACHED_GETS, SINGLE_THREAD, StorableVec, Version};
+use brk_vec::{StorableVec, Version};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     {
-        let mut vec: StorableVec<usize, u32, CACHED_GETS> =
-            StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
+        let mut vec: StorableVec<usize, u32> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
 
         vec.push(0);
         vec.push(1);
@@ -17,13 +16,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     {
-        let mut vec: StorableVec<usize, u32, SINGLE_THREAD> =
-            StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
+        let mut vec: StorableVec<usize, u32> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
 
-        dbg!(vec.get(0)?); // 0
-        dbg!(vec.get(1)?); // 0
-        dbg!(vec.get(2)?); // 0
-        dbg!(vec.get(0)?); // 0
+        dbg!(vec.read(0)?); // 0
+        dbg!(vec.read(1)?); // 0
+        dbg!(vec.read(2)?); // 0
+        dbg!(vec.read(0)?); // 0
     }
 
     Ok(())

crates/brk_vec/src/traits/any.rs

@@ -1,6 +1,6 @@
-use std::mem;
+use std::io;
 
-use crate::{Result, STATELESS, StorableVec};
+use crate::{Result, StorableVec};
 
 use super::{StoredIndex, StoredType};
 
@@ -9,10 +9,11 @@ pub trait AnyStorableVec: Send + Sync {
     fn index_type_to_string(&self) -> &str;
     fn len(&self) -> usize;
     fn is_empty(&self) -> bool;
-    // fn flush(&mut self) -> io::Result<()>;
+    fn collect_range_values(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<serde_json::Value>>;
+    fn flush(&mut self) -> io::Result<()>;
 }
 
-impl<I, T, const MODE: u8> AnyStorableVec for StorableVec<I, T, MODE>
+impl<I, T> AnyStorableVec for StorableVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,
@@ -33,33 +34,15 @@ where
         self.is_empty()
     }
 
-    // fn flush(&mut self) -> io::Result<()> {
-    //     self.flush()
-    // }
-}
+    fn flush(&mut self) -> io::Result<()> {
+        self.flush()
+    }
 
-#[cfg(feature = "json")]
-pub trait AnyJsonStorableVec: AnyStorableVec {
-    fn collect_range_values(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<serde_json::Value>>;
-}
-
-#[cfg(feature = "json")]
-impl<I, T, const MODE: u8> AnyJsonStorableVec for StorableVec<I, T, MODE>
-where
-    I: StoredIndex,
-    T: StoredType + serde::Serialize,
-{
     fn collect_range_values(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<serde_json::Value>> {
-        if MODE == STATELESS {
-            Ok(
-                unsafe { mem::transmute::<&StorableVec<I, T, MODE>, &StorableVec<I, T, STATELESS>>(self) }
-                    .collect_range(from, to)?
-                    .into_iter()
-                    .map(|v| serde_json::to_value(v).unwrap())
-                    .collect::<Vec<_>>(),
-            )
-        } else {
-            todo!("todo ?")
-        }
+        Ok(self
+            .collect_range(from, to)?
+            .into_iter()
+            .map(|v| serde_json::to_value(v).unwrap())
+            .collect::<Vec<_>>())
     }
 }

crates/brk_vec/src/traits/stored_type.rs

@@ -1,13 +1,14 @@
 use std::fmt::Debug;
 
+use serde::Serialize;
 use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
 
 pub trait StoredType
 where
-    Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync,
+    Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync + Serialize,
 {
 }
 impl<T> StoredType for T where
-    T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync
+    T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync + Serialize
 {
 }