global: big vec refactor + lazy

crates/brk_vec/src/traits/any.rs

@@ -1,24 +1,73 @@
 use std::time::Duration;
 
-use crate::Result;
+use crate::{Result, Version};
+
+use super::{BoxedVecIterator, StoredIndex, StoredType};
 
 pub trait AnyVec: Send + Sync {
+    fn version(&self) -> Version;
     fn name(&self) -> String;
-    fn index_type_to_string(&self) -> &str;
     fn len(&self) -> usize;
     fn is_empty(&self) -> bool {
         self.len() == 0
     }
-    fn collect_range_serde_json(
-        &self,
-        from: Option<i64>,
-        to: Option<i64>,
-    ) -> Result<Vec<serde_json::Value>>;
     fn modified_time(&self) -> Result<Duration>;
-    fn any_vec(&self) -> &dyn AnyVec
+    fn index_type_to_string(&self) -> &str;
+}
+
+pub trait AnyIterableVec<I, T>: AnyVec {
+    #[allow(clippy::wrong_self_convention)]
+    fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
     where
-        Self: Sized,
+        I: StoredIndex,
+        T: StoredType + 'a;
+
+    fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T>
+    where
+        I: StoredIndex,
+        T: StoredType + 'a,
     {
-        self
+        self.boxed_iter()
+    }
+
+    fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T>
+    where
+        I: StoredIndex,
+        T: StoredType + 'a,
+    {
+        let mut iter = self.boxed_iter();
+        iter.set(i);
+        iter
+    }
+
+    fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T>
+    where
+        I: StoredIndex,
+        T: StoredType + 'a,
+    {
+        let mut iter = self.boxed_iter();
+        iter.set_(i);
+        iter
     }
 }
+
+pub trait CloneableAnyIterableVec<I, T>: AnyIterableVec<I, T> {
+    fn boxed_clone(&self) -> Box<dyn CloneableAnyIterableVec<I, T>>;
+}
+
+impl<I, T, U> CloneableAnyIterableVec<I, T> for U
+where
+    U: 'static + AnyIterableVec<I, T> + Clone,
+{
+    fn boxed_clone(&self) -> Box<dyn CloneableAnyIterableVec<I, T>> {
+        Box::new(self.clone())
+    }
+}
+
+impl<I, T> Clone for Box<dyn CloneableAnyIterableVec<I, T>> {
+    fn clone(&self) -> Self {
+        self.boxed_clone()
+    }
+}
+
+pub type BoxedAnyIterableVec<I, T> = Box<dyn CloneableAnyIterableVec<I, T>>;

crates/brk_vec/src/traits/collectable.rs

@@ -0,0 +1,71 @@
+use crate::{Error, Result};
+
+use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType};
+
+pub trait CollectableVec<I, T>: AnyVec + AnyIterableVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
+        let len = self.len();
+        let from = from.unwrap_or_default();
+        let to = to.map_or(len, |i| i.min(len));
+
+        if from >= len || from >= to {
+            return Ok(vec![]);
+        }
+
+        Ok(self
+            .iter_at_(from)
+            .take(to - from)
+            .map(|(_, v)| v.into_inner())
+            .collect::<Vec<_>>())
+    }
+
+    #[inline]
+    fn i64_to_usize(i: i64, len: usize) -> usize {
+        if i >= 0 {
+            i as usize
+        } else {
+            let v = len as i64 + i;
+            if v < 0 { 0 } else { v as usize }
+        }
+    }
+
+    #[doc(hidden)]
+    fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
+        let len = self.len();
+        let from = from.map(|i| Self::i64_to_usize(i, len));
+        let to = to.map(|i| Self::i64_to_usize(i, len));
+        self.collect_range(from, to)
+    }
+
+    #[inline]
+    fn collect_range_serde_json(
+        &self,
+        from: Option<i64>,
+        to: Option<i64>,
+    ) -> Result<Vec<serde_json::Value>> {
+        self.collect_signed_range(from, to)?
+            .into_iter()
+            .map(|v| serde_json::to_value(v).map_err(Error::from))
+            .collect::<Result<Vec<_>>>()
+    }
+}
+
+impl<I, T, V> CollectableVec<I, T> for V
+where
+    V: AnyVec + AnyIterableVec<I, T>,
+    I: StoredIndex,
+    T: StoredType,
+{
+}
+
+pub trait AnyCollectableVec: AnyVec {
+    fn collect_range_serde_json(
+        &self,
+        from: Option<i64>,
+        to: Option<i64>,
+    ) -> Result<Vec<serde_json::Value>>;
+}

crates/brk_vec/src/traits/dynamic.rs

@@ -1,66 +0,0 @@
-use std::path::Path;
-
-use arc_swap::ArcSwap;
-use memmap2::Mmap;
-
-use crate::{Result, Value};
-
-use super::{StoredIndex, StoredType};
-
-pub trait DynamicVec: Send + Sync {
-    type I: StoredIndex;
-    type T: StoredType;
-    const SIZE_OF_T: usize = size_of::<Self::T>();
-
-    #[inline]
-    fn read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Self::T>> {
-        self.read_(index.to_usize()?, mmap)
-    }
-    fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
-
-    #[inline]
-    fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
-        self.get_or_read_(index.to_usize()?, mmap)
-    }
-    #[inline]
-    fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
-        let stored_len = mmap.len() / Self::SIZE_OF_T;
-
-        if index >= stored_len {
-            let pushed = self.pushed();
-            let j = index - stored_len;
-            if j >= pushed.len() {
-                return Ok(None);
-            }
-            Ok(pushed.get(j).map(Value::Ref))
-        } else {
-            Ok(self.read_(index, mmap)?.map(Value::Owned))
-        }
-    }
-
-    #[inline]
-    fn len(&self) -> usize {
-        self.stored_len() + self.pushed_len()
-    }
-    #[inline]
-    fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    fn mmap(&self) -> &ArcSwap<Mmap>;
-
-    fn stored_len(&self) -> usize;
-
-    fn pushed(&self) -> &[Self::T];
-    #[inline]
-    fn pushed_len(&self) -> usize {
-        self.pushed().len()
-    }
-    fn mut_pushed(&mut self) -> &mut Vec<Self::T>;
-    #[inline]
-    fn push(&mut self, value: Self::T) {
-        self.mut_pushed().push(value)
-    }
-
-    fn path(&self) -> &Path;
-}

crates/brk_vec/src/traits/generic.rs

@@ -6,17 +6,70 @@ use std::{
     time::{self, Duration},
 };
 
+use arc_swap::ArcSwap;
 use memmap2::Mmap;
 
-use crate::{Error, Result, Version};
+use crate::{Result, Value};
 
-use super::{DynamicVec, StoredIndex, StoredType};
+use super::{StoredIndex, StoredType};
 
-pub trait GenericVec<I, T>: DynamicVec<I = I, T = T>
+pub trait GenericStoredVec<I, T>: Send + Sync
 where
     I: StoredIndex,
     T: StoredType,
 {
+    const SIZE_OF_T: usize = size_of::<T>();
+
+    #[inline]
+    fn read(&self, index: I, mmap: &Mmap) -> Result<Option<T>> {
+        self.read_(index.to_usize()?, mmap)
+    }
+    fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>>;
+
+    #[inline]
+    fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
+        self.get_or_read_(index.to_usize()?, mmap)
+    }
+    #[inline]
+    fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<T>>> {
+        let stored_len = mmap.len() / Self::SIZE_OF_T;
+
+        if index >= stored_len {
+            let pushed = self.pushed();
+            let j = index - stored_len;
+            if j >= pushed.len() {
+                return Ok(None);
+            }
+            Ok(pushed.get(j).map(Value::Ref))
+        } else {
+            Ok(self.read_(index, mmap)?.map(Value::Owned))
+        }
+    }
+
+    #[inline]
+    fn len_(&self) -> usize {
+        self.stored_len() + self.pushed_len()
+    }
+
+    fn mmap(&self) -> &ArcSwap<Mmap>;
+
+    fn stored_len(&self) -> usize;
+
+    fn pushed(&self) -> &[T];
+    #[inline]
+    fn pushed_len(&self) -> usize {
+        self.pushed().len()
+    }
+    fn mut_pushed(&mut self) -> &mut Vec<T>;
+    #[inline]
+    fn push(&mut self, value: T) {
+        self.mut_pushed().push(value)
+    }
+
+    fn path(&self) -> &Path;
+
+    // ---
+
     fn open_file(&self) -> io::Result<File> {
         Self::open_file_(&self.path_vec())
     }
@@ -74,54 +127,17 @@ where
     }
 
     #[inline]
-    fn has(&self, index: Self::I) -> Result<bool> {
+    fn has(&self, index: I) -> Result<bool> {
         Ok(self.has_(index.to_usize()?))
     }
     #[inline]
     fn has_(&self, index: usize) -> bool {
-        index < self.len()
-    }
-
-    #[inline]
-    fn index_type_to_string(&self) -> &str {
-        Self::I::to_string()
+        index < self.len_()
     }
 
     fn flush(&mut self) -> Result<()>;
 
-    fn truncate_if_needed(&mut self, index: Self::I) -> Result<()>;
-
-    fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>>;
-
-    #[inline]
-    fn i64_to_usize(i: i64, len: usize) -> usize {
-        if i >= 0 {
-            i as usize
-        } else {
-            let v = len as i64 + i;
-            if v < 0 { 0 } else { v as usize }
-        }
-    }
-
-    #[doc(hidden)]
-    fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<Self::T>> {
-        let len = self.len();
-        let from = from.map(|i| Self::i64_to_usize(i, len));
-        let to = to.map(|i| Self::i64_to_usize(i, len));
-        self.collect_range(from, to)
-    }
-
-    #[inline]
-    fn collect_range_serde_json(
-        &self,
-        from: Option<i64>,
-        to: Option<i64>,
-    ) -> Result<Vec<serde_json::Value>> {
-        self.collect_signed_range(from, to)?
-            .into_iter()
-            .map(|v| serde_json::to_value(v).map_err(Error::from))
-            .collect::<Result<Vec<_>>>()
-    }
+    fn truncate_if_needed(&mut self, index: I) -> Result<()>;
 
     #[inline]
     fn path_vec(&self) -> PathBuf {
@@ -143,7 +159,7 @@ where
     }
 
     #[inline]
-    fn name(&self) -> String {
+    fn name_(&self) -> String {
         self.path()
             .file_name()
             .unwrap()
@@ -152,13 +168,11 @@ where
             .to_owned()
     }
 
-    fn modified_time(&self) -> Result<Duration> {
+    fn modified_time_(&self) -> Result<Duration> {
         Ok(self
             .path_vec()
             .metadata()?
             .modified()?
             .duration_since(time::UNIX_EPOCH)?)
     }
-
-    fn version(&self) -> Version;
 }

crates/brk_vec/src/traits/index.rs

@@ -23,6 +23,7 @@ where
     fn to_string<'a>() -> &'a str;
     fn decremented(self) -> Option<Self>;
 }
+
 impl<I> StoredIndex for I
 where
     I: Debug

crates/brk_vec/src/traits/iterator.rs

@@ -32,9 +32,9 @@ pub trait BaseVecIterator: Iterator {
     }
 }
 
-pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)> + 'a {
+pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)> {
     type I: StoredIndex;
-    type T: StoredType;
+    type T: StoredType + 'a;
 
     #[inline]
     fn set(&mut self, i: Self::I) {
@@ -77,10 +77,13 @@ pub trait VecIterator<'a>: BaseVecIterator<Item = (Self::I, Value<'a, Self::T>)>
 
 impl<'a, I, T, Iter> VecIterator<'a> for Iter
 where
-    Iter: BaseVecIterator<Item = (I, Value<'a, T>)> + 'a,
+    Iter: BaseVecIterator<Item = (I, Value<'a, T>)>,
     I: StoredIndex,
     T: StoredType + 'a,
 {
     type I = I;
     type T = T;
 }
+
+pub type BoxedVecIterator<'a, I, T> =
+    Box<dyn VecIterator<'a, I = I, T = T, Item = (I, Value<'a, T>)> + 'a>;

crates/brk_vec/src/traits/mod.rs

@@ -1,13 +1,13 @@
 mod any;
-mod dynamic;
+mod collectable;
 mod generic;
+mod index;
 mod iterator;
-mod stored_index;
-mod stored_type;
+mod r#type;
 
 pub use any::*;
-pub use dynamic::*;
+pub use collectable::*;
 pub use generic::*;
+pub use index::*;
 pub use iterator::*;
-pub use stored_index::*;
-pub use stored_type::*;
+pub use r#type::*;

crates/brk_vec/src/traits/type.rs

@@ -17,6 +17,7 @@ where
         + Serialize,
 {
 }
+
 impl<T> StoredType for T where
     T: Sized
         + Debug