bindex: snapshot

storable_vec/.gitignore

@@ -0,0 +1 @@
+/v

storable_vec/Cargo.lock

@@ -0,0 +1,25 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "libc"
+version = "0.2.169"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
+
+[[package]]
+name = "memmap2"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "storable_vec"
+version = "0.1.2"
+dependencies = [
+ "memmap2",
+]

storable_vec/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "storable_vec"
+description = "A very small, fast, efficient and simple storable Vec"
+version = "0.1.2"
+license = "MIT"
+keywords = ["vec", "disk", "data"]
+categories = ["database"]
+edition = "2021"
+
+[dependencies]
+memmap2 = "0.9.5"

storable_vec/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 storable_vec
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

storable_vec/README.md

@@ -0,0 +1,53 @@
+# storable_vec
+
+A very small, fast, efficient and simple storable `vec` which uses `mmap2` for speed.
+
+## Features
+
+- [x] Get (Rayon compatible)
+- [x] Push
+- [ ] Update
+- [ ] Insert
+- [ ] Remove
+
+## Example
+
+```rust
+use std::path::Path;
+
+use storable_vec::{AnyStorableVec, StorableVec};
+
+fn main() -> color_eyre::Result<()> {
+    color_eyre::install()?;
+
+    {
+        let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
+        vec.push(21);
+        dbg!(vec.get(0)?); // 21
+        vec.flush()?;
+    }
+
+    {
+        let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
+        dbg!(vec.get(0)?); // 21
+    }
+
+    Ok(())
+}
+```
+
+## Disclaimer
+
+Portability will depend on the type of values.
+
+Non bytes/slices types (`u8`, `u16`, ...) will be read as slice in an unsafe manner (using `std::slice::from_raw_parts`) and thus have the endianness of the system. On the other hand, `&[u8]` should be inserted as is.
+
+If portability is important to you, just create a wrapper struct which has custom `get`, `push`, ... methods and does something like:
+
+```rust
+impl StorableVecU64 {
+    pub fn push(&mut self, value: u64) {
+        self.push(&value.to_be_bytes())
+    }
+}
+```

storable_vec/src/lib.rs

@@ -0,0 +1,421 @@
+use std::{
+    cmp::Ordering,
+    fmt::{self, Debug},
+    fs::{File, OpenOptions},
+    io::{self, Read, Seek, SeekFrom, Write},
+    marker::PhantomData,
+    mem,
+    ops::{Deref, Range},
+    path::{Path, PathBuf},
+    sync::OnceLock,
+};
+
+use memmap2::{Mmap, MmapOptions};
+
+///
+/// A very small, fast, efficient and simple storable Vec
+///
+/// Reads (imports of Mmap) are lazy
+///
+/// Stores only raw data without any overhead, and doesn't even have a header (TODO: which it should, at least to Err if wrong endian)
+///
+/// The file isn't portable for speed reasons (TODO: but could be ?)
+///
+/// If you don't call `.flush()` it just acts as a normal Vec
+///
+#[derive(Debug)]
+pub struct StorableVec<I, T> {
+    pathbuf: PathBuf,
+    // file_for_reads: File,
+    file: File,
+    cache: Vec<OnceLock<Box<Mmap>>>, // Boxed to reduce the size of the lock (24 > 16)
+    disk_len: usize,
+    pushed: Vec<T>,
+    // updated: BTreeMap<usize, T>,
+    // inserted: BTreeMap<usize, T>,
+    // removed: BTreeSet<usize>,
+    phantom: PhantomData<I>,
+}
+
+/// In bytes
+const MAX_PAGE_SIZE: usize = 4 * 4096;
+const ONE_MB: usize = 1000 * 1024;
+const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
+
+#[derive(Debug, Clone)]
+pub enum Value<'a, T> {
+    Ref(&'a T),
+    Owned(T),
+}
+
+impl<T> Value<'_, T>
+where
+    T: Sized + Debug + Clone,
+{
+    pub fn into_inner(self) -> T {
+        match self {
+            Self::Ref(t) => t.to_owned(),
+            Self::Owned(t) => t,
+        }
+    }
+}
+
+impl<T> Deref for Value<'_, T> {
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Ref(t) => t,
+            Self::Owned(t) => t,
+        }
+    }
+}
+
+impl<T> AsRef<T> for Value<'_, T>
+where
+    T: Sized + Debug + Clone,
+{
+    fn as_ref(&self) -> &T {
+        match self {
+            Self::Ref(t) => t,
+            Self::Owned(t) => t,
+        }
+    }
+}
+
+impl<I, T> StorableVec<I, T>
+where
+    I: Into<usize>,
+    T: Sized + Debug + Clone,
+{
+    pub const SIZE_OF_T: usize = size_of::<T>();
+    pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
+    /// In bytes
+    pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
+    pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
+
+    pub fn import(path: &Path) -> Result<Self, io::Error> {
+        let file = Self::open_file(path)?;
+
+        let mut this = Self {
+            pathbuf: path.to_owned(),
+            disk_len: Self::byte_index_to_index(file.metadata()?.len() as usize),
+            file,
+            // file_for_reads: Self::open_file(path)?,
+            cache: vec![],
+            pushed: vec![],
+            // updated: BTreeMap::new(),
+            // inserted: BTreeMap::new(),
+            // removed: BTreeSet::new(),
+            phantom: PhantomData,
+        };
+
+        this.cache.resize_with(Self::CACHE_LENGTH, Default::default);
+        this.cache.shrink_to_fit();
+
+        this.reset_cache();
+
+        Ok(this)
+    }
+
+    fn reset_cache(&mut self) {
+        // let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
+        // self.cache.clear();
+        // self.cache.resize_with(len, Default::default);
+        self.cache.iter_mut().for_each(|lock| {
+            lock.take();
+        });
+    }
+
+    fn open_file(path: &Path) -> Result<File, io::Error> {
+        OpenOptions::new()
+            .read(true)
+            .create(true)
+            .truncate(false)
+            .append(true)
+            .open(path)
+    }
+
+    #[inline]
+    fn index_to_byte_range(index: usize) -> Range<usize> {
+        let index = Self::index_to_byte_index(index) % Self::PAGE_SIZE;
+        index..(index + Self::SIZE_OF_T)
+    }
+
+    #[inline]
+    fn index_to_byte_index(index: usize) -> usize {
+        index * Self::SIZE_OF_T
+    }
+
+    #[inline]
+    fn byte_index_to_index(byte_index: usize) -> usize {
+        byte_index / Self::SIZE_OF_T
+    }
+
+    fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
+        if index >= self.disk_len {
+            let index = index - self.disk_len;
+            if index >= self.pushed.len() {
+                Err(Error::IndexTooHigh)
+            } else {
+                Ok(Some(index))
+            }
+        } else {
+            Ok(None)
+        }
+    }
+
+    #[allow(unused)]
+    #[inline]
+    pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
+        self.get_(index.into())
+    }
+    pub fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
+        match self.index_to_pushed_index(index) {
+            Ok(index) => {
+                if let Some(index) = index {
+                    return Ok(self.pushed.get(index).map(|v| Value::Ref(v)));
+                }
+            }
+            Err(Error::IndexTooHigh) => return Ok(None),
+            Err(error) => return Err(error),
+        }
+
+        // if !self.updated.is_empty() {
+        //     if let Some(v) = self.updated.get(&index) {
+        //         return Ok(Some(v));
+        //     }
+        // }
+
+        let byte_index = Self::index_to_byte_index(index);
+        let page_index = index / Self::PER_PAGE;
+        let last_index = self.disk_len - 1;
+        let max_page_index = last_index / Self::PER_PAGE;
+        let min_page_index = (max_page_index + 1)
+            .checked_sub(Self::CACHE_LENGTH)
+            .unwrap_or_default();
+
+        if page_index >= min_page_index {
+            let mmap = &**self
+                .cache
+                .get(page_index - min_page_index)
+                .ok_or(Error::MmapsVecIsTooSmall)?
+                .get_or_init(|| {
+                    Box::new(unsafe {
+                        MmapOptions::new()
+                            .len(Self::PAGE_SIZE)
+                            .offset((page_index * Self::PAGE_SIZE) as u64)
+                            .map(&self.file)
+                            .unwrap()
+                    })
+                });
+
+            let range = Self::index_to_byte_range(index);
+
+            let slice = &mmap[range];
+
+            Ok(Some(Value::Ref(T::unsafe_try_from_slice(slice)?)))
+        } else {
+            // let mut file = &self.file_for_reads;
+            let mut file = Self::open_file(&self.pathbuf).unwrap();
+
+            file.seek(SeekFrom::Start(byte_index as u64)).unwrap();
+
+            let mut buf = vec![0; Self::SIZE_OF_T];
+            file.read_exact(&mut buf).unwrap();
+
+            let value = T::unsafe_try_from_slice(&buf[..])?;
+
+            Ok(Some(Value::Owned(value.to_owned())))
+        }
+    }
+
+    #[allow(unused)]
+    pub fn first(&self) -> Result<Option<Value<'_, T>>> {
+        self.get_(0)
+    }
+
+    #[allow(unused)]
+    pub fn last(&self) -> Result<Option<Value<'_, T>>> {
+        let len = self.len();
+        if len == 0 {
+            return Ok(None);
+        }
+        self.get_(len - 1)
+    }
+
+    pub fn push(&mut self, value: T) {
+        self.pushed.push(value)
+    }
+
+    pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
+        self.push_if_needed_(index.into(), value)
+    }
+    pub fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> {
+        let len = self.len();
+        match len.cmp(&index) {
+            Ordering::Greater => Ok(()),
+            Ordering::Equal => {
+                self.push(value);
+                Ok(())
+            }
+            Ordering::Less => Err(Error::IndexTooHigh),
+        }
+    }
+
+    // pub fn update(&mut self, index: I, value: T) -> Result<()> {
+    //     self._update(index.into(), value)
+    // }
+    // pub fn update_(&mut self, index: usize, value: T) -> Result<()> {
+    //     if let Some(index) = self.index_to_pushed_index(index) {
+    //         self.pushed[index] = value;
+    //     } else {
+    //         self.updated.insert(index, value);
+    //     }
+    //     Ok(())
+    // }
+
+    // pub fn fetch_update(&mut self, index: I, value: T) -> Result<T>
+    // where
+    //     T: Clone,
+    // {
+    //     self._fetch_update(index.into(), value)
+    // }
+    // pub fn fetch_update_(&mut self, index: usize, value: T) -> Result<T>
+    // where
+    //     T: Clone,
+    // {
+    //     let prev_opt = self.updated.insert(index, value);
+    //     if let Some(prev) = prev_opt {
+    //         Ok(prev)
+    //     } else {
+    //         Ok(self
+    //             ._get(index)?
+    //             .ok_or(Error::ExpectFileToHaveIndex)?
+    //             .clone())
+    //     }
+    // }
+
+    // pub fn remove(&mut self, index: I) {
+    //     self._remove(index.into())
+    // }
+    // pub fn remove_(&mut self, index: usize) {
+    //     self.removed.insert(index);
+    // }
+
+    pub fn len(&self) -> usize {
+        self.disk_len + self.pushed.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn has(&self, index: I) -> bool {
+        self.has_(index.into())
+    }
+    pub fn has_(&self, index: usize) -> bool {
+        index < self.len()
+    }
+
+    pub fn hasnt(&self, index: I) -> bool {
+        self.hasnt_(index.into())
+    }
+    pub fn hasnt_(&self, index: usize) -> bool {
+        !self.has_(index)
+    }
+
+    pub fn flush(&mut self) -> io::Result<()> {
+        self.disk_len += self.pushed.len();
+
+        self.reset_cache();
+
+        let mut bytes: Vec<u8> = vec![];
+
+        mem::take(&mut self.pushed)
+            .into_iter()
+            .for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice()));
+
+        // self.file.seek(SeekFrom::End(0))?;
+        self.file.write_all(&bytes)?;
+        // self.file_for_mmaps.write_all(&bytes)?;
+
+        // self.file_for_mmaps.sync_all()?;
+        // self.file_for_reads.sync_all()?;
+        // self.file_for_reads = Self::open_file(&self.pathbuf)?;
+
+        Ok(())
+    }
+}
+
+pub trait AnyStorableVec {
+    fn len(&self) -> usize;
+    fn is_empty(&self) -> bool;
+    fn flush(&mut self) -> io::Result<()>;
+}
+
+impl<I, T> AnyStorableVec for StorableVec<I, T>
+where
+    I: Into<usize>,
+    T: Sized + Debug + Clone,
+{
+    fn len(&self) -> usize {
+        self.len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.is_empty()
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        self.flush()
+    }
+}
+
+pub trait UnsafeSizedSerDe
+where
+    Self: Sized,
+{
+    const SIZE: usize = size_of::<Self>();
+
+    fn unsafe_try_from_slice(slice: &[u8]) -> Result<&Self> {
+        let (prefix, shorts, suffix) = unsafe { slice.align_to::<Self>() };
+
+        if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
+            // dbg!(&slice, &prefix, &shorts, &suffix);
+            return Err(Error::FailedToAlignToSelf);
+        }
+
+        Ok(&shorts[0])
+    }
+
+    fn unsafe_as_slice(&self) -> &[u8] {
+        let data: *const Self = self;
+        let data: *const u8 = data as *const u8;
+        unsafe { std::slice::from_raw_parts(data, Self::SIZE) }
+    }
+}
+impl<T> UnsafeSizedSerDe for T {}
+
+pub type Result<T, E = Error> = std::result::Result<T, E>;
+
+#[derive(Debug)]
+pub enum Error {
+    MmapsVecIsTooSmall,
+    FailedToAlignToSelf,
+    IndexTooHigh,
+    ExpectFileToHaveIndex,
+    ExpectVecToHaveIndex,
+}
+impl fmt::Display for Error {
+    // This trait requires `fmt` with this exact signature.
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
+            Error::FailedToAlignToSelf => write!(f, "Failed to align_to for T"),
+            Error::IndexTooHigh => write!(f, "Index too high"),
+            Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
+            Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"),
+        }
+    }
+}
+impl std::error::Error for Error {}

storable_vec/src/main.rs

@@ -0,0 +1,25 @@
+use std::path::Path;
+
+use storable_vec::StorableVec;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    {
+        let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
+
+        vec.push(0);
+        vec.push(1);
+        vec.push(2);
+        dbg!(vec.get(0)?); // Some(0)
+        dbg!(vec.get(21)?); // None
+
+        vec.flush()?;
+    }
+
+    {
+        let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
+
+        dbg!(vec.get(0)?); // 0
+    }
+
+    Ok(())
+}