vec: compression part 1

crates/brk_vec/src/structs/back.rs

@@ -0,0 +1,20 @@
+use std::{fs::File, sync::OnceLock};
+
+use super::CompressedPagesMetadata;
+
+type CompressedPage<T> = Option<(usize, Box<[T]>)>;
+
+pub enum Back<T> {
+    Raw {
+        raw_pages: Vec<OnceLock<Box<memmap2::Mmap>>>,
+        raw_page: memmap2::Mmap,
+        file: File,
+        file_position: u64,
+        buf: Vec<u8>,
+    },
+    Compressed {
+        decoded_pages: Option<Vec<OnceLock<Box<[T]>>>>,
+        decoded_page: CompressedPage<T>,
+        pages: CompressedPagesMetadata,
+    },
+}

crates/brk_vec/src/structs/compressed.rs

@@ -0,0 +1,67 @@
+use std::{
+    fs,
+    io::{self},
+    ops::Deref,
+    path::Path,
+};
+
+use crate::{Error, Result};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Compressed(bool);
+
+impl Compressed {
+    pub const YES: Self = Self(true);
+    pub const NO: Self = Self(false);
+
+    pub fn write(&self, path: &Path) -> Result<(), io::Error> {
+        fs::write(path, self.as_bytes())
+    }
+
+    fn as_bytes(&self) -> Vec<u8> {
+        if self.0 { vec![1] } else { vec![0] }
+    }
+
+    fn from_bytes(bytes: &[u8]) -> Self {
+        if bytes.len() != 1 {
+            panic!();
+        }
+        if bytes[0] == 1 {
+            Self(true)
+        } else if bytes[0] == 0 {
+            Self(false)
+        } else {
+            panic!()
+        }
+    }
+
+    pub fn validate(&self, path: &Path) -> Result<()> {
+        if let Ok(prev_compressed) = Compressed::try_from(path) {
+            if prev_compressed != *self {
+                return Err(Error::DifferentCompressionMode);
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl TryFrom<&Path> for Compressed {
+    type Error = Error;
+    fn try_from(value: &Path) -> Result<Self, Self::Error> {
+        Ok(Self::from_bytes(&fs::read(value)?))
+    }
+}
+
+impl From<bool> for Compressed {
+    fn from(value: bool) -> Self {
+        Self(value)
+    }
+}
+
+impl Deref for Compressed {
+    type Target = bool;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

crates/brk_vec/src/structs/length.rs

@@ -0,0 +1,71 @@
+use std::{
+    fs,
+    io::{self, Read},
+    ops::{AddAssign, Deref, DerefMut},
+    path::Path,
+};
+
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{Error, Result};
+
+#[derive(
+    Debug,
+    Default,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    FromBytes,
+    IntoBytes,
+    Immutable,
+    KnownLayout,
+)]
+pub struct Length(usize);
+
+impl Length {
+    pub fn write(&self, path: &Path) -> Result<(), io::Error> {
+        fs::write(path, self.as_bytes())
+    }
+}
+
+impl From<usize> for Length {
+    fn from(value: usize) -> Self {
+        Self(value)
+    }
+}
+
+impl Deref for Length {
+    type Target = usize;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for Length {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl TryFrom<&Path> for Length {
+    type Error = Error;
+    fn try_from(value: &Path) -> Result<Self, Self::Error> {
+        let mut buf = [0; 8];
+        if let Ok(bytes) = fs::read(value) {
+            bytes.as_slice().read_exact(&mut buf)?;
+            Ok(*(Self::ref_from_bytes(&buf)?))
+        } else {
+            Ok(Self::default())
+        }
+    }
+}
+
+impl AddAssign<usize> for Length {
+    fn add_assign(&mut self, rhs: usize) {
+        self.0 += rhs;
+    }
+}

crates/brk_vec/src/structs/mod.rs

@@ -1,5 +1,15 @@
+mod back;
+mod compressed;
+mod length;
+mod page;
+mod pages;
 mod unsafe_slice;
 mod version;
 
+pub use back::*;
+pub use compressed::*;
+pub use length::*;
+pub use page::*;
+pub use pages::*;
 pub use unsafe_slice::*;
 pub use version::*;

crates/brk_vec/src/structs/page.rs

@@ -0,0 +1,18 @@
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+#[derive(Debug, Clone, IntoBytes, Immutable, FromBytes, KnownLayout)]
+pub struct CompressedPageMetadata {
+    pub start: u64,
+    pub bytes_len: u32,
+    pub values_len: u32,
+}
+
+impl CompressedPageMetadata {
+    pub fn new(start: u64, bytes_len: u32, values_len: u32) -> Self {
+        Self {
+            start,
+            bytes_len,
+            values_len,
+        }
+    }
+}

crates/brk_vec/src/structs/pages.rs

@@ -0,0 +1,118 @@
+use std::{
+    fs::{self, OpenOptions},
+    io::{self, Seek, SeekFrom, Write},
+    path::{Path, PathBuf},
+};
+
+use rayon::prelude::*;
+use zerocopy::{IntoBytes, TryFromBytes};
+
+use crate::Result;
+
+use super::{CompressedPageMetadata, UnsafeSlice};
+
+#[derive(Debug, Clone)]
+pub struct CompressedPagesMetadata {
+    vec: Vec<CompressedPageMetadata>,
+    change_at: Option<usize>,
+    path: PathBuf,
+}
+
+impl CompressedPagesMetadata {
+    const PAGE_SIZE: usize = size_of::<CompressedPageMetadata>();
+
+    pub fn read(path: &Path) -> Result<CompressedPagesMetadata> {
+        let slf = Self {
+            vec: fs::read(path)
+                .unwrap_or_default()
+                .chunks(Self::PAGE_SIZE)
+                .map(|bytes| {
+                    if bytes.len() != Self::PAGE_SIZE {
+                        panic!()
+                    }
+                    CompressedPageMetadata::try_read_from_bytes(bytes).unwrap()
+                })
+                .collect::<Vec<_>>(),
+            path: path.to_owned(),
+            change_at: None,
+        };
+
+        Ok(slf)
+    }
+
+    pub fn write(&mut self) -> io::Result<()> {
+        if self.change_at.is_none() {
+            return Ok(());
+        }
+
+        let change_at = self.change_at.take().unwrap();
+
+        let len = (self.vec.len() - change_at) * Self::PAGE_SIZE;
+
+        let mut bytes: Vec<u8> = vec![0; len];
+
+        let unsafe_bytes = UnsafeSlice::new(&mut bytes);
+
+        self.vec[change_at..]
+            .par_iter()
+            .enumerate()
+            .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::PAGE_SIZE, v.as_bytes()));
+
+        let mut file = OpenOptions::new()
+            .read(true)
+            .create(true)
+            .truncate(false)
+            .append(true)
+            .open(&self.path)?;
+
+        file.set_len((change_at * Self::PAGE_SIZE) as u64)?;
+        file.seek(SeekFrom::End(0))?;
+
+        file.write_all(&bytes)?;
+
+        Ok(())
+    }
+
+    pub fn len(&self) -> usize {
+        self.vec.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn get(&self, page_index: usize) -> Option<&CompressedPageMetadata> {
+        self.vec.get(page_index)
+    }
+
+    pub fn last(&self) -> Option<&CompressedPageMetadata> {
+        self.vec.last()
+    }
+
+    pub fn pop(&mut self) -> Option<CompressedPageMetadata> {
+        self.vec.pop()
+    }
+
+    pub fn push(&mut self, page_index: usize, page: CompressedPageMetadata) {
+        if page_index != self.vec.len() {
+            panic!();
+        }
+
+        self.set_changed_at(page_index);
+
+        self.vec.push(page);
+    }
+
+    fn set_changed_at(&mut self, page_index: usize) {
+        if self.change_at.is_none_or(|pi| pi > page_index) {
+            self.change_at.replace(page_index);
+        }
+    }
+
+    pub fn truncate(&mut self, page_index: usize) -> Option<CompressedPageMetadata> {
+        let page = self.get(page_index).cloned();
+        self.vec.truncate(page_index);
+        self.set_changed_at(page_index);
+        page
+    }
+}