global: from custom unsafe_slice to zerocopy

This commit is contained in:
nym21
2025-02-05 16:42:53 +01:00
parent 138ca80c10
commit d86d614520
43 changed files with 684 additions and 366 deletions

View File

@@ -13,7 +13,8 @@ pub enum Error {
DifferentVersion { found: Version, expected: Version },
MmapsVecIsTooSmall,
IO(io::Error),
UnsafeSliceSerde(unsafe_slice_serde::Error),
// UnsafeSliceSerde(zerocopy::error::),
ZeroCopyError,
IndexTooHigh,
IndexTooLow,
ExpectFileToHaveIndex,
@@ -28,9 +29,15 @@ impl From<io::Error> for Error {
}
}
impl From<unsafe_slice_serde::Error> for Error {
fn from(value: unsafe_slice_serde::Error) -> Self {
Self::UnsafeSliceSerde(value)
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
}
}
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
Self::ZeroCopyError
}
}
@@ -43,7 +50,7 @@ impl fmt::Display for Error {
}
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
// Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::IndexTooLow => write!(f, "Index too low"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
@@ -52,6 +59,7 @@ impl fmt::Display for Error {
Error::UnsupportedUnflushedState => {
write!(f, "Unsupported unflush state, please flush before using this function")
}
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
}
}
}

View File

@@ -11,8 +11,8 @@ use std::{
sync::OnceLock,
};
use memmap2::{Mmap, MmapOptions};
use unsafe_slice_serde::UnsafeSliceSerde;
pub use memmap2;
pub use zerocopy;
mod enums;
mod structs;
@@ -60,7 +60,7 @@ pub struct StorableVec<I, T, const MODE: u8> {
file_position: u64,
buf: Buffer,
/// Only for CACHED_GETS
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
cache: Vec<OnceLock<Box<memmap2::Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
pushed: Vec<T>,
// updated: BTreeMap<usize, T>,
// inserted: BTreeMap<usize, T>,
@@ -207,7 +207,7 @@ where
fn read_exact<'a>(file: &'a mut File, buf: &'a mut [u8]) -> Result<&'a T> {
file.read_exact(buf)?;
let v = T::unsafe_try_from_slice(&buf[..])?;
let v = T::try_ref_from_bytes(&buf[..])?;
Ok(v)
}
@@ -277,7 +277,7 @@ where
mem::take(&mut self.pushed)
.into_iter()
.for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice()));
.for_each(|v| bytes.extend_from_slice(v.as_bytes()));
self.file.write_all(&bytes)?;
@@ -383,7 +383,7 @@ where
.ok_or(Error::MmapsVecIsTooSmall)?
.get_or_init(|| {
Box::new(unsafe {
MmapOptions::new()
memmap2::MmapOptions::new()
.len(Self::PAGE_SIZE)
.offset((page_index * Self::PAGE_SIZE) as u64)
.map(&self.file)
@@ -393,7 +393,7 @@ where
let range = Self::index_to_byte_range(index);
let slice = &mmap[range];
return Ok(Some(Value::Ref(T::unsafe_try_from_slice(slice)?)));
return Ok(Some(Value::Ref(T::try_ref_from_bytes(slice)?)));
}
Ok(Some(Value::Owned(self.open_file_at_then_read(index)?.to_owned())))
@@ -581,8 +581,9 @@ where
other_to_self: &mut StorableVec<A, I, SINGLE_THREAD>,
) -> Result<()>
where
A: StorableVecIndex + StorableVecType,
I: StorableVecType,
T: From<bool>,
A: StorableVecIndex + StorableVecType,
{
self_to_other.iter_from(I::from(self.len()), |(i, other)| {
self.push_if_needed(i, T::from(other_to_self.get(*other)? == &i))

View File

@@ -4,16 +4,16 @@ use std::{
path::Path,
};
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.0.unsafe_as_slice())
fs::write(path, self.as_bytes())
}
pub fn swap_bytes(self) -> Self {
@@ -32,6 +32,6 @@ impl TryFrom<&Path> for Version {
fn try_from(value: &Path) -> Result<Self, Self::Error> {
let mut buf = [0; 4];
fs::read(value)?.as_slice().read_exact(&mut buf)?;
Ok(*(Self::unsafe_try_from_slice(&buf)?))
Ok(*(Self::ref_from_bytes(&buf)?))
}
}

View File

@@ -2,10 +2,30 @@ use std::{fmt::Debug, ops::Add};
pub trait StorableVecIndex
where
Self: Debug + Default + Copy + Clone + PartialOrd + Ord + TryInto<usize> + From<usize> + Add<usize, Output = Self>,
Self: Debug
+ Default
+ Copy
+ Clone
+ PartialEq
+ Eq
+ PartialOrd
+ Ord
+ TryInto<usize>
+ From<usize>
+ Add<usize, Output = Self>,
{
}
impl<I> StorableVecIndex for I where
I: Debug + Default + Copy + Clone + PartialOrd + Ord + TryInto<usize> + From<usize> + Add<usize, Output = Self>
I: Debug
+ Default
+ Copy
+ Clone
+ PartialEq
+ Eq
+ PartialOrd
+ Ord
+ TryInto<usize>
+ From<usize>
+ Add<usize, Output = Self>
{
}

View File

@@ -1,8 +1,10 @@
use std::fmt::Debug;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
pub trait StorableVecType
where
Self: Sized + Debug + Clone,
Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout,
{
}
impl<T> StorableVecType for T where T: Sized + Debug + Clone {}
impl<T> StorableVecType for T where T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout {}