Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-26 17:54:44 -07:00
Code Issues Packages Projects Releases Wiki Activity

vec: rework part 2

Browse Source
This commit is contained in:
nym21
2025-04-09 22:59:18 +02:00
parent 962254e511
commit 41cf0225e3
21 changed files with 678 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/brk_vec/src/enums/mod.rs
View File

@@ -1,7 +1,5 @@
mod error;
mod value;
mod values;
pub use error::*;
pub use value::*;
pub use values::*;

86
crates/brk_vec/src/enums/values.rs
View File

@@ -1,86 +0,0 @@
use std::ops::Range;
use memmap2::Mmap;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
const ONE_KIB: usize = 1024;
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
use super::Result;
#[derive(Debug)]
pub enum Values<T> {
Owned(Box<[T]>),
Ref(Box<Mmap>),
}
impl<T> Values<T> {
const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
const SIZE_OF_T: usize = size_of::<T>();
pub fn get(&self, index: usize) -> Result<Option<&T>>
where
T: TryFromBytes + IntoBytes + Immutable + KnownLayout,
{
let index = Self::index_to_decoded_index(index);
Ok(match self {
Self::Owned(a) => a.get(index),
Self::Ref(m) => {
let range = Self::index_to_byte_range(index);
let source = &m[range];
Some(T::try_ref_from_bytes(source)?)
}
})
}
pub fn as_arr(&self) -> &[T] {
match self {
Self::Owned(a) => a,
Self::Ref(_) => unreachable!(),
}
}
pub fn as_mmap(&self) -> &Mmap {
match self {
Self::Owned(_) => unreachable!(),
Self::Ref(m) => m,
}
}
#[inline]
fn index_to_byte_range(index: usize) -> Range<usize> {
let index = Self::index_to_byte_index(index) as usize;
index..(index + Self::SIZE_OF_T)
}
#[inline]
fn index_to_byte_index(index: usize) -> u64 {
(index * Self::SIZE_OF_T) as u64
}
#[inline(always)]
fn index_to_decoded_index(index: usize) -> usize {
index % Self::PER_PAGE
}
}
impl<T> From<Box<[T]>> for Values<T> {
fn from(value: Box<[T]>) -> Self {
Self::Owned(value)
}
}
impl<T> From<Mmap> for Values<T> {
fn from(value: Mmap) -> Self {
Self::Ref(Box::new(value))
}
}
impl<T> Default for Values<T> {
fn default() -> Self {
Self::Owned(vec![].into_boxed_slice())
}
}

136
crates/brk_vec/src/lib.rs
View File

@@ -8,18 +8,25 @@ mod structs;
mod traits;
mod variants;
use std::{path::Path, sync::Arc};
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use arc_swap::{ArcSwap, Guard};
use axum::Json;
use axum::{Json, response::Response};
pub use enums::*;
use memmap2::Mmap;
pub use structs::*;
pub use traits::*;
pub use variants::*;
use variants::*;
#[derive(Debug)]
pub enum StoredVec<I, T> {
#[derive(Debug, Clone)]
pub enum StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
Raw(RawVec<I, T>),
Compressed(CompressedVec<I, T>),
}
@@ -38,30 +45,68 @@ where
Ok(Self::Raw(RawVec::forced_import(path, version)?))
}
}
pub fn enable_large_cache_if_needed(&mut self) {
match self {
StoredVec::Compressed(v) => v.enable_large_cache(),
Self::Raw(_) => {}
}
}
}
impl<I, T> AnyVec<I, T> for StoredVec<I, T>
impl<I, T> DynamicVec for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type I = I;
type T = T;
#[inline]
fn get_(&mut self, index: usize) -> Result<Option<Value<T>>> {
fn get_(&self, index: usize) -> Result<Option<Value<T>>> {
match self {
StoredVec::Raw(v) => v.get_(index),
StoredVec::Compressed(v) => v.get_(index),
}
}
fn iter_from<F>(&mut self, index: I, mut f: F) -> Result<()>
#[inline]
fn stored_len(&self) -> usize {
match self {
StoredVec::Raw(v) => v.stored_len(),
StoredVec::Compressed(v) => v.stored_len(),
}
}
#[inline]
fn pushed(&self) -> &[T] {
match self {
StoredVec::Raw(v) => v.pushed(),
StoredVec::Compressed(v) => v.pushed(),
}
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
match self {
StoredVec::Raw(v) => v.mut_pushed(),
StoredVec::Compressed(v) => v.mut_pushed(),
}
}
}
impl<I, T> GenericVec<I, T> for StoredVec<I, T>
where
I: StoredIndex + Send + Sync,
T: StoredType + Send + Sync,
{
fn iter_from<F>(&mut self, index: I, f: F) -> Result<()>
where
F: FnMut((I, T, &mut Self)) -> Result<()>,
F: FnMut((I, T, &mut dyn DynamicVec<I = I, T = T>)) -> Result<()>,
{
todo!();
// match self {
// StoredVec::Raw(v) => v.iter_from(index, |(i, t, inner)| f((i, t, self))),
// StoredVec::Compressed(v) => v.iter_from(index, |(i, t, inner)| f((i, t, self))),
// }
match self {
StoredVec::Raw(v) => v.iter_from(index, f),
StoredVec::Compressed(v) => v.iter_from(index, f),
}
}
fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Json<Vec<T>>> {
@@ -108,21 +153,6 @@ where
}
}
#[inline]
fn pushed(&self) -> &[T] {
match self {
StoredVec::Raw(v) => v.pushed(),
StoredVec::Compressed(v) => v.pushed(),
}
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
match self {
StoredVec::Raw(v) => v.mut_pushed(),
StoredVec::Compressed(v) => v.mut_pushed(),
}
}
#[inline]
fn path(&self) -> &Path {
match self {
@@ -139,3 +169,51 @@ where
}
}
}
pub trait AnyStoredVec: Send + Sync {
fn file_name(&self) -> String;
fn index_type_to_string(&self) -> &str;
fn len(&self) -> usize;
fn is_empty(&self) -> bool;
fn flush(&mut self) -> Result<()>;
fn collect_range_response(&self, from: Option<i64>, to: Option<i64>) -> Result<Response>;
fn path_vec(&self) -> PathBuf;
}
impl<I, T> AnyStoredVec for StoredVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
#[inline]
fn len(&self) -> usize {
DynamicVec::len(self)
}
#[inline]
fn is_empty(&self) -> bool {
DynamicVec::is_empty(self)
}
#[inline]
fn index_type_to_string(&self) -> &str {
GenericVec::index_type_to_string(self)
}
fn flush(&mut self) -> Result<()> {
GenericVec::flush(self)
}
fn collect_range_response(&self, from: Option<i64>, to: Option<i64>) -> Result<Response> {
GenericVec::collect_range_response(self, from, to)
}
#[inline]
fn path_vec(&self) -> PathBuf {
GenericVec::path_vec(self)
}
fn file_name(&self) -> String {
GenericVec::file_name(self)
}
}

2
crates/brk_vec/src/lib2.rs
View File

@@ -27,6 +27,8 @@ pub use enums::*;
pub use structs::*;
pub use traits::*;
use crate::{Compressed, CompressedPagesMetadata, Length, StoredIndex, StoredType, Version};
const ONE_KIB: usize = 1024;
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;

43
crates/brk_vec/src/traits/dynamic.rs Normal file
View File

@@ -0,0 +1,43 @@
use crate::{Result, Value};
use super::{StoredIndex, StoredType};
pub trait DynamicVec: Send + Sync {
type I: StoredIndex;
type T: StoredType;
#[inline]
fn get(&self, index: Self::I) -> Result<Option<Value<Self::T>>> {
self.get_(index.to_usize()?)
}
fn get_(&self, index: usize) -> Result<Option<Value<Self::T>>>;
fn get_last(&self) -> Result<Option<Value<Self::T>>> {
let len = self.len();
if len == 0 {
return Ok(None);
}
self.get_(len - 1)
}
#[inline]
fn len(&self) -> usize {
self.stored_len() + self.pushed_len()
}
#[inline]
fn is_empty(&self) -> bool {
self.len() == 0
}
fn stored_len(&self) -> usize;
fn pushed(&self) -> &[Self::T];
#[inline]
fn pushed_len(&self) -> usize {
self.pushed().len()
}
fn mut_pushed(&mut self) -> &mut Vec<Self::T>;
#[inline]
fn push(&mut self, value: Self::T) {
self.mut_pushed().push(value)
}
}

103
crates/brk_vec/src/traits/any.rs → crates/brk_vec/src/traits/generic.rs
View File

@@ -1,7 +1,3 @@
// use std::{io, path::PathBuf};
// use crate::{Result};
use std::{
fs::{File, OpenOptions},
io::{self, Seek, SeekFrom, Write},
@@ -16,17 +12,16 @@ use axum::{
};
use memmap2::Mmap;
use crate::{Error, Result, Value, Version};
use crate::{Error, Result, Version};
use super::{StoredIndex, StoredType};
use super::{DynamicVec, StoredIndex, StoredType};
pub trait AnyVec<I, T>: Send + Sync
pub trait GenericVec<I, T>: DynamicVec<I = I, T = T>
where
I: StoredIndex + Sized,
I: StoredIndex,
T: StoredType,
Self: Sized,
{
const SIZE_OF_T: usize = size_of::<T>();
const SIZE_OF_T: usize = size_of::<Self::T>();
fn open_file(&self) -> io::Result<File> {
Self::open_file_(&self.path_vec())
@@ -67,15 +62,6 @@ where
}
fn guard(&self) -> &Option<Guard<Arc<Mmap>>>;
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>>;
#[inline]
fn guard_to_value(guard: &Guard<Arc<Mmap>>, index: usize) -> T {
let index = index * Self::SIZE_OF_T;
let slice = &guard[index..(index + Self::SIZE_OF_T)];
let v = T::try_ref_from_bytes(slice).unwrap();
v.clone()
}
fn new_mmap(file: File) -> Result<Arc<Mmap>> {
Ok(Arc::new(unsafe { Mmap::map(&file)? }))
@@ -94,39 +80,6 @@ where
Ok(())
}
#[inline]
fn get(&mut self, index: I) -> Result<Option<Value<T>>> {
self.get_(index.to_usize()?)
}
fn get_(&mut self, index: usize) -> Result<Option<Value<T>>>;
fn get_last(&mut self) -> Result<Option<Value<T>>> {
let len = self.len();
if len == 0 {
return Ok(None);
}
self.get_(len - 1)
}
#[inline]
fn stored_len(&self) -> usize {
if let Some(guard) = self.guard() {
guard.len() / Self::SIZE_OF_T
} else {
self.new_guard().len() / Self::SIZE_OF_T
}
}
fn pushed(&self) -> &[T];
#[inline]
fn pushed_len(&self) -> usize {
self.pushed().len()
}
fn mut_pushed(&mut self) -> &mut Vec<T>;
#[inline]
fn push(&mut self, value: T) {
self.mut_pushed().push(value)
}
#[inline]
fn is_pushed_empty(&self) -> bool {
self.pushed_len() == 0
@@ -149,12 +102,7 @@ where
}
#[inline]
fn len(&self) -> usize {
self.stored_len() + self.pushed_len()
}
#[inline]
fn has(&self, index: I) -> Result<bool> {
fn has(&self, index: Self::I) -> Result<bool> {
Ok(self.has_(index.to_usize()?))
}
#[inline]
@@ -162,27 +110,34 @@ where
index < self.len()
}
#[inline]
fn is_empty(&self) -> bool {
self.len() == 0
}
#[inline]
fn index_type_to_string(&self) -> &str {
I::to_string()
Self::I::to_string()
}
#[inline]
fn iter<F>(&mut self, f: F) -> Result<()>
where
F: FnMut((I, T, &mut Self)) -> Result<()>,
F: FnMut(
(
Self::I,
Self::T,
&mut dyn DynamicVec<I = Self::I, T = Self::T>,
),
) -> Result<()>,
{
self.iter_from(I::default(), f)
self.iter_from(Self::I::default(), f)
}
fn iter_from<F>(&mut self, index: I, f: F) -> Result<()>
fn iter_from<F>(&mut self, index: Self::I, f: F) -> Result<()>
where
F: FnMut((I, T, &mut Self)) -> Result<()>;
F: FnMut(
(
Self::I,
Self::T,
&mut dyn DynamicVec<I = Self::I, T = Self::T>,
),
) -> Result<()>;
fn fix_i64(i: i64, len: usize, from: bool) -> usize {
if i >= 0 {
@@ -202,9 +157,9 @@ where
fn flush(&mut self) -> Result<()>;
fn truncate_if_needed(&mut self, index: I) -> Result<()>;
fn truncate_if_needed(&mut self, index: Self::I) -> Result<()>;
fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Json<Vec<T>>>;
fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Json<Vec<Self::T>>>;
fn collect_range_response(&self, from: Option<i64>, to: Option<i64>) -> Result<Response> {
Ok(self.collect_range(from, to)?.into_response())
@@ -236,12 +191,4 @@ where
}
fn version(&self) -> Version;
fn any(&self) -> &Self {
self
}
fn mut_any(&mut self) -> &mut Self {
self
}
}

6
crates/brk_vec/src/traits/mod.rs
View File

@@ -1,7 +1,9 @@
mod any;
mod dynamic;
mod generic;
mod stored_index;
mod stored_type;
pub use any::*;
pub use dynamic::*;
pub use generic::*;
pub use stored_index::*;
pub use stored_type::*;

22
crates/brk_vec/src/traits/stored_type.rs
View File

@@ -5,10 +5,28 @@ use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
pub trait StoredType
where
Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync + Serialize,
Self: Sized
+ Debug
+ Clone
+ TryFromBytes
+ IntoBytes
+ Immutable
+ KnownLayout
+ Send
+ Sync
+ Serialize,
{
}
impl<T> StoredType for T where
T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync + Serialize
T: Sized
+ Debug
+ Clone
+ TryFromBytes
+ IntoBytes
+ Immutable
+ KnownLayout
+ Send
+ Sync
+ Serialize
{
}

309
crates/brk_vec/src/variants/compressed.rs
View File

@@ -1,5 +1,5 @@
use std::{
fs,
fs, mem,
path::Path,
sync::{Arc, OnceLock},
};
@@ -7,17 +7,25 @@ use std::{
use arc_swap::{ArcSwap, Guard};
use axum::Json;
use memmap2::Mmap;
use rayon::prelude::*;
use zstd::DEFAULT_COMPRESSION_LEVEL;
use crate::{
AnyVec, CompressedPagesMetadata, Error, RawVec, Result, StoredIndex, StoredType, Value, Version,
CompressedPageMetadata, CompressedPagesMetadata, DynamicVec, Error, GenericVec, RawVec, Result,
StoredIndex, StoredType, UnsafeSlice, Value, Version,
};
const ONE_KIB: usize = 1024;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
#[derive(Debug)]
pub struct CompressedVec<I, T> {
inner: RawVec<I, T>,
decoded_page: Option<(usize, Box<[T]>)>,
pages_meta: CompressedPagesMetadata,
decoded_pages: Option<Vec<OnceLock<Box<[T]>>>>,
decoded_page: Option<(usize, Vec<T>)>,
decoded_pages: Option<Vec<OnceLock<Vec<T>>>>,
pages_meta: Arc<ArcSwap<CompressedPagesMetadata>>,
// pages: Option<Vec<OnceLock<Values<T>>>>,
// page: Option<(usize, Values<T>)>,
// length: Length
@@ -28,6 +36,10 @@ where
I: StoredIndex,
T: StoredType,
{
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
/// Same as import but will reset the folder under certain errors, so be careful !
pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
let res = Self::import(path, version);
@@ -47,39 +59,284 @@ where
inner: RawVec::import(path, version)?,
decoded_page: None,
decoded_pages: None,
pages_meta: CompressedPagesMetadata::read(path)?,
pages_meta: Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read(path)?))),
})
}
pub fn decode_page(&self, page_index: usize) -> Result<Vec<T>> {
Self::decode_page_(
self.stored_len(),
page_index,
self.guard().as_ref().unwrap(),
&self.pages_meta.load(),
)
}
fn decode_page_(
stored_len: usize,
page_index: usize,
mmap: &Mmap,
compressed_pages_meta: &CompressedPagesMetadata,
) -> Result<Vec<T>> {
if Self::page_index_to_index(page_index) >= stored_len {
return Err(Error::IndexTooHigh);
} else if compressed_pages_meta.len() <= page_index {
return Err(Error::ExpectVecToHaveIndex);
}
let page = compressed_pages_meta.get(page_index).unwrap();
let len = page.bytes_len as usize;
let offset = page.start as usize;
Ok(zstd::decode_all(&mmap[offset..offset + len])
.inspect_err(|_| {
dbg!((len, offset, page_index, &mmap[..], &mmap.len()));
})?
.chunks(Self::SIZE_OF_T)
.map(|slice| T::try_read_from_bytes(slice).unwrap())
.collect::<Vec<_>>())
}
fn compress_page(chunk: &[T]) -> Box<[u8]> {
if chunk.len() > Self::PER_PAGE {
panic!();
}
let mut bytes: Vec<u8> = vec![0; chunk.len() * Self::SIZE_OF_T];
let unsafe_bytes = UnsafeSlice::new(&mut bytes);
chunk
.into_par_iter()
.enumerate()
.for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes()));
zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL)
.unwrap()
.into_boxed_slice()
}
pub fn enable_large_cache(&mut self) {
self.decoded_pages.replace(vec![]);
self.reset_large_cache();
}
pub fn disable_large_cache(&mut self) {
self.decoded_pages.take();
}
fn reset_large_cache(&mut self) {
let stored_len = self.stored_len();
if let Some(pages) = self.decoded_pages.as_mut() {
pages.par_iter_mut().for_each(|lock| {
lock.take();
});
let len = (stored_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
let len = Self::CACHE_LENGTH.min(len);
if pages.len() != len {
pages.resize_with(len, Default::default);
}
}
}
pub fn large_cache_len(&self) -> usize {
self.decoded_pages.as_ref().map_or(0, |v| v.len())
}
fn reset_small_cache(&mut self) {
self.decoded_page.take();
}
fn reset_caches(&mut self) {
self.reset_small_cache();
self.reset_large_cache();
}
#[inline(always)]
fn index_to_page_index(index: usize) -> usize {
index / Self::PER_PAGE
}
#[inline(always)]
fn page_index_to_index(page_index: usize) -> usize {
page_index * Self::PER_PAGE
}
}
impl<I, T> AnyVec<I, T> for CompressedVec<I, T>
impl<I, T> DynamicVec for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type I = I;
type T = T;
#[inline]
fn get_(&mut self, index: usize) -> Result<Option<Value<T>>> {
self.inner.get_(index)
fn get_(&self, index: usize) -> Result<Option<Value<T>>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.pushed().get(index).map(|v| Value::Ref(v)));
}
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(Error::IndexTooLow) => {}
Err(error) => return Err(error),
}
let page_index = Self::index_to_page_index(index);
// if self.page().is_none_or(|b| b.0 != page_index) {
// let values = self.decode_page(page_index)?;
// self.mut_page().replace((page_index, values));
// }
// self.page().unwrap().1.get(index)
//
todo!();
// let v = self.inner.guard().as_ref().map_or_else(
// || Self::guard_to_value(&self.new_guard(), index),
// |guard| Self::guard_to_value(guard, index),
// );
// Ok(Some(Value::Owned(v)))
}
fn stored_len(&self) -> usize {
todo!()
}
#[inline]
fn pushed(&self) -> &[T] {
self.inner.pushed()
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
self.inner.mut_pushed()
}
}
impl<I, T> GenericVec<I, T> for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn iter_from<F>(&mut self, _index: I, _f: F) -> Result<()>
where
F: FnMut((I, T, &mut Self)) -> Result<()>,
F: FnMut((I, T, &mut dyn DynamicVec<I = Self::I, T = Self::T>)) -> Result<()>,
{
todo!()
// self.inner.iter_from(index, f)
}
fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Json<Vec<T>>> {
self.inner.collect_range(from, to)
todo!()
}
fn flush(&mut self) -> Result<()> {
self.inner.flush()
let pushed_len = self.pushed_len();
if pushed_len == 0 {
return Ok(());
}
let stored_len = self.stored_len();
let mut pages_meta = (**self.pages_meta.load()).clone();
let mut starting_page_index = pages_meta.len();
let mut values = vec![];
let mut truncate_at = None;
if self.stored_len() % Self::PER_PAGE != 0 {
if pages_meta.is_empty() {
unreachable!()
}
let last_page_index = pages_meta.len() - 1;
values = if let Some(values) = self
.decoded_pages
.as_mut()
.and_then(|big_cache| big_cache.last_mut().and_then(|lock| lock.take()))
{
values
} else if self
.decoded_page
.as_ref()
.is_some_and(|(page_index, _)| *page_index == last_page_index)
{
self.decoded_page.take().unwrap().1
} else {
Self::decode_page_(
stored_len,
last_page_index,
self.guard().as_ref().unwrap(),
&pages_meta,
)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap()
};
truncate_at.replace(pages_meta.pop().unwrap().start);
starting_page_index = last_page_index;
}
let compressed = values
.into_par_iter()
.chain(mem::take(self.mut_pushed()).into_par_iter())
.chunks(Self::PER_PAGE)
.map(|chunk| (Self::compress_page(chunk.as_ref()), chunk.len()))
.collect::<Vec<_>>();
compressed
.iter()
.enumerate()
.for_each(|(i, (compressed_bytes, values_len))| {
let page_index = starting_page_index + i;
let start = if page_index != 0 {
let prev = pages_meta.get(page_index - 1).unwrap();
prev.start + prev.bytes_len as u64
} else {
0
};
let bytes_len = compressed_bytes.len() as u32;
let values_len = *values_len as u32;
let page = CompressedPageMetadata::new(start, bytes_len, values_len);
pages_meta.push(page_index, page);
});
pages_meta.write()?;
let buf = compressed
.into_iter()
.flat_map(|(v, _)| v)
.collect::<Vec<_>>();
if let Some(truncate_at) = truncate_at {
self.file_set_len(truncate_at)?;
}
self.file_write_all(&buf)?;
self.pages_meta.store(Arc::new(pages_meta));
self.reset_caches();
Ok(())
}
fn truncate_if_needed(&mut self, index: I) -> Result<()> {
self.inner.truncate_if_needed(index)
todo!()
}
#[inline]
@@ -96,15 +353,6 @@ where
self.inner.mut_guard()
}
#[inline]
fn pushed(&self) -> &[T] {
self.inner.pushed()
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
self.inner.mut_pushed()
}
#[inline]
fn path(&self) -> &Path {
self.inner.path()
@@ -115,3 +363,18 @@ where
self.inner.version()
}
}
impl<I, T> Clone for CompressedVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
decoded_page: None,
decoded_pages: None,
pages_meta: self.pages_meta.clone(),
}
}
}

80
crates/brk_vec/src/variants/raw.rs
View File

@@ -11,7 +11,9 @@ use axum::Json;
use memmap2::Mmap;
use rayon::prelude::*;
use crate::{AnyVec, Error, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version};
use crate::{
DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version,
};
#[derive(Debug)]
pub struct RawVec<I, T> {
@@ -63,13 +65,16 @@ where
}
}
impl<I, T> AnyVec<I, T> for RawVec<I, T>
impl<I, T> DynamicVec for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
type I = I;
type T = T;
#[inline]
fn get_(&mut self, index: usize) -> Result<Option<Value<T>>> {
fn get_(&self, index: usize) -> Result<Option<Value<T>>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
@@ -81,18 +86,42 @@ where
Err(error) => return Err(error),
}
let v = if let Some(guard) = self.guard.as_ref() {
Self::guard_to_value(guard, index)
} else {
Self::guard_to_value(&self.new_guard(), index)
};
let guard = self.guard.as_ref().unwrap();
let index = index * Self::SIZE_OF_T;
let slice = &guard[index..(index + Self::SIZE_OF_T)];
let v = Self::T::try_read_from_bytes(slice)?;
Ok(Some(Value::Owned(v)))
}
#[inline]
fn stored_len(&self) -> usize {
if let Some(guard) = self.guard() {
guard.len() / Self::SIZE_OF_T
} else {
self.new_guard().len() / Self::SIZE_OF_T
}
}
#[inline]
fn pushed(&self) -> &[T] {
self.pushed.as_slice()
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
&mut self.pushed
}
}
impl<I, T> GenericVec<I, T> for RawVec<I, T>
where
I: StoredIndex + Send + Sync,
T: StoredType + Send + Sync,
{
fn iter_from<F>(&mut self, index: I, mut f: F) -> Result<()>
where
F: FnMut((I, T, &mut Self)) -> Result<()>,
F: FnMut((I, T, &mut dyn DynamicVec<I = Self::I, T = Self::T>)) -> Result<()>,
{
if !self.is_pushed_empty() {
return Err(Error::UnsupportedUnflushedState);
@@ -107,7 +136,7 @@ where
.enumerate()
.try_for_each(|(i, chunk)| {
let v = T::try_read_from_bytes(chunk).unwrap();
f((I::from(i), v, self))
f((I::from(i), v, self as &mut dyn DynamicVec<I = I, T = T>))
})?;
Ok(())
@@ -168,8 +197,8 @@ where
return Ok(Json(vec![]));
}
let from = from.map_or(0, |i| Self::fix_i64(i, len, true)) * Self::SIZE_OF_T;
let to = to.map_or(len, |i| Self::fix_i64(i, len, false)) * Self::SIZE_OF_T;
let from = from.map_or(0, |i| Self::fix_i64(i, len, true));
let to = to.map_or(len, |i| Self::fix_i64(i, len, false));
Ok(Json(
guard[from * Self::SIZE_OF_T..to * Self::SIZE_OF_T]
@@ -193,15 +222,6 @@ where
&mut self.guard
}
#[inline]
fn pushed(&self) -> &[T] {
self.pushed.as_slice()
}
#[inline]
fn mut_pushed(&mut self) -> &mut Vec<T> {
&mut self.pushed
}
#[inline]
fn path(&self) -> &Path {
self.pathbuf.as_path()
@@ -212,3 +232,21 @@ where
self.version
}
}
impl<I, T> Clone for RawVec<I, T>
where
I: StoredIndex,
T: StoredType,
{
fn clone(&self) -> Self {
Self {
version: self.version,
pathbuf: self.pathbuf.clone(),
// Consider Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
mmap: self.mmap.clone(),
guard: None,
pushed: vec![],
phantom: PhantomData,
}
}
}