Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-05 19:59:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

vec: don't store mmap in struct anymore

Browse Source
This commit is contained in:
nym21
2025-07-13 11:50:34 +02:00
parent a98546f605
commit 2dd608dfed
13 changed files with 204 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
crates/brk_vec/src/structs/header.rs
View File

@@ -1,13 +1,12 @@
use std::{
fs::File,
io::{self, Seek, SeekFrom},
io::{self, Read, Seek, SeekFrom},
os::unix::fs::FileExt,
sync::Arc,
};
use arc_swap::ArcSwap;
use brk_core::{Error, Height, Result, Version};
use memmap2::Mmap;
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -31,8 +30,12 @@ impl Header {
})
}
pub fn import_and_verify(mmap: &Mmap, vec_version: Version, format: Format) -> Result<Self> {
let inner = HeaderInner::import_and_verify(mmap, vec_version, format)?;
pub fn import_and_verify(
file: &mut File,
vec_version: Version,
format: Format,
) -> Result<Self> {
let inner = HeaderInner::import_and_verify(file, vec_version, format)?;
Ok(Self {
inner: Arc::new(ArcSwap::from_pointee(inner)),
modified: false,
@@ -109,13 +112,20 @@ impl HeaderInner {
file.write_all_at(self.as_bytes(), 0)
}
pub fn import_and_verify(mmap: &Mmap, vec_version: Version, format: Format) -> Result<Self> {
if mmap.len() < HEADER_OFFSET {
pub fn import_and_verify(
file: &mut File,
vec_version: Version,
format: Format,
) -> Result<Self> {
if file.metadata()?.len() < HEADER_OFFSET as u64 {
return Err(Error::WrongLength);
}
// dbg!(mmap.len());
let header = HeaderInner::read_from_bytes(&mmap[..HEADER_OFFSET])?;
// dbg!(&header);
let mut buf = [0; HEADER_OFFSET];
file.read_exact(&mut buf)?;
let header = HeaderInner::read_from_bytes(&buf)?;
if header.header_version != HEADER_VERSION {
return Err(Error::DifferentVersion {
found: header.header_version,

23
crates/brk_vec/src/traits/generic.rs
View File

@@ -3,10 +3,8 @@ use std::{
fs::{File, OpenOptions},
io::{self, Seek, SeekFrom, Write},
path::{Path, PathBuf},
sync::Arc,
};
use arc_swap::ArcSwap;
use brk_core::Result;
use memmap2::Mmap;
@@ -38,7 +36,7 @@ where
}
#[inline]
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Cow<T>>> {
let stored_len = self.stored_len_(mmap);
let stored_len = self.stored_len();
if index >= stored_len {
let pushed = self.pushed();
@@ -61,10 +59,7 @@ where
format!("{}_to_{}", I::to_string(), self.name())
}
fn mmap(&self) -> &ArcSwap<Mmap>;
fn stored_len(&self) -> usize;
fn stored_len_(&self, mmap: &Mmap) -> usize;
fn pushed(&self) -> &[T];
#[inline]
@@ -116,7 +111,7 @@ where
fn file_set_len(&mut self, file: &mut File, len: u64) -> Result<()> {
Self::file_set_len_(file, len)?;
self.update_mmap(file)
Ok(())
}
fn file_set_len_(file: &mut File, len: u64) -> Result<()> {
file.set_len(len)?;
@@ -127,7 +122,7 @@ where
fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> {
file.write_all(buf)?;
file.flush()?;
self.update_mmap(file)
Ok(())
}
fn file_truncate_and_write_all(&mut self, file: &mut File, len: u64, buf: &[u8]) -> Result<()> {
@@ -143,14 +138,10 @@ where
self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[])
}
fn new_mmap(file: &File) -> Result<Arc<Mmap>> {
Ok(Arc::new(unsafe { Mmap::map(file)? }))
}
fn update_mmap(&mut self, file: &File) -> Result<()> {
let mmap = Self::new_mmap(file)?;
self.mmap().store(mmap);
Ok(())
#[inline]
fn create_mmap(&self) -> Result<Mmap> {
let file = self.open_file()?;
unsafe { Mmap::map(&file).map_err(|e| e.into()) }
}
#[inline]

44
crates/brk_vec/src/variants/compressed.rs
View File

@@ -180,11 +180,6 @@ where
self.inner.mut_header()
}
#[inline]
fn mmap(&self) -> &ArcSwap<Mmap> {
self.inner.mmap()
}
fn parent(&self) -> &Path {
self.inner.parent()
}
@@ -193,10 +188,6 @@ where
fn stored_len(&self) -> usize {
Self::stored_len__(&self.pages_meta.load())
}
#[inline]
fn stored_len_(&self, _: &Mmap) -> usize {
self.stored_len()
}
#[inline]
fn pushed(&self) -> &[T] {
@@ -223,6 +214,8 @@ where
let stored_len = self.stored_len();
let mut file = file_opt.unwrap_or(self.open_file()?);
let mut pages_meta = (**self.pages_meta.load()).clone();
let mut starting_page_index = pages_meta.len();
@@ -236,16 +229,13 @@ where
let last_page_index = pages_meta.len() - 1;
values = Self::decode_page_(
stored_len,
last_page_index,
&self.mmap().load(),
&pages_meta,
)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap();
let mmap = unsafe { Mmap::map(&file)? };
values = Self::decode_page_(stored_len, last_page_index, &mmap, &pages_meta)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap();
truncate_at.replace(pages_meta.pop().unwrap().start);
starting_page_index = last_page_index;
@@ -287,8 +277,6 @@ where
pages_meta.write()?;
let mut file = file_opt.unwrap_or(self.open_file()?);
if let Some(truncate_at) = truncate_at {
self.file_set_len(&mut file, truncate_at)?;
}
@@ -324,7 +312,11 @@ where
let page_index = Self::index_to_page_index(index);
let values = self.decode_page(page_index, &self.mmap().load())?;
let mut file = self.open_file()?;
let mmap = unsafe { Mmap::map(&file)? };
let values = self.decode_page(page_index, &mmap)?;
let mut buf = vec![];
let mut page = pages_meta.truncate(page_index).unwrap();
@@ -348,8 +340,6 @@ where
self.pages_meta.store(Arc::new(pages_meta));
let mut file = self.open_file()?;
self.file_truncate_and_write_all(&mut file, len, &buf)?;
Ok(())
@@ -399,7 +389,7 @@ impl<I, T> Clone for CompressedVec<I, T> {
#[derive(Debug)]
pub struct CompressedVecIterator<'a, I, T> {
vec: &'a CompressedVec<I, T>,
guard: Guard<Arc<Mmap>>,
mmap: Mmap,
decoded_page: Option<(usize, Vec<T>)>,
// second_decoded_page?: Option<(usize, Vec<T>)>,
pages_meta: Guard<Arc<CompressedPagesMetadata>>,
@@ -445,7 +435,7 @@ where
type Item = (I, Cow<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let mmap = &self.guard;
let mmap = &self.mmap;
let i = self.index;
let stored_len = self.stored_len;
@@ -499,7 +489,7 @@ where
let stored_len = CompressedVec::<I, T>::stored_len__(&pages_meta);
CompressedVecIterator {
vec: self,
guard: self.mmap().load(),
mmap: self.create_mmap().unwrap(),
decoded_page: None,
pages_meta,
stored_len,

5
crates/brk_vec/src/variants/eager.rs
View File

@@ -8,7 +8,6 @@ use std::{
path::{Path, PathBuf},
};
use arc_swap::ArcSwap;
use brk_core::{
Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Error, Result, Sats, StoredF32,
StoredUsize, Version,
@@ -108,10 +107,6 @@ where
self.0.get_or_read(index, mmap)
}
pub fn mmap(&self) -> &ArcSwap<Mmap> {
self.0.mmap()
}
pub fn inner_version(&self) -> Version {
self.0.version()
}

5
crates/brk_vec/src/variants/indexed.rs
View File

@@ -1,6 +1,5 @@
use std::{borrow::Cow, cmp::Ordering, fmt::Debug, path::Path};
use arc_swap::ArcSwap;
use brk_core::{Error, Height, Result, Version};
use crate::{
@@ -78,8 +77,8 @@ where
self.0.header()
}
pub fn mmap(&self) -> &ArcSwap<Mmap> {
self.0.mmap()
pub fn create_mmap(&self) -> Result<Mmap> {
self.0.create_mmap()
}
#[inline]

61
crates/brk_vec/src/variants/raw.rs
View File

@@ -5,10 +5,12 @@ use std::{
marker::PhantomData,
mem,
path::{Path, PathBuf},
sync::Arc,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use arc_swap::{ArcSwap, Guard};
use brk_core::{Error, Result, Version};
use memmap2::Mmap;
use rayon::prelude::*;
@@ -25,10 +27,9 @@ pub struct RawVec<I, T> {
header: Header,
parent: PathBuf,
name: &'static str,
// Consider Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
mmap: Arc<ArcSwap<Mmap>>,
pushed: Vec<T>,
phantom: PhantomData<I>,
stored_len: Arc<AtomicUsize>,
}
impl<I, T> RawVec<I, T>
@@ -55,16 +56,18 @@ where
pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
let path = Self::path_(parent, name);
let (mmap, header) = match Self::open_file_(&path) {
let (header, file) = match Self::open_file_(&path) {
Ok(mut file) => {
if file.metadata()?.len() == 0 {
let header = Header::create_and_write(&mut file, version, Format::Raw)?;
let mmap = Self::new_mmap(&file)?;
(mmap, header)
(
Header::create_and_write(&mut file, version, Format::Raw)?,
Some(file),
)
} else {
let mmap = Self::new_mmap(&file)?;
let header = Header::import_and_verify(&mmap, version, Format::Raw)?;
(mmap, header)
(
Header::import_and_verify(&mut file, version, Format::Raw)?,
None,
)
}
}
Err(e) => match e.kind() {
@@ -72,22 +75,25 @@ where
fs::create_dir_all(Self::folder_(parent, name))?;
let mut file = Self::open_file_(&path)?;
let header = Header::create_and_write(&mut file, version, Format::Raw)?;
let mmap = Self::new_mmap(&file)?;
(mmap, header)
(header, None)
}
_ => return Err(e.into()),
},
};
let mmap = Arc::new(ArcSwap::new(mmap));
let stored_len = if let Some(file) = file {
(file.metadata()?.len() as usize - HEADER_OFFSET) / Self::SIZE_OF_T
} else {
0
};
Ok(Self {
mmap,
header,
name: Box::leak(Box::new(name.to_string())),
parent: parent.to_owned(),
pushed: vec![],
phantom: PhantomData,
stored_len: Arc::new(AtomicUsize::new(stored_len)),
})
}
@@ -141,18 +147,9 @@ where
&mut self.header
}
#[inline]
fn mmap(&self) -> &ArcSwap<Mmap> {
&self.mmap
}
#[inline]
fn stored_len(&self) -> usize {
self.stored_len_(&self.mmap.load())
}
#[inline]
fn stored_len_(&self, mmap: &Mmap) -> usize {
(mmap.len() - HEADER_OFFSET) / Self::SIZE_OF_T
self.stored_len.load(Ordering::SeqCst)
}
#[inline]
@@ -194,9 +191,10 @@ where
};
let mut file = file_opt.unwrap_or(self.open_file()?);
self.file_write_all(&mut file, &bytes)?;
self.stored_len.fetch_add(pushed_len, Ordering::SeqCst);
Ok(())
}
@@ -212,6 +210,8 @@ where
return Ok(());
}
self.stored_len.store(index, Ordering::SeqCst);
let len = index * Self::SIZE_OF_T + HEADER_OFFSET;
let mut file = self.open_file()?;
@@ -221,6 +221,7 @@ where
}
fn reset(&mut self) -> Result<()> {
self.stored_len.store(0, Ordering::SeqCst);
self.reset_()
}
}
@@ -262,9 +263,9 @@ impl<I, T> Clone for RawVec<I, T> {
header: self.header.clone(),
parent: self.parent.clone(),
name: self.name,
mmap: self.mmap.clone(),
pushed: vec![],
phantom: PhantomData,
stored_len: self.stored_len.clone(),
}
}
}
@@ -272,7 +273,7 @@ impl<I, T> Clone for RawVec<I, T> {
#[derive(Debug)]
pub struct RawVecIterator<'a, I, T> {
vec: &'a RawVec<I, T>,
guard: Guard<Arc<Mmap>>,
mmap: Mmap,
index: usize,
}
@@ -305,7 +306,7 @@ where
type Item = (I, Cow<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let mmap = &self.guard;
let mmap = &self.mmap;
let index = self.index;
let opt = self
@@ -333,7 +334,7 @@ where
fn into_iter(self) -> Self::IntoIter {
RawVecIterator {
vec: self,
guard: self.mmap.load(),
mmap: self.create_mmap().unwrap(),
index: 0,
}
}

16
crates/brk_vec/src/variants/stored.rs
View File

@@ -3,7 +3,6 @@ use std::{
path::{Path, PathBuf},
};
use arc_swap::ArcSwap;
use brk_core::{Result, Version};
use memmap2::Mmap;
@@ -75,14 +74,6 @@ where
}
}
#[inline]
fn mmap(&self) -> &ArcSwap<Mmap> {
match self {
StoredVec::Raw(v) => v.mmap(),
StoredVec::Compressed(v) => v.mmap(),
}
}
#[inline]
fn parent(&self) -> &Path {
match self {
@@ -98,13 +89,6 @@ where
StoredVec::Compressed(v) => v.stored_len(),
}
}
#[inline]
fn stored_len_(&self, mmap: &Mmap) -> usize {
match self {
StoredVec::Raw(v) => v.stored_len_(mmap),
StoredVec::Compressed(v) => v.stored_len_(mmap),
}
}
#[inline]
fn pushed(&self) -> &[T] {