Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 14:11:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-05 00:23:58 +01:00
parent d11a1622f8
commit 138ca80c10
5 changed files with 88 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
computer/src/lib.rs
+9 -9
View File
@@ -13,8 +13,8 @@ use structs::*;
pub struct Computer<const MODE: u8> {
outputs_dir: PathBuf,
vecs: StorableVecs<MODE>,
trees: Fjalls,
pub vecs: StorableVecs<MODE>,
pub trees: Fjalls,
}
impl<const MODE: u8> Computer<MODE> {
@@ -53,20 +53,20 @@ impl Computer<SINGLE_THREAD> {
self.vecs
.txindex_to_last_txinindex
.compute_last_index_from_first(&indexer.vecs.txindex_to_first_txinindex, txinindexes_count)?;
.compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txinindex, txinindexes_count)?;
self.vecs.txindex_to_inputcount.compute_count_from_indexes(
&indexer.vecs.txindex_to_first_txinindex,
&self.vecs.txindex_to_last_txinindex,
&mut indexer.vecs.txindex_to_first_txinindex,
&mut self.vecs.txindex_to_last_txinindex,
)?;
self.vecs
.txindex_to_last_txoutindex
.compute_last_index_from_first(&indexer.vecs.txindex_to_first_txoutindex, txoutindexes_count)?;
.compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txoutindex, txoutindexes_count)?;
self.vecs.txindex_to_outputcount.compute_count_from_indexes(
&indexer.vecs.txindex_to_first_txoutindex,
&self.vecs.txindex_to_last_txoutindex,
&mut indexer.vecs.txindex_to_first_txoutindex,
&mut self.vecs.txindex_to_last_txoutindex,
)?;
self.vecs
@@ -75,7 +75,7 @@ impl Computer<SINGLE_THREAD> {
self.vecs
.height_to_last_txindex
.compute_last_index_from_first(&indexer.vecs.height_to_first_txindex, height_count)?;
.compute_last_index_from_first(&mut indexer.vecs.height_to_first_txindex, height_count)?;
self.vecs.txindex_to_height.compute_inverse_less_to_more(
&mut indexer.vecs.height_to_first_txindex,
computer/src/storage/storable_vecs/base.rs
+2 -92
View File
@@ -1,19 +1,11 @@
use std::{
error,
fmt::Debug,
io,
ops::{Add, Sub},
path::Path,
};
use std::{fmt::Debug, path::Path};
use derive_deref::{Deref, DerefMut};
use storable_vec::{StorableVecIndex, StorableVecType, Version, SINGLE_THREAD};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
#[derive(Debug, Deref, DerefMut)]
pub struct StorableVec<I, T, const MODE: u8>(storable_vec::StorableVec<I, T, MODE>);
const FLUSH_EVERY: usize = 10_000;
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
where
I: StorableVecIndex,
@@ -23,85 +15,3 @@ where
Ok(Self(storable_vec::StorableVec::forced_import(path, version)?))
}
}
impl<I, T> StorableVec<I, T, SINGLE_THREAD>
where
I: StorableVecIndex,
T: StorableVecType,
{
fn flush_vec_if_needed(&mut self) -> io::Result<()> {
if self.pushed_len() == FLUSH_EVERY {
self.flush()
} else {
Ok(())
}
}
pub fn compute_is_first_ordered<A>(
&mut self,
self_to_other: &storable_vec::StorableVec<I, A, SINGLE_THREAD>,
other_to_self: &storable_vec::StorableVec<A, I, SINGLE_THREAD>,
) -> storable_vec::Result<()>
where
A: StorableVecIndex + StorableVecType,
{
// let mut prev_a_opt = None;
// self_to_other.iter_from(I::from(self.len()), |(i, a)| {
// if prev_a_opt.is_none() {
// prev_a_opt.replace(a);
// self.push_if_needed(i, other_to_self.read_at(a) == i);
// } else {
// let prev_a = prev_a_opt.unwrap();
// if a != prev_a
// }
// other_to_self.seek_read(a);
// self.push_if_needed(i, t(a));
// Ok(())
// })
Ok(())
}
pub fn compute_last_index_from_first(
&mut self,
first_index_vec: &storable_vec::StorableVec<I, T, SINGLE_THREAD>,
final_len: usize,
) -> color_eyre::Result<()>
where
T: Copy + From<usize> + Sub<T, Output = T> + StorableVecIndex,
{
let mut prev_index: Option<I> = None;
first_index_vec.iter_from(I::from(self.len()), |(i, v)| {
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, *v - T::from(1))?;
}
prev_index.replace(i);
Ok(self.flush_vec_if_needed()?)
})?;
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, T::from(final_len) - T::from(1))?;
}
self.flush()?;
Ok(())
}
pub fn compute_count_from_indexes<T2>(
&mut self,
first_indexes: &storable_vec::StorableVec<I, T2, SINGLE_THREAD>,
last_indexes: &storable_vec::StorableVec<I, T2, SINGLE_THREAD>,
) -> color_eyre::Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
{
let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?;
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let count = *last_index + 1_usize - *first_index;
self.push_if_needed(i, count.into())?;
Ok(self.flush_vec_if_needed()?)
})?;
self.flush()?;
Ok(())
}
}
indexer/src/storage/storable_vecs/base.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use std::{
path::{Path, PathBuf},
};
use storable_vec::{StorableVecIndex, StorableVecType, Version, CACHED_GETS};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use super::Height;
storable_vec/src/lib.rs
+70 -121
View File
@@ -1,11 +1,12 @@
use std::{
cmp::Ordering,
error,
fmt::Debug,
fs::{self, File, OpenOptions},
io::{self, Read, Seek, SeekFrom, Write},
marker::PhantomData,
mem,
ops::Range,
ops::{Add, Range, Sub},
path::{Path, PathBuf},
sync::OnceLock,
};
@@ -192,47 +193,18 @@ where
}
}
// #[inline]
// fn open_file_at_then_read(&self, index: I) -> Result<T> {
// self.open_file_at_then_read_(Self::i_to_usize(index)?)
// }
fn open_file_at_then_read(&self, index: usize) -> Result<T> {
// let (mut file, mut buf) = self.open_file_at(index)?;
let mut file = self.open_file()?;
let mut buf = Self::create_buffer();
let byte_index = Self::index_to_byte_index(index);
Self::seek(&mut file, byte_index)?;
Ok(Self::read_exact(&mut file, &mut buf)?.to_owned())
Self::seek(&mut file, Self::index_to_byte_index(index))?;
Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned())
}
// #[inline]
// fn open_file_at(&self, index: I) -> Result<(File, Buffer)> {
// self.open_file_at_(Self::i_to_usize(index)?)
// }
// fn open_file_at(&self, index: usize) -> Result<(File, Buffer)> {
// let mut file = self.open_file()?;
// let buf = Self::create_buffer();
// let byte_index = Self::index_to_byte_index(index);
// Self::seek(&mut file, byte_index)?;
// Ok((file, buf))
// }
// #[inline]
// fn seek_if_needed_(file: &mut File, index: I) -> Result<u64> {
// Self::seek_if_needed__(file, Self::i_to_usize(index)?).map_err(Error::IO)
// }
// #[inline]
// fn seek_if_needed(file: &mut File, index: usize) -> io::Result<u64> {
// let byte_index = Self::index_to_byte_index(index);
// if file.stream_position()? != byte_index {
// Self::seek(file, byte_index)?;
// }
// Ok(byte_index)
// }
#[inline]
fn seek(file: &mut File, byte_index: u64) -> io::Result<u64> {
file.seek(SeekFrom::Start(byte_index))
}
fn read_exact<'a>(file: &'a mut File, buf: &'a mut [u8]) -> Result<&'a T> {
file.read_exact(buf)?;
let v = T::unsafe_try_from_slice(&buf[..])?;
@@ -261,74 +233,7 @@ where
Err(Error::IndexTooHigh)
}
}
// Self::push_to_vec_if_needed(&mut self.pushed, index, value)
}
// #[inline]
// fn push_if_needed__(&mut self, index: usize, value: T) -> Result<()> {
// Self::push_to_vec_if_needed_(&mut self.pushed, index, value)
// }
// #[inline]
// fn push_to_vec_if_needed(vec: &mut Vec<T>, index: I, value: T) -> Result<()> {
// Self::push_to_vec_if_needed_(vec, Self::i_to_usize(index)?, value)
// }
// fn push_to_vec_if_needed_(vec: &mut Vec<T>, index: usize, value: T) -> Result<()> {
// let len = vec.len();
// match len.cmp(&index) {
// Ordering::Greater => {
// // dbg!(len, index, &self.pathbuf);
// // panic!();
// Ok(())
// }
// Ordering::Equal => {
// vec.push(value);
// Ok(())
// }
// Ordering::Less => {
// dbg!(index, value);
// Err(Error::IndexTooHigh)
// }
// }
// }
// pub fn update(&mut self, index: I, value: T) -> Result<()> {
// self._update(index.into(), value)
// }
// pub fn update_(&mut self, index: usize, value: T) -> Result<()> {
// if let Some(index) = self.index_to_pushed_index(index) {
// self.pushed[index] = value;
// } else {
// self.updated.insert(index, value);
// }
// Ok(())
// }
// pub fn fetch_update(&mut self, index: I, value: T) -> Result<T>
// where
// T: Clone,
// {
// self._fetch_update(index.into(), value)
// }
// pub fn fetch_update_(&mut self, index: usize, value: T) -> Result<T>
// where
// T: Clone,
// {
// let prev_opt = self.updated.insert(index, value);
// if let Some(prev) = prev_opt {
// Ok(prev)
// } else {
// Ok(self
// ._get(index)?
// .ok_or(Error::ExpectFileToHaveIndex)?
// .clone())
// }
// }
// pub fn remove(&mut self, index: I) {
// self._remove(index.into())
// }
// pub fn remove_(&mut self, index: usize) {
// self.removed.insert(index);
// }
#[inline]
pub fn len(&self) -> usize {
@@ -358,10 +263,6 @@ where
pub fn hasnt(&self, index: I) -> Result<bool> {
self.has(index).map(|b| !b)
}
#[inline]
fn hasnt_(&self, index: usize) -> bool {
!self.has_(index)
}
pub fn flush(&mut self) -> io::Result<()> {
self.reset_disk_related_state()?;
@@ -528,6 +429,7 @@ where
fn get_(&mut self, index: usize) -> Result<&T> {
let byte_index = Self::index_to_byte_index(index);
if self.file_position != byte_index {
println!("seek");
self.file_position = Self::seek(&mut self.file, byte_index)?;
}
let res = Self::read_exact(&mut self.file, &mut self.buf);
@@ -561,15 +463,6 @@ where
}
}
// #[inline]
// fn seek_if_needed(&mut self, index: I) -> Result<()> {
// if self.file_position == Self::index_to_byte_index(Self::i_to_usize(index)?) {
// return Ok(());
// }
// self.file_position = Self::seek_if_needed_(&mut self.file, index)?;
// Ok(())
// }
pub fn iter<F>(&mut self, f: F) -> Result<()>
where
F: FnMut((I, &T)) -> Result<()>,
@@ -605,6 +498,15 @@ where
Ok(())
}
pub fn compute_transform<A, F>(&mut self, other: &mut StorableVec<I, A, SINGLE_THREAD>, t: F) -> Result<()>
where
A: StorableVecType,
F: Fn(&A) -> T,
{
other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))?;
Ok(self.flush()?)
}
pub fn compute_inverse_more_to_less(&mut self, other: &mut StorableVec<T, I, SINGLE_THREAD>) -> Result<()>
where
I: StorableVecType,
@@ -632,12 +534,59 @@ where
Ok(self.flush()?)
}
pub fn compute_transform<A, F>(&mut self, other: &mut StorableVec<I, A, SINGLE_THREAD>, t: F) -> Result<()>
pub fn compute_last_index_from_first(
&mut self,
first_index_vec: &mut StorableVec<I, T, SINGLE_THREAD>,
final_len: usize,
) -> Result<()>
where
A: StorableVecType,
F: Fn(&A) -> T,
T: Copy + From<usize> + Sub<T, Output = T> + StorableVecIndex,
{
other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))?;
let one = T::from(1);
let mut prev_index: Option<I> = None;
first_index_vec.iter_from(I::from(self.len()), |(i, v)| {
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, *v - one)?;
}
prev_index.replace(i);
Ok(())
})?;
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, T::from(final_len) - one)?;
}
Ok(self.flush()?)
}
pub fn compute_count_from_indexes<T2>(
&mut self,
first_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
last_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
) -> Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
{
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
let last_index = last_indexes.get(i)?;
let count = *last_index + 1_usize - *first_index;
self.push_if_needed(i, count.into())
})?;
Ok(self.flush()?)
}
pub fn compute_is_first_ordered<A>(
&mut self,
self_to_other: &mut StorableVec<I, A, SINGLE_THREAD>,
other_to_self: &mut StorableVec<A, I, SINGLE_THREAD>,
) -> Result<()>
where
A: StorableVecIndex + StorableVecType,
T: From<bool>,
{
self_to_other.iter_from(I::from(self.len()), |(i, other)| {
self.push_if_needed(i, T::from(other_to_self.get(*other)? == &i))
})?;
Ok(self.flush()?)
}
}
@@ -648,12 +597,12 @@ where
T: StorableVecType,
{
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
pub fn get(&self, index: I) -> Result<Option<T>> {
self.get_(Self::i_to_usize(index)?)
}
#[inline]
fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
Ok(Some(Value::Owned(self.open_file_at_then_read(index)?.to_owned())))
fn get_(&self, index: usize) -> Result<Option<T>> {
Ok(Some(self.open_file_at_then_read(index)?.to_owned()))
}
// Add iter iter_from iter_range collect..
storable_vec/src/main.rs
+6 -2
View File
@@ -1,6 +1,6 @@
use std::path::Path;
use storable_vec::{StorableVec, Version, CACHED_GETS};
use storable_vec::{StorableVec, Version, ASYNC_READ_ONLY, CACHED_GETS, SINGLE_THREAD};
fn main() -> Result<(), Box<dyn std::error::Error>> {
{
@@ -17,9 +17,13 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
}
{
let vec: StorableVec<usize, u32, CACHED_GETS> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
let mut vec: StorableVec<usize, u32, SINGLE_THREAD> =
StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
dbg!(vec.get(0)?); // 0
dbg!(vec.get(1)?); // 0
dbg!(vec.get(2)?); // 0
dbg!(vec.get(0)?); // 0
}
Ok(())