mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-05-07 20:59:09 -07:00
vec: moved compute functions to computer
This commit is contained in:
291
crates/brk_computer/src/storage/vecs/base.rs
Normal file
291
crates/brk_computer/src/storage/vecs/base.rs
Normal file
@@ -0,0 +1,291 @@
|
||||
use core::error;
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
fmt::Debug,
|
||||
ops::{Add, Deref, DerefMut, Sub},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use brk_core::CheckedSub;
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{Error, Result, StoredIndex, StoredType, Version};
|
||||
|
||||
const FLUSH_EVERY: usize = 10_000;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct StorableVec<I, T> {
|
||||
computed_version: Option<Version>,
|
||||
vec: brk_vec::StorableVec<I, T>,
|
||||
}
|
||||
|
||||
impl<I, T> StorableVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
pub fn import(path: &Path, version: Version) -> brk_vec::Result<Self> {
|
||||
let vec = brk_vec::StorableVec::forced_import(path, version)?;
|
||||
|
||||
Ok(Self {
|
||||
computed_version: None,
|
||||
vec,
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn i_to_usize(index: I) -> Result<usize> {
|
||||
index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
|
||||
match self.len().cmp(&Self::i_to_usize(index)?) {
|
||||
Ordering::Less => {
|
||||
return Err(Error::IndexTooHigh);
|
||||
}
|
||||
ord => {
|
||||
if ord == Ordering::Greater {
|
||||
self.safe_truncate_if_needed(index, exit)?;
|
||||
}
|
||||
self.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.pushed_len() >= FLUSH_EVERY {
|
||||
Ok(self.safe_flush(exit)?)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path_computed_version(&self) -> PathBuf {
|
||||
self.path().join("computed_version")
|
||||
}
|
||||
|
||||
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
|
||||
let path = self.path_computed_version();
|
||||
if version.validate(path.as_ref()).is_err() {
|
||||
self.reset_file()?;
|
||||
}
|
||||
version.write(path.as_ref())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_transform<A, B, F>(
|
||||
&mut self,
|
||||
max_from: A,
|
||||
other: &mut brk_vec::StorableVec<A, B>,
|
||||
mut t: F,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
A: StoredIndex,
|
||||
B: StoredType,
|
||||
F: FnMut((A, &B, &mut Self, &mut brk_vec::StorableVec<A, B>)) -> (I, T),
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + other.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(A::from(self.len()));
|
||||
other.iter_from(index, |(a, b, other)| {
|
||||
let (i, v) = t((a, b, self, other));
|
||||
self.push_and_flush_if_needed(i, v, exit)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_inverse_more_to_less(
|
||||
&mut self,
|
||||
max_from: T,
|
||||
other: &mut brk_vec::StorableVec<T, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType + StoredIndex,
|
||||
T: StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + other.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
|
||||
other.iter_from(index, |(v, i, ..)| {
|
||||
let i = *i;
|
||||
if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) {
|
||||
self.push_and_flush_if_needed(i, v, exit)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_inverse_less_to_more(
|
||||
&mut self,
|
||||
max_from: T,
|
||||
first_indexes: &mut brk_vec::StorableVec<T, I>,
|
||||
last_indexes: &mut brk_vec::StorableVec<T, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType,
|
||||
T: StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(T::from(self.len()));
|
||||
first_indexes.iter_from(index, |(value, first_index, ..)| {
|
||||
let first_index = Self::i_to_usize(*first_index)?;
|
||||
let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
|
||||
(first_index..last_index)
|
||||
.try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit))
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_last_index_from_first(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &mut brk_vec::StorableVec<I, T>,
|
||||
final_len: usize,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: Copy + From<usize> + CheckedSub<T> + StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + first_indexes.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
let one = T::from(1);
|
||||
let mut prev_index: Option<I> = None;
|
||||
first_indexes.iter_from(index, |(i, v, ..)| {
|
||||
if let Some(prev_index) = prev_index.take() {
|
||||
self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?;
|
||||
}
|
||||
prev_index.replace(i);
|
||||
Ok(())
|
||||
})?;
|
||||
if let Some(prev_index) = prev_index {
|
||||
self.push_and_flush_if_needed(
|
||||
prev_index,
|
||||
T::from(final_len).checked_sub(one).unwrap(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_count_from_indexes<T2>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &mut brk_vec::StorableVec<I, T2>,
|
||||
last_indexes: &mut brk_vec::StorableVec<I, T2>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
first_indexes.iter_from(index, |(i, first_index, ..)| {
|
||||
let last_index = last_indexes.read(i)?.unwrap();
|
||||
let count = *last_index + 1_usize - *first_index;
|
||||
self.push_and_flush_if_needed(i, count.into(), exit)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_is_first_ordered<A>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
self_to_other: &mut brk_vec::StorableVec<I, A>,
|
||||
other_to_self: &mut brk_vec::StorableVec<A, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType,
|
||||
T: From<bool>,
|
||||
A: StoredIndex + StoredType,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + self_to_other.version() + other_to_self.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
self_to_other.iter_from(index, |(i, other, ..)| {
|
||||
self.push_and_flush_if_needed(
|
||||
i,
|
||||
T::from(other_to_self.read(*other)?.unwrap() == &i),
|
||||
exit,
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_sum_from_indexes<T2>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &mut brk_vec::StorableVec<I, T2>,
|
||||
last_indexes: &mut brk_vec::StorableVec<I, T2>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
first_indexes.iter_from(index, |(index, first_index, ..)| {
|
||||
let last_index = last_indexes.read(index)?.unwrap();
|
||||
let count = *last_index + 1_usize - *first_index;
|
||||
self.push_and_flush_if_needed(index, count.into(), exit)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> Deref for StorableVec<I, T> {
|
||||
type Target = brk_vec::StorableVec<I, T>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.vec
|
||||
}
|
||||
}
|
||||
impl<I, T> DerefMut for StorableVec<I, T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.vec
|
||||
}
|
||||
}
|
||||
impl<I, T> Clone for StorableVec<I, T>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
computed_version: self.computed_version,
|
||||
vec: self.vec.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ use std::{fs, ops::Deref, path::Path};
|
||||
use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyStorableVec, StorableVec, Value, Version};
|
||||
use brk_vec::{AnyStorableVec, Value, Version};
|
||||
|
||||
use super::StorableVec;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
@@ -27,48 +29,48 @@ impl Vecs {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self {
|
||||
dateindex_to_date: StorableVec::forced_import(
|
||||
dateindex_to_date: StorableVec::import(
|
||||
&path.join("dateindex_to_date"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
dateindex_to_dateindex: StorableVec::forced_import(
|
||||
dateindex_to_dateindex: StorableVec::import(
|
||||
&path.join("dateindex_to_dateindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
dateindex_to_first_height: StorableVec::forced_import(
|
||||
dateindex_to_first_height: StorableVec::import(
|
||||
&path.join("dateindex_to_first_height"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
dateindex_to_last_height: StorableVec::forced_import(
|
||||
dateindex_to_last_height: StorableVec::import(
|
||||
&path.join("dateindex_to_last_height"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_real_date: StorableVec::forced_import(
|
||||
height_to_real_date: StorableVec::import(
|
||||
&path.join("height_to_real_date"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_fixed_date: StorableVec::forced_import(
|
||||
height_to_fixed_date: StorableVec::import(
|
||||
&path.join("height_to_fixed_date"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_dateindex: StorableVec::forced_import(
|
||||
height_to_dateindex: StorableVec::import(
|
||||
&path.join("height_to_dateindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_height: StorableVec::forced_import(
|
||||
height_to_height: StorableVec::import(
|
||||
&path.join("height_to_height"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_last_txindex: StorableVec::forced_import(
|
||||
height_to_last_txindex: StorableVec::import(
|
||||
&path.join("height_to_last_txindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
|
||||
txindex_to_last_txinindex: StorableVec::forced_import(
|
||||
txindex_to_last_txinindex: StorableVec::import(
|
||||
&path.join("txindex_to_last_txinindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
txindex_to_last_txoutindex: StorableVec::forced_import(
|
||||
txindex_to_last_txoutindex: StorableVec::import(
|
||||
&path.join("txindex_to_last_txoutindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
@@ -186,17 +188,17 @@ impl Vecs {
|
||||
|
||||
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
|
||||
vec![
|
||||
&self.dateindex_to_date,
|
||||
&self.dateindex_to_dateindex,
|
||||
&self.dateindex_to_first_height,
|
||||
&self.dateindex_to_last_height,
|
||||
&self.height_to_dateindex,
|
||||
&self.height_to_fixed_date,
|
||||
&self.height_to_height,
|
||||
&self.height_to_last_txindex,
|
||||
&self.height_to_real_date,
|
||||
&self.txindex_to_last_txinindex,
|
||||
&self.txindex_to_last_txoutindex,
|
||||
&*self.dateindex_to_date,
|
||||
&*self.dateindex_to_dateindex,
|
||||
&*self.dateindex_to_first_height,
|
||||
&*self.dateindex_to_last_height,
|
||||
&*self.height_to_dateindex,
|
||||
&*self.height_to_fixed_date,
|
||||
&*self.height_to_height,
|
||||
&*self.height_to_last_txindex,
|
||||
&*self.height_to_real_date,
|
||||
&*self.txindex_to_last_txinindex,
|
||||
&*self.txindex_to_last_txoutindex,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,9 +6,9 @@ use brk_core::{
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyStorableVec, StorableVec, Value, Version};
|
||||
use brk_vec::{AnyStorableVec, Value, Version};
|
||||
|
||||
use super::indexes::{self, Indexes};
|
||||
use super::{Indexes, StorableVec, indexes};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
@@ -304,28 +304,28 @@ impl Vecs {
|
||||
|
||||
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
|
||||
vec![
|
||||
&self.dateindex_to_close,
|
||||
&self.dateindex_to_close_in_cents,
|
||||
&self.dateindex_to_high,
|
||||
&self.dateindex_to_high_in_cents,
|
||||
&self.dateindex_to_low,
|
||||
&self.dateindex_to_low_in_cents,
|
||||
&self.dateindex_to_ohlc,
|
||||
&self.dateindex_to_ohlc_in_cents,
|
||||
&self.dateindex_to_open,
|
||||
&self.dateindex_to_open_in_cents,
|
||||
&self.dateindex_to_sats_per_dollar,
|
||||
&self.height_to_close,
|
||||
&self.height_to_close_in_cents,
|
||||
&self.height_to_high,
|
||||
&self.height_to_high_in_cents,
|
||||
&self.height_to_low,
|
||||
&self.height_to_low_in_cents,
|
||||
&self.height_to_ohlc,
|
||||
&self.height_to_ohlc_in_cents,
|
||||
&self.height_to_open,
|
||||
&self.height_to_open_in_cents,
|
||||
&self.height_to_sats_per_dollar,
|
||||
&*self.dateindex_to_close,
|
||||
&*self.dateindex_to_close_in_cents,
|
||||
&*self.dateindex_to_high,
|
||||
&*self.dateindex_to_high_in_cents,
|
||||
&*self.dateindex_to_low,
|
||||
&*self.dateindex_to_low_in_cents,
|
||||
&*self.dateindex_to_ohlc,
|
||||
&*self.dateindex_to_ohlc_in_cents,
|
||||
&*self.dateindex_to_open,
|
||||
&*self.dateindex_to_open_in_cents,
|
||||
&*self.dateindex_to_sats_per_dollar,
|
||||
&*self.height_to_close,
|
||||
&*self.height_to_close_in_cents,
|
||||
&*self.height_to_high,
|
||||
&*self.height_to_high_in_cents,
|
||||
&*self.height_to_low,
|
||||
&*self.height_to_low_in_cents,
|
||||
&*self.height_to_ohlc,
|
||||
&*self.height_to_ohlc_in_cents,
|
||||
&*self.height_to_open,
|
||||
&*self.height_to_open_in_cents,
|
||||
&*self.height_to_sats_per_dollar,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,14 @@ use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::AnyStorableVec;
|
||||
|
||||
mod base;
|
||||
mod indexes;
|
||||
mod marketprice;
|
||||
mod transactions;
|
||||
|
||||
use base::*;
|
||||
use indexes::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub indexes: indexes::Vecs,
|
||||
|
||||
@@ -3,9 +3,9 @@ use std::{fs, path::Path};
|
||||
use brk_core::Txindex;
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyStorableVec, StorableVec, Version};
|
||||
use brk_vec::{AnyStorableVec, Version};
|
||||
|
||||
use super::indexes::{self, Indexes};
|
||||
use super::{Indexes, StorableVec, indexes};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
@@ -47,7 +47,7 @@ impl Vecs {
|
||||
// &path.join("txindex_to_fee"),
|
||||
// Version::from(1),
|
||||
// )?,
|
||||
txindex_to_is_coinbase: StorableVec::forced_import(
|
||||
txindex_to_is_coinbase: StorableVec::import(
|
||||
&path.join("txindex_to_is_coinbase"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
@@ -127,6 +127,6 @@ impl Vecs {
|
||||
}
|
||||
|
||||
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
|
||||
vec![&self.txindex_to_is_coinbase]
|
||||
vec![&*self.txindex_to_is_coinbase]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ color-eyre = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
minreq = { workspace = true }
|
||||
oxc = { version = "0.56.5", features = ["codegen", "minifier"] }
|
||||
oxc = { version = "0.57.0", features = ["codegen", "minifier"] }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { version = "1.44.0", features = ["full"] }
|
||||
|
||||
@@ -16,7 +16,9 @@ pub fn minify_js(path: &Path) -> String {
|
||||
|
||||
let allocator = Allocator::default();
|
||||
|
||||
let mut program = Parser::new(&allocator, &source_text, source_type).parse().program;
|
||||
let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
|
||||
|
||||
let mut program = parser_return.program;
|
||||
|
||||
let minifier_return = Minifier::new(MinifierOptions {
|
||||
mangle: Some(MangleOptions::default()),
|
||||
@@ -33,7 +35,7 @@ pub fn minify_js(path: &Path) -> String {
|
||||
source_map_path: None,
|
||||
legal_comments: LegalComment::None,
|
||||
})
|
||||
.with_symbol_table(minifier_return.symbol_table)
|
||||
.with_scoping(minifier_return.scoping)
|
||||
.build(&program)
|
||||
.code
|
||||
}
|
||||
|
||||
2
crates/brk_vec/.gitignore
vendored
2
crates/brk_vec/.gitignore
vendored
@@ -1 +1 @@
|
||||
/v
|
||||
/vec
|
||||
|
||||
@@ -16,3 +16,4 @@ rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
zerocopy = { workspace = true }
|
||||
zstd = "0.13.3"
|
||||
|
||||
@@ -4,7 +4,8 @@ use brk_vec::{StorableVec, Version};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
{
|
||||
let mut vec: StorableVec<usize, u32> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
|
||||
let mut vec: StorableVec<usize, u32> =
|
||||
StorableVec::forced_import(Path::new("./vec"), Version::from(1))?;
|
||||
|
||||
vec.push(0);
|
||||
vec.push(1);
|
||||
@@ -16,7 +17,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
}
|
||||
|
||||
{
|
||||
let mut vec: StorableVec<usize, u32> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
|
||||
let mut vec: StorableVec<usize, u32> =
|
||||
StorableVec::forced_import(Path::new("./vec"), Version::from(1))?;
|
||||
|
||||
dbg!(vec.read(0)?); // 0
|
||||
dbg!(vec.read(1)?); // 0
|
||||
|
||||
@@ -5,18 +5,16 @@
|
||||
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
error,
|
||||
fmt::Debug,
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{self, Read, Seek, SeekFrom, Write},
|
||||
marker::PhantomData,
|
||||
mem,
|
||||
ops::{Add, Range, Sub},
|
||||
ops::Range,
|
||||
path::{Path, PathBuf},
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use brk_core::CheckedSub;
|
||||
use brk_exit::Exit;
|
||||
pub use memmap2;
|
||||
use rayon::prelude::*;
|
||||
@@ -59,7 +57,6 @@ pub struct StorableVec<I, T> {
|
||||
const MAX_PAGE_SIZE: usize = 4 * 4096;
|
||||
const ONE_MB: usize = 1024 * 1024;
|
||||
const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
|
||||
const FLUSH_EVERY: usize = 10_000;
|
||||
|
||||
impl<I, T> StorableVec<I, T>
|
||||
where
|
||||
@@ -272,7 +269,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn read_last(&mut self) -> Result<Option<&T>> {
|
||||
pub fn read_last(&mut self) -> Result<Option<&T>> {
|
||||
let len = self.len();
|
||||
if len == 0 {
|
||||
return Ok(None);
|
||||
@@ -376,27 +373,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
|
||||
match self.len().cmp(&Self::i_to_usize(index)?) {
|
||||
Ordering::Less => {
|
||||
return Err(Error::IndexTooHigh);
|
||||
}
|
||||
ord => {
|
||||
if ord == Ordering::Greater {
|
||||
self.safe_truncate_if_needed(index, exit)?;
|
||||
}
|
||||
self.pushed.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
if self.pushed_len() >= FLUSH_EVERY {
|
||||
Ok(self.safe_flush(exit)?)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.file_len + self.pushed_len()
|
||||
@@ -462,15 +438,6 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
|
||||
let path = self.path_computed_version();
|
||||
if version.validate(path.as_ref()).is_err() {
|
||||
self.reset_file()?;
|
||||
}
|
||||
version.write(path.as_ref())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn truncate_if_needed(&mut self, index: I) -> Result<Option<T>> {
|
||||
let index = Self::i_to_usize(index)?;
|
||||
|
||||
@@ -497,7 +464,7 @@ where
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn i_to_usize(index: I) -> Result<usize> {
|
||||
pub fn i_to_usize(index: I) -> Result<usize> {
|
||||
index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
|
||||
}
|
||||
|
||||
@@ -558,207 +525,12 @@ where
|
||||
path.join("version")
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path_computed_version(&self) -> PathBuf {
|
||||
self.path().join("computed_version")
|
||||
}
|
||||
|
||||
pub fn index_type_to_string(&self) -> &str {
|
||||
std::any::type_name::<I>()
|
||||
}
|
||||
|
||||
pub fn compute_transform<A, B, F>(
|
||||
&mut self,
|
||||
max_from: A,
|
||||
other: &mut StorableVec<A, B>,
|
||||
mut t: F,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
A: StoredIndex,
|
||||
B: StoredType,
|
||||
F: FnMut((A, &B, &mut Self, &mut StorableVec<A, B>)) -> (I, T),
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + other.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(A::from(self.len()));
|
||||
other.iter_from(index, |(a, b, other)| {
|
||||
let (i, v) = t((a, b, self, other));
|
||||
self.push_and_flush_if_needed(i, v, exit)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_inverse_more_to_less(
|
||||
&mut self,
|
||||
max_from: T,
|
||||
other: &mut StorableVec<T, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType + StoredIndex,
|
||||
T: StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + other.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
|
||||
other.iter_from(index, |(v, i, ..)| {
|
||||
let i = *i;
|
||||
if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) {
|
||||
self.push_and_flush_if_needed(i, v, exit)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_inverse_less_to_more(
|
||||
&mut self,
|
||||
max_from: T,
|
||||
first_indexes: &mut StorableVec<T, I>,
|
||||
last_indexes: &mut StorableVec<T, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType,
|
||||
T: StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + first_indexes.version + last_indexes.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(T::from(self.len()));
|
||||
first_indexes.iter_from(index, |(value, first_index, ..)| {
|
||||
let first_index = Self::i_to_usize(*first_index)?;
|
||||
let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
|
||||
(first_index..last_index)
|
||||
.try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit))
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_last_index_from_first(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &mut StorableVec<I, T>,
|
||||
final_len: usize,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: Copy + From<usize> + CheckedSub<T> + StoredIndex,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + first_indexes.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
let one = T::from(1);
|
||||
let mut prev_index: Option<I> = None;
|
||||
first_indexes.iter_from(index, |(i, v, ..)| {
|
||||
if let Some(prev_index) = prev_index.take() {
|
||||
self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?;
|
||||
}
|
||||
prev_index.replace(i);
|
||||
Ok(())
|
||||
})?;
|
||||
if let Some(prev_index) = prev_index {
|
||||
self.push_and_flush_if_needed(
|
||||
prev_index,
|
||||
T::from(final_len).checked_sub(one).unwrap(),
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_count_from_indexes<T2>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &mut StorableVec<I, T2>,
|
||||
last_indexes: &mut StorableVec<I, T2>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + first_indexes.version + last_indexes.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
first_indexes.iter_from(index, |(i, first_index, ..)| {
|
||||
let last_index = last_indexes.read(i)?.unwrap();
|
||||
let count = *last_index + 1_usize - *first_index;
|
||||
self.push_and_flush_if_needed(i, count.into(), exit)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_is_first_ordered<A>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
self_to_other: &mut StorableVec<I, A>,
|
||||
other_to_self: &mut StorableVec<A, I>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
I: StoredType,
|
||||
T: From<bool>,
|
||||
A: StoredIndex + StoredType,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + self_to_other.version + other_to_self.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
self_to_other.iter_from(index, |(i, other, ..)| {
|
||||
self.push_and_flush_if_needed(
|
||||
i,
|
||||
T::from(other_to_self.read(*other)?.unwrap() == &i),
|
||||
exit,
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
}
|
||||
|
||||
pub fn compute_sum_from_indexes<T2, F>(
|
||||
&mut self,
|
||||
max_from: I,
|
||||
first_indexes: &mut StorableVec<I, T2>,
|
||||
last_indexes: &mut StorableVec<I, T2>,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<T2>,
|
||||
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
|
||||
<T2 as TryInto<T>>::Error: error::Error + 'static,
|
||||
F: Fn(&T2) -> T,
|
||||
{
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::from(0) + self.version + first_indexes.version + last_indexes.version,
|
||||
)?;
|
||||
|
||||
let index = max_from.min(I::from(self.len()));
|
||||
first_indexes.iter_from(index, |(index, first_index, ..)| {
|
||||
let last_index = last_indexes.read(index)?.unwrap();
|
||||
let count = *last_index + 1_usize - *first_index;
|
||||
self.push_and_flush_if_needed(index, count.into(), exit)
|
||||
})?;
|
||||
|
||||
Ok(self.safe_flush(exit)?)
|
||||
pub fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::Result;
|
||||
|
||||
pub trait Bytes: Sized {
|
||||
const LEN: usize = size_of::<Self>();
|
||||
fn to_bytes(&self) -> Arc<[u8]>;
|
||||
fn try_from_bytes(bytes: &[u8]) -> Result<Self>;
|
||||
}
|
||||
|
||||
pub trait UnsafeBytes {}
|
||||
Reference in New Issue
Block a user