Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-04 19:29:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: wip

Browse Source
This commit is contained in:
nym21
2025-06-02 18:22:42 +02:00
parent 98a312701f
commit 93ee5e480b
54 changed files with 3914 additions and 1076 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
crates/brk_vec/src/structs/compressed.rs → crates/brk_vec/src/structs/format.rs
View File

@@ -1,20 +1,33 @@
use std::{fs, io, ops::Deref, path::Path};
use std::{fs, io, path::Path};
use brk_core::{Error, Result};
use clap_derive::ValueEnum;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Compressed(bool);
impl Compressed {
pub const YES: Self = Self(true);
pub const NO: Self = Self(false);
#[derive(
Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum,
)]
pub enum Format {
#[default]
Compressed,
Raw,
}
impl Format {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.as_bytes())
}
pub fn is_compressed(&self) -> bool {
*self == Self::Compressed
}
fn as_bytes(&self) -> Vec<u8> {
if self.0 { vec![1] } else { vec![0] }
if self.is_compressed() {
vec![1]
} else {
vec![0]
}
}
fn from_bytes(bytes: &[u8]) -> Self {
@@ -22,16 +35,16 @@ impl Compressed {
panic!();
}
if bytes[0] == 1 {
Self(true)
Self::Compressed
} else if bytes[0] == 0 {
Self(false)
Self::Raw
} else {
panic!()
}
}
pub fn validate(&self, path: &Path) -> Result<()> {
if let Ok(prev_compressed) = Compressed::try_from(path) {
if let Ok(prev_compressed) = Format::try_from(path) {
if prev_compressed != *self {
return Err(Error::DifferentCompressionMode);
}
@@ -41,22 +54,9 @@ impl Compressed {
}
}
impl TryFrom<&Path> for Compressed {
impl TryFrom<&Path> for Format {
type Error = Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::from_bytes(&fs::read(value)?))
}
}
impl From<bool> for Compressed {
fn from(value: bool) -> Self {
Self(value)
}
}
impl Deref for Compressed {
type Target = bool;
fn deref(&self) -> &Self::Target {
&self.0
}
}

4
crates/brk_vec/src/structs/mod.rs
View File

@@ -1,11 +1,11 @@
mod compressed;
mod compressed_page_meta;
mod compressed_pages_meta;
mod format;
mod length;
mod unsafe_slice;
pub use compressed::*;
pub use compressed_page_meta::*;
pub use compressed_pages_meta::*;
pub use format::*;
pub use length::*;
pub use unsafe_slice::*;

14
crates/brk_vec/src/variants/computed.rs
View File

@@ -8,7 +8,7 @@ use brk_core::{Result, StoredPhantom, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Compressed, StoredIndex, StoredType,
BoxedVecIterator, CollectableVec, Format, StoredIndex, StoredType,
};
use super::{
@@ -86,13 +86,13 @@ where
path: &Path,
value_name: &str,
version: Version,
compressed: Compressed,
format: Format,
source: BoxedAnyIterableVec<S1I, S1T>,
compute: ComputeFrom1<I, T, S1I, S1T>,
) -> Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, value_name, version, compressed)?,
vec: EagerVec::forced_import(path, value_name, version, format)?,
deps: Dependencies::From1(source, compute),
},
Computation::Lazy => {
@@ -108,14 +108,14 @@ where
path: &Path,
value_name: &str,
version: Version,
compressed: Compressed,
format: Format,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
compute: ComputeFrom2<I, T, S1I, S1T, S2I, S2T>,
) -> Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, value_name, version, compressed)?,
vec: EagerVec::forced_import(path, value_name, version, format)?,
deps: Dependencies::From2((source1, source2), compute),
},
Computation::Lazy => {
@@ -133,7 +133,7 @@ where
path: &Path,
value_name: &str,
version: Version,
compressed: Compressed,
format: Format,
source1: BoxedAnyIterableVec<S1I, S1T>,
source2: BoxedAnyIterableVec<S2I, S2T>,
source3: BoxedAnyIterableVec<S3I, S3T>,
@@ -141,7 +141,7 @@ where
) -> Result<Self> {
Ok(match mode {
Computation::Eager => Self::Eager {
vec: EagerVec::forced_import(path, value_name, version, compressed)?,
vec: EagerVec::forced_import(path, value_name, version, format)?,
deps: Dependencies::From3((source1, source2, source3), compute),
},
Computation::Lazy => {

222
crates/brk_vec/src/variants/eager.rs
View File

@@ -10,15 +10,15 @@ use std::{
use arc_swap::ArcSwap;
use brk_core::{
Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Error, Height, Result, Sats, StoredUsize,
TxIndex, Value, Version,
Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Error, Height, Result, Sats, StoredF32,
StoredUsize, TxIndex, Value, Version,
};
use brk_exit::Exit;
use log::info;
use memmap2::Mmap;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed,
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Format,
GenericStoredVec, StoredIndex, StoredType, StoredVec, StoredVecIterator, VecIterator,
};
@@ -44,9 +44,9 @@ where
path: &Path,
value_name: &str,
version: Version,
compressed: Compressed,
format: Format,
) -> Result<Self> {
let inner = StoredVec::forced_import(path, value_name, version, compressed)?;
let inner = StoredVec::forced_import(path, value_name, version, format)?;
Ok(Self {
computed_version: None,
@@ -219,10 +219,10 @@ where
)?;
let index = max_from.min(I::from(self.len()));
let mut added_iter = adder.iter();
let mut adder_iter = adder.iter();
added.iter_at(index).try_for_each(|(i, v)| {
let v = v.into_inner() + added_iter.unwrap_get_inner(i);
let v = v.into_inner() + adder_iter.unwrap_get_inner(i);
self.forced_push_at(i, v, exit)
})?;
@@ -245,12 +245,12 @@ where
)?;
let index = max_from.min(I::from(self.len()));
let mut subtracted_iter = subtracter.iter();
let mut subtracter_iter = subtracter.iter();
subtracted.iter_at(index).try_for_each(|(i, v)| {
let v = v
.into_inner()
.checked_sub(subtracted_iter.unwrap_get_inner(i))
.checked_sub(subtracter_iter.unwrap_get_inner(i))
.unwrap();
self.forced_push_at(i, v, exit)
@@ -259,6 +259,71 @@ where
self.safe_flush(exit)
}
pub fn compute_max<T2>(
&mut self,
max_from: I,
source: &impl AnyIterableVec<I, T2>,
exit: &Exit,
) -> Result<()>
where
T: From<T2> + Ord,
T2: StoredType,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + source.version(),
)?;
let index = max_from.min(I::from(self.len()));
let mut prev = None;
source.iter_at(index).try_for_each(|(i, v)| {
if prev.is_none() {
let i = i.unwrap_to_usize();
prev.replace(if i > 0 {
self.into_iter().unwrap_get_inner_(i - 1)
} else {
T::from(source.iter().unwrap_get_inner_(0))
});
}
let max = prev.clone().unwrap().max(T::from(v.into_inner()));
prev.replace(max.clone());
self.forced_push_at(i, max, exit)
})?;
self.safe_flush(exit)
}
pub fn compute_multiply<T2, T3, T4>(
&mut self,
max_from: I,
multiplied: &impl AnyIterableVec<I, T2>,
multiplier: &impl AnyIterableVec<I, T3>,
exit: &Exit,
) -> Result<()>
where
T2: StoredType + Mul<T3, Output = T4>,
T3: StoredType,
T4: StoredType,
T: From<T4>,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + multiplied.version() + multiplier.version(),
)?;
let index = max_from.min(I::from(self.len()));
let mut multiplier_iter = multiplier.iter();
multiplied.iter_at(index).try_for_each(|(i, v)| {
let v = v.into_inner() * multiplier_iter.unwrap_get_inner(i);
self.forced_push_at(i, v.into(), exit)
})?;
self.safe_flush(exit)
}
pub fn compute_divide<T2, T3, T4>(
&mut self,
max_from: I,
@@ -343,6 +408,36 @@ where
self.safe_flush(exit)
}
pub fn compute_drawdown(
&mut self,
max_from: I,
close: &impl AnyIterableVec<I, Close<Dollars>>,
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()>
where
T: From<StoredF32>,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + ath.version() + close.version(),
)?;
let index = max_from.min(I::from(self.len()));
let mut close_iter = close.iter();
ath.iter_at(index).try_for_each(|(i, ath)| {
let ath = ath.into_inner();
if ath == Dollars::ZERO {
self.forced_push_at(i, T::from(StoredF32::default()), exit)
} else {
let close = *close_iter.unwrap_get_inner(i);
let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0);
self.forced_push_at(i, T::from(drawdown), exit)
}
})?;
self.safe_flush(exit)
}
pub fn compute_inverse_more_to_less(
&mut self,
max_from: T,
@@ -579,6 +674,115 @@ where
self.safe_flush(exit)
}
pub fn compute_sum_of_others(
&mut self,
max_from: I,
others: &[&impl AnyIterableVec<I, T>],
exit: &Exit,
) -> Result<()>
where
T: From<usize> + Add<T, Output = T>,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + others.iter().map(|v| v.version()).sum(),
)?;
if others.is_empty() {
unreachable!("others should've length of 1 at least");
}
let mut others_iter = others[1..].iter().map(|v| v.iter()).collect::<Vec<_>>();
let index = max_from.min(I::from(self.len()));
others
.first()
.unwrap()
.iter_at(index)
.try_for_each(|(i, v)| {
let mut sum = v.into_inner();
others_iter.iter_mut().for_each(|iter| {
sum = sum.clone() + iter.unwrap_get_inner(i);
});
self.forced_push_at(i, sum, exit)
})?;
self.safe_flush(exit)
}
pub fn compute_min_of_others(
&mut self,
max_from: I,
others: &[&impl AnyIterableVec<I, T>],
exit: &Exit,
) -> Result<()>
where
T: From<usize> + Add<T, Output = T> + Ord,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + others.iter().map(|v| v.version()).sum(),
)?;
if others.is_empty() {
unreachable!("others should've length of 1 at least");
}
let mut others_iter = others[1..].iter().map(|v| v.iter()).collect::<Vec<_>>();
let index = max_from.min(I::from(self.len()));
others
.first()
.unwrap()
.iter_at(index)
.try_for_each(|(i, v)| {
let min = v.into_inner();
let min = others_iter
.iter_mut()
.map(|iter| iter.unwrap_get_inner(i))
.min()
.map_or(min.clone(), |min2| min.min(min2));
self.forced_push_at(i, min, exit)
})?;
self.safe_flush(exit)
}
pub fn compute_max_of_others(
&mut self,
max_from: I,
others: &[&impl AnyIterableVec<I, T>],
exit: &Exit,
) -> Result<()>
where
T: From<usize> + Add<T, Output = T> + Ord,
{
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + others.iter().map(|v| v.version()).sum(),
)?;
if others.is_empty() {
unreachable!("others should've length of 1 at least");
}
let mut others_iter = others[1..].iter().map(|v| v.iter()).collect::<Vec<_>>();
let index = max_from.min(I::from(self.len()));
others
.first()
.unwrap()
.iter_at(index)
.try_for_each(|(i, v)| {
let max = v.into_inner();
let max = others_iter
.iter_mut()
.map(|iter| iter.unwrap_get_inner(i))
.max()
.map_or(max.clone(), |max2| max.max(max2));
self.forced_push_at(i, max, exit)
})?;
self.safe_flush(exit)
}
pub fn compute_sma<T2>(
&mut self,
max_from: I,

6
crates/brk_vec/src/variants/indexed.rs
View File

@@ -9,7 +9,7 @@ use arc_swap::ArcSwap;
use brk_core::{Error, Height, Result, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed,
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Format,
GenericStoredVec, Mmap, StoredIndex, StoredType, StoredVec,
};
@@ -30,9 +30,9 @@ where
path: &Path,
value_name: &str,
version: Version,
compressed: Compressed,
format: Format,
) -> Result<Self> {
let inner = StoredVec::forced_import(path, value_name, version, compressed)?;
let inner = StoredVec::forced_import(path, value_name, version, format)?;
Ok(Self {
height: Height::try_from(Self::path_height_(inner.path()).as_path()).ok(),

6
crates/brk_vec/src/variants/stored.rs
View File

@@ -6,7 +6,7 @@ use memmap2::Mmap;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
Compressed, GenericStoredVec, StoredIndex, StoredType,
Format, GenericStoredVec, StoredIndex, StoredType,
};
use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator};
@@ -26,7 +26,7 @@ where
path: &Path,
value_name: &str,
version: Version,
compressed: Compressed,
format: Format,
) -> Result<Self> {
let path = I::path(path, value_name);
@@ -35,7 +35,7 @@ where
panic!("Version must be at least 1, can't verify endianess otherwise");
}
if *compressed {
if format.is_compressed() {
Ok(Self::Compressed(CompressedVec::forced_import(
&path, version,
)?))