From b9e679a5145a13e0e19d1dd4f4372f87e0184fb0 Mon Sep 17 00:00:00 2001 From: nym21 Date: Tue, 11 Mar 2025 16:27:45 +0100 Subject: [PATCH] vec: moved compute functions to computer --- Cargo.lock | 94 +++--- Cargo.toml | 4 +- crates/brk_computer/src/storage/vecs/base.rs | 291 ++++++++++++++++++ .../brk_computer/src/storage/vecs/indexes.rs | 48 +-- .../src/storage/vecs/marketprice.rs | 48 +-- crates/brk_computer/src/storage/vecs/mod.rs | 4 + .../src/storage/vecs/transactions.rs | 8 +- crates/brk_server/Cargo.toml | 2 +- crates/brk_server/src/files/minify.rs | 6 +- crates/brk_vec/.gitignore | 2 +- crates/brk_vec/Cargo.toml | 1 + crates/brk_vec/examples/main.rs | 6 +- crates/brk_vec/src/lib.rs | 238 +------------- crates/brk_vec/src/traits/bytes.rs | 11 - 14 files changed, 413 insertions(+), 350 deletions(-) create mode 100644 crates/brk_computer/src/storage/vecs/base.rs delete mode 100644 crates/brk_vec/src/traits/bytes.rs diff --git a/Cargo.lock b/Cargo.lock index 5d432b949..1ccd43e04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -560,6 +560,7 @@ dependencies = [ "serde", "serde_json", "zerocopy", + "zstd", ] [[package]] @@ -1476,9 +1477,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c163c633eb184a4ad2a5e7a5dacf12a58c830d717a7963563d4eceb4ced079f" +checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1491,9 +1492,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc3e0019b0f5f43038cf46471b1312136f29e36f54436c6042c8f155fec8789" +checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" dependencies = [ "proc-macro2", "quote", @@ -1554,9 +1555,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.170" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "linux-raw-sys" @@ -1673,12 +1674,11 @@ dependencies = [ [[package]] name = "minreq" -version = "2.13.2" +version = "2.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0c420feb01b9fb5061f8c8f452534361dd783756dcf38ec45191ce55e7a161" +checksum = "567496f13503d6cae8c9f961f34536850275f396307d7a6b981eef1464032f53" dependencies = [ "log", - "once_cell", "rustls", "rustls-webpki", "serde", @@ -1784,9 +1784,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae6276febbc5abc1f1e4cf49167d54ab341818656ae4f622d5992b65fcdd371" +checksum = "918d8f6deffeb380427e17bfc2d63c38f30e485af3a285faa6ca87d8c8a37a0d" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1827,9 +1827,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cc5cd078806a1b7061fa146dc4228a57d0765da6c85e99500d069b86f57e94" +checksum = "d92b3f2a5e042d3e7d21aab5896d3f02d657c4b3cae42aa2a5842ebd018ec194" dependencies = [ "allocator-api2", "bumpalo", @@ -1840,9 +1840,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4722414ac21a2e28a16b76de8390672c01a39adcb703d405b848149cfaeeaf7" +checksum = "864754f59a7965a83bf47c376f97ac1937dedd415c367f7e6b7d6499453aaad4" dependencies = [ "bitflags", "cow-utils", @@ -1857,9 +1857,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1505d8622b2ea6ed0274f355bd5e4ee3f09df5d9b39c8a3a673f344d87b82a" +checksum = "0f1b369ef5a746cfd3400047f0ec972d66fe75ee5ab5fbfa8f1699e7a3b86724" dependencies = [ "proc-macro2", "quote", @@ -1868,9 +1868,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf45370c6da0dd142a70468e5b25127d6a34caa71056105c85087559c8ee9afb" +checksum = "7874fe470308c6882fbeb42953896a21ea727d76165bfa80613dab0abbf72254" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1880,9 +1880,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96be30717d29eb7d1780758d033e92fcc208b8cce83b3b4869d2155fa4c9b7bd" +checksum = "ff4ba4ecfa3296e937a3173f3a6b7e98446fed48e182d03f0955729f74b219c1" dependencies = [ "bitflags", "itertools", @@ -1895,9 +1895,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5fd891e06aa19c7d22e129d366e55644ffeed99167e3495dcaec8149a0631b3" +checksum = "6eb495fd82a33f1d40ea0c27d1db81772946b26a4a236de411a83511aa628a96" dependencies = [ "bitflags", "cow-utils", @@ -1916,9 +1916,9 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb49a2ee880952c2079b61851ecc35d7671d9d3509e306f5e704ccacd2783984" +checksum = "3454bded31b396d75d4ebad5fbdc547ad544b47556b540925be24a874e1d87d8" dependencies = [ "assert-unchecked", "ropey", @@ -1926,9 +1926,9 @@ dependencies = [ [[package]] name = "oxc_diagnostics" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea83fe2415b0580980ac83364c1ae943f8ee9c00becf5395a89e800a9526a080" +checksum = "27cd28f66608e282c1384a877505d3678e099bc3cef6856bc55e2ef60f5344b2" dependencies = [ "cow-utils", "oxc-miette", @@ -1936,9 +1936,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c7e7bcc382cf901e93a16f86e70f2737c507aaa833656b0d8484d64f8ae358a" +checksum = "43b9a66f69c6dcc78bf1c164e1e98b4494ba71fbf1b17ebd8940b936902e68ec" dependencies = [ "cow-utils", "num-bigint", @@ -1950,9 +1950,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cb66484735d21f096b07c894badc96d89a0c6d31b4bdd46b33b3e44da9b97ac" +checksum = "5aa7a068d758764707466b733f92ad66bbe91d5c66ce562d868dff8a7ae72b09" [[package]] name = "oxc_index" @@ -1962,9 +1962,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8069aaf088ccaee383e008ff29d407c2c88b4ff8694f7e34a9d6aaf202c81e71" +checksum = "f4f5ae1f1f842c714f3144036a644ee275c8a360fd50ac6f295ce52e468cee53" dependencies = [ "fixedbitset", "itertools", @@ -1979,9 +1979,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db16b19f75f29b9502b651c6ffae09366d43fd91decf5535d80c634f8e7e21d" +checksum = "9e4030f603ae314c93b761753969328e96960a853f443fdeaf8c9b2273f7971e" dependencies = [ "cow-utils", "oxc_allocator", @@ -2001,9 +2001,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7414e779b9723b0bd2880fe69b0ee517e583f88c5118a7c9053cf3317b95b1" +checksum = "71d48cff62f816c27be1912d1d173140602214c8eec38f1e98e77270f39e0ce6" dependencies = [ "assert-unchecked", "bitflags", @@ -2024,9 +2024,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18196c212eac24a7faf613e373e5f9317d3542578a088d01dc1a548fa1e1cb3" +checksum = "3a49d5d6ede8d8740b25743a78c2fb45859099ba89d88425b1b3d6a433dda352" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -2040,9 +2040,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4556f1c700baaec8589872ddf2af41d9a964db52fdbec9a5d07d7477dce45cf8" +checksum = "c7b84c302f11c87b330b5e3f02f3efc4cd1ea44e8553551d6e245cb161dcc6ef" dependencies = [ "assert-unchecked", "itertools", @@ -2077,9 +2077,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318f925e26bd118adc082d290538d07611fe2434987a5c60cf5084381ecb42e6" +checksum = "cf5d0e27b520397a9d0a99dfa21d9c29fa389f34719aff30f6c5468498d603d9" dependencies = [ "compact_str", "oxc-miette", @@ -2090,9 +2090,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb68ceb7c6902f3043fe8fe49bb886826b1d7741dc2904337297d53692b1b9c" +checksum = "e91fdded3cdce1641b8ee44a223be03dca3def248120cbe3fe0492a14bd8a744" dependencies = [ "assert-unchecked", "bitflags", @@ -2111,9 +2111,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.56.5" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a090c9cd461d468f5faf3fe3f56378a4e27d80febe34ad39f2a04920f594d4" +checksum = "77703db3bd925f63fd688e3858ecd48d4b1a455a64f4c54bec6baae50af03cab" dependencies = [ "compact_str", "itoa", diff --git a/Cargo.toml b/Cargo.toml index fb9d53cca..c8097921c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,9 @@ clap = { version = "4.5.32", features = ["derive", "string"] } color-eyre = "0.6.3" derive_deref = "1.1.1" fjall = "2.6.7" -jiff = "0.2.3" +jiff = "0.2.4" log = { version = "0.4.26" } -minreq = { version = "2.13.2", features = ["https", "serde_json"] } +minreq = { version = "2.13.3", features = ["https", "serde_json"] } rayon = "1.10.0" serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.140", features = ["float_roundtrip"] } diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs new file mode 100644 index 000000000..65ef0812d --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -0,0 +1,291 @@ +use core::error; +use std::{ + cmp::Ordering, + fmt::Debug, + ops::{Add, Deref, DerefMut, Sub}, + path::{Path, PathBuf}, +}; + +use brk_core::CheckedSub; +use brk_exit::Exit; +use brk_vec::{Error, Result, StoredIndex, StoredType, Version}; + +const FLUSH_EVERY: usize = 10_000; + +#[derive(Debug)] +pub struct StorableVec { + computed_version: Option, + vec: brk_vec::StorableVec, +} + +impl StorableVec +where + I: StoredIndex, + T: StoredType, +{ + pub fn import(path: &Path, version: Version) -> brk_vec::Result { + let vec = brk_vec::StorableVec::forced_import(path, version)?; + + Ok(Self { + computed_version: None, + vec, + }) + } + + #[inline] + pub fn i_to_usize(index: I) -> Result { + index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) + } + + #[inline] + fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { + match self.len().cmp(&Self::i_to_usize(index)?) { + Ordering::Less => { + return Err(Error::IndexTooHigh); + } + ord => { + if ord == Ordering::Greater { + self.safe_truncate_if_needed(index, exit)?; + } + self.push(value); + } + } + + if self.pushed_len() >= FLUSH_EVERY { + Ok(self.safe_flush(exit)?) + } else { + Ok(()) + } + } + + #[inline] + fn path_computed_version(&self) -> PathBuf { + self.path().join("computed_version") + } + + fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { + let path = self.path_computed_version(); + if version.validate(path.as_ref()).is_err() { + self.reset_file()?; + } + version.write(path.as_ref())?; + Ok(()) + } + + pub fn compute_transform( + &mut self, + max_from: A, + other: &mut brk_vec::StorableVec, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + A: StoredIndex, + B: StoredType, + F: FnMut((A, &B, &mut Self, &mut brk_vec::StorableVec)) -> (I, T), + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + other.version(), + )?; + + let index = max_from.min(A::from(self.len())); + other.iter_from(index, |(a, b, other)| { + let (i, v) = t((a, b, self, other)); + self.push_and_flush_if_needed(i, v, exit) + })?; + + Ok(self.safe_flush(exit)?) + } + + pub fn compute_inverse_more_to_less( + &mut self, + max_from: T, + other: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + I: StoredType + StoredIndex, + T: StoredIndex, + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + other.version(), + )?; + + let index = max_from.min(self.read_last()?.cloned().unwrap_or_default()); + other.iter_from(index, |(v, i, ..)| { + let i = *i; + if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) { + self.push_and_flush_if_needed(i, v, exit) + } else { + Ok(()) + } + })?; + + Ok(self.safe_flush(exit)?) + } + + pub fn compute_inverse_less_to_more( + &mut self, + max_from: T, + first_indexes: &mut brk_vec::StorableVec, + last_indexes: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + I: StoredType, + T: StoredIndex, + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(), + )?; + + let index = max_from.min(T::from(self.len())); + first_indexes.iter_from(index, |(value, first_index, ..)| { + let first_index = Self::i_to_usize(*first_index)?; + let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?; + (first_index..last_index) + .try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit)) + })?; + + Ok(self.safe_flush(exit)?) + } + + pub fn compute_last_index_from_first( + &mut self, + max_from: I, + first_indexes: &mut brk_vec::StorableVec, + final_len: usize, + exit: &Exit, + ) -> Result<()> + where + T: Copy + From + CheckedSub + StoredIndex, + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + first_indexes.version(), + )?; + + let index = max_from.min(I::from(self.len())); + let one = T::from(1); + let mut prev_index: Option = None; + first_indexes.iter_from(index, |(i, v, ..)| { + if let Some(prev_index) = prev_index.take() { + self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?; + } + prev_index.replace(i); + Ok(()) + })?; + if let Some(prev_index) = prev_index { + self.push_and_flush_if_needed( + prev_index, + T::from(final_len).checked_sub(one).unwrap(), + exit, + )?; + } + + Ok(self.safe_flush(exit)?) + } + + pub fn compute_count_from_indexes( + &mut self, + max_from: I, + first_indexes: &mut brk_vec::StorableVec, + last_indexes: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + T: From, + T2: StoredType + Copy + Add + Sub + TryInto, + >::Error: error::Error + 'static, + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(), + )?; + + let index = max_from.min(I::from(self.len())); + first_indexes.iter_from(index, |(i, first_index, ..)| { + let last_index = last_indexes.read(i)?.unwrap(); + let count = *last_index + 1_usize - *first_index; + self.push_and_flush_if_needed(i, count.into(), exit) + })?; + + Ok(self.safe_flush(exit)?) + } + + pub fn compute_is_first_ordered( + &mut self, + max_from: I, + self_to_other: &mut brk_vec::StorableVec, + other_to_self: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + I: StoredType, + T: From, + A: StoredIndex + StoredType, + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + self_to_other.version() + other_to_self.version(), + )?; + + let index = max_from.min(I::from(self.len())); + self_to_other.iter_from(index, |(i, other, ..)| { + self.push_and_flush_if_needed( + i, + T::from(other_to_self.read(*other)?.unwrap() == &i), + exit, + ) + })?; + + Ok(self.safe_flush(exit)?) + } + + pub fn compute_sum_from_indexes( + &mut self, + max_from: I, + first_indexes: &mut brk_vec::StorableVec, + last_indexes: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + T: From, + T2: StoredType + Copy + Add + Sub + TryInto, + >::Error: error::Error + 'static, + { + self.validate_computed_version_or_reset_file( + Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(), + )?; + + let index = max_from.min(I::from(self.len())); + first_indexes.iter_from(index, |(index, first_index, ..)| { + let last_index = last_indexes.read(index)?.unwrap(); + let count = *last_index + 1_usize - *first_index; + self.push_and_flush_if_needed(index, count.into(), exit) + })?; + + Ok(self.safe_flush(exit)?) + } +} + +impl Deref for StorableVec { + type Target = brk_vec::StorableVec; + fn deref(&self) -> &Self::Target { + &self.vec + } +} +impl DerefMut for StorableVec { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.vec + } +} +impl Clone for StorableVec +where + I: StoredIndex, + T: StoredType, +{ + fn clone(&self) -> Self { + Self { + computed_version: self.computed_version, + vec: self.vec.clone(), + } + } +} diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 778e2f66c..ecb5a7b71 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -3,7 +3,9 @@ use std::{fs, ops::Deref, path::Path}; use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, StorableVec, Value, Version}; +use brk_vec::{AnyStorableVec, Value, Version}; + +use super::StorableVec; #[derive(Clone)] pub struct Vecs { @@ -27,48 +29,48 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - dateindex_to_date: StorableVec::forced_import( + dateindex_to_date: StorableVec::import( &path.join("dateindex_to_date"), Version::from(1), )?, - dateindex_to_dateindex: StorableVec::forced_import( + dateindex_to_dateindex: StorableVec::import( &path.join("dateindex_to_dateindex"), Version::from(1), )?, - dateindex_to_first_height: StorableVec::forced_import( + dateindex_to_first_height: StorableVec::import( &path.join("dateindex_to_first_height"), Version::from(1), )?, - dateindex_to_last_height: StorableVec::forced_import( + dateindex_to_last_height: StorableVec::import( &path.join("dateindex_to_last_height"), Version::from(1), )?, - height_to_real_date: StorableVec::forced_import( + height_to_real_date: StorableVec::import( &path.join("height_to_real_date"), Version::from(1), )?, - height_to_fixed_date: StorableVec::forced_import( + height_to_fixed_date: StorableVec::import( &path.join("height_to_fixed_date"), Version::from(1), )?, - height_to_dateindex: StorableVec::forced_import( + height_to_dateindex: StorableVec::import( &path.join("height_to_dateindex"), Version::from(1), )?, - height_to_height: StorableVec::forced_import( + height_to_height: StorableVec::import( &path.join("height_to_height"), Version::from(1), )?, - height_to_last_txindex: StorableVec::forced_import( + height_to_last_txindex: StorableVec::import( &path.join("height_to_last_txindex"), Version::from(1), )?, - txindex_to_last_txinindex: StorableVec::forced_import( + txindex_to_last_txinindex: StorableVec::import( &path.join("txindex_to_last_txinindex"), Version::from(1), )?, - txindex_to_last_txoutindex: StorableVec::forced_import( + txindex_to_last_txoutindex: StorableVec::import( &path.join("txindex_to_last_txoutindex"), Version::from(1), )?, @@ -186,17 +188,17 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { vec![ - &self.dateindex_to_date, - &self.dateindex_to_dateindex, - &self.dateindex_to_first_height, - &self.dateindex_to_last_height, - &self.height_to_dateindex, - &self.height_to_fixed_date, - &self.height_to_height, - &self.height_to_last_txindex, - &self.height_to_real_date, - &self.txindex_to_last_txinindex, - &self.txindex_to_last_txoutindex, + &*self.dateindex_to_date, + &*self.dateindex_to_dateindex, + &*self.dateindex_to_first_height, + &*self.dateindex_to_last_height, + &*self.height_to_dateindex, + &*self.height_to_fixed_date, + &*self.height_to_height, + &*self.height_to_last_txindex, + &*self.height_to_real_date, + &*self.txindex_to_last_txinindex, + &*self.txindex_to_last_txoutindex, ] } } diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index fd4257d90..0c63f808a 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -6,9 +6,9 @@ use brk_core::{ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, StorableVec, Value, Version}; +use brk_vec::{AnyStorableVec, Value, Version}; -use super::indexes::{self, Indexes}; +use super::{Indexes, StorableVec, indexes}; #[derive(Clone)] pub struct Vecs { @@ -304,28 +304,28 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { vec![ - &self.dateindex_to_close, - &self.dateindex_to_close_in_cents, - &self.dateindex_to_high, - &self.dateindex_to_high_in_cents, - &self.dateindex_to_low, - &self.dateindex_to_low_in_cents, - &self.dateindex_to_ohlc, - &self.dateindex_to_ohlc_in_cents, - &self.dateindex_to_open, - &self.dateindex_to_open_in_cents, - &self.dateindex_to_sats_per_dollar, - &self.height_to_close, - &self.height_to_close_in_cents, - &self.height_to_high, - &self.height_to_high_in_cents, - &self.height_to_low, - &self.height_to_low_in_cents, - &self.height_to_ohlc, - &self.height_to_ohlc_in_cents, - &self.height_to_open, - &self.height_to_open_in_cents, - &self.height_to_sats_per_dollar, + &*self.dateindex_to_close, + &*self.dateindex_to_close_in_cents, + &*self.dateindex_to_high, + &*self.dateindex_to_high_in_cents, + &*self.dateindex_to_low, + &*self.dateindex_to_low_in_cents, + &*self.dateindex_to_ohlc, + &*self.dateindex_to_ohlc_in_cents, + &*self.dateindex_to_open, + &*self.dateindex_to_open_in_cents, + &*self.dateindex_to_sats_per_dollar, + &*self.height_to_close, + &*self.height_to_close_in_cents, + &*self.height_to_high, + &*self.height_to_high_in_cents, + &*self.height_to_low, + &*self.height_to_low_in_cents, + &*self.height_to_ohlc, + &*self.height_to_ohlc_in_cents, + &*self.height_to_open, + &*self.height_to_open_in_cents, + &*self.height_to_sats_per_dollar, ] } } diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 9d362470e..d777eaaa8 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -5,10 +5,14 @@ use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_vec::AnyStorableVec; +mod base; mod indexes; mod marketprice; mod transactions; +use base::*; +use indexes::*; + #[derive(Clone)] pub struct Vecs { pub indexes: indexes::Vecs, diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 7d28ee33e..84177c347 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -3,9 +3,9 @@ use std::{fs, path::Path}; use brk_core::Txindex; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStorableVec, StorableVec, Version}; +use brk_vec::{AnyStorableVec, Version}; -use super::indexes::{self, Indexes}; +use super::{Indexes, StorableVec, indexes}; #[derive(Clone)] pub struct Vecs { @@ -47,7 +47,7 @@ impl Vecs { // &path.join("txindex_to_fee"), // Version::from(1), // )?, - txindex_to_is_coinbase: StorableVec::forced_import( + txindex_to_is_coinbase: StorableVec::import( &path.join("txindex_to_is_coinbase"), Version::from(1), )?, @@ -127,6 +127,6 @@ impl Vecs { } pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { - vec![&self.txindex_to_is_coinbase] + vec![&*self.txindex_to_is_coinbase] } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index f9d14a393..49d775650 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -22,7 +22,7 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.56.5", features = ["codegen", "minifier"] } +oxc = { version = "0.57.0", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } tokio = { version = "1.44.0", features = ["full"] } diff --git a/crates/brk_server/src/files/minify.rs b/crates/brk_server/src/files/minify.rs index 327800bcc..d591e3e82 100644 --- a/crates/brk_server/src/files/minify.rs +++ b/crates/brk_server/src/files/minify.rs @@ -16,7 +16,9 @@ pub fn minify_js(path: &Path) -> String { let allocator = Allocator::default(); - let mut program = Parser::new(&allocator, &source_text, source_type).parse().program; + let parser_return = Parser::new(&allocator, &source_text, source_type).parse(); + + let mut program = parser_return.program; let minifier_return = Minifier::new(MinifierOptions { mangle: Some(MangleOptions::default()), @@ -33,7 +35,7 @@ pub fn minify_js(path: &Path) -> String { source_map_path: None, legal_comments: LegalComment::None, }) - .with_symbol_table(minifier_return.symbol_table) + .with_scoping(minifier_return.scoping) .build(&program) .code } diff --git a/crates/brk_vec/.gitignore b/crates/brk_vec/.gitignore index a7ab01af7..709561520 100644 --- a/crates/brk_vec/.gitignore +++ b/crates/brk_vec/.gitignore @@ -1 +1 @@ -/v +/vec diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index 45d841536..6e1fc08db 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -16,3 +16,4 @@ rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } zerocopy = { workspace = true } +zstd = "0.13.3" diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index 5083360f9..efaeef671 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -4,7 +4,8 @@ use brk_vec::{StorableVec, Version}; fn main() -> Result<(), Box> { { - let mut vec: StorableVec = StorableVec::forced_import(Path::new("./v"), Version::from(1))?; + let mut vec: StorableVec = + StorableVec::forced_import(Path::new("./vec"), Version::from(1))?; vec.push(0); vec.push(1); @@ -16,7 +17,8 @@ fn main() -> Result<(), Box> { } { - let mut vec: StorableVec = StorableVec::forced_import(Path::new("./v"), Version::from(1))?; + let mut vec: StorableVec = + StorableVec::forced_import(Path::new("./vec"), Version::from(1))?; dbg!(vec.read(0)?); // 0 dbg!(vec.read(1)?); // 0 diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 300c81c59..f376b70e8 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -5,18 +5,16 @@ use std::{ cmp::Ordering, - error, fmt::Debug, fs::{self, File, OpenOptions}, io::{self, Read, Seek, SeekFrom, Write}, marker::PhantomData, mem, - ops::{Add, Range, Sub}, + ops::Range, path::{Path, PathBuf}, sync::OnceLock, }; -use brk_core::CheckedSub; use brk_exit::Exit; pub use memmap2; use rayon::prelude::*; @@ -59,7 +57,6 @@ pub struct StorableVec { const MAX_PAGE_SIZE: usize = 4 * 4096; const ONE_MB: usize = 1024 * 1024; const MAX_CACHE_SIZE: usize = 100 * ONE_MB; -const FLUSH_EVERY: usize = 10_000; impl StorableVec where @@ -272,7 +269,7 @@ where } } - fn read_last(&mut self) -> Result> { + pub fn read_last(&mut self) -> Result> { let len = self.len(); if len == 0 { return Ok(None); @@ -376,27 +373,6 @@ where } } - #[inline] - fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> { - match self.len().cmp(&Self::i_to_usize(index)?) { - Ordering::Less => { - return Err(Error::IndexTooHigh); - } - ord => { - if ord == Ordering::Greater { - self.safe_truncate_if_needed(index, exit)?; - } - self.pushed.push(value); - } - } - - if self.pushed_len() >= FLUSH_EVERY { - Ok(self.safe_flush(exit)?) - } else { - Ok(()) - } - } - #[inline] pub fn len(&self) -> usize { self.file_len + self.pushed_len() @@ -462,15 +438,6 @@ where Ok(()) } - fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { - let path = self.path_computed_version(); - if version.validate(path.as_ref()).is_err() { - self.reset_file()?; - } - version.write(path.as_ref())?; - Ok(()) - } - pub fn truncate_if_needed(&mut self, index: I) -> Result> { let index = Self::i_to_usize(index)?; @@ -497,7 +464,7 @@ where } #[inline] - fn i_to_usize(index: I) -> Result { + pub fn i_to_usize(index: I) -> Result { index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) } @@ -558,207 +525,12 @@ where path.join("version") } - #[inline] - fn path_computed_version(&self) -> PathBuf { - self.path().join("computed_version") - } - pub fn index_type_to_string(&self) -> &str { std::any::type_name::() } - pub fn compute_transform( - &mut self, - max_from: A, - other: &mut StorableVec, - mut t: F, - exit: &Exit, - ) -> Result<()> - where - A: StoredIndex, - B: StoredType, - F: FnMut((A, &B, &mut Self, &mut StorableVec)) -> (I, T), - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + other.version, - )?; - - let index = max_from.min(A::from(self.len())); - other.iter_from(index, |(a, b, other)| { - let (i, v) = t((a, b, self, other)); - self.push_and_flush_if_needed(i, v, exit) - })?; - - Ok(self.safe_flush(exit)?) - } - - pub fn compute_inverse_more_to_less( - &mut self, - max_from: T, - other: &mut StorableVec, - exit: &Exit, - ) -> Result<()> - where - I: StoredType + StoredIndex, - T: StoredIndex, - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + other.version, - )?; - - let index = max_from.min(self.read_last()?.cloned().unwrap_or_default()); - other.iter_from(index, |(v, i, ..)| { - let i = *i; - if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) { - self.push_and_flush_if_needed(i, v, exit) - } else { - Ok(()) - } - })?; - - Ok(self.safe_flush(exit)?) - } - - pub fn compute_inverse_less_to_more( - &mut self, - max_from: T, - first_indexes: &mut StorableVec, - last_indexes: &mut StorableVec, - exit: &Exit, - ) -> Result<()> - where - I: StoredType, - T: StoredIndex, - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + first_indexes.version + last_indexes.version, - )?; - - let index = max_from.min(T::from(self.len())); - first_indexes.iter_from(index, |(value, first_index, ..)| { - let first_index = Self::i_to_usize(*first_index)?; - let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?; - (first_index..last_index) - .try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit)) - })?; - - Ok(self.safe_flush(exit)?) - } - - pub fn compute_last_index_from_first( - &mut self, - max_from: I, - first_indexes: &mut StorableVec, - final_len: usize, - exit: &Exit, - ) -> Result<()> - where - T: Copy + From + CheckedSub + StoredIndex, - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + first_indexes.version, - )?; - - let index = max_from.min(I::from(self.len())); - let one = T::from(1); - let mut prev_index: Option = None; - first_indexes.iter_from(index, |(i, v, ..)| { - if let Some(prev_index) = prev_index.take() { - self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?; - } - prev_index.replace(i); - Ok(()) - })?; - if let Some(prev_index) = prev_index { - self.push_and_flush_if_needed( - prev_index, - T::from(final_len).checked_sub(one).unwrap(), - exit, - )?; - } - - Ok(self.safe_flush(exit)?) - } - - pub fn compute_count_from_indexes( - &mut self, - max_from: I, - first_indexes: &mut StorableVec, - last_indexes: &mut StorableVec, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredType + Copy + Add + Sub + TryInto, - >::Error: error::Error + 'static, - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + first_indexes.version + last_indexes.version, - )?; - - let index = max_from.min(I::from(self.len())); - first_indexes.iter_from(index, |(i, first_index, ..)| { - let last_index = last_indexes.read(i)?.unwrap(); - let count = *last_index + 1_usize - *first_index; - self.push_and_flush_if_needed(i, count.into(), exit) - })?; - - Ok(self.safe_flush(exit)?) - } - - pub fn compute_is_first_ordered( - &mut self, - max_from: I, - self_to_other: &mut StorableVec, - other_to_self: &mut StorableVec, - exit: &Exit, - ) -> Result<()> - where - I: StoredType, - T: From, - A: StoredIndex + StoredType, - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + self_to_other.version + other_to_self.version, - )?; - - let index = max_from.min(I::from(self.len())); - self_to_other.iter_from(index, |(i, other, ..)| { - self.push_and_flush_if_needed( - i, - T::from(other_to_self.read(*other)?.unwrap() == &i), - exit, - ) - })?; - - Ok(self.safe_flush(exit)?) - } - - pub fn compute_sum_from_indexes( - &mut self, - max_from: I, - first_indexes: &mut StorableVec, - last_indexes: &mut StorableVec, - exit: &Exit, - ) -> Result<()> - where - T: From, - T2: StoredType + Copy + Add + Sub + TryInto, - >::Error: error::Error + 'static, - F: Fn(&T2) -> T, - { - self.validate_computed_version_or_reset_file( - Version::from(0) + self.version + first_indexes.version + last_indexes.version, - )?; - - let index = max_from.min(I::from(self.len())); - first_indexes.iter_from(index, |(index, first_index, ..)| { - let last_index = last_indexes.read(index)?.unwrap(); - let count = *last_index + 1_usize - *first_index; - self.push_and_flush_if_needed(index, count.into(), exit) - })?; - - Ok(self.safe_flush(exit)?) + pub fn version(&self) -> Version { + self.version } } diff --git a/crates/brk_vec/src/traits/bytes.rs b/crates/brk_vec/src/traits/bytes.rs deleted file mode 100644 index 569d9dd32..000000000 --- a/crates/brk_vec/src/traits/bytes.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::sync::Arc; - -use crate::Result; - -pub trait Bytes: Sized { - const LEN: usize = size_of::(); - fn to_bytes(&self) -> Arc<[u8]>; - fn try_from_bytes(bytes: &[u8]) -> Result; -} - -pub trait UnsafeBytes {}