vec: moved compute functions to computer

Cargo.lock

@@ -560,6 +560,7 @@ dependencies = [
  "serde",
  "serde_json",
  "zerocopy",
+ "zstd",
 ]
 
 [[package]]
@@ -1476,9 +1477,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "jiff"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c163c633eb184a4ad2a5e7a5dacf12a58c830d717a7963563d4eceb4ced079f"
+checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e"
 dependencies = [
  "jiff-static",
  "jiff-tzdb-platform",
@@ -1491,9 +1492,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbc3e0019b0f5f43038cf46471b1312136f29e36f54436c6042c8f155fec8789"
+checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1554,9 +1555,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "libc"
-version = "0.2.170"
+version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "linux-raw-sys"
@@ -1673,12 +1674,11 @@ dependencies = [
 
 [[package]]
 name = "minreq"
-version = "2.13.2"
+version = "2.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da0c420feb01b9fb5061f8c8f452534361dd783756dcf38ec45191ce55e7a161"
+checksum = "567496f13503d6cae8c9f961f34536850275f396307d7a6b981eef1464032f53"
 dependencies = [
  "log",
- "once_cell",
  "rustls",
  "rustls-webpki",
  "serde",
@@ -1784,9 +1784,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
 
 [[package]]
 name = "oxc"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cae6276febbc5abc1f1e4cf49167d54ab341818656ae4f622d5992b65fcdd371"
+checksum = "918d8f6deffeb380427e17bfc2d63c38f30e485af3a285faa6ca87d8c8a37a0d"
 dependencies = [
  "oxc_allocator",
  "oxc_ast",
@@ -1827,9 +1827,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_allocator"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cc5cd078806a1b7061fa146dc4228a57d0765da6c85e99500d069b86f57e94"
+checksum = "d92b3f2a5e042d3e7d21aab5896d3f02d657c4b3cae42aa2a5842ebd018ec194"
 dependencies = [
  "allocator-api2",
  "bumpalo",
@@ -1840,9 +1840,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ast"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4722414ac21a2e28a16b76de8390672c01a39adcb703d405b848149cfaeeaf7"
+checksum = "864754f59a7965a83bf47c376f97ac1937dedd415c367f7e6b7d6499453aaad4"
 dependencies = [
  "bitflags",
  "cow-utils",
@@ -1857,9 +1857,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ast_macros"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f1505d8622b2ea6ed0274f355bd5e4ee3f09df5d9b39c8a3a673f344d87b82a"
+checksum = "0f1b369ef5a746cfd3400047f0ec972d66fe75ee5ab5fbfa8f1699e7a3b86724"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1868,9 +1868,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ast_visit"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf45370c6da0dd142a70468e5b25127d6a34caa71056105c85087559c8ee9afb"
+checksum = "7874fe470308c6882fbeb42953896a21ea727d76165bfa80613dab0abbf72254"
 dependencies = [
  "oxc_allocator",
  "oxc_ast",
@@ -1880,9 +1880,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_cfg"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96be30717d29eb7d1780758d033e92fcc208b8cce83b3b4869d2155fa4c9b7bd"
+checksum = "ff4ba4ecfa3296e937a3173f3a6b7e98446fed48e182d03f0955729f74b219c1"
 dependencies = [
  "bitflags",
  "itertools",
@@ -1895,9 +1895,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_codegen"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5fd891e06aa19c7d22e129d366e55644ffeed99167e3495dcaec8149a0631b3"
+checksum = "6eb495fd82a33f1d40ea0c27d1db81772946b26a4a236de411a83511aa628a96"
 dependencies = [
  "bitflags",
  "cow-utils",
@@ -1916,9 +1916,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_data_structures"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb49a2ee880952c2079b61851ecc35d7671d9d3509e306f5e704ccacd2783984"
+checksum = "3454bded31b396d75d4ebad5fbdc547ad544b47556b540925be24a874e1d87d8"
 dependencies = [
  "assert-unchecked",
  "ropey",
@@ -1926,9 +1926,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_diagnostics"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea83fe2415b0580980ac83364c1ae943f8ee9c00becf5395a89e800a9526a080"
+checksum = "27cd28f66608e282c1384a877505d3678e099bc3cef6856bc55e2ef60f5344b2"
 dependencies = [
  "cow-utils",
  "oxc-miette",
@@ -1936,9 +1936,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ecmascript"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c7e7bcc382cf901e93a16f86e70f2737c507aaa833656b0d8484d64f8ae358a"
+checksum = "43b9a66f69c6dcc78bf1c164e1e98b4494ba71fbf1b17ebd8940b936902e68ec"
 dependencies = [
  "cow-utils",
  "num-bigint",
@@ -1950,9 +1950,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_estree"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cb66484735d21f096b07c894badc96d89a0c6d31b4bdd46b33b3e44da9b97ac"
+checksum = "5aa7a068d758764707466b733f92ad66bbe91d5c66ce562d868dff8a7ae72b09"
 
 [[package]]
 name = "oxc_index"
@@ -1962,9 +1962,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
 
 [[package]]
 name = "oxc_mangler"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8069aaf088ccaee383e008ff29d407c2c88b4ff8694f7e34a9d6aaf202c81e71"
+checksum = "f4f5ae1f1f842c714f3144036a644ee275c8a360fd50ac6f295ce52e468cee53"
 dependencies = [
  "fixedbitset",
  "itertools",
@@ -1979,9 +1979,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_minifier"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1db16b19f75f29b9502b651c6ffae09366d43fd91decf5535d80c634f8e7e21d"
+checksum = "9e4030f603ae314c93b761753969328e96960a853f443fdeaf8c9b2273f7971e"
 dependencies = [
  "cow-utils",
  "oxc_allocator",
@@ -2001,9 +2001,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_parser"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a7414e779b9723b0bd2880fe69b0ee517e583f88c5118a7c9053cf3317b95b1"
+checksum = "71d48cff62f816c27be1912d1d173140602214c8eec38f1e98e77270f39e0ce6"
 dependencies = [
  "assert-unchecked",
  "bitflags",
@@ -2024,9 +2024,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_regular_expression"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d18196c212eac24a7faf613e373e5f9317d3542578a088d01dc1a548fa1e1cb3"
+checksum = "3a49d5d6ede8d8740b25743a78c2fb45859099ba89d88425b1b3d6a433dda352"
 dependencies = [
  "oxc_allocator",
  "oxc_ast_macros",
@@ -2040,9 +2040,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_semantic"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4556f1c700baaec8589872ddf2af41d9a964db52fdbec9a5d07d7477dce45cf8"
+checksum = "c7b84c302f11c87b330b5e3f02f3efc4cd1ea44e8553551d6e245cb161dcc6ef"
 dependencies = [
  "assert-unchecked",
  "itertools",
@@ -2077,9 +2077,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_span"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "318f925e26bd118adc082d290538d07611fe2434987a5c60cf5084381ecb42e6"
+checksum = "cf5d0e27b520397a9d0a99dfa21d9c29fa389f34719aff30f6c5468498d603d9"
 dependencies = [
  "compact_str",
  "oxc-miette",
@@ -2090,9 +2090,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_syntax"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cb68ceb7c6902f3043fe8fe49bb886826b1d7741dc2904337297d53692b1b9c"
+checksum = "e91fdded3cdce1641b8ee44a223be03dca3def248120cbe3fe0492a14bd8a744"
 dependencies = [
  "assert-unchecked",
  "bitflags",
@@ -2111,9 +2111,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_traverse"
-version = "0.56.5"
+version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a090c9cd461d468f5faf3fe3f56378a4e27d80febe34ad39f2a04920f594d4"
+checksum = "77703db3bd925f63fd688e3858ecd48d4b1a455a64f4c54bec6baae50af03cab"
 dependencies = [
  "compact_str",
  "itoa",

Cargo.toml

@@ -26,9 +26,9 @@ clap = { version = "4.5.32", features = ["derive", "string"] }
 color-eyre = "0.6.3"
 derive_deref = "1.1.1"
 fjall = "2.6.7"
-jiff = "0.2.3"
+jiff = "0.2.4"
 log = { version = "0.4.26" }
-minreq = { version = "2.13.2", features = ["https", "serde_json"] }
+minreq = { version = "2.13.3", features = ["https", "serde_json"] }
 rayon = "1.10.0"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = { version = "1.0.140", features = ["float_roundtrip"] }

crates/brk_computer/src/storage/vecs/base.rs

@@ -0,0 +1,291 @@
+use core::error;
+use std::{
+    cmp::Ordering,
+    fmt::Debug,
+    ops::{Add, Deref, DerefMut, Sub},
+    path::{Path, PathBuf},
+};
+
+use brk_core::CheckedSub;
+use brk_exit::Exit;
+use brk_vec::{Error, Result, StoredIndex, StoredType, Version};
+
+const FLUSH_EVERY: usize = 10_000;
+
+#[derive(Debug)]
+pub struct StorableVec<I, T> {
+    computed_version: Option<Version>,
+    vec: brk_vec::StorableVec<I, T>,
+}
+
+impl<I, T> StorableVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    pub fn import(path: &Path, version: Version) -> brk_vec::Result<Self> {
+        let vec = brk_vec::StorableVec::forced_import(path, version)?;
+
+        Ok(Self {
+            computed_version: None,
+            vec,
+        })
+    }
+
+    #[inline]
+    pub fn i_to_usize(index: I) -> Result<usize> {
+        index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
+    }
+
+    #[inline]
+    fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
+        match self.len().cmp(&Self::i_to_usize(index)?) {
+            Ordering::Less => {
+                return Err(Error::IndexTooHigh);
+            }
+            ord => {
+                if ord == Ordering::Greater {
+                    self.safe_truncate_if_needed(index, exit)?;
+                }
+                self.push(value);
+            }
+        }
+
+        if self.pushed_len() >= FLUSH_EVERY {
+            Ok(self.safe_flush(exit)?)
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn path_computed_version(&self) -> PathBuf {
+        self.path().join("computed_version")
+    }
+
+    fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
+        let path = self.path_computed_version();
+        if version.validate(path.as_ref()).is_err() {
+            self.reset_file()?;
+        }
+        version.write(path.as_ref())?;
+        Ok(())
+    }
+
+    pub fn compute_transform<A, B, F>(
+        &mut self,
+        max_from: A,
+        other: &mut brk_vec::StorableVec<A, B>,
+        mut t: F,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        A: StoredIndex,
+        B: StoredType,
+        F: FnMut((A, &B, &mut Self, &mut brk_vec::StorableVec<A, B>)) -> (I, T),
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + other.version(),
+        )?;
+
+        let index = max_from.min(A::from(self.len()));
+        other.iter_from(index, |(a, b, other)| {
+            let (i, v) = t((a, b, self, other));
+            self.push_and_flush_if_needed(i, v, exit)
+        })?;
+
+        Ok(self.safe_flush(exit)?)
+    }
+
+    pub fn compute_inverse_more_to_less(
+        &mut self,
+        max_from: T,
+        other: &mut brk_vec::StorableVec<T, I>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        I: StoredType + StoredIndex,
+        T: StoredIndex,
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + other.version(),
+        )?;
+
+        let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
+        other.iter_from(index, |(v, i, ..)| {
+            let i = *i;
+            if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) {
+                self.push_and_flush_if_needed(i, v, exit)
+            } else {
+                Ok(())
+            }
+        })?;
+
+        Ok(self.safe_flush(exit)?)
+    }
+
+    pub fn compute_inverse_less_to_more(
+        &mut self,
+        max_from: T,
+        first_indexes: &mut brk_vec::StorableVec<T, I>,
+        last_indexes: &mut brk_vec::StorableVec<T, I>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        I: StoredType,
+        T: StoredIndex,
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
+        )?;
+
+        let index = max_from.min(T::from(self.len()));
+        first_indexes.iter_from(index, |(value, first_index, ..)| {
+            let first_index = Self::i_to_usize(*first_index)?;
+            let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
+            (first_index..last_index)
+                .try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit))
+        })?;
+
+        Ok(self.safe_flush(exit)?)
+    }
+
+    pub fn compute_last_index_from_first(
+        &mut self,
+        max_from: I,
+        first_indexes: &mut brk_vec::StorableVec<I, T>,
+        final_len: usize,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        T: Copy + From<usize> + CheckedSub<T> + StoredIndex,
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + first_indexes.version(),
+        )?;
+
+        let index = max_from.min(I::from(self.len()));
+        let one = T::from(1);
+        let mut prev_index: Option<I> = None;
+        first_indexes.iter_from(index, |(i, v, ..)| {
+            if let Some(prev_index) = prev_index.take() {
+                self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?;
+            }
+            prev_index.replace(i);
+            Ok(())
+        })?;
+        if let Some(prev_index) = prev_index {
+            self.push_and_flush_if_needed(
+                prev_index,
+                T::from(final_len).checked_sub(one).unwrap(),
+                exit,
+            )?;
+        }
+
+        Ok(self.safe_flush(exit)?)
+    }
+
+    pub fn compute_count_from_indexes<T2>(
+        &mut self,
+        max_from: I,
+        first_indexes: &mut brk_vec::StorableVec<I, T2>,
+        last_indexes: &mut brk_vec::StorableVec<I, T2>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        T: From<T2>,
+        T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
+        <T2 as TryInto<T>>::Error: error::Error + 'static,
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
+        )?;
+
+        let index = max_from.min(I::from(self.len()));
+        first_indexes.iter_from(index, |(i, first_index, ..)| {
+            let last_index = last_indexes.read(i)?.unwrap();
+            let count = *last_index + 1_usize - *first_index;
+            self.push_and_flush_if_needed(i, count.into(), exit)
+        })?;
+
+        Ok(self.safe_flush(exit)?)
+    }
+
+    pub fn compute_is_first_ordered<A>(
+        &mut self,
+        max_from: I,
+        self_to_other: &mut brk_vec::StorableVec<I, A>,
+        other_to_self: &mut brk_vec::StorableVec<A, I>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        I: StoredType,
+        T: From<bool>,
+        A: StoredIndex + StoredType,
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + self_to_other.version() + other_to_self.version(),
+        )?;
+
+        let index = max_from.min(I::from(self.len()));
+        self_to_other.iter_from(index, |(i, other, ..)| {
+            self.push_and_flush_if_needed(
+                i,
+                T::from(other_to_self.read(*other)?.unwrap() == &i),
+                exit,
+            )
+        })?;
+
+        Ok(self.safe_flush(exit)?)
+    }
+
+    pub fn compute_sum_from_indexes<T2>(
+        &mut self,
+        max_from: I,
+        first_indexes: &mut brk_vec::StorableVec<I, T2>,
+        last_indexes: &mut brk_vec::StorableVec<I, T2>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        T: From<T2>,
+        T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
+        <T2 as TryInto<T>>::Error: error::Error + 'static,
+    {
+        self.validate_computed_version_or_reset_file(
+            Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
+        )?;
+
+        let index = max_from.min(I::from(self.len()));
+        first_indexes.iter_from(index, |(index, first_index, ..)| {
+            let last_index = last_indexes.read(index)?.unwrap();
+            let count = *last_index + 1_usize - *first_index;
+            self.push_and_flush_if_needed(index, count.into(), exit)
+        })?;
+
+        Ok(self.safe_flush(exit)?)
+    }
+}
+
+impl<I, T> Deref for StorableVec<I, T> {
+    type Target = brk_vec::StorableVec<I, T>;
+    fn deref(&self) -> &Self::Target {
+        &self.vec
+    }
+}
+impl<I, T> DerefMut for StorableVec<I, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.vec
+    }
+}
+impl<I, T> Clone for StorableVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    fn clone(&self) -> Self {
+        Self {
+            computed_version: self.computed_version,
+            vec: self.vec.clone(),
+        }
+    }
+}

crates/brk_computer/src/storage/vecs/indexes.rs

@@ -3,7 +3,9 @@ use std::{fs, ops::Deref, path::Path};
 use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, StorableVec, Value, Version};
+use brk_vec::{AnyStorableVec, Value, Version};
+
+use super::StorableVec;
 
 #[derive(Clone)]
 pub struct Vecs {
@@ -27,48 +29,48 @@ impl Vecs {
         fs::create_dir_all(path)?;
 
         Ok(Self {
-            dateindex_to_date: StorableVec::forced_import(
+            dateindex_to_date: StorableVec::import(
                 &path.join("dateindex_to_date"),
                 Version::from(1),
             )?,
-            dateindex_to_dateindex: StorableVec::forced_import(
+            dateindex_to_dateindex: StorableVec::import(
                 &path.join("dateindex_to_dateindex"),
                 Version::from(1),
             )?,
-            dateindex_to_first_height: StorableVec::forced_import(
+            dateindex_to_first_height: StorableVec::import(
                 &path.join("dateindex_to_first_height"),
                 Version::from(1),
             )?,
-            dateindex_to_last_height: StorableVec::forced_import(
+            dateindex_to_last_height: StorableVec::import(
                 &path.join("dateindex_to_last_height"),
                 Version::from(1),
             )?,
-            height_to_real_date: StorableVec::forced_import(
+            height_to_real_date: StorableVec::import(
                 &path.join("height_to_real_date"),
                 Version::from(1),
             )?,
-            height_to_fixed_date: StorableVec::forced_import(
+            height_to_fixed_date: StorableVec::import(
                 &path.join("height_to_fixed_date"),
                 Version::from(1),
             )?,
-            height_to_dateindex: StorableVec::forced_import(
+            height_to_dateindex: StorableVec::import(
                 &path.join("height_to_dateindex"),
                 Version::from(1),
             )?,
-            height_to_height: StorableVec::forced_import(
+            height_to_height: StorableVec::import(
                 &path.join("height_to_height"),
                 Version::from(1),
             )?,
-            height_to_last_txindex: StorableVec::forced_import(
+            height_to_last_txindex: StorableVec::import(
                 &path.join("height_to_last_txindex"),
                 Version::from(1),
             )?,
 
-            txindex_to_last_txinindex: StorableVec::forced_import(
+            txindex_to_last_txinindex: StorableVec::import(
                 &path.join("txindex_to_last_txinindex"),
                 Version::from(1),
             )?,
-            txindex_to_last_txoutindex: StorableVec::forced_import(
+            txindex_to_last_txoutindex: StorableVec::import(
                 &path.join("txindex_to_last_txoutindex"),
                 Version::from(1),
             )?,
@@ -186,17 +188,17 @@ impl Vecs {
 
     pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
         vec![
-            &self.dateindex_to_date,
+            &*self.dateindex_to_date,
-            &self.dateindex_to_dateindex,
+            &*self.dateindex_to_dateindex,
-            &self.dateindex_to_first_height,
+            &*self.dateindex_to_first_height,
-            &self.dateindex_to_last_height,
+            &*self.dateindex_to_last_height,
-            &self.height_to_dateindex,
+            &*self.height_to_dateindex,
-            &self.height_to_fixed_date,
+            &*self.height_to_fixed_date,
-            &self.height_to_height,
+            &*self.height_to_height,
-            &self.height_to_last_txindex,
+            &*self.height_to_last_txindex,
-            &self.height_to_real_date,
+            &*self.height_to_real_date,
-            &self.txindex_to_last_txinindex,
+            &*self.txindex_to_last_txinindex,
-            &self.txindex_to_last_txoutindex,
+            &*self.txindex_to_last_txoutindex,
         ]
     }
 }

crates/brk_computer/src/storage/vecs/marketprice.rs

@@ -6,9 +6,9 @@ use brk_core::{
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, StorableVec, Value, Version};
+use brk_vec::{AnyStorableVec, Value, Version};
 
-use super::indexes::{self, Indexes};
+use super::{Indexes, StorableVec, indexes};
 
 #[derive(Clone)]
 pub struct Vecs {
@@ -304,28 +304,28 @@ impl Vecs {
 
     pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
         vec![
-            &self.dateindex_to_close,
+            &*self.dateindex_to_close,
-            &self.dateindex_to_close_in_cents,
+            &*self.dateindex_to_close_in_cents,
-            &self.dateindex_to_high,
+            &*self.dateindex_to_high,
-            &self.dateindex_to_high_in_cents,
+            &*self.dateindex_to_high_in_cents,
-            &self.dateindex_to_low,
+            &*self.dateindex_to_low,
-            &self.dateindex_to_low_in_cents,
+            &*self.dateindex_to_low_in_cents,
-            &self.dateindex_to_ohlc,
+            &*self.dateindex_to_ohlc,
-            &self.dateindex_to_ohlc_in_cents,
+            &*self.dateindex_to_ohlc_in_cents,
-            &self.dateindex_to_open,
+            &*self.dateindex_to_open,
-            &self.dateindex_to_open_in_cents,
+            &*self.dateindex_to_open_in_cents,
-            &self.dateindex_to_sats_per_dollar,
+            &*self.dateindex_to_sats_per_dollar,
-            &self.height_to_close,
+            &*self.height_to_close,
-            &self.height_to_close_in_cents,
+            &*self.height_to_close_in_cents,
-            &self.height_to_high,
+            &*self.height_to_high,
-            &self.height_to_high_in_cents,
+            &*self.height_to_high_in_cents,
-            &self.height_to_low,
+            &*self.height_to_low,
-            &self.height_to_low_in_cents,
+            &*self.height_to_low_in_cents,
-            &self.height_to_ohlc,
+            &*self.height_to_ohlc,
-            &self.height_to_ohlc_in_cents,
+            &*self.height_to_ohlc_in_cents,
-            &self.height_to_open,
+            &*self.height_to_open,
-            &self.height_to_open_in_cents,
+            &*self.height_to_open_in_cents,
-            &self.height_to_sats_per_dollar,
+            &*self.height_to_sats_per_dollar,
         ]
     }
 }

crates/brk_computer/src/storage/vecs/mod.rs

@@ -5,10 +5,14 @@ use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
 use brk_vec::AnyStorableVec;
 
+mod base;
 mod indexes;
 mod marketprice;
 mod transactions;
 
+use base::*;
+use indexes::*;
+
 #[derive(Clone)]
 pub struct Vecs {
     pub indexes: indexes::Vecs,

crates/brk_computer/src/storage/vecs/transactions.rs

@@ -3,9 +3,9 @@ use std::{fs, path::Path};
 use brk_core::Txindex;
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, StorableVec, Version};
+use brk_vec::{AnyStorableVec, Version};
 
-use super::indexes::{self, Indexes};
+use super::{Indexes, StorableVec, indexes};
 
 #[derive(Clone)]
 pub struct Vecs {
@@ -47,7 +47,7 @@ impl Vecs {
             //     &path.join("txindex_to_fee"),
             //     Version::from(1),
             // )?,
-            txindex_to_is_coinbase: StorableVec::forced_import(
+            txindex_to_is_coinbase: StorableVec::import(
                 &path.join("txindex_to_is_coinbase"),
                 Version::from(1),
             )?,
@@ -127,6 +127,6 @@ impl Vecs {
     }
 
     pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
-        vec![&self.txindex_to_is_coinbase]
+        vec![&*self.txindex_to_is_coinbase]
     }
 }

crates/brk_server/Cargo.toml

@@ -22,7 +22,7 @@ color-eyre = { workspace = true }
 jiff = { workspace = true }
 log = { workspace = true }
 minreq = { workspace = true }
-oxc = { version = "0.56.5", features = ["codegen", "minifier"] }
+oxc = { version = "0.57.0", features = ["codegen", "minifier"] }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tokio = { version = "1.44.0", features = ["full"] }

crates/brk_server/src/files/minify.rs

@@ -16,7 +16,9 @@ pub fn minify_js(path: &Path) -> String {
 
     let allocator = Allocator::default();
 
-    let mut program = Parser::new(&allocator, &source_text, source_type).parse().program;
+    let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
+
+    let mut program = parser_return.program;
 
     let minifier_return = Minifier::new(MinifierOptions {
         mangle: Some(MangleOptions::default()),
@@ -33,7 +35,7 @@ pub fn minify_js(path: &Path) -> String {
             source_map_path: None,
             legal_comments: LegalComment::None,
         })
-        .with_symbol_table(minifier_return.symbol_table)
+        .with_scoping(minifier_return.scoping)
         .build(&program)
         .code
 }

crates/brk_vec/.gitignore

@@ -1 +1 @@
-/v
+/vec

crates/brk_vec/Cargo.toml

@@ -16,3 +16,4 @@ rayon = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 zerocopy = { workspace = true }
+zstd = "0.13.3"

crates/brk_vec/examples/main.rs

@@ -4,7 +4,8 @@ use brk_vec::{StorableVec, Version};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     {
-        let mut vec: StorableVec<usize, u32> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
+        let mut vec: StorableVec<usize, u32> =
+            StorableVec::forced_import(Path::new("./vec"), Version::from(1))?;
 
         vec.push(0);
         vec.push(1);
@@ -16,7 +17,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     {
-        let mut vec: StorableVec<usize, u32> = StorableVec::forced_import(Path::new("./v"), Version::from(1))?;
+        let mut vec: StorableVec<usize, u32> =
+            StorableVec::forced_import(Path::new("./vec"), Version::from(1))?;
 
         dbg!(vec.read(0)?); // 0
         dbg!(vec.read(1)?); // 0

crates/brk_vec/src/lib.rs

@@ -5,18 +5,16 @@
 
 use std::{
     cmp::Ordering,
-    error,
     fmt::Debug,
     fs::{self, File, OpenOptions},
     io::{self, Read, Seek, SeekFrom, Write},
     marker::PhantomData,
     mem,
-    ops::{Add, Range, Sub},
+    ops::Range,
     path::{Path, PathBuf},
     sync::OnceLock,
 };
 
-use brk_core::CheckedSub;
 use brk_exit::Exit;
 pub use memmap2;
 use rayon::prelude::*;
@@ -59,7 +57,6 @@ pub struct StorableVec<I, T> {
 const MAX_PAGE_SIZE: usize = 4 * 4096;
 const ONE_MB: usize = 1024 * 1024;
 const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
-const FLUSH_EVERY: usize = 10_000;
 
 impl<I, T> StorableVec<I, T>
 where
@@ -272,7 +269,7 @@ where
         }
     }
 
-    fn read_last(&mut self) -> Result<Option<&T>> {
+    pub fn read_last(&mut self) -> Result<Option<&T>> {
         let len = self.len();
         if len == 0 {
             return Ok(None);
@@ -376,27 +373,6 @@ where
         }
     }
 
-    #[inline]
-    fn push_and_flush_if_needed(&mut self, index: I, value: T, exit: &Exit) -> Result<()> {
-        match self.len().cmp(&Self::i_to_usize(index)?) {
-            Ordering::Less => {
-                return Err(Error::IndexTooHigh);
-            }
-            ord => {
-                if ord == Ordering::Greater {
-                    self.safe_truncate_if_needed(index, exit)?;
-                }
-                self.pushed.push(value);
-            }
-        }
-
-        if self.pushed_len() >= FLUSH_EVERY {
-            Ok(self.safe_flush(exit)?)
-        } else {
-            Ok(())
-        }
-    }
-
     #[inline]
     pub fn len(&self) -> usize {
         self.file_len + self.pushed_len()
@@ -462,15 +438,6 @@ where
         Ok(())
     }
 
-    fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
-        let path = self.path_computed_version();
-        if version.validate(path.as_ref()).is_err() {
-            self.reset_file()?;
-        }
-        version.write(path.as_ref())?;
-        Ok(())
-    }
-
     pub fn truncate_if_needed(&mut self, index: I) -> Result<Option<T>> {
         let index = Self::i_to_usize(index)?;
 
@@ -497,7 +464,7 @@ where
     }
 
     #[inline]
-    fn i_to_usize(index: I) -> Result<usize> {
+    pub fn i_to_usize(index: I) -> Result<usize> {
         index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)
     }
 
@@ -558,207 +525,12 @@ where
         path.join("version")
     }
 
-    #[inline]
-    fn path_computed_version(&self) -> PathBuf {
-        self.path().join("computed_version")
-    }
-
     pub fn index_type_to_string(&self) -> &str {
         std::any::type_name::<I>()
     }
 
-    pub fn compute_transform<A, B, F>(
+    pub fn version(&self) -> Version {
-        &mut self,
+        self.version
-        max_from: A,
-        other: &mut StorableVec<A, B>,
-        mut t: F,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        A: StoredIndex,
-        B: StoredType,
-        F: FnMut((A, &B, &mut Self, &mut StorableVec<A, B>)) -> (I, T),
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + other.version,
-        )?;
-
-        let index = max_from.min(A::from(self.len()));
-        other.iter_from(index, |(a, b, other)| {
-            let (i, v) = t((a, b, self, other));
-            self.push_and_flush_if_needed(i, v, exit)
-        })?;
-
-        Ok(self.safe_flush(exit)?)
-    }
-
-    pub fn compute_inverse_more_to_less(
-        &mut self,
-        max_from: T,
-        other: &mut StorableVec<T, I>,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        I: StoredType + StoredIndex,
-        T: StoredIndex,
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + other.version,
-        )?;
-
-        let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
-        other.iter_from(index, |(v, i, ..)| {
-            let i = *i;
-            if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) {
-                self.push_and_flush_if_needed(i, v, exit)
-            } else {
-                Ok(())
-            }
-        })?;
-
-        Ok(self.safe_flush(exit)?)
-    }
-
-    pub fn compute_inverse_less_to_more(
-        &mut self,
-        max_from: T,
-        first_indexes: &mut StorableVec<T, I>,
-        last_indexes: &mut StorableVec<T, I>,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        I: StoredType,
-        T: StoredIndex,
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + first_indexes.version + last_indexes.version,
-        )?;
-
-        let index = max_from.min(T::from(self.len()));
-        first_indexes.iter_from(index, |(value, first_index, ..)| {
-            let first_index = Self::i_to_usize(*first_index)?;
-            let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
-            (first_index..last_index)
-                .try_for_each(|index| self.push_and_flush_if_needed(I::from(index), value, exit))
-        })?;
-
-        Ok(self.safe_flush(exit)?)
-    }
-
-    pub fn compute_last_index_from_first(
-        &mut self,
-        max_from: I,
-        first_indexes: &mut StorableVec<I, T>,
-        final_len: usize,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        T: Copy + From<usize> + CheckedSub<T> + StoredIndex,
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + first_indexes.version,
-        )?;
-
-        let index = max_from.min(I::from(self.len()));
-        let one = T::from(1);
-        let mut prev_index: Option<I> = None;
-        first_indexes.iter_from(index, |(i, v, ..)| {
-            if let Some(prev_index) = prev_index.take() {
-                self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?;
-            }
-            prev_index.replace(i);
-            Ok(())
-        })?;
-        if let Some(prev_index) = prev_index {
-            self.push_and_flush_if_needed(
-                prev_index,
-                T::from(final_len).checked_sub(one).unwrap(),
-                exit,
-            )?;
-        }
-
-        Ok(self.safe_flush(exit)?)
-    }
-
-    pub fn compute_count_from_indexes<T2>(
-        &mut self,
-        max_from: I,
-        first_indexes: &mut StorableVec<I, T2>,
-        last_indexes: &mut StorableVec<I, T2>,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        T: From<T2>,
-        T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
-        <T2 as TryInto<T>>::Error: error::Error + 'static,
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + first_indexes.version + last_indexes.version,
-        )?;
-
-        let index = max_from.min(I::from(self.len()));
-        first_indexes.iter_from(index, |(i, first_index, ..)| {
-            let last_index = last_indexes.read(i)?.unwrap();
-            let count = *last_index + 1_usize - *first_index;
-            self.push_and_flush_if_needed(i, count.into(), exit)
-        })?;
-
-        Ok(self.safe_flush(exit)?)
-    }
-
-    pub fn compute_is_first_ordered<A>(
-        &mut self,
-        max_from: I,
-        self_to_other: &mut StorableVec<I, A>,
-        other_to_self: &mut StorableVec<A, I>,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        I: StoredType,
-        T: From<bool>,
-        A: StoredIndex + StoredType,
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + self_to_other.version + other_to_self.version,
-        )?;
-
-        let index = max_from.min(I::from(self.len()));
-        self_to_other.iter_from(index, |(i, other, ..)| {
-            self.push_and_flush_if_needed(
-                i,
-                T::from(other_to_self.read(*other)?.unwrap() == &i),
-                exit,
-            )
-        })?;
-
-        Ok(self.safe_flush(exit)?)
-    }
-
-    pub fn compute_sum_from_indexes<T2, F>(
-        &mut self,
-        max_from: I,
-        first_indexes: &mut StorableVec<I, T2>,
-        last_indexes: &mut StorableVec<I, T2>,
-        exit: &Exit,
-    ) -> Result<()>
-    where
-        T: From<T2>,
-        T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
-        <T2 as TryInto<T>>::Error: error::Error + 'static,
-        F: Fn(&T2) -> T,
-    {
-        self.validate_computed_version_or_reset_file(
-            Version::from(0) + self.version + first_indexes.version + last_indexes.version,
-        )?;
-
-        let index = max_from.min(I::from(self.len()));
-        first_indexes.iter_from(index, |(index, first_index, ..)| {
-            let last_index = last_indexes.read(index)?.unwrap();
-            let count = *last_index + 1_usize - *first_index;
-            self.push_and_flush_if_needed(index, count.into(), exit)
-        })?;
-
-        Ok(self.safe_flush(exit)?)
     }
 }
 

crates/brk_vec/src/traits/bytes.rs

@@ -1,11 +0,0 @@
-use std::sync::Arc;
-
-use crate::Result;
-
-pub trait Bytes: Sized {
-    const LEN: usize = size_of::<Self>();
-    fn to_bytes(&self) -> Arc<[u8]>;
-    fn try_from_bytes(bytes: &[u8]) -> Result<Self>;
-}
-
-pub trait UnsafeBytes {}