From a8901b1097cebf92ce1bef44948a5ce023ae264b Mon Sep 17 00:00:00 2001 From: nym21 Date: Wed, 5 Aug 2026 00:20:26 +0200 Subject: [PATCH] global: part X + 3 --- Cargo.lock | 14 +- crates/brk_computer/examples/computer_read.rs | 9 +- .../src/distribution/metrics/realized/core.rs | 6 +- crates/brk_computer/src/distribution/vecs.rs | 11 +- crates/brk_computer/src/indexes/addr.rs | 82 ++++---- crates/brk_computer/src/indexes/height.rs | 38 ++-- crates/brk_computer/src/indexes/timestamp.rs | 32 ++-- crates/brk_computer/src/indexes/tx_index.rs | 6 +- crates/brk_computer/src/indexes/txin_index.rs | 6 +- .../brk_computer/src/indexes/txout_index.rs | 6 +- .../src/inputs/by_type/compute.rs | 5 +- .../src/internal/containers/constant.rs | 38 ++-- .../brk_computer/src/internal/daily_metric.rs | 66 ++++++- .../brk_computer/src/internal/indexes/lazy.rs | 34 ++-- .../cached_count_cumulative_rolling.rs | 6 +- .../src/internal/per_block/computed/full.rs | 7 +- .../per_block/computed/lazy_distribution.rs | 32 ++-- .../src/internal/per_block/fiat/lazy_block.rs | 6 +- .../per_block/fiat/lazy_rolling_sum.rs | 4 +- .../src/internal/per_block/lazy/base.rs | 28 +-- .../internal/per_block/lazy/transform_last.rs | 6 +- .../internal/per_block/percent/lazy_vec.rs | 10 +- .../src/internal/per_block/rolling/delta.rs | 19 +- .../internal/per_block/value/lazy_block.rs | 10 +- .../per_block/value/lazy_rolling_avg.rs | 6 +- .../per_block/value/lazy_rolling_sum.rs | 6 +- .../per_tx/lazy_distribution_transformed.rs | 6 +- crates/brk_computer/src/internal/value.rs | 18 +- crates/brk_computer/src/lib.rs | 1 + .../brk_computer/src/models/bedrock/price.rs | 2 +- .../src/models/capital_sentiment/compute.rs | 179 +++++++++--------- .../src/models/capital_sentiment/import.rs | 10 +- crates/brk_computer/src/models/compute.rs | 13 +- .../rarity_meter/cached_component_price.rs | 7 +- crates/brk_computer/src/price/compute.rs | 6 +- crates/brk_computer/src/price/ohlcs.rs | 36 ++-- .../src/transactions/size/import.rs | 4 +- crates/brk_query/src/impl/urpd.rs | 4 +- crates/brk_query/src/vecs.rs | 16 +- crates/brk_traversable/README.md | 2 +- crates/brk_traversable/src/lib.rs | 46 +---- crates/brk_types/src/series_count.rs | 10 +- crates/brk_types/src/tree_node.rs | 6 +- crates/lsm-tree/Cargo.toml | 1 + .../src/table/filter/bit_array/reader.rs | 13 +- .../src/table/filter/standard_bloom/mod.rs | 35 +++- crates/lsm-tree/src/tree/inner.rs | 10 +- crates/lsm-tree/src/tree/mod.rs | 13 +- crates/lsm-tree/src/version/super_version.rs | 99 +++++++--- crates/lsm-tree/tests/exclusive_point_read.rs | 44 +++++ crates/vecdb/README.md | 8 +- .../binary_transform}/divide.rs | 0 .../binary_transform}/minus.rs | 0 .../transform => ops/binary_transform}/mod.rs | 2 +- .../binary_transform}/plus.rs | 0 .../binary_transform}/times.rs | 0 crates/vecdb/src/ops/mod.rs | 2 + crates/vecdb/src/variants/cached/mod.rs | 2 +- .../src/variants/eager/compute/aggregates.rs | 39 ++-- .../vecdb/src/variants/lazy/from2/any_vec.rs | 54 ------ crates/vecdb/src/variants/lazy/from2/mod.rs | 95 ---------- .../vecdb/src/variants/lazy/from2/readable.rs | 92 --------- crates/vecdb/src/variants/lazy/from2/typed.rs | 16 -- .../vecdb/src/variants/lazy/from3/any_vec.rs | 61 ------ crates/vecdb/src/variants/lazy/from3/mod.rs | 85 --------- .../vecdb/src/variants/lazy/from3/readable.rs | 101 ---------- crates/vecdb/src/variants/lazy/from3/typed.rs | 19 -- crates/vecdb/src/variants/lazy/mod.rs | 8 +- .../variants/lazy/{from1 => vec}/any_vec.rs | 4 +- .../src/variants/lazy/{from1 => vec}/mod.rs | 16 +- .../lazy/{from1 => vec}/read_only_clone.rs | 4 +- .../variants/lazy/{from1 => vec}/readable.rs | 4 +- .../lazy/{from1 => vec}/transform/halve.rs | 0 .../lazy/{from1 => vec}/transform/ident.rs | 0 .../lazy/{from1 => vec}/transform/mod.rs | 0 .../lazy/{from1 => vec}/transform/negate.rs | 0 .../src/variants/lazy/{from1 => vec}/typed.rs | 4 +- modules/brk-client/index.js | 18 +- packages/brk_client/brk_client/__init__.py | 34 ++-- website/scripts/options/market.js | 17 +- .../options/models/capital-sentiment.js | 28 ++- website/scripts/options/partial.js | 4 +- 82 files changed, 703 insertions(+), 1088 deletions(-) create mode 100644 crates/lsm-tree/tests/exclusive_point_read.rs rename crates/vecdb/src/{variants/lazy/from2/transform => ops/binary_transform}/divide.rs (100%) rename crates/vecdb/src/{variants/lazy/from2/transform => ops/binary_transform}/minus.rs (100%) rename crates/vecdb/src/{variants/lazy/from2/transform => ops/binary_transform}/mod.rs (82%) rename crates/vecdb/src/{variants/lazy/from2/transform => ops/binary_transform}/plus.rs (100%) rename crates/vecdb/src/{variants/lazy/from2/transform => ops/binary_transform}/times.rs (100%) delete mode 100644 crates/vecdb/src/variants/lazy/from2/any_vec.rs delete mode 100644 crates/vecdb/src/variants/lazy/from2/mod.rs delete mode 100644 crates/vecdb/src/variants/lazy/from2/readable.rs delete mode 100644 crates/vecdb/src/variants/lazy/from2/typed.rs delete mode 100644 crates/vecdb/src/variants/lazy/from3/any_vec.rs delete mode 100644 crates/vecdb/src/variants/lazy/from3/mod.rs delete mode 100644 crates/vecdb/src/variants/lazy/from3/readable.rs delete mode 100644 crates/vecdb/src/variants/lazy/from3/typed.rs rename crates/vecdb/src/variants/lazy/{from1 => vec}/any_vec.rs (89%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/mod.rs (79%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/read_only_clone.rs (70%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/readable.rs (95%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/transform/halve.rs (100%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/transform/ident.rs (100%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/transform/mod.rs (100%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/transform/negate.rs (100%) rename crates/vecdb/src/variants/lazy/{from1 => vec}/typed.rs (64%) diff --git a/Cargo.lock b/Cargo.lock index 6d036783f..25b66a377 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,6 +155,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayvec" version = "0.7.8" @@ -2317,6 +2326,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" name = "lsm-tree" version = "0.3.6" dependencies = [ + "arc-swap", "byteorder-lite", "byteview", "criterion", @@ -2967,9 +2977,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", diff --git a/crates/brk_computer/examples/computer_read.rs b/crates/brk_computer/examples/computer_read.rs index e7270158e..126333947 100644 --- a/crates/brk_computer/examples/computer_read.rs +++ b/crates/brk_computer/examples/computer_read.rs @@ -27,12 +27,9 @@ pub fn main() -> Result<()> { ); println!("Time for BytesVec write_json: {:?}", start.elapsed()); - // Test empty_addr_index (LazyVecFrom1 wrapper) - computed access + // Test empty_addr_index (LazyVec wrapper) - computed access let empty_index = &computer.distribution.addrs.empty_index; - println!( - "\nempty_addr_index (LazyVecFrom1) len: {}", - empty_index.len() - ); + println!("\nempty_addr_index (LazyVec) len: {}", empty_index.len()); let start = Instant::now(); let mut buf = Vec::new(); @@ -45,7 +42,7 @@ pub fn main() -> Result<()> { "empty_addr_index last item JSON: {}", String::from_utf8_lossy(&buf) ); - println!("Time for LazyVecFrom1 write_json: {:?}", start.elapsed()); + println!("Time for LazyVec write_json: {:?}", start.elapsed()); // Compare with funded versions let funded_data = &computer.distribution.addrs_data.funded; diff --git a/crates/brk_computer/src/distribution/metrics/realized/core.rs b/crates/brk_computer/src/distribution/metrics/realized/core.rs index 7addefc82..1504fa097 100644 --- a/crates/brk_computer/src/distribution/metrics/realized/core.rs +++ b/crates/brk_computer/src/distribution/metrics/realized/core.rs @@ -5,7 +5,7 @@ use brk_types::{ Bitcoin, Cents, CentsSigned, Dollars, Height, PartsPerMillionSigned64, StoredF64, Version, }; use derive_more::{Deref, DerefMut}; -use vecdb::{AnyStoredVec, Exit, LazyVecFrom1, ReadableCloneableVec, ReadableVec, Rw, StorageMode}; +use vecdb::{AnyStoredVec, Exit, LazyVec, ReadableCloneableVec, ReadableVec, Rw, StorageMode}; use crate::{ distribution::state::{CohortState, CostBasisOps, RealizedOps}, @@ -23,7 +23,7 @@ use super::RealizedMinimal; #[derive(Clone, Traversable)] pub struct NegRealizedLoss { #[traversable(flatten)] - pub base: LazyVecFrom1, + pub base: LazyVec, pub sum: Windows>, } @@ -57,7 +57,7 @@ impl RealizedCore { let minimal = RealizedMinimal::forced_import(cfg)?; - let neg_loss_base = LazyVecFrom1::transformed::( + let neg_loss_base = LazyVec::transformed::( &cfg.name("realized_loss_neg"), cfg.version + Version::ONE, minimal.loss.block.cents.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/distribution/vecs.rs b/crates/brk_computer/src/distribution/vecs.rs index 4c0fa762f..da801421e 100644 --- a/crates/brk_computer/src/distribution/vecs.rs +++ b/crates/brk_computer/src/distribution/vecs.rs @@ -11,7 +11,7 @@ use brk_types::{ use rayon::prelude::*; use tracing::{debug, info}; use vecdb::{ - AnyStoredVec, AnyVec, BytesVec, Database, Exit, ImportOptions, ImportableVec, LazyVecFrom1, + AnyStoredVec, AnyVec, BytesVec, Database, Exit, ImportOptions, ImportableVec, LazyVec, ReadableCloneableVec, ReadableVec, Rw, Stamp, StorageMode, WritableVec, }; @@ -55,10 +55,9 @@ pub struct AddrMetricsVecs { pub delta: DeltaVecs, pub avg_amount: AvgAmountVecs, #[traversable(wrap = "indexes", rename = "funded")] - pub funded_index: - LazyVecFrom1, + pub funded_index: LazyVec, #[traversable(wrap = "indexes", rename = "empty")] - pub empty_index: LazyVecFrom1, + pub empty_index: LazyVec, } impl AddrMetricsVecs { @@ -230,13 +229,13 @@ impl Vecs { )?; // Identity mappings for traversable - let funded_addr_index = LazyVecFrom1::init( + let funded_addr_index = LazyVec::init( "funded_addr_index", funded_addr_data_version, funded_addr_index_to_funded_addr_data.read_only_boxed_clone(), |index, _| index, ); - let empty_addr_index = LazyVecFrom1::init( + let empty_addr_index = LazyVec::init( "empty_addr_index", version, empty_addr_index_to_empty_addr_data.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/indexes/addr.rs b/crates/brk_computer/src/indexes/addr.rs index b1c8f7463..2fbdc4019 100644 --- a/crates/brk_computer/src/indexes/addr.rs +++ b/crates/brk_computer/src/indexes/addr.rs @@ -6,7 +6,7 @@ use brk_types::{ P2SHAddrIndex, P2SHBytes, P2TRAddrIndex, P2TRBytes, P2WPKHAddrIndex, P2WPKHBytes, P2WSHAddrIndex, P2WSHBytes, TxIndex, UnknownOutputIndex, Version, }; -use vecdb::{LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{LazyVec, ReadableCloneableVec}; #[derive(Clone, Traversable)] pub struct Vecs { @@ -26,83 +26,83 @@ pub struct Vecs { #[derive(Clone, Traversable)] pub struct P2PK33Vecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2PK65Vecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2PKHVecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2SHVecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2TRVecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2WPKHVecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2WSHVecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2AVecs { - pub identity: LazyVecFrom1, - pub addr: LazyVecFrom1, + pub identity: LazyVec, + pub addr: LazyVec, } #[derive(Clone, Traversable)] pub struct P2MSVecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, } #[derive(Clone, Traversable)] pub struct EmptyVecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, } #[derive(Clone, Traversable)] pub struct UnknownVecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, } #[derive(Clone, Traversable)] pub struct OpReturnVecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, } impl Vecs { pub(crate) fn forced_import(version: Version, indexer: &Indexer) -> Self { Self { p2pk33: P2PK33Vecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2pk33_addr_index", version, indexer.vecs.addrs.p2pk33.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2pk33_addr", version, indexer.vecs.addrs.p2pk33.bytes.read_only_boxed_clone(), @@ -110,13 +110,13 @@ impl Vecs { ), }, p2pk65: P2PK65Vecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2pk65_addr_index", version, indexer.vecs.addrs.p2pk65.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2pk65_addr", version, indexer.vecs.addrs.p2pk65.bytes.read_only_boxed_clone(), @@ -124,13 +124,13 @@ impl Vecs { ), }, p2pkh: P2PKHVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2pkh_addr_index", version, indexer.vecs.addrs.p2pkh.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2pkh_addr", version, indexer.vecs.addrs.p2pkh.bytes.read_only_boxed_clone(), @@ -138,13 +138,13 @@ impl Vecs { ), }, p2sh: P2SHVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2sh_addr_index", version, indexer.vecs.addrs.p2sh.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2sh_addr", version, indexer.vecs.addrs.p2sh.bytes.read_only_boxed_clone(), @@ -152,13 +152,13 @@ impl Vecs { ), }, p2tr: P2TRVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2tr_addr_index", version, indexer.vecs.addrs.p2tr.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2tr_addr", version, indexer.vecs.addrs.p2tr.bytes.read_only_boxed_clone(), @@ -166,13 +166,13 @@ impl Vecs { ), }, p2wpkh: P2WPKHVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2wpkh_addr_index", version, indexer.vecs.addrs.p2wpkh.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2wpkh_addr", version, indexer.vecs.addrs.p2wpkh.bytes.read_only_boxed_clone(), @@ -180,13 +180,13 @@ impl Vecs { ), }, p2wsh: P2WSHVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2wsh_addr_index", version, indexer.vecs.addrs.p2wsh.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2wsh_addr", version, indexer.vecs.addrs.p2wsh.bytes.read_only_boxed_clone(), @@ -194,13 +194,13 @@ impl Vecs { ), }, p2a: P2AVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2a_addr_index", version, indexer.vecs.addrs.p2a.bytes.read_only_boxed_clone(), |index, _| index, ), - addr: LazyVecFrom1::init( + addr: LazyVec::init( "p2a_addr", version, indexer.vecs.addrs.p2a.bytes.read_only_boxed_clone(), @@ -208,7 +208,7 @@ impl Vecs { ), }, p2ms: P2MSVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "p2ms_output_index", version, indexer @@ -221,7 +221,7 @@ impl Vecs { ), }, empty: EmptyVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "empty_output_index", version, indexer @@ -234,7 +234,7 @@ impl Vecs { ), }, unknown: UnknownVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "unknown_output_index", version, indexer @@ -247,7 +247,7 @@ impl Vecs { ), }, op_return: OpReturnVecs { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "op_return_index", version, indexer.vecs.op_return.to_tx_index.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/indexes/height.rs b/crates/brk_computer/src/indexes/height.rs index 678445a5a..14a74f8ee 100644 --- a/crates/brk_computer/src/indexes/height.rs +++ b/crates/brk_computer/src/indexes/height.rs @@ -3,29 +3,27 @@ use brk_types::{ Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Minute10, Minute30, Month1, Month3, Month6, StoredU64, Timestamp, Version, Week1, Year1, Year10, }; -use vecdb::{ - CachedBoxedVec, CachedReadableVec, CachedVec, LazyVecFrom1, ReadableBoxedVec, VecValue, -}; +use vecdb::{CachedBoxedVec, CachedReadableVec, CachedVec, LazyVec, ReadableBoxedVec, VecValue}; use crate::internal::LazyPreviousDeltaVec; #[derive(Clone, Traversable)] pub struct Vecs { - pub minute10: LazyVecFrom1, - pub minute30: LazyVecFrom1, - pub hour1: LazyVecFrom1, - pub hour4: LazyVecFrom1, - pub hour12: LazyVecFrom1, - pub day1: CachedVec>, - pub day3: LazyVecFrom1, - pub epoch: LazyVecFrom1, - pub halving: LazyVecFrom1, - pub week1: LazyVecFrom1, - pub month1: LazyVecFrom1, - pub month3: LazyVecFrom1, - pub month6: LazyVecFrom1, - pub year1: LazyVecFrom1, - pub year10: LazyVecFrom1, + pub minute10: LazyVec, + pub minute30: LazyVec, + pub hour1: LazyVec, + pub hour4: LazyVec, + pub hour12: LazyVec, + pub day1: CachedVec>, + pub day3: LazyVec, + pub epoch: LazyVec, + pub halving: LazyVec, + pub week1: LazyVec, + pub month1: LazyVec, + pub month3: LazyVec, + pub month6: LazyVec, + pub year1: LazyVec, + pub year10: LazyVec, pub tx_index_count: LazyPreviousDeltaVec, } @@ -91,8 +89,8 @@ impl Vecs { name: &str, timestamps: ReadableBoxedVec, compute: fn(Height, Timestamp) -> T, - ) -> LazyVecFrom1 { - LazyVecFrom1::init(name, Version::ZERO, timestamps, compute) + ) -> LazyVec { + LazyVec::init(name, Version::ZERO, timestamps, compute) } pub(crate) fn day1_from_timestamp(timestamp: Timestamp) -> Day1 { diff --git a/crates/brk_computer/src/indexes/timestamp.rs b/crates/brk_computer/src/indexes/timestamp.rs index 60f528858..79e64f4e4 100644 --- a/crates/brk_computer/src/indexes/timestamp.rs +++ b/crates/brk_computer/src/indexes/timestamp.rs @@ -8,8 +8,8 @@ use brk_types::{ }; use derive_more::{Deref, DerefMut}; use vecdb::{ - AnyVec, CachedVec, Database, EagerVec, Exit, ImportableVec, LazyVecFrom1, PcoVec, - ReadableBoxedVec, ReadableVec, Rw, StorageMode, Version, + AnyVec, CachedVec, Database, EagerVec, Exit, ImportableVec, LazyVec, PcoVec, ReadableBoxedVec, + ReadableVec, Rw, StorageMode, Version, }; use crate::internal::PerResolution; @@ -30,19 +30,19 @@ pub struct Timestamps { #[traversable(flatten)] #[allow(clippy::type_complexity)] pub resolutions: PerResolution< - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, BoundaryTimestampVec, BoundaryTimestampVec, >, @@ -81,7 +81,7 @@ impl Timestamps { ) -> Self { macro_rules! period { ($field:ident) => { - LazyVecFrom1::init( + LazyVec::init( "timestamp", version, $field.first_height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/indexes/tx_index.rs b/crates/brk_computer/src/indexes/tx_index.rs index 3050bfac8..23b033c29 100644 --- a/crates/brk_computer/src/indexes/tx_index.rs +++ b/crates/brk_computer/src/indexes/tx_index.rs @@ -1,13 +1,13 @@ use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{TxIndex, Txid, Version}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{LazyVec, ReadableCloneableVec}; use crate::internal::LazyIndexCountVec; #[derive(Clone, Traversable)] pub struct Vecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, pub input_count: LazyIndexCountVec, pub output_count: LazyIndexCountVec, } @@ -15,7 +15,7 @@ pub struct Vecs { impl Vecs { pub(crate) fn new(version: Version, indexer: &Indexer) -> Self { Self { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "tx_index", version, indexer.vecs.transactions.txid.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/indexes/txin_index.rs b/crates/brk_computer/src/indexes/txin_index.rs index ddf6e0f94..a89d970b7 100644 --- a/crates/brk_computer/src/indexes/txin_index.rs +++ b/crates/brk_computer/src/indexes/txin_index.rs @@ -1,17 +1,17 @@ use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{OutPoint, TxInIndex, Version}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{LazyVec, ReadableCloneableVec}; #[derive(Clone, Traversable)] pub struct Vecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, } impl Vecs { pub(crate) fn forced_import(version: Version, indexer: &Indexer) -> Self { Self { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "txin_index", version, indexer.vecs.inputs.outpoint.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/indexes/txout_index.rs b/crates/brk_computer/src/indexes/txout_index.rs index f4d33ba5c..a613cc797 100644 --- a/crates/brk_computer/src/indexes/txout_index.rs +++ b/crates/brk_computer/src/indexes/txout_index.rs @@ -1,17 +1,17 @@ use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{Sats, TxOutIndex, Version}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{LazyVec, ReadableCloneableVec}; #[derive(Clone, Traversable)] pub struct Vecs { - pub identity: LazyVecFrom1, + pub identity: LazyVec, } impl Vecs { pub(crate) fn forced_import(version: Version, indexer: &Indexer) -> Self { Self { - identity: LazyVecFrom1::init( + identity: LazyVec::init( "txout_index", version, indexer.vecs.outputs.value.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/inputs/by_type/compute.rs b/crates/brk_computer/src/inputs/by_type/compute.rs index f2f361f14..e0a104ca7 100644 --- a/crates/brk_computer/src/inputs/by_type/compute.rs +++ b/crates/brk_computer/src/inputs/by_type/compute.rs @@ -53,10 +53,9 @@ impl Vecs { }; itype_cursor.advance(fi_in - itype_cursor.position()); - for _ in fi_in..next_fi_in { - let otype = itype_cursor.next().unwrap(); + itype_cursor.for_each(next_fi_in - fi_in, |otype| { per_tx[otype as usize] += 1; - } + }); Ok(()) }, |agg| { diff --git a/crates/brk_computer/src/internal/containers/constant.rs b/crates/brk_computer/src/internal/containers/constant.rs index 269b2b643..fdc7c0ca9 100644 --- a/crates/brk_computer/src/internal/containers/constant.rs +++ b/crates/brk_computer/src/internal/containers/constant.rs @@ -5,7 +5,7 @@ use brk_types::{ }; use schemars::JsonSchema; use serde::Serialize; -use vecdb::{Formattable, LazyVecFrom1, ReadableCloneableVec, UnaryTransform, VecValue}; +use vecdb::{Formattable, LazyVec, ReadableCloneableVec, UnaryTransform, VecValue}; use crate::indexes; @@ -15,22 +15,22 @@ pub struct ConstantVecs where T: VecValue + Formattable + Serialize + JsonSchema, { - pub height: LazyVecFrom1, - pub minute10: LazyVecFrom1, - pub minute30: LazyVecFrom1, - pub hour1: LazyVecFrom1, - pub hour4: LazyVecFrom1, - pub hour12: LazyVecFrom1, - pub day1: LazyVecFrom1, - pub day3: LazyVecFrom1, - pub week1: LazyVecFrom1, - pub month1: LazyVecFrom1, - pub month3: LazyVecFrom1, - pub month6: LazyVecFrom1, - pub year1: LazyVecFrom1, - pub year10: LazyVecFrom1, - pub halving: LazyVecFrom1, - pub epoch: LazyVecFrom1, + pub height: LazyVec, + pub minute10: LazyVec, + pub minute30: LazyVec, + pub hour1: LazyVec, + pub hour4: LazyVec, + pub hour12: LazyVec, + pub day1: LazyVec, + pub day3: LazyVec, + pub week1: LazyVec, + pub month1: LazyVec, + pub month3: LazyVec, + pub month6: LazyVec, + pub year1: LazyVec, + pub year10: LazyVec, + pub halving: LazyVec, + pub epoch: LazyVec, } impl ConstantVecs { @@ -55,7 +55,7 @@ impl ConstantVecs { { macro_rules! period { ($idx:ident) => { - LazyVecFrom1::init( + LazyVec::init( name, version, indexes.$idx.first_height.read_only_boxed_clone(), @@ -65,7 +65,7 @@ impl ConstantVecs { } Self { - height: LazyVecFrom1::init( + height: LazyVec::init( name, version, indexes.height.minute10.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/daily_metric.rs b/crates/brk_computer/src/internal/daily_metric.rs index db7d9cd9b..adfda6df4 100644 --- a/crates/brk_computer/src/internal/daily_metric.rs +++ b/crates/brk_computer/src/internal/daily_metric.rs @@ -9,15 +9,15 @@ use brk_types::{ use schemars::JsonSchema; use serde::Serialize; use vecdb::{ - AnyExportableVec, AnyVec, Database, EagerVec, Formattable, ImportableVec, LazyVecFrom1, - PcoVec, PcoVecValue, ReadableBoxedVec, ReadableCloneableVec, ReadableVec, Rw, StorageMode, - TypedVec, UnaryTransform, VecIndex, VecValue, short_type_name, + AnyExportableVec, AnyVec, Database, EagerVec, Formattable, ImportableVec, LazyVec, PcoVec, + PcoVecValue, ReadableBoxedVec, ReadableCloneableVec, ReadableVec, Rw, StorageMode, TypedVec, + UnaryTransform, VecIndex, VecValue, short_type_name, }; use crate::indexes; type StoredDay = ::Stored>>; -type DayMapping = LazyVecFrom1; +type DayMapping = LazyVec; type Repeated = DailyView; type Last = DailyView; @@ -49,7 +49,7 @@ pub(crate) struct DailyMappings { impl DailyMappings { pub(crate) fn new(indexes: &indexes::Vecs) -> Self { - let height = LazyVecFrom1::init( + let height = LazyVec::init( "day1", Version::ZERO, indexes.height.day1_read_only_boxed_clone(), @@ -79,13 +79,13 @@ impl DailyMappings { fn timestamp_mapping( source: ReadableBoxedVec, ) -> DayMapping { - LazyVecFrom1::init("day1", Version::ZERO, source, |_, timestamp| { + LazyVec::init("day1", Version::ZERO, source, |_, timestamp| { Day1::try_from(Date::from(timestamp)).unwrap_or_default() }) } fn date_mapping(source: ReadableBoxedVec) -> DayMapping { - LazyVecFrom1::init("day1", Version::ZERO, source, |_, date| { + LazyVec::init("day1", Version::ZERO, source, |_, date| { Day1::try_from(date).unwrap_or_default() }) } @@ -172,7 +172,7 @@ where } } -type LazyDay = LazyVecFrom1; +type LazyDay = LazyVec; #[derive(Clone, Traversable)] #[traversable(merge)] @@ -200,7 +200,7 @@ where where F: UnaryTransform, { - let day1 = LazyVecFrom1::transformed::(name, version, source); + let day1 = LazyVec::transformed::(name, version, source); let views = Box::new(DailyViews::new( name, day1.read_only_boxed_clone(), @@ -540,7 +540,7 @@ fn last_source_index(mapping: &[Day1], index: usize, source_len: usize) -> Optio #[cfg(test)] mod tests { use super::*; - use brk_types::StoredF64; + use brk_types::{StoredBool, StoredF64}; use vecdb::{AnyStoredVec, WritableVec}; #[test] @@ -611,4 +611,50 @@ mod tests { drop(db); std::fs::remove_dir_all(path).unwrap(); } + + #[test] + fn repeated_view_supports_stored_booleans() { + let suffix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = + std::env::temp_dir().join(format!("brk-daily-bool-{}-{suffix}", std::process::id())); + let db = Database::open(&path).unwrap(); + + let mut source: EagerVec> = + EagerVec::forced_import(&db, "source", Version::ONE).unwrap(); + let mut mapping: EagerVec> = + EagerVec::forced_import(&db, "mapping", Version::ONE).unwrap(); + source.push(StoredBool::FALSE); + source.push(StoredBool::TRUE); + for day in [0, 0, 1, 1] { + mapping.push(Day1::from(day)); + } + source.write().unwrap(); + mapping.write().unwrap(); + + let view = DailyView::::new( + "test", + Version::ONE, + source.read_only_boxed_clone(), + mapping.read_only_boxed_clone(), + ); + + assert_eq!( + view.collect_range_at(0, 4), + vec![ + Some(StoredBool::FALSE), + Some(StoredBool::FALSE), + Some(StoredBool::TRUE), + Some(StoredBool::TRUE), + ] + ); + + drop(view); + drop(mapping); + drop(source); + drop(db); + std::fs::remove_dir_all(path).unwrap(); + } } diff --git a/crates/brk_computer/src/internal/indexes/lazy.rs b/crates/brk_computer/src/internal/indexes/lazy.rs index 6959f1caa..7ad9dfa08 100644 --- a/crates/brk_computer/src/internal/indexes/lazy.rs +++ b/crates/brk_computer/src/internal/indexes/lazy.rs @@ -5,7 +5,7 @@ use brk_types::{ }; use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; -use vecdb::{LazyVecFrom1, ReadableCloneableVec, UnaryTransform, VecValue}; +use vecdb::{LazyVec, ReadableCloneableVec, UnaryTransform, VecValue}; use crate::internal::{ComputedVecValue, PerResolution}; @@ -14,21 +14,21 @@ use crate::internal::{ComputedVecValue, PerResolution}; pub struct LazyIndexes( #[allow(clippy::type_complexity)] pub PerResolution< - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, >, ) where @@ -52,7 +52,7 @@ where { macro_rules! period { ($idx:ident) => { - LazyVecFrom1::transformed::( + LazyVec::transformed::( name, version, source.$idx.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/computed/cached_count_cumulative_rolling.rs b/crates/brk_computer/src/internal/per_block/computed/cached_count_cumulative_rolling.rs index d4366d459..47e8bd169 100644 --- a/crates/brk_computer/src/internal/per_block/computed/cached_count_cumulative_rolling.rs +++ b/crates/brk_computer/src/internal/per_block/computed/cached_count_cumulative_rolling.rs @@ -2,7 +2,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Height, StoredU16, StoredU64, Version}; use vecdb::{ - AnyStoredVec, AnyVec, CachedVec, Database, EagerVec, ImportableVec, LazyVecFrom1, PcoVec, Rw, + AnyStoredVec, AnyVec, CachedVec, Database, EagerVec, ImportableVec, LazyVec, PcoVec, Rw, StorageMode, WritableVec, }; @@ -16,7 +16,7 @@ use crate::{ #[derive(Traversable)] pub struct CachedCountPerBlockCumulativeRolling { - pub block: LazyVecFrom1, + pub block: LazyVec, pub cumulative: LazyPerBlock, pub sum: LazyRollingSumsFromHeight, pub average: LazyRollingAvgsFromHeight, @@ -36,7 +36,7 @@ impl CachedCountPerBlockCumulativeRolling { ) -> Result { let source = CachedVec::wrap(EagerVec::forced_import(db, name, version)?); let cached_cumulative = CachedBlockCountReader::new(source.read_only_cached_boxed_clone()); - let block = LazyVecFrom1::transformed::( + let block = LazyVec::transformed::( name, version, source.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/computed/full.rs b/crates/brk_computer/src/internal/per_block/computed/full.rs index 06e5507dd..258302eb2 100644 --- a/crates/brk_computer/src/internal/per_block/computed/full.rs +++ b/crates/brk_computer/src/internal/per_block/computed/full.rs @@ -5,7 +5,7 @@ use brk_traversable::Traversable; use brk_types::{Height, Version}; use schemars::JsonSchema; use vecdb::{ - Database, Exit, LazyVecFrom1, ReadableCloneableVec, ReadableVec, Rw, StorageMode, VecValue, + Database, Exit, LazyVec, ReadableCloneableVec, ReadableVec, Rw, StorageMode, VecValue, }; use crate::{ @@ -21,7 +21,7 @@ where T: NumericValue + JsonSchema, S: VecValue, { - pub block: LazyVecFrom1, + pub block: LazyVec, pub cumulative: PerBlock, #[traversable(flatten)] pub rolling: RollingComplete, @@ -41,8 +41,7 @@ where indexes: &indexes::Vecs, cached_starts: &Windows<&CachedWindowStartVec>, ) -> Result { - let block = - LazyVecFrom1::init(name, version, source.read_only_boxed_clone(), compute_block); + let block = LazyVec::init(name, version, source.read_only_boxed_clone(), compute_block); let cumulative = PerBlock::forced_import(db, &format!("{name}_cumulative"), version, indexes)?; let rolling = RollingComplete::forced_import( diff --git a/crates/brk_computer/src/internal/per_block/computed/lazy_distribution.rs b/crates/brk_computer/src/internal/per_block/computed/lazy_distribution.rs index e21d6957b..c05042700 100644 --- a/crates/brk_computer/src/internal/per_block/computed/lazy_distribution.rs +++ b/crates/brk_computer/src/internal/per_block/computed/lazy_distribution.rs @@ -1,11 +1,11 @@ use brk_traversable::Traversable; use brk_types::{Height, Version}; use schemars::JsonSchema; -use vecdb::{LazyVecFrom1, ReadableCloneableVec, UnaryTransform, VecIndex}; +use vecdb::{LazyVec, ReadableCloneableVec, UnaryTransform, VecIndex}; use crate::internal::{ComputedVecValue, DistributionStats, PerBlockDistribution}; -/// Lazy analog of `Distribution`: 7 `LazyVecFrom1` fields, +/// Lazy analog of `Distribution`: 7 `LazyVec` fields, /// each derived by transforming the corresponding field of a source `PerBlockDistribution`. #[derive(Clone, Traversable)] pub struct LazyDistribution @@ -14,13 +14,13 @@ where T: ComputedVecValue + JsonSchema, S1T: ComputedVecValue, { - pub min: LazyVecFrom1, - pub max: LazyVecFrom1, - pub pct10: LazyVecFrom1, - pub pct25: LazyVecFrom1, - pub median: LazyVecFrom1, - pub pct75: LazyVecFrom1, - pub pct90: LazyVecFrom1, + pub min: LazyVec, + pub max: LazyVec, + pub pct10: LazyVec, + pub pct25: LazyVec, + pub median: LazyVec, + pub pct75: LazyVec, + pub pct90: LazyVec, } impl LazyDistribution @@ -35,37 +35,37 @@ where ) -> Self { let s = DistributionStats::<()>::SUFFIXES; Self { - min: LazyVecFrom1::transformed::( + min: LazyVec::transformed::( &format!("{name}_{}", s[0]), version, source.min.height.read_only_boxed_clone(), ), - max: LazyVecFrom1::transformed::( + max: LazyVec::transformed::( &format!("{name}_{}", s[1]), version, source.max.height.read_only_boxed_clone(), ), - pct10: LazyVecFrom1::transformed::( + pct10: LazyVec::transformed::( &format!("{name}_{}", s[2]), version, source.pct10.height.read_only_boxed_clone(), ), - pct25: LazyVecFrom1::transformed::( + pct25: LazyVec::transformed::( &format!("{name}_{}", s[3]), version, source.pct25.height.read_only_boxed_clone(), ), - median: LazyVecFrom1::transformed::( + median: LazyVec::transformed::( &format!("{name}_{}", s[4]), version, source.median.height.read_only_boxed_clone(), ), - pct75: LazyVecFrom1::transformed::( + pct75: LazyVec::transformed::( &format!("{name}_{}", s[5]), version, source.pct75.height.read_only_boxed_clone(), ), - pct90: LazyVecFrom1::transformed::( + pct90: LazyVec::transformed::( &format!("{name}_{}", s[6]), version, source.pct90.height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/fiat/lazy_block.rs b/crates/brk_computer/src/internal/per_block/fiat/lazy_block.rs index 19364c3ef..8428b74c4 100644 --- a/crates/brk_computer/src/internal/per_block/fiat/lazy_block.rs +++ b/crates/brk_computer/src/internal/per_block/fiat/lazy_block.rs @@ -1,13 +1,13 @@ use brk_traversable::Traversable; use brk_types::{Dollars, Height, Version}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{LazyVec, ReadableCloneableVec}; use crate::internal::{FiatPerBlock, FiatType, LazyPreviousDeltaVec}; /// Per-block fiat data derived from stored cumulative cents. #[derive(Clone, Traversable)] pub struct LazyFiatBlock { - pub usd: LazyVecFrom1, + pub usd: LazyVec, pub cents: LazyPreviousDeltaVec, } @@ -23,7 +23,7 @@ impl LazyFiatBlock { cumulative.cents.height.read_only_boxed_clone(), ); let usd = - LazyVecFrom1::transformed::(name, version, cents.read_only_boxed_clone()); + LazyVec::transformed::(name, version, cents.read_only_boxed_clone()); Self { usd, cents } } diff --git a/crates/brk_computer/src/internal/per_block/fiat/lazy_rolling_sum.rs b/crates/brk_computer/src/internal/per_block/fiat/lazy_rolling_sum.rs index c98ca756a..19783f350 100644 --- a/crates/brk_computer/src/internal/per_block/fiat/lazy_rolling_sum.rs +++ b/crates/brk_computer/src/internal/per_block/fiat/lazy_rolling_sum.rs @@ -1,7 +1,7 @@ use brk_traversable::Traversable; use brk_types::{Dollars, Height, Version}; use derive_more::{Deref, DerefMut}; -use vecdb::{DeltaSub, LazyDeltaVec, LazyVecFrom1, ReadOnlyClone, ReadableCloneableVec}; +use vecdb::{DeltaSub, LazyDeltaVec, LazyVec, ReadOnlyClone, ReadableCloneableVec}; use crate::{ indexes, @@ -55,7 +55,7 @@ impl LazyRollingSumsFiatFromHeight { }; let usd = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &full_name, version, cents.height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/lazy/base.rs b/crates/brk_computer/src/internal/per_block/lazy/base.rs index 64ea9cb92..8ecdb779c 100644 --- a/crates/brk_computer/src/internal/per_block/lazy/base.rs +++ b/crates/brk_computer/src/internal/per_block/lazy/base.rs @@ -3,7 +3,7 @@ use brk_types::{Height, Version}; use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; use vecdb::{ - LazyVecFrom1, PcoVecValue, ReadOnlyClone, ReadableBoxedVec, ReadableCloneableVec, ReadableVec, + LazyVec, PcoVecValue, ReadOnlyClone, ReadableBoxedVec, ReadableCloneableVec, ReadableVec, TypedVec, UnaryTransform, VecValue, }; @@ -21,7 +21,7 @@ where T: VecValue + PartialOrd + JsonSchema, S1T: VecValue, { - pub height: LazyVecFrom1, + pub height: LazyVec, #[deref] #[deref_mut] #[traversable(flatten)] @@ -40,7 +40,7 @@ where resolutions: &Resolutions, ) -> Self { Self { - height: LazyVecFrom1::transformed::(name, version, height_source), + height: LazyVec::transformed::(name, version, height_source), resolutions: Box::new(DerivedResolutions::from_derived_computed::( name, version, @@ -84,11 +84,7 @@ where V: TypedVec + ReadableVec + Clone + 'static, { Self { - height: LazyVecFrom1::transformed::( - name, - version, - height_source.read_only_boxed_clone(), - ), + height: LazyVec::transformed::(name, version, height_source.read_only_boxed_clone()), resolutions: Box::new(DerivedResolutions::from_height_source::( name, version, @@ -114,11 +110,7 @@ where Resolutions::forced_import_uncached(name, height_source.clone(), version, indexes); Self { - height: LazyVecFrom1::transformed::( - name, - version, - height_source.read_only_boxed_clone(), - ), + height: LazyVec::transformed::(name, version, height_source.read_only_boxed_clone()), resolutions: Box::new(DerivedResolutions::from_derived_computed::( name, version, @@ -138,11 +130,7 @@ where S2T: ComputedVecValue + JsonSchema, { Self { - height: LazyVecFrom1::transformed::( - name, - version, - source.height.read_only_boxed_clone(), - ), + height: LazyVec::transformed::(name, version, source.height.read_only_boxed_clone()), resolutions: Box::new(DerivedResolutions::from_lazy::( name, version, @@ -167,7 +155,7 @@ where where S: VecValue, { - let indexed = LazyVecFrom1::init( + let indexed = LazyVec::init( &format!("{name}_source"), version, source.read_only_boxed_clone(), @@ -188,7 +176,7 @@ where where S: VecValue, { - let indexed = LazyVecFrom1::init( + let indexed = LazyVec::init( &format!("{name}_source"), version, source.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/lazy/transform_last.rs b/crates/brk_computer/src/internal/per_block/lazy/transform_last.rs index 6d496cb6e..9ca856176 100644 --- a/crates/brk_computer/src/internal/per_block/lazy/transform_last.rs +++ b/crates/brk_computer/src/internal/per_block/lazy/transform_last.rs @@ -1,13 +1,13 @@ use brk_traversable::Traversable; use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; -use vecdb::{LazyVecFrom1, ReadableBoxedVec, UnaryTransform, VecIndex, VecValue}; +use vecdb::{LazyVec, ReadableBoxedVec, UnaryTransform, VecIndex, VecValue}; use brk_types::Version; #[derive(Clone, Deref, DerefMut, Traversable)] #[traversable(transparent)] -pub struct LazyTransformLast(pub LazyVecFrom1) +pub struct LazyTransformLast(pub LazyVec) where I: VecIndex, T: VecValue + PartialOrd + JsonSchema, @@ -24,6 +24,6 @@ where version: Version, source: ReadableBoxedVec, ) -> Self { - Self(LazyVecFrom1::transformed::(name, version, source)) + Self(LazyVec::transformed::(name, version, source)) } } diff --git a/crates/brk_computer/src/internal/per_block/percent/lazy_vec.rs b/crates/brk_computer/src/internal/per_block/percent/lazy_vec.rs index 7074a5fec..0fd55817b 100644 --- a/crates/brk_computer/src/internal/per_block/percent/lazy_vec.rs +++ b/crates/brk_computer/src/internal/per_block/percent/lazy_vec.rs @@ -1,7 +1,7 @@ use brk_traversable::Traversable; use brk_types::{Height, StoredF32, Version}; use derive_more::{Deref, DerefMut}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec, VecValue}; +use vecdb::{LazyVec, ReadableCloneableVec, VecValue}; use crate::internal::{FixedRatio, Percent}; @@ -10,7 +10,7 @@ use crate::internal::{FixedRatio, Percent}; #[traversable(transparent)] #[allow(clippy::type_complexity)] pub struct LazyPercentVec( - pub Percent, LazyVecFrom1>, + pub Percent, LazyVec>, ); impl LazyPercentVec { @@ -20,19 +20,19 @@ impl LazyPercentVec { source: &(impl ReadableCloneableVec + 'static), compute: fn(Height, S) -> B, ) -> Self { - let ppm = LazyVecFrom1::init( + let ppm = LazyVec::init( &format!("{name}_{}", B::SUFFIX), version, source.read_only_boxed_clone(), compute, ); let ppm_source = ppm.read_only_boxed_clone(); - let ratio = LazyVecFrom1::transformed::( + let ratio = LazyVec::transformed::( &format!("{name}_ratio"), version, ppm_source.clone(), ); - let percent = LazyVecFrom1::transformed::(name, version, ppm_source); + let percent = LazyVec::transformed::(name, version, ppm_source); Self(Percent { ppm, diff --git a/crates/brk_computer/src/internal/per_block/rolling/delta.rs b/crates/brk_computer/src/internal/per_block/rolling/delta.rs index b4ccb8731..11ac89893 100644 --- a/crates/brk_computer/src/internal/per_block/rolling/delta.rs +++ b/crates/brk_computer/src/internal/per_block/rolling/delta.rs @@ -3,8 +3,7 @@ use brk_types::{Bitcoin, Dollars, Height, StoredF32, Version}; use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; use vecdb::{ - DeltaChange, DeltaRate, LazyDeltaVec, LazyVecFrom1, ReadOnlyClone, ReadableCloneableVec, - VecValue, + DeltaChange, DeltaRate, LazyDeltaVec, LazyVec, ReadOnlyClone, ReadableCloneableVec, VecValue, }; use crate::{ @@ -117,7 +116,7 @@ where let rate_ratio_name = format!("{full_name}_rate_ratio"); let ratio = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &rate_ratio_name, version, ppm.height.read_only_boxed_clone(), @@ -131,7 +130,7 @@ where let rate_name = format!("{full_name}_rate"); let percent = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &rate_name, version, ppm.height.read_only_boxed_clone(), @@ -231,7 +230,7 @@ where // Absolute change (btc): lazy from sats delta let btc = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &full_name, version, sats.height.read_only_boxed_clone(), @@ -263,7 +262,7 @@ where let rate_ratio_name = format!("{full_name}_rate_ratio"); let ratio = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &rate_ratio_name, version, ppm.height.read_only_boxed_clone(), @@ -277,7 +276,7 @@ where let rate_name = format!("{full_name}_rate"); let percent = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &rate_name, version, ppm.height.read_only_boxed_clone(), @@ -376,7 +375,7 @@ where // Absolute change (usd): lazy from cents delta let usd = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &full_name, version, cents.height.read_only_boxed_clone(), @@ -408,7 +407,7 @@ where let rate_ratio_name = format!("{full_name}_rate_ratio"); let ratio = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &rate_ratio_name, version, ppm.height.read_only_boxed_clone(), @@ -422,7 +421,7 @@ where let rate_name = format!("{full_name}_rate"); let percent = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &rate_name, version, ppm.height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/value/lazy_block.rs b/crates/brk_computer/src/internal/per_block/value/lazy_block.rs index 934e9cdd6..68f1372b1 100644 --- a/crates/brk_computer/src/internal/per_block/value/lazy_block.rs +++ b/crates/brk_computer/src/internal/per_block/value/lazy_block.rs @@ -1,15 +1,15 @@ use brk_traversable::Traversable; use brk_types::{Bitcoin, Cents, Dollars, Height, Sats, Version}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{LazyVec, ReadableCloneableVec}; use crate::internal::{CentsUnsignedToDollars, LazyPreviousDeltaVec, SatsToBitcoin, ValuePerBlock}; /// Per-block amount data derived from stored cumulative sats and cents. #[derive(Clone, Traversable)] pub struct LazyValueBlock { - pub btc: LazyVecFrom1, + pub btc: LazyVec, pub sats: LazyPreviousDeltaVec, - pub usd: LazyVecFrom1, + pub usd: LazyVec, pub cents: LazyPreviousDeltaVec, } @@ -39,13 +39,13 @@ impl LazyValueBlock { cumulative_sats.read_only_boxed_clone(), ); let btc = - LazyVecFrom1::transformed::(name, version, sats.read_only_boxed_clone()); + LazyVec::transformed::(name, version, sats.read_only_boxed_clone()); let cents = LazyPreviousDeltaVec::new( &format!("{name}_cents"), version, cumulative_cents.read_only_boxed_clone(), ); - let usd = LazyVecFrom1::transformed::( + let usd = LazyVec::transformed::( &format!("{name}_usd"), version, cents.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/value/lazy_rolling_avg.rs b/crates/brk_computer/src/internal/per_block/value/lazy_rolling_avg.rs index 5e8da9bc9..8669c927a 100644 --- a/crates/brk_computer/src/internal/per_block/value/lazy_rolling_avg.rs +++ b/crates/brk_computer/src/internal/per_block/value/lazy_rolling_avg.rs @@ -1,7 +1,7 @@ use brk_traversable::Traversable; use brk_types::{Bitcoin, Cents, Dollars, Height, Sats, StoredF32, Version}; use derive_more::{Deref, DerefMut}; -use vecdb::{DeltaAvg, LazyDeltaVec, LazyVecFrom1, ReadOnlyClone, ReadableCloneableVec}; +use vecdb::{DeltaAvg, LazyDeltaVec, LazyVec, ReadOnlyClone, ReadableCloneableVec}; use crate::{ indexes, @@ -66,7 +66,7 @@ impl LazyRollingAvgsAmountFromHeight { // Btc: f64 sats avg / 1e8 let btc = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &full_name, version, sats.height.read_only_boxed_clone(), @@ -99,7 +99,7 @@ impl LazyRollingAvgsAmountFromHeight { // Usd: f64 cents avg / 100 let usd = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &format!("{full_name}_usd"), version, cents.height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_block/value/lazy_rolling_sum.rs b/crates/brk_computer/src/internal/per_block/value/lazy_rolling_sum.rs index 6b7cfe883..95f59bb9a 100644 --- a/crates/brk_computer/src/internal/per_block/value/lazy_rolling_sum.rs +++ b/crates/brk_computer/src/internal/per_block/value/lazy_rolling_sum.rs @@ -1,7 +1,7 @@ use brk_traversable::Traversable; use brk_types::{Bitcoin, Cents, Dollars, Height, Sats, Version}; use derive_more::{Deref, DerefMut}; -use vecdb::{DeltaSub, LazyDeltaVec, LazyVecFrom1, ReadOnlyClone, ReadableCloneableVec}; +use vecdb::{DeltaSub, LazyDeltaVec, LazyVec, ReadOnlyClone, ReadableCloneableVec}; use crate::{ indexes, @@ -66,7 +66,7 @@ impl LazyRollingSumsAmountFromHeight { // Btc lazy from sats let btc = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &full_name, version, sats.height.read_only_boxed_clone(), @@ -99,7 +99,7 @@ impl LazyRollingSumsAmountFromHeight { // Usd lazy from cents let usd = LazyPerBlock { - height: LazyVecFrom1::transformed::( + height: LazyVec::transformed::( &format!("{full_name}_usd"), version, cents.height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs b/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs index 65a4ed2ff..a0eb18850 100644 --- a/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs +++ b/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs @@ -1,7 +1,7 @@ use brk_traversable::Traversable; use brk_types::{TxIndex, Version}; use schemars::JsonSchema; -use vecdb::{LazyVecFrom1, UnaryTransform}; +use vecdb::{LazyVec, UnaryTransform}; use crate::internal::{ComputedVecValue, LazyTxDerivedDistribution, TxDerivedDistribution}; @@ -13,7 +13,7 @@ where S: ComputedVecValue, DSource: ComputedVecValue, { - pub tx_index: LazyVecFrom1, + pub tx_index: LazyVec, #[traversable(flatten)] pub distribution: LazyTxDerivedDistribution, } @@ -27,7 +27,7 @@ where pub(crate) fn new>( name: &str, version: Version, - tx_index: LazyVecFrom1, + tx_index: LazyVec, source_distribution: &TxDerivedDistribution, ) -> Self { let distribution = diff --git a/crates/brk_computer/src/internal/value.rs b/crates/brk_computer/src/internal/value.rs index ba7c6234f..10c2eff5f 100644 --- a/crates/brk_computer/src/internal/value.rs +++ b/crates/brk_computer/src/internal/value.rs @@ -1,6 +1,6 @@ use brk_traversable::Traversable; use brk_types::{Bitcoin, Cents, Dollars, Height, Sats, Version}; -use vecdb::{LazyVecFrom1, ReadableCloneableVec, UnaryTransform, VecIndex}; +use vecdb::{LazyVec, ReadableCloneableVec, UnaryTransform, VecIndex}; use crate::internal::SpotValuePerBlock; @@ -9,10 +9,10 @@ use crate::internal::SpotValuePerBlock; /// All fields are lazy transforms from existing sources - no storage. #[derive(Clone, Traversable)] pub struct LazyValue { - pub btc: LazyVecFrom1, - pub sats: LazyVecFrom1, - pub usd: LazyVecFrom1, - pub cents: LazyVecFrom1, + pub btc: LazyVec, + pub sats: LazyVec, + pub usd: LazyVec, + pub cents: LazyVec, } impl LazyValue { @@ -32,25 +32,25 @@ impl LazyValue { CentsTransform: UnaryTransform, DollarsTransform: UnaryTransform, { - let sats = LazyVecFrom1::transformed::( + let sats = LazyVec::transformed::( &format!("{name}_sats"), version, source.sats.height.read_only_boxed_clone(), ); - let btc = LazyVecFrom1::transformed::( + let btc = LazyVec::transformed::( name, version, source.sats.height.read_only_boxed_clone(), ); - let cents = LazyVecFrom1::transformed::( + let cents = LazyVec::transformed::( &format!("{name}_cents"), version, source.cents.height.read_only_boxed_clone(), ); - let usd = LazyVecFrom1::transformed::( + let usd = LazyVec::transformed::( &format!("{name}_usd"), version, source.usd.height.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 47776bd4d..5da73d876 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -469,6 +469,7 @@ impl Computer { &self.price, &self.distribution, &self.frameworks, + &self.market.moving_average, exit, ) })?; diff --git a/crates/brk_computer/src/models/bedrock/price.rs b/crates/brk_computer/src/models/bedrock/price.rs index 87cf3372d..0c0cf6499 100644 --- a/crates/brk_computer/src/models/bedrock/price.rs +++ b/crates/brk_computer/src/models/bedrock/price.rs @@ -4,8 +4,8 @@ //! custom daily repeat/last-day views: cents are stored, USD is derived from cents, //! and sats are derived from USD. -use brk_types::{Cents, Day1, Dollars, SatsFract, Version}; use brk_traversable::Traversable; +use brk_types::{Cents, Dollars, SatsFract, Version}; use vecdb::{ReadableCloneableVec, Rw, StorageMode}; use crate::internal::{ diff --git a/crates/brk_computer/src/models/capital_sentiment/compute.rs b/crates/brk_computer/src/models/capital_sentiment/compute.rs index f6c89f3ec..bc537c214 100644 --- a/crates/brk_computer/src/models/capital_sentiment/compute.rs +++ b/crates/brk_computer/src/models/capital_sentiment/compute.rs @@ -1,16 +1,14 @@ -use std::collections::VecDeque; - use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{CapitalSentimentPhase, Cents, Day1, StoredBool, StoredU8, Version}; +use brk_types::{CapitalSentimentPhase, Cents, Day1, Height, StoredBool, StoredU8, Version}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::Vecs; use crate::{ - distribution, indexes, internal::db_utils::validate_any_computed_version_or_reset, price, + distribution, indexes, internal::db_utils::validate_any_computed_version_or_reset, market, + price, }; -const PRICE_SMA_DAYS: usize = 365; const WRITE_INTERVAL_DAYS: usize = 1_000; impl Vecs { @@ -20,9 +18,11 @@ impl Vecs { indexes: &indexes::Vecs, prices: &price::Vecs, distribution: &distribution::Vecs, + moving_average: &market::MovingAverageVecs, exit: &Exit, ) -> Result<()> { - let close = &prices.split.close.cents.day1; + let spot = &prices.spot.cents.height; + let sma = &moving_average.sma._1y.cents.height; let all = &distribution .utxo_cohorts .all @@ -31,7 +31,7 @@ impl Vecs { .capitalized .price .cents - .day1; + .height; let sth = &distribution .utxo_cohorts .sth @@ -40,7 +40,7 @@ impl Vecs { .capitalized .price .cents - .day1; + .height; let lth = &distribution .utxo_cohorts .lth @@ -49,24 +49,28 @@ impl Vecs { .capitalized .price .cents - .day1; + .height; + let first_height = &indexes.day1.first_height; - let source_version: Version = [close.version(), all.version(), sth.version(), lth.version()] + let source_version: Version = [ + spot.version(), + sma.version(), + all.version(), + sth.version(), + lth.version(), + first_height.version(), + ] .into_iter() .sum(); validate_any_computed_version_or_reset(&mut self.phase_code.day1, source_version)?; validate_any_computed_version_or_reset(&mut self.is_long.day1, source_version)?; - let source_end = [ - indexes.day1.date.len(), - close.len(), - all.len(), - sth.len(), - lth.len(), - ] + let height_end = [spot.len(), sma.len(), all.len(), sth.len(), lth.len()] .into_iter() .min() .unwrap_or_default(); + let first_heights = first_height.collect(); + let source_end = indexes.day1.date.len().min(first_heights.len()); let recompute_from = recompute_day(indexer, indexes) .map(usize::from) .unwrap_or_default(); @@ -85,28 +89,24 @@ impl Vecs { .map(Day1::from) .and_then(|day| self.is_long.day1.collect_one(day)) .is_some_and(|value| value.is_true()); - let mut previous_over_sth = start - .checked_sub(1) - .map(Day1::from) - .map(|day| { - is_over_sth( - close.collect_one(day).flatten(), - sth.collect_one(day).flatten(), - ) - }); - let mut sma = RollingSma::from_history(close, start); - + let mut previous_over_sth = start.checked_sub(1).map(|day| { + let height = last_height_of_day(&first_heights, day, height_end); + is_over_sth( + height.and_then(|height| spot.collect_one(height)), + height.and_then(|height| sth.collect_one(height)), + ) + }); for day_index in start..source_end { - let day = Day1::from(day_index); - let price = close.collect_one(day).flatten(); - let sth = sth.collect_one(day).flatten(); - let over_sth = is_over_sth(price, sth); + let height = last_height_of_day(&first_heights, day_index, height_end); + let price = height.and_then(|height| spot.collect_one(height)); + let sth_price = height.and_then(|height| sth.collect_one(height)); + let over_sth = is_over_sth(price, sth_price); let code = classify_phase_code( price, - all.collect_one(day).flatten(), - sth, - lth.collect_one(day).flatten(), - sma.observe(price), + height.and_then(|height| all.collect_one(height)), + sth_price, + height.and_then(|height| lth.collect_one(height)), + height.and_then(|height| sma.collect_one(height)), ); is_long = next_is_long(is_long, previous_over_sth, over_sth, code); @@ -114,9 +114,7 @@ impl Vecs { self.is_long.day1.push(StoredBool::from(is_long)); previous_over_sth = Some(over_sth); - if (day_index + 1).is_multiple_of(WRITE_INTERVAL_DAYS) - || day_index + 1 == source_end - { + if (day_index + 1).is_multiple_of(WRITE_INTERVAL_DAYS) || day_index + 1 == source_end { let _lock = exit.lock(); self.phase_code.day1.write()?; self.is_long.day1.write()?; @@ -127,36 +125,14 @@ impl Vecs { } } -#[derive(Default)] -struct RollingSma { - values: VecDeque, - sum: u128, -} - -impl RollingSma { - fn from_history( - source: &impl ReadableVec>, - end: usize, - ) -> Self { - let mut sma = Self::default(); - source.for_each_range_at(0, end, |price| { - let _ = sma.observe(price); - }); - sma - } - - /// Observe one daily close and return the sum of the latest 365 valid closes. - fn observe(&mut self, price: Option) -> Option { - if let Some(price) = price.filter(|price| is_finite_positive(*price)) { - let price = price.inner(); - self.values.push_back(price); - self.sum += u128::from(price); - if self.values.len() > PRICE_SMA_DAYS { - self.sum -= u128::from(self.values.pop_front().unwrap()); - } - } - (self.values.len() == PRICE_SMA_DAYS).then_some(self.sum) - } +fn last_height_of_day(first_heights: &[Height], day: usize, height_end: usize) -> Option { + let first = first_heights.get(day)?.to_usize().min(height_end); + let end = first_heights + .get(day + 1) + .map(|height| height.to_usize()) + .unwrap_or(height_end) + .min(height_end); + (first < end).then(|| Height::from(end - 1)) } /// Advance the stateful short/long strategy used by BRK Signal. @@ -166,8 +142,7 @@ fn next_is_long( over_sth: bool, phase_code: StoredU8, ) -> bool { - let crossed_above_sth = - previous_over_sth.is_some_and(|previous| !previous && over_sth); + let crossed_above_sth = previous_over_sth.is_some_and(|previous| !previous && over_sth); if !is_long && crossed_above_sth { return true; @@ -186,11 +161,9 @@ fn is_finite_positive(value: Cents) -> bool { #[inline] fn is_over_sth(price: Option, sth: Option) -> bool { - price - .zip(sth) - .is_some_and(|(price, sth)| { - is_finite_positive(price) && is_finite_positive(sth) && price >= sth - }) + price.zip(sth).is_some_and(|(price, sth)| { + is_finite_positive(price) && is_finite_positive(sth) && price >= sth + }) } /// Code `0` means the capitalized-price references are not all available yet. @@ -199,7 +172,7 @@ fn classify_phase_code( all: Option, sth: Option, lth: Option, - sma_sum: Option, + sma: Option, ) -> StoredU8 { let Some((price, all, sth, lth)) = price .zip(all) @@ -215,26 +188,24 @@ fn classify_phase_code( return StoredU8::ZERO; }; - StoredU8::new(classify_phase(price, all, sth, lth, sma_sum).code()) + StoredU8::new(classify_phase(price, all, sth, lth, sma).code()) } /// Classify investor sentiment from the three capitalized-price references, -/// using the 365-daily-close SMA only as confirmation and disambiguation. +/// using the one-year price SMA only as confirmation and disambiguation. fn classify_phase( price: Cents, all: Cents, sth: Cents, lth: Cents, - sma_sum: Option, + sma: Option, ) -> CapitalSentimentPhase { use CapitalSentimentPhase as Phase; - let price_x_days = price.as_u128() * PRICE_SMA_DAYS as u128; - let all_x_days = all.as_u128() * PRICE_SMA_DAYS as u128; let above_all = price >= all; let above_sth = price >= sth; let above_lth = price >= lth; - let above_sma = sma_sum.is_some_and(|sma| price_x_days >= sma); + let above_sma = sma.is_some_and(|sma| price >= sma); let bull_structure = sth > lth; let above_slow_refs = above_all && above_lth; let above_any_slow_ref = above_all || above_lth; @@ -242,9 +213,9 @@ fn classify_phase( .into_iter() .filter(|reference| *reference > price) .count() - + usize::from(sma_sum.is_some_and(|sma| sma > price_x_days)); + + usize::from(sma.is_some_and(|sma| sma > price)); let price_in_middle = references_above_price == 2; - let core_bull_phase = if sma_sum.is_some_and(|sma| all_x_days > sma) { + let core_bull_phase = if sma.is_some_and(|sma| all > sma) { Phase::RagingBull } else { Phase::Bull @@ -255,7 +226,7 @@ fn classify_phase( Phase::Bear }; - if sma_sum.is_none() { + if sma.is_none() { if bull_structure { if above_sth { return if above_slow_refs { @@ -338,13 +309,43 @@ mod tests { Cents::new(value) } + #[test] + fn samples_each_days_last_available_block() { + let first_heights = [0_usize, 2, 2, 5].map(Height::from); + + assert_eq!( + last_height_of_day(&first_heights, 0, 7), + Some(Height::from(1_usize)) + ); + assert_eq!(last_height_of_day(&first_heights, 1, 7), None); + assert_eq!( + last_height_of_day(&first_heights, 2, 7), + Some(Height::from(4_usize)) + ); + assert_eq!( + last_height_of_day(&first_heights, 3, 7), + Some(Height::from(6_usize)) + ); + } + + #[test] + fn sampling_clamps_the_current_day_to_the_shared_source_length() { + let first_heights = [0_usize, 2, 5].map(Height::from); + + assert_eq!( + last_height_of_day(&first_heights, 1, 4), + Some(Height::from(3_usize)) + ); + assert_eq!(last_height_of_day(&first_heights, 2, 4), None); + } + fn classify(price: u64, all: u64, sth: u64, lth: u64, sma: u64) -> CapitalSentimentPhase { classify_phase( cents(price), cents(all), cents(sth), cents(lth), - Some(u128::from(sma) * PRICE_SMA_DAYS as u128), + Some(cents(sma)), ) } @@ -407,14 +408,14 @@ mod tests { Some(cents(70)), Some(cents(80)), None, - Some(u128::from(50_u64) * PRICE_SMA_DAYS as u128), + Some(cents(50)), ), StoredU8::ZERO ); } #[test] - fn phase_is_available_before_the_sma_window_is_full() { + fn phase_is_available_without_sma() { assert_eq!( classify_phase(cents(100), cents(70), cents(80), cents(60), None), CapitalSentimentPhase::Bull diff --git a/crates/brk_computer/src/models/capital_sentiment/import.rs b/crates/brk_computer/src/models/capital_sentiment/import.rs index 3b790ed67..4e42ba00f 100644 --- a/crates/brk_computer/src/models/capital_sentiment/import.rs +++ b/crates/brk_computer/src/models/capital_sentiment/import.rs @@ -8,7 +8,7 @@ use crate::{ internal::{DailyMappings, DailyMetric, LazyDailyMetric}, }; -const VERSION: Version = Version::new(4); +const VERSION: Version = Version::new(5); struct CodeToPhase; @@ -53,12 +53,8 @@ impl Vecs { let version = parent_version + VERSION; let mappings = DailyMappings::new(indexes); - let phase_code = DailyMetric::forced_import( - db, - "capital_sentiment_phase_code", - version, - &mappings, - )?; + let phase_code = + DailyMetric::forced_import(db, "capital_sentiment_phase_code", version, &mappings)?; let is_long = DailyMetric::::forced_import( db, "capital_sentiment_is_long", diff --git a/crates/brk_computer/src/models/compute.rs b/crates/brk_computer/src/models/compute.rs index 9a359e4ba..2ea555ea4 100644 --- a/crates/brk_computer/src/models/compute.rs +++ b/crates/brk_computer/src/models/compute.rs @@ -3,7 +3,7 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::{distribution, frameworks, indexes, price}; +use crate::{distribution, frameworks, indexes, market, price}; impl Vecs { #[allow(clippy::too_many_arguments)] @@ -14,6 +14,7 @@ impl Vecs { prices: &price::Vecs, distribution: &distribution::Vecs, frameworks: &frameworks::Vecs, + moving_average: &market::MovingAverageVecs, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; @@ -26,8 +27,14 @@ impl Vecs { &frameworks.coinflow, exit, )?; - self.capital_sentiment - .compute(indexer, indexes, prices, distribution, exit)?; + self.capital_sentiment.compute( + indexer, + indexes, + prices, + distribution, + moving_average, + exit, + )?; self.rarity_meter.compute( indexer, distribution, diff --git a/crates/brk_computer/src/models/rarity_meter/cached_component_price.rs b/crates/brk_computer/src/models/rarity_meter/cached_component_price.rs index acad7c6c4..2361c6b2d 100644 --- a/crates/brk_computer/src/models/rarity_meter/cached_component_price.rs +++ b/crates/brk_computer/src/models/rarity_meter/cached_component_price.rs @@ -1,7 +1,6 @@ use brk_types::{Cents, CentsCompact, Height, PartsPerMillion32, Version}; use vecdb::{ - AnyVec, BinaryTransform, CachedReadableVec, CachedVec, LazyVecFrom1, ReadableCloneableVec, - VecIndex, + AnyVec, BinaryTransform, CachedReadableVec, CachedVec, LazyVec, ReadableCloneableVec, VecIndex, }; use crate::{ @@ -11,7 +10,7 @@ use crate::{ #[derive(Clone)] pub(super) struct CachedComponentPrice { - cache: CachedVec>, + cache: CachedVec>, } impl CachedComponentPrice { @@ -20,7 +19,7 @@ impl CachedComponentPrice { version: Version, source: &(impl ReadableCloneableVec + 'static), ) -> Self { - let compact = LazyVecFrom1::init( + let compact = LazyVec::init( &format!("{name}_cached_price"), version, source.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/price/compute.rs b/crates/brk_computer/src/price/compute.rs index bb2915efe..50b2c1d80 100644 --- a/crates/brk_computer/src/price/compute.rs +++ b/crates/brk_computer/src/price/compute.rs @@ -243,9 +243,9 @@ impl Vecs { txout_cursor.advance(block_first_tx - txout_cursor.position()); tx_starts.clear(); - for _ in 0..tx_count { - tx_starts.push(txout_cursor.next().unwrap().to_usize()); - } + txout_cursor.for_each(tx_count, |txout_index| { + tx_starts.push(txout_index.to_usize()); + }); let out_start = tx_starts.first().copied().unwrap_or(out_end); indexer diff --git a/crates/brk_computer/src/price/ohlcs.rs b/crates/brk_computer/src/price/ohlcs.rs index 24332cc80..0c5f63376 100644 --- a/crates/brk_computer/src/price/ohlcs.rs +++ b/crates/brk_computer/src/price/ohlcs.rs @@ -7,7 +7,7 @@ use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; use serde::Serialize; use vecdb::{ - BytesVecValue, CachedBoxedVec, Formattable, LazyVecFrom1, ReadableCloneableVec, UnaryTransform, + BytesVecValue, CachedBoxedVec, Formattable, LazyVec, ReadableCloneableVec, UnaryTransform, }; use crate::{ @@ -88,7 +88,7 @@ where ) -> Self { macro_rules! period { ($idx:ident) => { - LazyVecFrom1::transformed::( + LazyVec::transformed::( name, version, source.$idx.read_only_boxed_clone(), @@ -121,21 +121,21 @@ where pub struct LazyOhlcVecs( #[allow(clippy::type_complexity)] pub PerResolution< - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, - LazyVecFrom1, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, + LazyVec, >, ) where @@ -153,7 +153,7 @@ where ) -> Self { macro_rules! period { ($idx:ident) => { - LazyVecFrom1::transformed::( + LazyVec::transformed::( name, version, source.$idx.read_only_boxed_clone(), diff --git a/crates/brk_computer/src/transactions/size/import.rs b/crates/brk_computer/src/transactions/size/import.rs index 2f6e1b57e..910f1e497 100644 --- a/crates/brk_computer/src/transactions/size/import.rs +++ b/crates/brk_computer/src/transactions/size/import.rs @@ -1,7 +1,7 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_types::Version; -use vecdb::{Database, LazyVecFrom1, ReadableCloneableVec}; +use vecdb::{Database, LazyVec, ReadableCloneableVec}; use super::Vecs; use crate::{ @@ -18,7 +18,7 @@ impl Vecs { ) -> Result { let weight = TxDerivedDistribution::forced_import(db, "tx_weight", version, indexes)?; - let tx_index_to_vsize = LazyVecFrom1::transformed::( + let tx_index_to_vsize = LazyVec::transformed::( "tx_vsize", version, indexer.vecs.transactions.weight.read_only_boxed_clone(), diff --git a/crates/brk_query/src/impl/urpd.rs b/crates/brk_query/src/impl/urpd.rs index 24ea65c0a..cc0df0817 100644 --- a/crates/brk_query/src/impl/urpd.rs +++ b/crates/brk_query/src/impl/urpd.rs @@ -186,9 +186,7 @@ impl Query { } fn is_aggregate_cohort(cohort: &Cohort) -> bool { - UTXO_AGGREGATE_NAMES - .iter() - .any(|name| name.id == &**cohort) + UTXO_AGGREGATE_NAMES.iter().any(|name| name.id == &**cohort) } fn dates_in_dir(dir: &Path) -> Result> { diff --git a/crates/brk_query/src/vecs.rs b/crates/brk_query/src/vecs.rs index 3f9225f4b..2e97fb313 100644 --- a/crates/brk_query/src/vecs.rs +++ b/crates/brk_query/src/vecs.rs @@ -51,7 +51,7 @@ impl<'a> Vecs<'a> { let mut builder = Builder::default(); indexed_vecs.for_each(|vec| builder.insert(vec, "indexed")); computed_vecs.for_each(|(db, vec)| builder.insert(vec, db)); - builder.counts.distinct_series = builder.series_to_index_to_vec.len(); + builder.counts.distinct = builder.series_to_index_to_vec.len(); let Builder { series_to_index_to_vec, index_to_series_to_vec, @@ -205,17 +205,17 @@ impl<'a> Builder<'a> { let is_lazy = vec.region_names().is_empty(); let by_db = self.counts_by_db.entry(db.to_string()).or_default(); - self.counts.total_endpoints += 1; - by_db.total_endpoints += 1; + self.counts.total += 1; + by_db.total += 1; if is_lazy { - self.counts.lazy_endpoints += 1; - by_db.lazy_endpoints += 1; + self.counts.lazy += 1; + by_db.lazy += 1; } else { - self.counts.stored_endpoints += 1; - by_db.stored_endpoints += 1; + self.counts.stored += 1; + by_db.stored += 1; } if self.seen_by_db.entry(db).or_default().insert(name) { - by_db.distinct_series += 1; + by_db.distinct += 1; } } } diff --git a/crates/brk_traversable/README.md b/crates/brk_traversable/README.md index 4699d8ca8..13ec6c5c4 100644 --- a/crates/brk_traversable/README.md +++ b/crates/brk_traversable/README.md @@ -29,7 +29,7 @@ All vecdb vector types implement `Traversable`: - `BytesVec`, `EagerVec`, `PcoVec` (with `pco` feature) - `ZeroCopyVec` (with `zerocopy` feature) - `LZ4Vec`, `ZstdVec` (with respective features) -- `LazyVecFrom1/2/3` for derived vectors +- `LazyVec` for single-source derived vectors ## Feature Flags diff --git a/crates/brk_traversable/src/lib.rs b/crates/brk_traversable/src/lib.rs index 6c4349310..18d392c3c 100644 --- a/crates/brk_traversable/src/lib.rs +++ b/crates/brk_traversable/src/lib.rs @@ -9,9 +9,8 @@ use schemars::JsonSchema; use serde::Serialize; use vecdb::{ AggFold, AnyExportableVec, AnyVec, BytesVec, BytesVecValue, CachedVec, CompressionStrategy, - DeltaOp, EagerVec, Formattable, LazyAggVec, LazyDeltaVec, LazyVecFrom1, LazyVecFrom2, - LazyVecFrom3, RawStrategy, ReadOnlyCompressedVec, ReadOnlyRawVec, StoredVec, TypedVec, - VecIndex, VecValue, + DeltaOp, EagerVec, Formattable, LazyAggVec, LazyDeltaVec, LazyVec, RawStrategy, + ReadOnlyCompressedVec, ReadOnlyRawVec, StoredVec, TypedVec, VecIndex, VecValue, }; pub trait Traversable { @@ -168,7 +167,7 @@ where } } -impl Traversable for LazyVecFrom1 +impl Traversable for LazyVec where I: VecIndex, T: VecValue + Formattable + Serialize + JsonSchema, @@ -184,45 +183,6 @@ where } } -impl Traversable for LazyVecFrom2 -where - I: VecIndex, - T: VecValue + Formattable + Serialize + JsonSchema, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, -{ - fn iter_any_exportable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyExportableVec) - } - - fn to_tree_node(&self) -> TreeNode { - make_leaf::(self) - } -} - -impl Traversable - for LazyVecFrom3 -where - I: VecIndex, - T: VecValue + Formattable + Serialize + JsonSchema, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, - S3I: VecIndex, - S3T: VecValue, -{ - fn iter_any_exportable(&self) -> impl Iterator { - std::iter::once(self as &dyn AnyExportableVec) - } - - fn to_tree_node(&self) -> TreeNode { - make_leaf::(self) - } -} - impl Traversable for LazyAggVec where I: VecIndex, diff --git a/crates/brk_types/src/series_count.rs b/crates/brk_types/src/series_count.rs index 48a931ade..9a516d305 100644 --- a/crates/brk_types/src/series_count.rs +++ b/crates/brk_types/src/series_count.rs @@ -3,21 +3,21 @@ use std::collections::BTreeMap; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -/// Series count statistics - distinct series and total series-index combinations +/// Series count statistics #[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)] pub struct SeriesCount { /// Number of unique series available (e.g., realized_price, market_cap) #[schemars(example = 3141)] - pub distinct_series: usize, + pub distinct: usize, /// Total number of series-index combinations across all timeframes #[schemars(example = 21000)] - pub total_endpoints: usize, + pub total: usize, /// Number of lazy (computed on-the-fly) series-index combinations #[schemars(example = 5000)] - pub lazy_endpoints: usize, + pub lazy: usize, /// Number of eager (stored on disk) series-index combinations #[schemars(example = 16000)] - pub stored_endpoints: usize, + pub stored: usize, } /// Detailed series count with per-database breakdown diff --git a/crates/brk_types/src/tree_node.rs b/crates/brk_types/src/tree_node.rs index 6c5ca74fe..4177b4a8c 100644 --- a/crates/brk_types/src/tree_node.rs +++ b/crates/brk_types/src/tree_node.rs @@ -936,10 +936,10 @@ mod tests { // ========== Case 7: LazyBlockValue ========== // LazyBlockValue (no merge): - // - sats: LazyVecFrom1 with wrap="sats" + // - sats: LazyVec with wrap="sats" // - rest: LazyDerivedBlockValue with flatten - // - bitcoin: LazyVecFrom1 (plain field) - // - dollars: Option (plain field) + // - bitcoin: LazyVec (plain field) + // - dollars: optional derived vector (plain field) #[test] fn case7_lazy_block_value() { diff --git a/crates/lsm-tree/Cargo.toml b/crates/lsm-tree/Cargo.toml index d58813838..31d629520 100644 --- a/crates/lsm-tree/Cargo.toml +++ b/crates/lsm-tree/Cargo.toml @@ -17,6 +17,7 @@ default = [] lz4 = ["dep:lz4_flex"] [dependencies] +arc-swap = "1.9.2" byteorder = { package = "byteorder-lite", version = "0.1.0" } byteview = { workspace = true } crossbeam-skiplist = "0.1.3" diff --git a/crates/lsm-tree/src/table/filter/bit_array/reader.rs b/crates/lsm-tree/src/table/filter/bit_array/reader.rs index bf2ea88d9..f9fcffc0b 100644 --- a/crates/lsm-tree/src/table/filter/bit_array/reader.rs +++ b/crates/lsm-tree/src/table/filter/bit_array/reader.rs @@ -27,13 +27,16 @@ impl<'a> BitArrayReader<'a> { self.0 } - /// Gets the i-th bit. + /// Gets the i-th bit without checking the byte index. + /// + /// # Safety + /// + /// `idx` must be less than the number of bits in this array. #[must_use] - pub fn get(&self, idx: usize) -> bool { + pub unsafe fn get_unchecked(&self, idx: usize) -> bool { let byte_idx = idx / 8; - - #[expect(clippy::expect_used, reason = "we trust the caller")] - let byte = self.0.get(byte_idx).expect("should be in bounds"); + debug_assert!(byte_idx < self.0.len()); + let byte = unsafe { self.0.get_unchecked(byte_idx) }; let bit_idx = idx % 8; get_bit(*byte, bit_idx) diff --git a/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs b/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs index e47d000ad..c3aa0aff7 100644 --- a/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs +++ b/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs @@ -77,14 +77,21 @@ impl<'a> StandardBloomFilterReader<'a> { )] let offset = reader.position() as usize; - #[expect( - clippy::expect_used, - reason = "offset is expected to be with slice bounds" - )] + let bytes = slice + .get(offset..) + .ok_or(crate::Error::InvalidHeader("BloomFilter"))?; + let bit_len = bytes + .len() + .checked_mul(8) + .ok_or(crate::Error::InvalidHeader("BloomFilter"))?; + if m == 0 || m != bit_len { + return Err(crate::Error::InvalidHeader("BloomFilter")); + } + Ok(Self { k, m, - inner: BitArrayReader::new(slice.get(offset..).expect("should be in bounds")), + inner: BitArrayReader::new(bytes), }) } @@ -130,7 +137,11 @@ impl<'a> StandardBloomFilterReader<'a> { /// Returns `true` if the bit at `idx` is `1`. fn has_bit(&self, idx: usize) -> bool { - self.inner.get(idx) + debug_assert!(idx < self.m); + + // SAFETY: construction validates that `m` matches the bit array, and + // callers derive `idx` modulo `m`. + unsafe { self.inner.get_unchecked(idx) } } /// Gets the hash of a key. @@ -169,6 +180,18 @@ mod tests { Ok(()) } + #[test] + fn filter_bloom_standard_rejects_truncated_bits() { + let filter = Builder::with_fp_rate(10, 0.0001); + let mut filter_bytes = filter.build(); + filter_bytes.pop(); + + assert!(matches!( + StandardBloomFilterReader::new(&filter_bytes), + Err(crate::Error::InvalidHeader("BloomFilter")) + )); + } + #[test] fn filter_bloom_standard_basic() -> crate::Result<()> { let mut filter = Builder::with_fp_rate(10, 0.0001); diff --git a/crates/lsm-tree/src/tree/inner.rs b/crates/lsm-tree/src/tree/inner.rs index 9acf8b3c3..1d1ef56ac 100644 --- a/crates/lsm-tree/src/tree/inner.rs +++ b/crates/lsm-tree/src/tree/inner.rs @@ -8,8 +8,9 @@ use crate::{ config::Config, stop_signal::StopSignal, table::next_table_id, - version::{SuperVersions, Version, persist_version}, + version::{SuperVersion, SuperVersions, Version, persist_version}, }; +use arc_swap::ArcSwap; use std::sync::{Arc, Mutex, RwLock, atomic::AtomicU64}; /// Unique tree ID @@ -45,6 +46,8 @@ pub struct TreeInner { pub(crate) version_history: Arc>, + pub(crate) latest_version: Arc>, + pub(crate) compaction_state: Arc>, /// Tree configuration @@ -68,13 +71,16 @@ impl TreeInner { pub(crate) fn create_new(config: Config) -> crate::Result { let version = Version::new(0); persist_version(&config.path, &version)?; + let version_history = SuperVersions::new(version); + let latest_version = version_history.latest_version_reader(); Ok(Self { id: get_next_tree_id(), memtable_id_counter: SequenceNumberCounter::new(1), table_id_counter: SequenceNumberCounter::default(), config: Arc::new(config), - version_history: Arc::new(RwLock::new(SuperVersions::new(version))), + version_history: Arc::new(RwLock::new(version_history)), + latest_version, stop_signal: StopSignal::default(), major_compaction_lock: RwLock::default(), flush_lock: Mutex::default(), diff --git a/crates/lsm-tree/src/tree/mod.rs b/crates/lsm-tree/src/tree/mod.rs index 85d52e783..e59e612c9 100644 --- a/crates/lsm-tree/src/tree/mod.rs +++ b/crates/lsm-tree/src/tree/mod.rs @@ -873,12 +873,7 @@ impl Tree { #[doc(hidden)] pub fn get_exclusive>(&self, key: K) -> crate::Result> { let key = key.as_ref(); - #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] - let super_version = self - .version_history - .read() - .expect("lock is poisoned") - .latest_version_arc(); + let super_version = self.latest_version.load(); Self::get_value_from_tables(&super_version.version, key, SeqNo::MAX) } @@ -970,17 +965,19 @@ impl Tree { } let version = Self::recover_levels(&config.path, tree_id, &config)?; - let highest_table_id = version .iter_tables() .map(Table::id) .max() .unwrap_or_default(); + let version_history = SuperVersions::new(version); + let latest_version = version_history.latest_version_reader(); let inner = TreeInner { id: tree_id, memtable_id_counter: SequenceNumberCounter::new(1), table_id_counter: SequenceNumberCounter::new(u64::from(highest_table_id) + 1), - version_history: Arc::new(RwLock::new(SuperVersions::new(version))), + version_history: Arc::new(RwLock::new(version_history)), + latest_version, stop_signal: StopSignal::default(), config: Arc::new(config), major_compaction_lock: RwLock::default(), diff --git a/crates/lsm-tree/src/version/super_version.rs b/crates/lsm-tree/src/version/super_version.rs index 4c00b196d..8d240b3ea 100644 --- a/crates/lsm-tree/src/version/super_version.rs +++ b/crates/lsm-tree/src/version/super_version.rs @@ -8,6 +8,7 @@ use crate::{ tree::sealed::SealedMemtables, version::{Version, persist_version}, }; +use arc_swap::ArcSwap; use std::{collections::VecDeque, path::Path, sync::Arc}; /// A super version is a point-in-time snapshot of memtables and a [`Version`] (list of disk files) @@ -26,25 +27,30 @@ pub struct SuperVersion { pub(crate) seqno: SeqNo, } -pub struct SuperVersions(VecDeque>); +pub struct SuperVersions { + versions: VecDeque>, + latest: Arc>, +} impl SuperVersions { pub fn new(version: Version) -> Self { - Self( - vec![Arc::new(SuperVersion { - active_memtable: Arc::new(Memtable::new(0)), - sealed_memtables: Arc::default(), - version, - seqno: 0, - })] - .into(), - ) + let version = Arc::new(SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version, + seqno: 0, + }); + + Self { + versions: vec![version.clone()].into(), + latest: Arc::new(ArcSwap::from(version)), + } } pub fn memtable_size_sum(&self) -> u64 { let mut set = crate::HashMap::default(); - for super_version in &self.0 { + for super_version in &self.versions { set.entry(super_version.active_memtable.id) .and_modify(|bytes| *bytes += super_version.active_memtable.size()) .or_insert_with(|| super_version.active_memtable.size()); @@ -60,7 +66,7 @@ impl SuperVersions { } pub fn len(&self) -> usize { - self.0.len() + self.versions.len() } pub fn free_list_len(&self) -> usize { @@ -78,9 +84,9 @@ impl SuperVersions { log::trace!("Running manifest GC with watermark={gc_watermark}"); - if let Some(hi_idx) = self.0.iter().rposition(|x| x.seqno < gc_watermark) { + if let Some(hi_idx) = self.versions.iter().rposition(|x| x.seqno < gc_watermark) { for _ in 0..hi_idx { - let Some(head) = self.0.front() else { + let Some(head) = self.versions.front() else { break; }; @@ -95,11 +101,14 @@ impl SuperVersions { crate::file::retry_transient_io(|| std::fs::remove_file(&path))?; } - self.0.pop_front(); + self.versions.pop_front(); } } - log::trace!("Manifest GC done, version length now {}", self.0.len()); + log::trace!( + "Manifest GC done, version length now {}", + self.versions.len() + ); Ok(()) } @@ -146,43 +155,43 @@ impl SuperVersions { } pub fn append_version(&mut self, version: SuperVersion) { - self.0.push_back(Arc::new(version)); + let version = Arc::new(version); + self.versions.push_back(version.clone()); + self.latest.store(version); } pub fn replace_latest_version(&mut self, version: SuperVersion) { - if self.0.pop_back().is_some() { - self.0.push_back(Arc::new(version)); + if let Some(latest) = self.versions.back_mut() { + let version = Arc::new(version); + *latest = version.clone(); + self.latest.store(version); } } pub fn latest_version(&self) -> SuperVersion { #[expect(clippy::expect_used, reason = "SuperVersion is expected to exist")] - self.0 + self.versions .back() .map(|version| version.as_ref().clone()) .expect("should always have a SuperVersion") } - pub(crate) fn latest_version_arc(&self) -> Arc { - #[expect(clippy::expect_used, reason = "SuperVersion is expected to exist")] - self.0 - .back() - .cloned() - .expect("should always have a SuperVersion") + pub(crate) fn latest_version_reader(&self) -> Arc> { + self.latest.clone() } pub(crate) fn get_version_arc_for_snapshot(&self, seqno: SeqNo) -> Arc { if seqno == 0 { #[expect(clippy::expect_used, reason = "SuperVersion is expected to exist")] return self - .0 + .versions .front() .cloned() .expect("should always find a SuperVersion"); } let version = self - .0 + .versions .iter() .rev() .find(|version| version.seqno < seqno) @@ -192,7 +201,7 @@ impl SuperVersions { log::error!("Failed to find a SuperVersion for snapshot with seqno={seqno}"); log::error!("SuperVersions:"); - for version in self.0.iter().rev() { + for version in self.versions.iter().rev() { log::error!("-> {}, seqno={}", version.version.id(), version.seqno); } } @@ -207,7 +216,17 @@ impl SuperVersions { #[cfg(test)] fn from_versions(versions: VecDeque) -> Self { - Self(versions.into_iter().map(Arc::new).collect()) + let versions: VecDeque<_> = versions.into_iter().map(Arc::new).collect(); + #[expect(clippy::expect_used, reason = "test histories are non-empty")] + let latest = versions + .back() + .cloned() + .expect("history should not be empty"); + + Self { + versions, + latest: Arc::new(ArcSwap::from(latest)), + } } } @@ -216,6 +235,26 @@ mod tests { use super::*; use test_log::test; + #[test] + fn latest_reader_tracks_publications() { + let mut history = SuperVersions::new(Version::new(0)); + let reader = history.latest_version_reader(); + let original = reader.load_full(); + + let mut appended = history.latest_version(); + appended.version = Version::new(1); + history.append_version(appended); + + assert_eq!(original.version.id(), 0); + assert_eq!(reader.load().version.id(), 1); + + let mut replacement = history.latest_version(); + replacement.version = Version::new(2); + history.replace_latest_version(replacement); + + assert_eq!(reader.load().version.id(), 2); + } + #[test] fn super_version_gc_above_watermark() -> crate::Result<()> { let mut history = SuperVersions::from_versions( diff --git a/crates/lsm-tree/tests/exclusive_point_read.rs b/crates/lsm-tree/tests/exclusive_point_read.rs new file mode 100644 index 000000000..bc80760e7 --- /dev/null +++ b/crates/lsm-tree/tests/exclusive_point_read.rs @@ -0,0 +1,44 @@ +use lsm_tree::{AbstractTree, Config, SequenceNumberCounter, get_tmp_folder}; +use test_log::test; + +#[test] +fn exclusive_point_read_tracks_latest_version() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let seqno = SequenceNumberCounter::default(); + let tree = Config::new( + folder.path(), + seqno.clone(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "first", seqno.next()); + tree.flush_active_memtable(0)?; + assert_eq!( + tree.get_exclusive("a")?.as_deref(), + Some(b"first".as_slice()) + ); + + tree.insert("b", "second", seqno.next()); + tree.flush_active_memtable(0)?; + assert_eq!( + tree.get_exclusive("a")?.as_deref(), + Some(b"first".as_slice()) + ); + assert_eq!( + tree.get_exclusive("b")?.as_deref(), + Some(b"second".as_slice()) + ); + + tree.major_compact(u64::MAX, 1_000)?; + assert_eq!( + tree.get_exclusive("a")?.as_deref(), + Some(b"first".as_slice()) + ); + assert_eq!( + tree.get_exclusive("b")?.as_deref(), + Some(b"second".as_slice()) + ); + + Ok(()) +} diff --git a/crates/vecdb/README.md b/crates/vecdb/README.md index 25eb8ad25..41139c708 100644 --- a/crates/vecdb/README.md +++ b/crates/vecdb/README.md @@ -8,7 +8,7 @@ High-performance mutable persistent vectors built on [`rawdb`](../rawdb/README.m - **Multiple storage formats**: - **Raw**: `BytesVec`, `ZeroCopyVec` (uncompressed) - **Compressed**: `PcoVec`, `LZ4Vec`, `ZstdVec` -- **Computed vectors**: `EagerVec` (stored computations), `LazyVecFrom1/2/3` (on-the-fly computation) +- **Computed vectors**: `EagerVec` (stored computations), `LazyVec` (single-source on-the-fly computation) - **Rollback support**: Time-travel via stamped change deltas without full snapshots - **Sparse deletions**: Delete elements leaving holes, no reindexing required - **Thread-safe**: Concurrent reads with exclusive writes @@ -144,14 +144,14 @@ let mut derived: EagerVec> = // derived.compute_sma(&source, 20)?; ``` -**`LazyVecFrom1/2/3<...>`** - Lazily computed vectors from 1-3 source vectors +**`LazyVec<...>`** - Lazily computed vector from one source vector Values computed on-the-fly during iteration, nothing stored on disk. Use for temporary views or simple transformations. ```rust,ignore -use vecdb::LazyVecFrom1; +use vecdb::LazyVec; -let lazy = LazyVecFrom1::init( +let lazy = LazyVec::init( "computed", Version::TWO, Box::new(source.clone()), // ScannableBoxedVec diff --git a/crates/vecdb/src/variants/lazy/from2/transform/divide.rs b/crates/vecdb/src/ops/binary_transform/divide.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from2/transform/divide.rs rename to crates/vecdb/src/ops/binary_transform/divide.rs diff --git a/crates/vecdb/src/variants/lazy/from2/transform/minus.rs b/crates/vecdb/src/ops/binary_transform/minus.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from2/transform/minus.rs rename to crates/vecdb/src/ops/binary_transform/minus.rs diff --git a/crates/vecdb/src/variants/lazy/from2/transform/mod.rs b/crates/vecdb/src/ops/binary_transform/mod.rs similarity index 82% rename from crates/vecdb/src/variants/lazy/from2/transform/mod.rs rename to crates/vecdb/src/ops/binary_transform/mod.rs index 8019fe52f..2e4aea464 100644 --- a/crates/vecdb/src/variants/lazy/from2/transform/mod.rs +++ b/crates/vecdb/src/ops/binary_transform/mod.rs @@ -8,7 +8,7 @@ pub use minus::Minus; pub use plus::Plus; pub use times::Times; -/// Trait for binary transforms applied lazily during iteration. +/// Trait for binary transforms used by vector computations. /// Zero-sized types implementing this get monomorphized (zero runtime cost). pub trait BinaryTransform { fn apply(lhs: In1, rhs: In2) -> Out; diff --git a/crates/vecdb/src/variants/lazy/from2/transform/plus.rs b/crates/vecdb/src/ops/binary_transform/plus.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from2/transform/plus.rs rename to crates/vecdb/src/ops/binary_transform/plus.rs diff --git a/crates/vecdb/src/variants/lazy/from2/transform/times.rs b/crates/vecdb/src/ops/binary_transform/times.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from2/transform/times.rs rename to crates/vecdb/src/ops/binary_transform/times.rs diff --git a/crates/vecdb/src/ops/mod.rs b/crates/vecdb/src/ops/mod.rs index 664c30ca0..85d8d68bd 100644 --- a/crates/vecdb/src/ops/mod.rs +++ b/crates/vecdb/src/ops/mod.rs @@ -1,7 +1,9 @@ //! Numeric traits (overflow-safe arithmetic) shared across the crate. +mod binary_transform; mod checked_sub; mod saturating_add; +pub use binary_transform::*; pub use checked_sub::*; pub use saturating_add::*; diff --git a/crates/vecdb/src/variants/cached/mod.rs b/crates/vecdb/src/variants/cached/mod.rs index 0f904ffbd..5a5a57517 100644 --- a/crates/vecdb/src/variants/cached/mod.rs +++ b/crates/vecdb/src/variants/cached/mod.rs @@ -176,7 +176,7 @@ impl> CachedVec { } impl CachedVec { - /// Boxes a read-only clone for use with type-erased APIs (e.g. LazyVecFrom1). + /// Boxes a read-only clone for use with type-erased APIs (e.g. LazyVec). #[inline] pub fn read_only_boxed_clone(&self) -> crate::ReadableBoxedVec { Box::new(self.read_only_clone()) diff --git a/crates/vecdb/src/variants/eager/compute/aggregates.rs b/crates/vecdb/src/variants/eager/compute/aggregates.rs index 7b253c82f..1aa7a4c8c 100644 --- a/crates/vecdb/src/variants/eager/compute/aggregates.rs +++ b/crates/vecdb/src/variants/eager/compute/aggregates.rs @@ -20,7 +20,7 @@ where ) -> Result<()> where O: ReadableVec, - F: Fn(&[Vec], usize) -> V::T, + F: Fn(&mut V::T, V::T), { if others.is_empty() { return Err(Error::InvalidArgument( @@ -40,13 +40,18 @@ where return Ok(()); } - let batches: Vec> = others - .iter() - .map(|v| v.collect_range_at(skip, end)) - .collect(); + let mut batches = others.iter().map(|v| v.collect_range_at(skip, end)); + let mut aggregated = batches.next().unwrap(); - for j in 0..(end - skip) { - this.push(aggregate(&batches, j)); + for batch in batches { + debug_assert_eq!(aggregated.len(), batch.len()); + for (value, other) in aggregated.iter_mut().zip(batch) { + aggregate(value, other); + } + } + + for value in aggregated { + this.push(value); } Ok(()) @@ -64,12 +69,8 @@ where O: ReadableVec, V::T: Add, { - self.compute_aggregate_of_others(max_from, others, exit, |batches, j| { - batches - .iter() - .map(|b| b[j].clone()) - .reduce(|sum, v| sum + v) - .unwrap() + self.compute_aggregate_of_others(max_from, others, exit, |sum, value| { + *sum = sum.clone() + value; }) } @@ -83,8 +84,10 @@ where O: ReadableVec, V::T: Add + Ord, { - self.compute_aggregate_of_others(max_from, others, exit, |batches, j| { - batches.iter().map(|b| &b[j]).min().unwrap().clone() + self.compute_aggregate_of_others(max_from, others, exit, |min, value| { + if value.lt(min) { + *min = value; + } }) } @@ -98,8 +101,10 @@ where O: ReadableVec, V::T: Add + Ord, { - self.compute_aggregate_of_others(max_from, others, exit, |batches, j| { - batches.iter().map(|b| &b[j]).max().unwrap().clone() + self.compute_aggregate_of_others(max_from, others, exit, |max, value| { + if value.gt(max) { + *max = value; + } }) } diff --git a/crates/vecdb/src/variants/lazy/from2/any_vec.rs b/crates/vecdb/src/variants/lazy/from2/any_vec.rs deleted file mode 100644 index 44eeb4588..000000000 --- a/crates/vecdb/src/variants/lazy/from2/any_vec.rs +++ /dev/null @@ -1,54 +0,0 @@ -use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; - -use super::LazyVecFrom2; - -impl AnyVec for LazyVecFrom2 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, -{ - fn version(&self) -> Version { - self.base_version + self.source1.version() + self.source2.version() - } - - fn name(&self) -> &str { - &self.name - } - - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - fn len(&self) -> usize { - let len1 = if self.s1_counts { - self.source1.len() - } else { - usize::MAX - }; - let len2 = if self.s2_counts { - self.source2.len() - } else { - usize::MAX - }; - len1.min(len2) - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } - - #[inline] - fn value_type_to_string(&self) -> &'static str { - short_type_name::() - } - - #[inline] - fn region_names(&self) -> Vec { - Vec::new() - } -} diff --git a/crates/vecdb/src/variants/lazy/from2/mod.rs b/crates/vecdb/src/variants/lazy/from2/mod.rs deleted file mode 100644 index 681f1a107..000000000 --- a/crates/vecdb/src/variants/lazy/from2/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::sync::Arc; - -mod any_vec; -mod readable; -mod transform; -mod typed; - -pub use transform::*; - -use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; - -pub type ComputeFrom2 = fn(I, S1T, S2T) -> T; - -/// Lazily computed vector deriving values from two source vectors. -/// -/// Values are computed on-the-fly during iteration using a provided function. -/// Nothing is stored on disk - all values are recomputed each time they're accessed. -#[derive(Clone)] -pub struct LazyVecFrom2 -where - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, -{ - pub(super) name: Arc, - pub(super) base_version: Version, - pub(super) source1: ReadableBoxedVec, - pub(super) source2: ReadableBoxedVec, - pub(super) compute: ComputeFrom2, - pub(super) s1_counts: bool, - pub(super) s2_counts: bool, -} - -impl LazyVecFrom2 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, -{ - pub fn init( - name: &str, - version: Version, - source1: ReadableBoxedVec, - source2: ReadableBoxedVec, - compute: ComputeFrom2, - ) -> Self { - let target = I::to_string(); - let s1 = source1.index_type_to_string(); - let s2 = source2.index_type_to_string(); - - assert!( - s1 == target || s2 == target, - "LazyVecFrom2: at least one source must have index type {}, got {} and {}", - target, - s1, - s2 - ); - - let s1_counts = s1 == target; - let s2_counts = s2 == target; - - Self { - name: Arc::from(name), - base_version: version, - source1, - source2, - compute, - s1_counts, - s2_counts, - } - } -} - -impl LazyVecFrom2 -where - I: VecIndex, - T: VecValue, - S1T: VecValue, - S2T: VecValue, -{ - /// Create a lazy vec with a generic binary transform. - /// Usage: `LazyVecFrom2::transformed::(name, v, source1, source2)` - pub fn transformed>( - name: &str, - version: Version, - source1: ReadableBoxedVec, - source2: ReadableBoxedVec, - ) -> Self { - Self::init(name, version, source1, source2, |_, a, b| F::apply(a, b)) - } -} diff --git a/crates/vecdb/src/variants/lazy/from2/readable.rs b/crates/vecdb/src/variants/lazy/from2/readable.rs deleted file mode 100644 index 5b3adec78..000000000 --- a/crates/vecdb/src/variants/lazy/from2/readable.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; - -use super::LazyVecFrom2; - -impl ReadableVec for LazyVecFrom2 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, -{ - #[inline] - fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { - let to = to.min(self.len()); - buf.reserve(to.saturating_sub(from)); - self.for_each_range_dyn_at(from, to, &mut |v| buf.push(v)); - } - - #[inline] - fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { - let compute = self.compute; - let to = to.min(self.len()); - let buf1 = self.source1.collect_range_dyn(from, to); - let buf2 = self.source2.collect_range_dyn(from, to); - buf1.into_iter() - .zip(buf2) - .enumerate() - .for_each(|(local, (v1, v2))| { - f(compute(I::from(from + local), v1, v2)); - }); - } - - #[inline] - fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B - where - Self: Sized, - { - self.try_fold_range_at(from, to, init, |acc, v| { - Ok::<_, std::convert::Infallible>(f(acc, v)) - }) - .unwrap_or_else(|e: std::convert::Infallible| match e {}) - } - - #[inline] - fn try_fold_range_at std::result::Result>( - &self, - from: usize, - to: usize, - init: B, - mut f: F, - ) -> std::result::Result - where - Self: Sized, - { - let to = to.min(self.len()); - if from >= to { - return Ok(init); - } - let compute = self.compute; - let buf1 = self.source1.collect_range_dyn(from, to); - let buf2 = self.source2.collect_range_dyn(from, to); - buf1.into_iter() - .zip(buf2) - .enumerate() - .try_fold(init, |acc, (local, (v1, v2))| { - f(acc, compute(I::from(from + local), v1, v2)) - }) - } - - #[inline] - fn collect_one_at(&self, index: usize) -> Option { - if index >= self.len() { - return None; - } - let v1 = self.source1.collect_one_at(index)?; - let v2 = self.source2.collect_one_at(index)?; - Some((self.compute)(I::from(index), v1, v2)) - } - - fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { - let compute = self.compute; - let vals1 = self.source1.read_sorted_at(indices); - let vals2 = self.source2.read_sorted_at(indices); - out.reserve(vals1.len().min(vals2.len())); - indices - .iter() - .zip(vals1.into_iter().zip(vals2)) - .for_each(|(&i, (v1, v2))| out.push(compute(I::from(i), v1, v2))); - } -} diff --git a/crates/vecdb/src/variants/lazy/from2/typed.rs b/crates/vecdb/src/variants/lazy/from2/typed.rs deleted file mode 100644 index 27c8e05a9..000000000 --- a/crates/vecdb/src/variants/lazy/from2/typed.rs +++ /dev/null @@ -1,16 +0,0 @@ -use crate::{TypedVec, VecIndex, VecValue}; - -use super::LazyVecFrom2; - -impl TypedVec for LazyVecFrom2 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, -{ - type I = I; - type T = T; -} diff --git a/crates/vecdb/src/variants/lazy/from3/any_vec.rs b/crates/vecdb/src/variants/lazy/from3/any_vec.rs deleted file mode 100644 index 5b5ab6dfb..000000000 --- a/crates/vecdb/src/variants/lazy/from3/any_vec.rs +++ /dev/null @@ -1,61 +0,0 @@ -use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; - -use super::LazyVecFrom3; - -impl AnyVec for LazyVecFrom3 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, - S3I: VecIndex, - S3T: VecValue, -{ - fn version(&self) -> Version { - self.base_version + self.source1.version() + self.source2.version() + self.source3.version() - } - - fn name(&self) -> &str { - &self.name - } - - fn index_type_to_string(&self) -> &'static str { - I::to_string() - } - - fn len(&self) -> usize { - let len1 = if self.s1_counts { - self.source1.len() - } else { - usize::MAX - }; - let len2 = if self.s2_counts { - self.source2.len() - } else { - usize::MAX - }; - let len3 = if self.s3_counts { - self.source3.len() - } else { - usize::MAX - }; - len1.min(len2).min(len3) - } - - #[inline] - fn value_type_to_size_of(&self) -> usize { - size_of::() - } - - #[inline] - fn value_type_to_string(&self) -> &'static str { - short_type_name::() - } - - #[inline] - fn region_names(&self) -> Vec { - Vec::new() - } -} diff --git a/crates/vecdb/src/variants/lazy/from3/mod.rs b/crates/vecdb/src/variants/lazy/from3/mod.rs deleted file mode 100644 index aa31645f6..000000000 --- a/crates/vecdb/src/variants/lazy/from3/mod.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::sync::Arc; - -mod any_vec; -mod readable; -mod typed; - -use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; - -pub type ComputeFrom3 = fn(I, S1T, S2T, S3T) -> T; - -/// Lazily computed vector deriving values from three source vectors. -/// -/// Values are computed on-the-fly during iteration using a provided function. -/// Nothing is stored on disk - all values are recomputed each time they're accessed. -#[derive(Clone)] -pub struct LazyVecFrom3 -where - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, - S3I: VecIndex, - S3T: VecValue, -{ - pub(super) name: Arc, - pub(super) base_version: Version, - pub(super) source1: ReadableBoxedVec, - pub(super) source2: ReadableBoxedVec, - pub(super) source3: ReadableBoxedVec, - pub(super) compute: ComputeFrom3, - pub(super) s1_counts: bool, - pub(super) s2_counts: bool, - pub(super) s3_counts: bool, -} - -impl LazyVecFrom3 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, - S3I: VecIndex, - S3T: VecValue, -{ - pub fn init( - name: &str, - version: Version, - source1: ReadableBoxedVec, - source2: ReadableBoxedVec, - source3: ReadableBoxedVec, - compute: ComputeFrom3, - ) -> Self { - let target = I::to_string(); - let s1 = source1.index_type_to_string(); - let s2 = source2.index_type_to_string(); - let s3 = source3.index_type_to_string(); - - assert!( - s1 == target || s2 == target || s3 == target, - "LazyVecFrom3: at least one source must have index type {}, got {}, {}, and {}", - target, - s1, - s2, - s3 - ); - - let s1_counts = s1 == target; - let s2_counts = s2 == target; - let s3_counts = s3 == target; - - Self { - name: Arc::from(name), - base_version: version, - source1, - source2, - source3, - compute, - s1_counts, - s2_counts, - s3_counts, - } - } -} diff --git a/crates/vecdb/src/variants/lazy/from3/readable.rs b/crates/vecdb/src/variants/lazy/from3/readable.rs deleted file mode 100644 index c2f607d53..000000000 --- a/crates/vecdb/src/variants/lazy/from3/readable.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; - -use super::LazyVecFrom3; - -impl ReadableVec - for LazyVecFrom3 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, - S3I: VecIndex, - S3T: VecValue, -{ - #[inline] - fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { - let to = to.min(self.len()); - buf.reserve(to.saturating_sub(from)); - self.for_each_range_dyn_at(from, to, &mut |v| buf.push(v)); - } - - #[inline] - fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { - let compute = self.compute; - let to = to.min(self.len()); - let buf1 = self.source1.collect_range_dyn(from, to); - let buf2 = self.source2.collect_range_dyn(from, to); - let buf3 = self.source3.collect_range_dyn(from, to); - buf1.into_iter() - .zip(buf2) - .zip(buf3) - .enumerate() - .for_each(|(local, ((v1, v2), v3))| { - f(compute(I::from(from + local), v1, v2, v3)); - }); - } - - #[inline] - fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B - where - Self: Sized, - { - self.try_fold_range_at(from, to, init, |acc, v| { - Ok::<_, std::convert::Infallible>(f(acc, v)) - }) - .unwrap_or_else(|e: std::convert::Infallible| match e {}) - } - - #[inline] - fn try_fold_range_at std::result::Result>( - &self, - from: usize, - to: usize, - init: B, - mut f: F, - ) -> std::result::Result - where - Self: Sized, - { - let to = to.min(self.len()); - if from >= to { - return Ok(init); - } - let compute = self.compute; - let buf1 = self.source1.collect_range_dyn(from, to); - let buf2 = self.source2.collect_range_dyn(from, to); - let buf3 = self.source3.collect_range_dyn(from, to); - buf1.into_iter() - .zip(buf2) - .zip(buf3) - .enumerate() - .try_fold(init, |acc, (local, ((v1, v2), v3))| { - f(acc, compute(I::from(from + local), v1, v2, v3)) - }) - } - - #[inline] - fn collect_one_at(&self, index: usize) -> Option { - if index >= self.len() { - return None; - } - let v1 = self.source1.collect_one_at(index)?; - let v2 = self.source2.collect_one_at(index)?; - let v3 = self.source3.collect_one_at(index)?; - Some((self.compute)(I::from(index), v1, v2, v3)) - } - - fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { - let compute = self.compute; - let vals1 = self.source1.read_sorted_at(indices); - let vals2 = self.source2.read_sorted_at(indices); - let vals3 = self.source3.read_sorted_at(indices); - out.reserve(vals1.len().min(vals2.len()).min(vals3.len())); - indices - .iter() - .zip(vals1.into_iter().zip(vals2).zip(vals3)) - .for_each(|(&i, ((v1, v2), v3))| out.push(compute(I::from(i), v1, v2, v3))); - } -} diff --git a/crates/vecdb/src/variants/lazy/from3/typed.rs b/crates/vecdb/src/variants/lazy/from3/typed.rs deleted file mode 100644 index 84fc50603..000000000 --- a/crates/vecdb/src/variants/lazy/from3/typed.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::{TypedVec, VecIndex, VecValue}; - -use super::LazyVecFrom3; - -impl TypedVec - for LazyVecFrom3 -where - I: VecIndex, - T: VecValue, - S1I: VecIndex, - S1T: VecValue, - S2I: VecIndex, - S2T: VecValue, - S3I: VecIndex, - S3T: VecValue, -{ - type I = I; - type T = T; -} diff --git a/crates/vecdb/src/variants/lazy/mod.rs b/crates/vecdb/src/variants/lazy/mod.rs index d017f44ef..2a7fd331a 100644 --- a/crates/vecdb/src/variants/lazy/mod.rs +++ b/crates/vecdb/src/variants/lazy/mod.rs @@ -1,11 +1,7 @@ mod agg; mod delta; -mod from1; -mod from2; -mod from3; +mod vec; pub use agg::*; pub use delta::*; -pub use from1::*; -pub use from2::*; -pub use from3::*; +pub use vec::*; diff --git a/crates/vecdb/src/variants/lazy/from1/any_vec.rs b/crates/vecdb/src/variants/lazy/vec/any_vec.rs similarity index 89% rename from crates/vecdb/src/variants/lazy/from1/any_vec.rs rename to crates/vecdb/src/variants/lazy/vec/any_vec.rs index 6faddc593..a79bcdd5c 100644 --- a/crates/vecdb/src/variants/lazy/from1/any_vec.rs +++ b/crates/vecdb/src/variants/lazy/vec/any_vec.rs @@ -1,8 +1,8 @@ use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; -use super::LazyVecFrom1; +use super::LazyVec; -impl AnyVec for LazyVecFrom1 +impl AnyVec for LazyVec where I: VecIndex, T: VecValue, diff --git a/crates/vecdb/src/variants/lazy/from1/mod.rs b/crates/vecdb/src/variants/lazy/vec/mod.rs similarity index 79% rename from crates/vecdb/src/variants/lazy/from1/mod.rs rename to crates/vecdb/src/variants/lazy/vec/mod.rs index 2728f1973..679d08656 100644 --- a/crates/vecdb/src/variants/lazy/from1/mod.rs +++ b/crates/vecdb/src/variants/lazy/vec/mod.rs @@ -10,8 +10,6 @@ pub use transform::*; use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; -pub type ComputeFrom1 = fn(I, S1T) -> T; - /// Lazily computed vector deriving values on-the-fly from one source vector. /// /// Unlike `EagerVec`, no data is stored on disk. Values are computed during @@ -22,7 +20,7 @@ pub type ComputeFrom1 = fn(I, S1T) -> T; /// /// For frequently accessed derived data, prefer `EagerVec` for better performance. #[derive(Clone)] -pub struct LazyVecFrom1 +pub struct LazyVec where S1I: VecIndex, S1T: VecValue, @@ -30,10 +28,10 @@ where pub(super) name: Arc, pub(super) base_version: Version, pub(super) source: ReadableBoxedVec, - pub(super) compute: ComputeFrom1, + pub(super) compute: fn(I, S1T) -> T, } -impl LazyVecFrom1 +impl LazyVec where I: VecIndex, T: VecValue, @@ -44,12 +42,12 @@ where name: &str, version: Version, source: ReadableBoxedVec, - compute: ComputeFrom1, + compute: fn(I, S1T) -> T, ) -> Self { assert_eq!( I::to_string(), S1I::to_string(), - "LazyVecFrom1 index type mismatch: expected {}, got {}", + "LazyVec index type mismatch: expected {}, got {}", I::to_string(), S1I::to_string() ); @@ -63,14 +61,14 @@ where } } -impl LazyVecFrom1 +impl LazyVec where I: VecIndex, T: VecValue, S1T: VecValue, { /// Create a lazy vec with a generic transform. - /// Usage: `LazyVecFrom1::transformed::(name, v, source)` + /// Usage: `LazyVec::transformed::(name, v, source)` pub fn transformed>( name: &str, version: Version, diff --git a/crates/vecdb/src/variants/lazy/from1/read_only_clone.rs b/crates/vecdb/src/variants/lazy/vec/read_only_clone.rs similarity index 70% rename from crates/vecdb/src/variants/lazy/from1/read_only_clone.rs rename to crates/vecdb/src/variants/lazy/vec/read_only_clone.rs index 96ea7a4a6..01e3672c2 100644 --- a/crates/vecdb/src/variants/lazy/from1/read_only_clone.rs +++ b/crates/vecdb/src/variants/lazy/vec/read_only_clone.rs @@ -1,8 +1,8 @@ use crate::{ReadOnlyClone, VecIndex, VecValue}; -use super::LazyVecFrom1; +use super::LazyVec; -impl ReadOnlyClone for LazyVecFrom1 +impl ReadOnlyClone for LazyVec where I: VecIndex, T: VecValue, diff --git a/crates/vecdb/src/variants/lazy/from1/readable.rs b/crates/vecdb/src/variants/lazy/vec/readable.rs similarity index 95% rename from crates/vecdb/src/variants/lazy/from1/readable.rs rename to crates/vecdb/src/variants/lazy/vec/readable.rs index 5979efabe..b6af66610 100644 --- a/crates/vecdb/src/variants/lazy/from1/readable.rs +++ b/crates/vecdb/src/variants/lazy/vec/readable.rs @@ -1,8 +1,8 @@ use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; -use super::LazyVecFrom1; +use super::LazyVec; -impl ReadableVec for LazyVecFrom1 +impl ReadableVec for LazyVec where I: VecIndex, T: VecValue, diff --git a/crates/vecdb/src/variants/lazy/from1/transform/halve.rs b/crates/vecdb/src/variants/lazy/vec/transform/halve.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from1/transform/halve.rs rename to crates/vecdb/src/variants/lazy/vec/transform/halve.rs diff --git a/crates/vecdb/src/variants/lazy/from1/transform/ident.rs b/crates/vecdb/src/variants/lazy/vec/transform/ident.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from1/transform/ident.rs rename to crates/vecdb/src/variants/lazy/vec/transform/ident.rs diff --git a/crates/vecdb/src/variants/lazy/from1/transform/mod.rs b/crates/vecdb/src/variants/lazy/vec/transform/mod.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from1/transform/mod.rs rename to crates/vecdb/src/variants/lazy/vec/transform/mod.rs diff --git a/crates/vecdb/src/variants/lazy/from1/transform/negate.rs b/crates/vecdb/src/variants/lazy/vec/transform/negate.rs similarity index 100% rename from crates/vecdb/src/variants/lazy/from1/transform/negate.rs rename to crates/vecdb/src/variants/lazy/vec/transform/negate.rs diff --git a/crates/vecdb/src/variants/lazy/from1/typed.rs b/crates/vecdb/src/variants/lazy/vec/typed.rs similarity index 64% rename from crates/vecdb/src/variants/lazy/from1/typed.rs rename to crates/vecdb/src/variants/lazy/vec/typed.rs index b7a090674..8569990c7 100644 --- a/crates/vecdb/src/variants/lazy/from1/typed.rs +++ b/crates/vecdb/src/variants/lazy/vec/typed.rs @@ -1,8 +1,8 @@ use crate::{TypedVec, VecIndex, VecValue}; -use super::LazyVecFrom1; +use super::LazyVec; -impl TypedVec for LazyVecFrom1 +impl TypedVec for LazyVec where I: VecIndex, T: VecValue, diff --git a/modules/brk-client/index.js b/modules/brk-client/index.js index 44470c07b..b91aa0467 100644 --- a/modules/brk-client/index.js +++ b/modules/brk-client/index.js @@ -493,10 +493,10 @@ ancestors and no descendants (matches mempool.space). * Detailed series count with per-database breakdown * * @typedef {Object} DetailedSeriesCount - * @property {number} distinctSeries - Number of unique series available (e.g., realized_price, market_cap) - * @property {number} totalEndpoints - Total number of series-index combinations across all timeframes - * @property {number} lazyEndpoints - Number of lazy (computed on-the-fly) series-index combinations - * @property {number} storedEndpoints - Number of eager (stored on disk) series-index combinations + * @property {number} distinct - Number of unique series available (e.g., realized_price, market_cap) + * @property {number} total - Total number of series-index combinations across all timeframes + * @property {number} lazy - Number of lazy (computed on-the-fly) series-index combinations + * @property {number} stored - Number of eager (stored on disk) series-index combinations * @property {{ [key: string]: SeriesCount }} byDb - Per-database breakdown of counts */ /** @@ -1092,13 +1092,13 @@ on serialization otherwise. * @property {Limit=} limit - Maximum number of results */ /** - * Series count statistics - distinct series and total series-index combinations + * Series count statistics * * @typedef {Object} SeriesCount - * @property {number} distinctSeries - Number of unique series available (e.g., realized_price, market_cap) - * @property {number} totalEndpoints - Total number of series-index combinations across all timeframes - * @property {number} lazyEndpoints - Number of lazy (computed on-the-fly) series-index combinations - * @property {number} storedEndpoints - Number of eager (stored on disk) series-index combinations + * @property {number} distinct - Number of unique series available (e.g., realized_price, market_cap) + * @property {number} total - Total number of series-index combinations across all timeframes + * @property {number} lazy - Number of lazy (computed on-the-fly) series-index combinations + * @property {number} stored - Number of eager (stored on disk) series-index combinations */ /** * Metadata about a series diff --git a/packages/brk_client/brk_client/__init__.py b/packages/brk_client/brk_client/__init__.py index 455635f10..54b98296e 100644 --- a/packages/brk_client/brk_client/__init__.py +++ b/packages/brk_client/brk_client/__init__.py @@ -944,34 +944,34 @@ class DataRangeFormat(TypedDict): class SeriesCount(TypedDict): """ - Series count statistics - distinct series and total series-index combinations + Series count statistics Attributes: - distinct_series: Number of unique series available (e.g., realized_price, market_cap) - total_endpoints: Total number of series-index combinations across all timeframes - lazy_endpoints: Number of lazy (computed on-the-fly) series-index combinations - stored_endpoints: Number of eager (stored on disk) series-index combinations + distinct: Number of unique series available (e.g., realized_price, market_cap) + total: Total number of series-index combinations across all timeframes + lazy: Number of lazy (computed on-the-fly) series-index combinations + stored: Number of eager (stored on disk) series-index combinations """ - distinct_series: int - total_endpoints: int - lazy_endpoints: int - stored_endpoints: int + distinct: int + total: int + lazy: int + stored: int class DetailedSeriesCount(TypedDict): """ Detailed series count with per-database breakdown Attributes: - distinct_series: Number of unique series available (e.g., realized_price, market_cap) - total_endpoints: Total number of series-index combinations across all timeframes - lazy_endpoints: Number of lazy (computed on-the-fly) series-index combinations - stored_endpoints: Number of eager (stored on disk) series-index combinations + distinct: Number of unique series available (e.g., realized_price, market_cap) + total: Total number of series-index combinations across all timeframes + lazy: Number of lazy (computed on-the-fly) series-index combinations + stored: Number of eager (stored on disk) series-index combinations by_db: Per-database breakdown of counts """ - distinct_series: int - total_endpoints: int - lazy_endpoints: int - stored_endpoints: int + distinct: int + total: int + lazy: int + stored: int by_db: dict[str, SeriesCount] class DifficultyAdjustment(TypedDict): diff --git a/website/scripts/options/market.js b/website/scripts/options/market.js index c47b761db..be2965086 100644 --- a/website/scripts/options/market.js +++ b/website/scripts/options/market.js @@ -81,15 +81,16 @@ function createMaSubSection(label, averages) { const more = averages.filter((a) => !includes(commonMaIds, a.id)); /** @param {MaPeriod} a */ - const toFolder = (a) => ({ - name: periodIdToName(a.id, true), - tree: simplePriceRatioTree({ + const toChart = (a) => { + const name = periodIdToName(a.id, true); + const [chart] = simplePriceRatioTree({ pattern: a.ratio, - title: `${periodIdToName(a.id, true)} ${label}`, + title: `${name} ${label}`, legend: "Average", color: a.color, - }), - }); + }); + return { ...chart, name }; + }; return { name: label, @@ -106,8 +107,8 @@ function createMaSubSection(label, averages) { }), ), }, - ...common.map(toFolder), - { name: "More...", tree: more.map(toFolder) }, + ...common.map(toChart), + { name: "More...", tree: more.map(toChart) }, ], }; } diff --git a/website/scripts/options/models/capital-sentiment.js b/website/scripts/options/models/capital-sentiment.js index 8140d4e92..57eb94e09 100644 --- a/website/scripts/options/models/capital-sentiment.js +++ b/website/scripts/options/models/capital-sentiment.js @@ -1,7 +1,7 @@ import { brk } from "../../utils/client.js"; import { colors } from "../../utils/colors.js"; import { Unit } from "../../utils/units.js"; -import { histogram } from "../series.js"; +import { histogram, price } from "../series.js"; /** * Create Capital Sentiment model section. @@ -9,6 +9,30 @@ import { histogram } from "../series.js"; */ export function createCapitalSentimentSection() { const { capitalSentiment } = brk.series.models; + const { all, sth, lth } = brk.series.cohorts.utxo; + const sma = brk.series.market.movingAverage.sma._1y; + const references = () => [ + price({ + series: all.realized.capitalized.price, + name: "All", + color: colors.capitalized, + }), + price({ + series: sth.realized.capitalized.price, + name: "STH", + color: colors.term.short, + }), + price({ + series: lth.realized.capitalized.price, + name: "LTH", + color: colors.term.long, + }), + price({ + series: sma, + name: "1Y SMA", + color: colors.time._1y, + }), + ]; return { name: "Capital Sentiment", @@ -16,6 +40,7 @@ export function createCapitalSentimentSection() { { name: "Score", title: "Capital Sentiment Score", + top: references(), bottom: [ histogram({ series: capitalSentiment.score, @@ -35,6 +60,7 @@ export function createCapitalSentimentSection() { { name: "Position", title: "Capital Sentiment Position", + top: references(), bottom: [ histogram({ series: capitalSentiment.isLong, diff --git a/website/scripts/options/partial.js b/website/scripts/options/partial.js index e8bfcfa6f..ea3c61a90 100644 --- a/website/scripts/options/partial.js +++ b/website/scripts/options/partial.js @@ -325,9 +325,9 @@ export function createPartialOptions() { { name: "Models", tree: [ - createCapitalSentimentSection(), - createRarityMeterSection(), createBedrockSection(), + createRarityMeterSection(), + createCapitalSentimentSection(), ], }, ],