diff --git a/Cargo.lock b/Cargo.lock index 063d3aded..d158d27e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -483,7 +483,6 @@ dependencies = [ "rustc-hash", "schemars", "serde", - "smallvec", "tracing", "vecdb", ] @@ -2388,8 +2387,6 @@ dependencies = [ [[package]] name = "rawdb" version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ec6d90b48955942ae9aa12b8630c8100798bf79e73dbdffd8115a3e71e4915" dependencies = [ "libc", "log", @@ -3281,8 +3278,6 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ca57cedd42c0c7d8a343c06ab9c311be28a731e5d1e4101ef671d9a9af409a8" dependencies = [ "itoa", "libc", @@ -3304,8 +3299,6 @@ dependencies = [ [[package]] name = "vecdb_derive" version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8840b74c96d5888e1a5846adcd2ec8355778052a07dd5f6d30d67ef0fbc33b7e" dependencies = [ "quote", "syn", diff --git a/Cargo.toml b/Cargo.toml index 34e7e893f..ebc69edd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,8 +86,8 @@ tower-http = { version = "0.6.9", features = ["catch-panic", "compression-br", " tower-layer = "0.3" tracing = { version = "0.1", default-features = false, features = ["std"] } ureq = { version = "3.3.0", features = ["json"] } -vecdb = { version = "0.10.2", features = ["derive", "serde_json", "pco", "schemars"] } -# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } +# vecdb = { version = "=0.10.2", features = ["derive", "serde_json", "pco", "schemars"] } +vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } [workspace.metadata.release] shared-version = true diff --git a/crates/brk_bindgen/src/openapi/mod.rs b/crates/brk_bindgen/src/openapi/mod.rs index 74cb76178..088e8856d 100644 --- a/crates/brk_bindgen/src/openapi/mod.rs +++ b/crates/brk_bindgen/src/openapi/mod.rs @@ -270,9 +270,7 @@ fn extract_response_kind(operation: &Operation, spec: &Spec) -> ResponseKind { fn schema_name_from_content(content: &oas3::spec::MediaType) -> Option { match content.schema.as_ref()? { - ObjectOrReference::Ref { ref_path, .. } => { - Some(ref_to_type_name(ref_path)?.to_string()) - } + ObjectOrReference::Ref { ref_path, .. } => Some(ref_to_type_name(ref_path)?.to_string()), ObjectOrReference::Object(schema) => schema_to_type_name(schema), } } @@ -288,7 +286,9 @@ fn is_numeric_schema(spec: &Spec, name: &str) -> bool { }; matches!( schema.schema_type.as_ref(), - Some(SchemaTypeSet::Single(SchemaType::Integer | SchemaType::Number)) + Some(SchemaTypeSet::Single( + SchemaType::Integer | SchemaType::Number + )) ) } diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs index 59f39bfe1..1bbda9d94 100644 --- a/crates/brk_cli/src/main.rs +++ b/crates/brk_cli/src/main.rs @@ -42,7 +42,7 @@ pub fn main() -> anyhow::Result<()> { { // Pre-run indexer if too far behind, then drop and reimport to reduce memory let chain_height = client.get_last_height()?; - let indexed_height = indexer.vecs.starting_height(); + let indexed_height = indexer.vecs.next_height(); let blocks_behind = chain_height.saturating_sub(*indexed_height); if blocks_behind > 10_000 { info!("---"); @@ -105,15 +105,17 @@ pub fn main() -> anyhow::Result<()> { let total_start = Instant::now(); - let starting_indexes = if cfg!(debug_assertions) { - indexer.checked_index(&reader, &client, &exit)? + if cfg!(debug_assertions) { + indexer.checked_index(&reader, &client, &exit)?; } else { - indexer.index(&reader, &client, &exit)? - }; + indexer.index(&reader, &client, &exit)?; + } Mimalloc::collect(); - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, &exit)?; + + indexer.advance_safe_lengths()?; info!("Total time: {:?}", total_start.elapsed()); info!("Waiting for new blocks..."); diff --git a/crates/brk_cli/src/paths.rs b/crates/brk_cli/src/paths.rs index 4f476ac25..0602dc383 100644 --- a/crates/brk_cli/src/paths.rs +++ b/crates/brk_cli/src/paths.rs @@ -6,7 +6,7 @@ pub fn dot_brk_path() -> PathBuf { } pub fn dot_brk_log_path() -> PathBuf { - dot_brk_path().join("log") + dot_brk_path().join("logs") } pub fn default_brk_path() -> PathBuf { diff --git a/crates/brk_computer/README.md b/crates/brk_computer/README.md index a75dc1013..37f93d34a 100644 --- a/crates/brk_computer/README.md +++ b/crates/brk_computer/README.md @@ -22,7 +22,7 @@ Compute 1000+ on-chain metrics from indexed blockchain data: supply breakdowns, let mut computer = Computer::forced_import(&outputs_path, &indexer)?; // Compute all metrics for new blocks -computer.compute(&indexer, starting_indexes, &reader, &exit)?; +computer.compute(&indexer, &exit)?; // Access computed data via traversable vecs let supply = computer.distribution.utxo_cohorts.all.metrics.supply.total.sats.height; diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 6373a86f3..9366fa016 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -37,7 +37,7 @@ pub fn main() -> color_eyre::Result<()> { // Pre-run indexer if too far behind, then drop and reimport to reduce memory let chain_height = client.get_last_height()?; - let indexed_height = indexer.vecs.starting_height(); + let indexed_height = indexer.vecs.next_height(); if u32::from(chain_height).saturating_sub(u32::from(indexed_height)) > 1000 { indexer.checked_index(&reader, &client, &exit)?; drop(indexer); @@ -49,11 +49,11 @@ pub fn main() -> color_eyre::Result<()> { loop { let i = Instant::now(); - let starting_indexes = indexer.checked_index(&reader, &client, &exit)?; + indexer.checked_index(&reader, &client, &exit)?; Mimalloc::collect(); - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, &exit)?; dbg!(i.elapsed()); sleep(Duration::from_secs(10)); } diff --git a/crates/brk_computer/examples/computer_bench.rs b/crates/brk_computer/examples/computer_bench.rs index c60d80a52..60d840252 100644 --- a/crates/brk_computer/examples/computer_bench.rs +++ b/crates/brk_computer/examples/computer_bench.rs @@ -44,13 +44,13 @@ pub fn main() -> Result<()> { }); let i = Instant::now(); - let starting_indexes = indexer.index(&reader, &client, &exit)?; + indexer.index(&reader, &client, &exit)?; info!("Done in {:?}", i.elapsed()); Mimalloc::collect(); let i = Instant::now(); - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, &exit)?; info!("Done in {:?}", i.elapsed()); // We want to benchmark the drop too diff --git a/crates/brk_computer/examples/full_bench.rs b/crates/brk_computer/examples/full_bench.rs index 03edd98a7..19150d4df 100644 --- a/crates/brk_computer/examples/full_bench.rs +++ b/crates/brk_computer/examples/full_bench.rs @@ -48,7 +48,7 @@ pub fn main() -> color_eyre::Result<()> { // Pre-run indexer if too far behind, then drop and reimport to reduce memory let chain_height = client.get_last_height()?; - let indexed_height = indexer.vecs.starting_height(); + let indexed_height = indexer.vecs.next_height(); if chain_height.saturating_sub(*indexed_height) > 1000 { indexer.index(&reader, &client, &exit)?; drop(indexer); @@ -60,13 +60,13 @@ pub fn main() -> color_eyre::Result<()> { loop { let i = Instant::now(); - let starting_indexes = indexer.index(&reader, &client, &exit)?; + indexer.index(&reader, &client, &exit)?; info!("Done in {:?}", i.elapsed()); Mimalloc::collect(); let i = Instant::now(); - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, &exit)?; info!("Done in {:?}", i.elapsed()); sleep(Duration::from_secs(60)); diff --git a/crates/brk_computer/src/blocks/compute.rs b/crates/brk_computer/src/blocks/compute.rs index 80cfd06e6..f7af05dab 100644 --- a/crates/brk_computer/src/blocks/compute.rs +++ b/crates/brk_computer/src/blocks/compute.rs @@ -2,7 +2,6 @@ use std::thread; use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use crate::indexes; @@ -14,13 +13,12 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; // lookback depends on indexes.timestamp.monotonic - self.lookback.compute(indexes, starting_indexes, exit)?; + self.lookback.compute(indexer, indexes, exit)?; // Parallel: remaining sub-modules are independent of each other. // size depends on lookback (already computed above). @@ -35,12 +33,12 @@ impl Vecs { .. } = self; thread::scope(|s| -> Result<()> { - let r1 = s.spawn(|| count.compute(indexer, starting_indexes, exit)); - let r2 = s.spawn(|| interval.compute(indexer, starting_indexes, exit)); - let r3 = s.spawn(|| weight.compute(indexer, starting_indexes, exit)); - let r4 = s.spawn(|| difficulty.compute(indexer, indexes, starting_indexes, exit)); - let r5 = s.spawn(|| halving.compute(indexes, starting_indexes, exit)); - size.compute(indexer, &*lookback, starting_indexes, exit)?; + let r1 = s.spawn(|| count.compute(indexer, exit)); + let r2 = s.spawn(|| interval.compute(indexer, exit)); + let r3 = s.spawn(|| weight.compute(indexer, exit)); + let r4 = s.spawn(|| difficulty.compute(indexer, indexes, exit)); + let r5 = s.spawn(|| halving.compute(indexer, indexes, exit)); + size.compute(indexer, &*lookback, exit)?; r1.join().unwrap()?; r2.join().unwrap()?; r3.join().unwrap()?; diff --git a/crates/brk_computer/src/blocks/count/compute.rs b/crates/brk_computer/src/blocks/count/compute.rs index b674eb09e..38ec22ef1 100644 --- a/crates/brk_computer/src/blocks/count/compute.rs +++ b/crates/brk_computer/src/blocks/count/compute.rs @@ -1,25 +1,20 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Indexes, StoredU32}; +use brk_types::StoredU32; use vecdb::Exit; use super::Vecs; impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { - // Block count raw + cumulative + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; self.total.block.compute_range( - starting_indexes.height, + starting_height, &indexer.vecs.blocks.weight, |h| (h, StoredU32::from(1_u32)), exit, )?; - self.total.compute_rest(starting_indexes.height, exit)?; + self.total.compute_rest(starting_height, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/blocks/difficulty/compute.rs b/crates/brk_computer/src/blocks/difficulty/compute.rs index af1650e1a..23395d50c 100644 --- a/crates/brk_computer/src/blocks/difficulty/compute.rs +++ b/crates/brk_computer/src/blocks/difficulty/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Indexes, StoredU32}; +use brk_types::StoredU32; use vecdb::Exit; use super::Vecs; @@ -11,25 +11,25 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; self.adjustment.bps.height.compute_ratio_change( - starting_indexes.height, + starting_height, &indexer.vecs.blocks.difficulty, 2016, exit, )?; self.epoch.height.compute_transform( - starting_indexes.height, + starting_height, &indexes.height.epoch, |(h, epoch, ..)| (h, epoch), exit, )?; self.blocks_to_retarget.height.compute_transform( - starting_indexes.height, + starting_height, &indexes.height.epoch, |(h, ..)| (h, StoredU32::from(h.left_before_next_diff_adj())), exit, diff --git a/crates/brk_computer/src/blocks/halving/compute.rs b/crates/brk_computer/src/blocks/halving/compute.rs index 93914b9af..41c957a45 100644 --- a/crates/brk_computer/src/blocks/halving/compute.rs +++ b/crates/brk_computer/src/blocks/halving/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, StoredU32}; +use brk_indexer::Indexer; +use brk_types::StoredU32; use vecdb::Exit; use super::Vecs; @@ -8,19 +9,20 @@ use crate::indexes; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; self.epoch.height.compute_transform( - starting_indexes.height, + starting_height, &indexes.height.halving, |(h, epoch, ..)| (h, epoch), exit, )?; self.blocks_to_halving.height.compute_transform( - starting_indexes.height, + starting_height, &indexes.height.halving, |(h, ..)| (h, StoredU32::from(h.left_before_next_halving())), exit, diff --git a/crates/brk_computer/src/blocks/interval/compute.rs b/crates/brk_computer/src/blocks/interval/compute.rs index 6a05f8f76..664daaaec 100644 --- a/crates/brk_computer/src/blocks/interval/compute.rs +++ b/crates/brk_computer/src/blocks/interval/compute.rs @@ -1,21 +1,17 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{CheckedSub, Indexes, Timestamp}; +use brk_types::{CheckedSub, Timestamp}; use vecdb::{Exit, ReadableVec}; use super::Vecs; impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let mut prev_timestamp = None; - self.0.compute(starting_indexes.height, exit, |vec| { + self.0.compute(starting_height, exit, |vec| { vec.compute_transform( - starting_indexes.height, + starting_height, &indexer.vecs.blocks.timestamp, |(h, timestamp, ..)| { let interval = if let Some(prev_h) = h.decremented() { diff --git a/crates/brk_computer/src/blocks/lookback.rs b/crates/brk_computer/src/blocks/lookback.rs index 73f11ac7e..01cf254d0 100644 --- a/crates/brk_computer/src/blocks/lookback.rs +++ b/crates/brk_computer/src/blocks/lookback.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Timestamp, Version}; +use brk_types::{Height, Timestamp, Version}; use vecdb::{ AnyVec, CachedVec, Cursor, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw, StorageMode, VecIndex, @@ -219,53 +220,54 @@ impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { - self.compute_rolling_start_hours(indexes, starting_indexes, exit, 1, |s| &mut s._1h)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 1, |s| &mut s._24h.inner)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 3, |s| &mut s._3d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 7, |s| &mut s._1w.inner)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 8, |s| &mut s._8d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 9, |s| &mut s._9d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 12, |s| &mut s._12d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 13, |s| &mut s._13d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 14, |s| &mut s._2w)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 21, |s| &mut s._21d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 26, |s| &mut s._26d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 30, |s| &mut s._1m.inner)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 34, |s| &mut s._34d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 55, |s| &mut s._55d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 60, |s| &mut s._2m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 63, |s| &mut s._9w)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 84, |s| &mut s._12w)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 89, |s| &mut s._89d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 90, |s| &mut s._3m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 98, |s| &mut s._14w)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 111, |s| &mut s._111d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 144, |s| &mut s._144d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 180, |s| &mut s._6m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 182, |s| &mut s._26w)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 200, |s| &mut s._200d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 270, |s| &mut s._9m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 350, |s| &mut s._350d)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 360, |s| &mut s._12m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 365, |s| &mut s._1y.inner)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 420, |s| &mut s._14m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 730, |s| &mut s._2y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 780, |s| &mut s._26m)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 1095, |s| &mut s._3y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 1400, |s| &mut s._200w)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 1460, |s| &mut s._4y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 1825, |s| &mut s._5y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 2190, |s| &mut s._6y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 2920, |s| &mut s._8y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 3285, |s| &mut s._9y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 3650, |s| &mut s._10y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 4380, |s| &mut s._12y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 5110, |s| &mut s._14y)?; - self.compute_rolling_start(indexes, starting_indexes, exit, 9490, |s| &mut s._26y)?; + let starting_height = indexer.safe_lengths().height; + self.compute_rolling_start_hours(indexes, starting_height, exit, 1, |s| &mut s._1h)?; + self.compute_rolling_start(indexes, starting_height, exit, 1, |s| &mut s._24h.inner)?; + self.compute_rolling_start(indexes, starting_height, exit, 3, |s| &mut s._3d)?; + self.compute_rolling_start(indexes, starting_height, exit, 7, |s| &mut s._1w.inner)?; + self.compute_rolling_start(indexes, starting_height, exit, 8, |s| &mut s._8d)?; + self.compute_rolling_start(indexes, starting_height, exit, 9, |s| &mut s._9d)?; + self.compute_rolling_start(indexes, starting_height, exit, 12, |s| &mut s._12d)?; + self.compute_rolling_start(indexes, starting_height, exit, 13, |s| &mut s._13d)?; + self.compute_rolling_start(indexes, starting_height, exit, 14, |s| &mut s._2w)?; + self.compute_rolling_start(indexes, starting_height, exit, 21, |s| &mut s._21d)?; + self.compute_rolling_start(indexes, starting_height, exit, 26, |s| &mut s._26d)?; + self.compute_rolling_start(indexes, starting_height, exit, 30, |s| &mut s._1m.inner)?; + self.compute_rolling_start(indexes, starting_height, exit, 34, |s| &mut s._34d)?; + self.compute_rolling_start(indexes, starting_height, exit, 55, |s| &mut s._55d)?; + self.compute_rolling_start(indexes, starting_height, exit, 60, |s| &mut s._2m)?; + self.compute_rolling_start(indexes, starting_height, exit, 63, |s| &mut s._9w)?; + self.compute_rolling_start(indexes, starting_height, exit, 84, |s| &mut s._12w)?; + self.compute_rolling_start(indexes, starting_height, exit, 89, |s| &mut s._89d)?; + self.compute_rolling_start(indexes, starting_height, exit, 90, |s| &mut s._3m)?; + self.compute_rolling_start(indexes, starting_height, exit, 98, |s| &mut s._14w)?; + self.compute_rolling_start(indexes, starting_height, exit, 111, |s| &mut s._111d)?; + self.compute_rolling_start(indexes, starting_height, exit, 144, |s| &mut s._144d)?; + self.compute_rolling_start(indexes, starting_height, exit, 180, |s| &mut s._6m)?; + self.compute_rolling_start(indexes, starting_height, exit, 182, |s| &mut s._26w)?; + self.compute_rolling_start(indexes, starting_height, exit, 200, |s| &mut s._200d)?; + self.compute_rolling_start(indexes, starting_height, exit, 270, |s| &mut s._9m)?; + self.compute_rolling_start(indexes, starting_height, exit, 350, |s| &mut s._350d)?; + self.compute_rolling_start(indexes, starting_height, exit, 360, |s| &mut s._12m)?; + self.compute_rolling_start(indexes, starting_height, exit, 365, |s| &mut s._1y.inner)?; + self.compute_rolling_start(indexes, starting_height, exit, 420, |s| &mut s._14m)?; + self.compute_rolling_start(indexes, starting_height, exit, 730, |s| &mut s._2y)?; + self.compute_rolling_start(indexes, starting_height, exit, 780, |s| &mut s._26m)?; + self.compute_rolling_start(indexes, starting_height, exit, 1095, |s| &mut s._3y)?; + self.compute_rolling_start(indexes, starting_height, exit, 1400, |s| &mut s._200w)?; + self.compute_rolling_start(indexes, starting_height, exit, 1460, |s| &mut s._4y)?; + self.compute_rolling_start(indexes, starting_height, exit, 1825, |s| &mut s._5y)?; + self.compute_rolling_start(indexes, starting_height, exit, 2190, |s| &mut s._6y)?; + self.compute_rolling_start(indexes, starting_height, exit, 2920, |s| &mut s._8y)?; + self.compute_rolling_start(indexes, starting_height, exit, 3285, |s| &mut s._9y)?; + self.compute_rolling_start(indexes, starting_height, exit, 3650, |s| &mut s._10y)?; + self.compute_rolling_start(indexes, starting_height, exit, 4380, |s| &mut s._12y)?; + self.compute_rolling_start(indexes, starting_height, exit, 5110, |s| &mut s._14y)?; + self.compute_rolling_start(indexes, starting_height, exit, 9490, |s| &mut s._26y)?; Ok(()) } @@ -273,7 +275,7 @@ impl Vecs { fn compute_rolling_start( &mut self, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_height: Height, exit: &Exit, days: usize, get_field: F, @@ -281,19 +283,15 @@ impl Vecs { where F: FnOnce(&mut Self) -> &mut EagerVec>, { - self.compute_rolling_start_inner( - indexes, - starting_indexes, - exit, - get_field, - |t, prev_ts| t.difference_in_days_between(prev_ts) >= days, - ) + self.compute_rolling_start_inner(indexes, starting_height, exit, get_field, |t, prev_ts| { + t.difference_in_days_between(prev_ts) >= days + }) } fn compute_rolling_start_hours( &mut self, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_height: Height, exit: &Exit, hours: usize, get_field: F, @@ -301,19 +299,15 @@ impl Vecs { where F: FnOnce(&mut Self) -> &mut EagerVec>, { - self.compute_rolling_start_inner( - indexes, - starting_indexes, - exit, - get_field, - |t, prev_ts| t.difference_in_hours_between(prev_ts) >= hours, - ) + self.compute_rolling_start_inner(indexes, starting_height, exit, get_field, |t, prev_ts| { + t.difference_in_hours_between(prev_ts) >= hours + }) } fn compute_rolling_start_inner( &mut self, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_height: Height, exit: &Exit, get_field: F, expired: D, @@ -323,7 +317,7 @@ impl Vecs { D: Fn(Timestamp, Timestamp) -> bool, { let field = get_field(self); - let resume_from = field.len().min(starting_indexes.height.to_usize()); + let resume_from = field.len().min(starting_height.to_usize()); let mut prev = if resume_from > 0 { field.collect_one_at(resume_from - 1).unwrap() } else { @@ -333,7 +327,7 @@ impl Vecs { cursor.advance(prev.to_usize()); let mut prev_ts = cursor.next().unwrap(); Ok(field.compute_transform( - starting_indexes.height, + starting_height, &indexes.timestamp.monotonic, |(h, t, ..)| { while expired(t, prev_ts) { diff --git a/crates/brk_computer/src/blocks/size/compute.rs b/crates/brk_computer/src/blocks/size/compute.rs index 81e2f9c91..17934b94b 100644 --- a/crates/brk_computer/src/blocks/size/compute.rs +++ b/crates/brk_computer/src/blocks/size/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Indexes, StoredU64}; +use brk_types::StoredU64; use vecdb::Exit; use super::Vecs; @@ -11,25 +11,23 @@ impl Vecs { &mut self, indexer: &Indexer, lookback: &blocks::LookbackVecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let window_starts = lookback.window_starts(); - // vbytes = floor(weight / 4), stored at height level self.vbytes - .compute(starting_indexes.height, &window_starts, exit, |height| { + .compute(starting_height, &window_starts, exit, |height| { Ok(height.compute_transform( - starting_indexes.height, + starting_height, &indexer.vecs.blocks.weight, |(h, weight, ..)| (h, StoredU64::from(weight.to_vbytes_floor())), exit, )?) })?; - // size from indexer total_size self.size.compute( - starting_indexes.height, + starting_height, &window_starts, &indexer.vecs.blocks.total, exit, diff --git a/crates/brk_computer/src/blocks/weight/compute.rs b/crates/brk_computer/src/blocks/weight/compute.rs index b7b95cacd..e39261c12 100644 --- a/crates/brk_computer/src/blocks/weight/compute.rs +++ b/crates/brk_computer/src/blocks/weight/compute.rs @@ -1,19 +1,15 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{BasisPoints16, Indexes}; +use brk_types::BasisPoints16; use vecdb::Exit; use super::Vecs; impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; self.fullness.bps.compute_transform( - starting_indexes.height, + starting_height, &indexer.vecs.blocks.weight, |(h, weight, ..)| (h, BasisPoints16::from(weight.fullness())), exit, diff --git a/crates/brk_computer/src/cointime/activity/compute.rs b/crates/brk_computer/src/cointime/activity/compute.rs index 2225f8ccd..4cdee08d6 100644 --- a/crates/brk_computer/src/cointime/activity/compute.rs +++ b/crates/brk_computer/src/cointime/activity/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Bitcoin, Indexes, StoredF64}; +use brk_indexer::Indexer; +use brk_types::{Bitcoin, StoredF64}; use vecdb::Exit; use super::Vecs; @@ -8,17 +9,18 @@ use crate::distribution; impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, distribution: &distribution::Vecs, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let all_metrics = &distribution.utxo_cohorts.all.metrics; let circulating_supply = &all_metrics.supply.total.sats.height; self.coinblocks_created - .compute(starting_indexes.height, exit, |vec| { + .compute(starting_height, exit, |vec| { vec.compute_transform( - starting_indexes.height, + starting_height, circulating_supply, |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), exit, @@ -27,9 +29,9 @@ impl Vecs { })?; self.coinblocks_stored - .compute(starting_indexes.height, exit, |vec| { + .compute(starting_height, exit, |vec| { vec.compute_subtract( - starting_indexes.height, + starting_height, &self.coinblocks_created.block, &distribution.coinblocks_destroyed.block, exit, @@ -38,14 +40,14 @@ impl Vecs { })?; self.liveliness.height.compute_divide( - starting_indexes.height, + starting_height, &distribution.coinblocks_destroyed.cumulative.height, &self.coinblocks_created.cumulative.height, exit, )?; self.ratio.height.compute_divide( - starting_indexes.height, + starting_height, &self.liveliness.height, &self.vaultedness.height, exit, diff --git a/crates/brk_computer/src/cointime/adjusted/compute.rs b/crates/brk_computer/src/cointime/adjusted/compute.rs index 69456946e..4a5856f9b 100644 --- a/crates/brk_computer/src/cointime/adjusted/compute.rs +++ b/crates/brk_computer/src/cointime/adjusted/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{BasisPointsSigned32, Indexes}; +use brk_indexer::Indexer; +use brk_types::BasisPointsSigned32; use vecdb::Exit; use super::super::activity; @@ -9,13 +10,15 @@ use crate::supply; impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, supply: &supply::Vecs, activity: &activity::Vecs, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + self.inflation_rate.bps.height.compute_transform2( - starting_indexes.height, + starting_height, &activity.ratio.height, &supply.inflation_rate.bps.height, |(h, a2vr, inflation, ..)| { @@ -28,14 +31,14 @@ impl Vecs { )?; self.tx_velocity_native.height.compute_multiply( - starting_indexes.height, + starting_height, &activity.ratio.height, &supply.velocity.native.height, exit, )?; self.tx_velocity_fiat.height.compute_multiply( - starting_indexes.height, + starting_height, &activity.ratio.height, &supply.velocity.fiat.height, exit, diff --git a/crates/brk_computer/src/cointime/cap/compute.rs b/crates/brk_computer/src/cointime/cap/compute.rs index ca1b2f581..32c00e88b 100644 --- a/crates/brk_computer/src/cointime/cap/compute.rs +++ b/crates/brk_computer/src/cointime/cap/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Dollars, Indexes}; +use brk_indexer::Indexer; +use brk_types::Dollars; use vecdb::Exit; use super::super::{activity, value}; @@ -10,40 +11,41 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, mining: &mining::Vecs, distribution: &distribution::Vecs, activity: &activity::Vecs, value: &value::Vecs, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); let all_metrics = &distribution.utxo_cohorts.all.metrics; let realized_cap_cents = &all_metrics.realized.cap.cents.height; let circulating_supply = &all_metrics.supply.total.btc.height; self.thermo.cents.height.compute_transform( - starting_indexes.height, + starting_lengths.height, &mining.rewards.subsidy.cumulative.cents.height, |(i, v, ..)| (i, v), exit, )?; self.investor.cents.height.compute_subtract( - starting_indexes.height, + starting_lengths.height, realized_cap_cents, &self.thermo.cents.height, exit, )?; self.vaulted.cents.height.compute_multiply( - starting_indexes.height, + starting_lengths.height, realized_cap_cents, &activity.vaultedness.height, exit, )?; self.active.cents.height.compute_multiply( - starting_indexes.height, + starting_lengths.height, realized_cap_cents, &activity.liveliness.height, exit, @@ -51,7 +53,7 @@ impl Vecs { // cointime_cap = (cointime_value_destroyed_cumulative * circulating_supply) / coinblocks_stored_cumulative self.cointime.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, &value.destroyed.cumulative.height, circulating_supply, &activity.coinblocks_stored.cumulative.height, @@ -67,7 +69,7 @@ impl Vecs { // AVIV = active_cap / investor_cap self.aviv.compute_ratio( - starting_indexes, + &starting_lengths, &self.active.cents.height, &self.investor.cents.height, exit, diff --git a/crates/brk_computer/src/cointime/compute.rs b/crates/brk_computer/src/cointime/compute.rs index 270da0898..ebcb69156 100644 --- a/crates/brk_computer/src/cointime/compute.rs +++ b/crates/brk_computer/src/cointime/compute.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; @@ -9,7 +9,7 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, prices: &prices::Vecs, blocks: &blocks::Vecs, mining: &mining::Vecs, @@ -20,29 +20,23 @@ impl Vecs { self.db.sync_bg_tasks()?; // Activity computes first (liveliness, vaultedness, etc.) - self.activity - .compute(starting_indexes, distribution, exit)?; + self.activity.compute(indexer, distribution, exit)?; // Phase 2: supply, adjusted, value are independent (all depend only on activity) let (r1, r2) = rayon::join( || { self.supply - .compute(starting_indexes, prices, distribution, &self.activity, exit) + .compute(indexer, prices, distribution, &self.activity, exit) }, || { rayon::join( || { self.adjusted - .compute(starting_indexes, supply_vecs, &self.activity, exit) + .compute(indexer, supply_vecs, &self.activity, exit) }, || { - self.value.compute( - starting_indexes, - prices, - distribution, - &self.activity, - exit, - ) + self.value + .compute(indexer, prices, distribution, &self.activity, exit) }, ) }, @@ -53,7 +47,7 @@ impl Vecs { // Cap depends on activity + value self.cap.compute( - starting_indexes, + indexer, mining, distribution, &self.activity, @@ -65,7 +59,7 @@ impl Vecs { let (r3, r4) = rayon::join( || { self.prices.compute( - starting_indexes, + indexer, prices, distribution, &self.activity, @@ -76,7 +70,7 @@ impl Vecs { }, || { self.reserve_risk - .compute(starting_indexes, blocks, prices, &self.value, exit) + .compute(indexer, blocks, prices, &self.value, exit) }, ); r3?; diff --git a/crates/brk_computer/src/cointime/prices/compute.rs b/crates/brk_computer/src/cointime/prices/compute.rs index d92776729..ea5b82614 100644 --- a/crates/brk_computer/src/cointime/prices/compute.rs +++ b/crates/brk_computer/src/cointime/prices/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Cents, Indexes}; +use brk_indexer::Indexer; +use brk_types::Cents; use vecdb::Exit; use super::super::{activity, cap, supply}; @@ -10,7 +11,7 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, prices: &prices::Vecs, distribution: &distribution::Vecs, activity: &activity::Vecs, @@ -18,14 +19,15 @@ impl Vecs { cap: &cap::Vecs, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); let all_metrics = &distribution.utxo_cohorts.all.metrics; let circulating_supply = &all_metrics.supply.total.btc.height; let realized_price = &all_metrics.realized.price.cents.height; self.vaulted - .compute_all(prices, starting_indexes, exit, |v| { + .compute_all(prices, &starting_lengths, exit, |v| { Ok(v.compute_transform2( - starting_indexes.height, + starting_lengths.height, realized_price, &activity.vaultedness.height, |(i, price, vaultedness, ..)| { @@ -36,9 +38,9 @@ impl Vecs { })?; self.active - .compute_all(prices, starting_indexes, exit, |v| { + .compute_all(prices, &starting_lengths, exit, |v| { Ok(v.compute_transform2( - starting_indexes.height, + starting_lengths.height, realized_price, &activity.liveliness.height, |(i, price, liveliness, ..)| { @@ -49,9 +51,9 @@ impl Vecs { })?; self.true_market_mean - .compute_all(prices, starting_indexes, exit, |v| { + .compute_all(prices, &starting_lengths, exit, |v| { Ok(v.compute_transform2( - starting_indexes.height, + starting_lengths.height, &cap.investor.cents.height, &supply.active.btc.height, |(i, cap_cents, supply_btc, ..)| { @@ -62,9 +64,9 @@ impl Vecs { })?; self.cointime - .compute_all(prices, starting_indexes, exit, |v| { + .compute_all(prices, &starting_lengths, exit, |v| { Ok(v.compute_transform2( - starting_indexes.height, + starting_lengths.height, &cap.cointime.cents.height, circulating_supply, |(i, cap_cents, supply_btc, ..)| { diff --git a/crates/brk_computer/src/cointime/reserve_risk/compute.rs b/crates/brk_computer/src/cointime/reserve_risk/compute.rs index 71281dc2f..18c87efdf 100644 --- a/crates/brk_computer/src/cointime/reserve_risk/compute.rs +++ b/crates/brk_computer/src/cointime/reserve_risk/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, StoredF64}; +use brk_indexer::Indexer; +use brk_types::StoredF64; use vecdb::Exit; use super::{super::value, Vecs}; @@ -8,21 +9,23 @@ use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, prices}; impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, blocks: &blocks::Vecs, prices: &prices::Vecs, value: &value::Vecs, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + self.vocdd_median_1y.compute_rolling_median_from_starts( - starting_indexes.height, + starting_height, &blocks.lookback._1y, &value.vocdd.block, exit, )?; self.hodl_bank.compute_cumulative_transformed_binary( - starting_indexes.height, + starting_height, &prices.spot.usd.height, &self.vocdd_median_1y, |price, median| StoredF64::from(f64::from(price) - f64::from(median)), @@ -30,7 +33,7 @@ impl Vecs { )?; self.value.height.compute_divide( - starting_indexes.height, + starting_height, &prices.spot.usd.height, &self.hodl_bank, exit, diff --git a/crates/brk_computer/src/cointime/supply/compute.rs b/crates/brk_computer/src/cointime/supply/compute.rs index 601e9af93..512102f9b 100644 --- a/crates/brk_computer/src/cointime/supply/compute.rs +++ b/crates/brk_computer/src/cointime/supply/compute.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use super::super::activity; @@ -9,12 +9,13 @@ use crate::{distribution, prices}; impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, prices: &prices::Vecs, distribution: &distribution::Vecs, activity: &activity::Vecs, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let circulating_supply = &distribution .utxo_cohorts .all @@ -25,22 +26,21 @@ impl Vecs { .height; self.vaulted.sats.height.compute_multiply( - starting_indexes.height, + starting_height, circulating_supply, &activity.vaultedness.height, exit, )?; self.active.sats.height.compute_multiply( - starting_indexes.height, + starting_height, circulating_supply, &activity.liveliness.height, exit, )?; - self.vaulted - .compute(prices, starting_indexes.height, exit)?; - self.active.compute(prices, starting_indexes.height, exit)?; + self.vaulted.compute(prices, starting_height, exit)?; + self.active.compute(prices, starting_height, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/cointime/value/compute.rs b/crates/brk_computer/src/cointime/value/compute.rs index 14110e9e9..18e105395 100644 --- a/crates/brk_computer/src/cointime/value/compute.rs +++ b/crates/brk_computer/src/cointime/value/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Bitcoin, Dollars, Indexes, StoredF64}; +use brk_indexer::Indexer; +use brk_types::{Bitcoin, Dollars, StoredF64}; use vecdb::Exit; use super::super::activity; @@ -9,31 +10,31 @@ use crate::{distribution, prices}; impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, prices: &prices::Vecs, distribution: &distribution::Vecs, activity: &activity::Vecs, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let all_metrics = &distribution.utxo_cohorts.all.metrics; let coinblocks_destroyed = &distribution.coinblocks_destroyed; let coindays_destroyed = &all_metrics.activity.coindays_destroyed; let circulating_supply = &all_metrics.supply.total.btc.height; - self.destroyed - .compute(starting_indexes.height, exit, |vec| { - vec.compute_multiply( - starting_indexes.height, - &prices.spot.usd.height, - &coinblocks_destroyed.block, - exit, - )?; - Ok(()) - })?; - - self.created.compute(starting_indexes.height, exit, |vec| { + self.destroyed.compute(starting_height, exit, |vec| { vec.compute_multiply( - starting_indexes.height, + starting_height, + &prices.spot.usd.height, + &coinblocks_destroyed.block, + exit, + )?; + Ok(()) + })?; + + self.created.compute(starting_height, exit, |vec| { + vec.compute_multiply( + starting_height, &prices.spot.usd.height, &activity.coinblocks_created.block, exit, @@ -41,9 +42,9 @@ impl Vecs { Ok(()) })?; - self.stored.compute(starting_indexes.height, exit, |vec| { + self.stored.compute(starting_height, exit, |vec| { vec.compute_multiply( - starting_indexes.height, + starting_height, &prices.spot.usd.height, &activity.coinblocks_stored.block, exit, @@ -54,9 +55,9 @@ impl Vecs { // VOCDD: Value of Coin Days Destroyed = price × (CDD / circulating_supply) // Supply-adjusted to account for growing supply over time // This is a key input for Reserve Risk / HODL Bank calculation - self.vocdd.compute(starting_indexes.height, exit, |vec| { + self.vocdd.compute(starting_height, exit, |vec| { vec.compute_transform3( - starting_indexes.height, + starting_height, &prices.spot.usd.height, &coindays_destroyed.block, circulating_supply, diff --git a/crates/brk_computer/src/distribution/addr/count/funded_total_vecs.rs b/crates/brk_computer/src/distribution/addr/count/funded_total_vecs.rs index 64af73ed9..cdf9b3708 100644 --- a/crates/brk_computer/src/distribution/addr/count/funded_total_vecs.rs +++ b/crates/brk_computer/src/distribution/addr/count/funded_total_vecs.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Indexes, Version}; +use brk_types::Version; use rayon::prelude::*; use vecdb::{AnyStoredVec, Database, Exit, Rw, StorageMode}; @@ -70,9 +71,9 @@ impl AddrCountFundedTotalVecs { self.total.push_counts(total); } - pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> { - self.funded.compute_rest(starting_indexes, exit)?; - self.total.compute_rest(starting_indexes, exit)?; + pub(crate) fn compute_rest(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()> { + self.funded.compute_rest(starting_lengths, exit)?; + self.total.compute_rest(starting_lengths, exit)?; Ok(()) } } diff --git a/crates/brk_computer/src/distribution/addr/exposed/mod.rs b/crates/brk_computer/src/distribution/addr/exposed/mod.rs index 9e4b7b96d..2bcd2b5e7 100644 --- a/crates/brk_computer/src/distribution/addr/exposed/mod.rs +++ b/crates/brk_computer/src/distribution/addr/exposed/mod.rs @@ -35,8 +35,9 @@ use brk_cohort::ByAddrType; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, Version}; +use brk_types::{Height, Sats, Version}; use rayon::prelude::*; use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode}; @@ -102,17 +103,17 @@ impl ExposedAddrVecs { pub(crate) fn compute_rest( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, prices: &prices::Vecs, all_supply_sats: &impl ReadableVec, type_supply_sats: &ByAddrType<&impl ReadableVec>, exit: &Exit, ) -> Result<()> { - self.count.compute_rest(starting_indexes, exit)?; + self.count.compute_rest(starting_lengths, exit)?; self.supply - .compute_rest(starting_indexes.height, prices, exit)?; + .compute_rest(starting_lengths.height, prices, exit)?; self.supply_share.compute_rest( - starting_indexes.height, + starting_lengths.height, &self.supply, all_supply_sats, type_supply_sats, diff --git a/crates/brk_computer/src/distribution/addr/reused/events/vecs.rs b/crates/brk_computer/src/distribution/addr/reused/events/vecs.rs index f1a0fb1ff..c48bb04f4 100644 --- a/crates/brk_computer/src/distribution/addr/reused/events/vecs.rs +++ b/crates/brk_computer/src/distribution/addr/reused/events/vecs.rs @@ -1,7 +1,8 @@ use brk_cohort::ByAddrType; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPoints16, Indexes, OutputType, StoredF32, StoredU32, StoredU64, Version}; +use brk_types::{BasisPoints16, OutputType, StoredF32, StoredU32, StoredU64, Version}; use rayon::prelude::*; use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec}; @@ -205,37 +206,37 @@ impl AddrEventsVecs { pub(crate) fn compute_rest( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, outputs_by_type: &outputs::ByTypeVecs, inputs_by_type: &inputs::ByTypeVecs, exit: &Exit, ) -> Result<()> { self.output_to_reused_addr_count - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.input_from_reused_addr_count - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.active_reused_addr_count - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.active_reused_addr_share - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.output_to_reused_addr_share.all.compute_count_ratio( &self.output_to_reused_addr_count.all, &outputs_by_type.output_count.all, - starting_indexes.height, + starting_lengths.height, exit, )?; self.spendable_output_to_reused_addr_share .compute_count_ratio( &self.output_to_reused_addr_count.all, &outputs_by_type.spendable_output_count, - starting_indexes.height, + starting_lengths.height, exit, )?; self.input_from_reused_addr_share.all.compute_count_ratio( &self.input_from_reused_addr_count.all, &inputs_by_type.input_count.all, - starting_indexes.height, + starting_lengths.height, exit, )?; for otype in OutputType::ADDR_TYPES { @@ -247,7 +248,7 @@ impl AddrEventsVecs { .by_addr_type .get_unwrap(otype), outputs_by_type.output_count.by_type.get(otype), - starting_indexes.height, + starting_lengths.height, exit, )?; self.input_from_reused_addr_share @@ -258,7 +259,7 @@ impl AddrEventsVecs { .by_addr_type .get_unwrap(otype), inputs_by_type.input_count.by_type.get(otype), - starting_indexes.height, + starting_lengths.height, exit, )?; } diff --git a/crates/brk_computer/src/distribution/addr/reused/mod.rs b/crates/brk_computer/src/distribution/addr/reused/mod.rs index f0867933d..bd68e3b89 100644 --- a/crates/brk_computer/src/distribution/addr/reused/mod.rs +++ b/crates/brk_computer/src/distribution/addr/reused/mod.rs @@ -22,8 +22,9 @@ pub use events::{AddrEventsVecs, AddrTypeToAddrEventCount}; use brk_cohort::ByAddrType; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, Version}; +use brk_types::{Height, Sats, Version}; use rayon::prelude::*; use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode}; @@ -108,7 +109,7 @@ impl ReusedAddrVecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute_rest( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, outputs_by_type: &outputs::ByTypeVecs, inputs_by_type: &inputs::ByTypeVecs, prices: &prices::Vecs, @@ -116,13 +117,13 @@ impl ReusedAddrVecs { type_supply_sats: &ByAddrType<&impl ReadableVec>, exit: &Exit, ) -> Result<()> { - self.count.compute_rest(starting_indexes, exit)?; + self.count.compute_rest(starting_lengths, exit)?; self.events - .compute_rest(starting_indexes, outputs_by_type, inputs_by_type, exit)?; + .compute_rest(starting_lengths, outputs_by_type, inputs_by_type, exit)?; self.supply - .compute_rest(starting_indexes.height, prices, exit)?; + .compute_rest(starting_lengths.height, prices, exit)?; self.supply_share.compute_rest( - starting_indexes.height, + starting_lengths.height, &self.supply, all_supply_sats, type_supply_sats, diff --git a/crates/brk_computer/src/distribution/cohorts/addr/groups.rs b/crates/brk_computer/src/distribution/cohorts/addr/groups.rs index 7b3ccd44e..60a3c6a48 100644 --- a/crates/brk_computer/src/distribution/cohorts/addr/groups.rs +++ b/crates/brk_computer/src/distribution/cohorts/addr/groups.rs @@ -2,8 +2,9 @@ use std::path::Path; use brk_cohort::{AddrGroups, AmountRange, Filter, Filtered, OverAmount, UnderAmount}; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, StoredU64, Version}; +use brk_types::{Height, Sats, StoredU64, Version}; use derive_more::{Deref, DerefMut}; use rayon::prelude::*; use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode}; @@ -83,11 +84,11 @@ impl AddrCohorts { /// Compute overlapping cohorts from component amount_range cohorts. pub(crate) fn compute_overlapping_vecs( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.for_each_aggregate(|vecs, sources| { - vecs.compute_from_stateful(starting_indexes, &sources, exit) + vecs.compute_from_stateful(starting_lengths, &sources, exit) }) } @@ -95,11 +96,11 @@ impl AddrCohorts { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.par_iter_mut() - .try_for_each(|v| v.compute_rest_part1(prices, starting_indexes, exit))?; + .try_for_each(|v| v.compute_rest_part1(prices, starting_lengths, exit))?; Ok(()) } @@ -108,7 +109,7 @@ impl AddrCohorts { pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, @@ -116,7 +117,7 @@ impl AddrCohorts { self.0.par_iter_mut().try_for_each(|v| { v.compute_rest_part2( prices, - starting_indexes, + starting_lengths, all_supply_sats, all_utxo_count, exit, diff --git a/crates/brk_computer/src/distribution/cohorts/addr/vecs.rs b/crates/brk_computer/src/distribution/cohorts/addr/vecs.rs index a141bbd61..07fc7cac5 100644 --- a/crates/brk_computer/src/distribution/cohorts/addr/vecs.rs +++ b/crates/brk_computer/src/distribution/cohorts/addr/vecs.rs @@ -2,8 +2,9 @@ use std::path::Path; use brk_cohort::{CohortContext, Filter, Filtered}; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPointsSigned32, Cents, Height, Indexes, Sats, StoredI64, StoredU64, Version}; +use brk_types::{BasisPointsSigned32, Cents, Height, Sats, StoredI64, StoredU64, Version}; use rayon::prelude::*; use vecdb::{AnyStoredVec, AnyVec, Database, Exit, ReadableVec, Rw, StorageMode, WritableVec}; @@ -174,11 +175,11 @@ impl DynCohortVecs for AddrCohortVecs { fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.metrics - .compute_rest_part1(prices, starting_indexes, exit) + .compute_rest_part1(prices, starting_lengths, exit) } fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> { @@ -205,12 +206,12 @@ impl DynCohortVecs for AddrCohortVecs { impl CohortVecs for AddrCohortVecs { fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { self.addr_count.height.compute_sum_of_others( - starting_indexes.height, + starting_lengths.height, others .iter() .map(|v| &v.addr_count.height) @@ -219,7 +220,7 @@ impl CohortVecs for AddrCohortVecs { exit, )?; self.metrics.compute_from_sources( - starting_indexes, + starting_lengths, &others.iter().map(|v| &v.metrics).collect::>(), exit, )?; @@ -229,14 +230,14 @@ impl CohortVecs for AddrCohortVecs { fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.metrics.compute_rest_part2( prices, - starting_indexes, + starting_lengths, all_supply_sats, all_utxo_count, exit, diff --git a/crates/brk_computer/src/distribution/cohorts/traits.rs b/crates/brk_computer/src/distribution/cohorts/traits.rs index 67d4118d5..7906d02d6 100644 --- a/crates/brk_computer/src/distribution/cohorts/traits.rs +++ b/crates/brk_computer/src/distribution/cohorts/traits.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Cents, Height, Indexes, Sats, StoredU64, Version}; +use brk_indexer::Lengths; +use brk_types::{Cents, Height, Sats, StoredU64, Version}; use vecdb::{Exit, ReadableVec}; use crate::prices; @@ -31,7 +32,7 @@ pub trait DynCohortVecs: Send + Sync { fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()>; @@ -52,7 +53,7 @@ pub trait CohortVecs: DynCohortVecs { /// Compute aggregate cohort from component cohorts. fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()>; @@ -61,7 +62,7 @@ pub trait CohortVecs: DynCohortVecs { fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs b/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs index d498e1a09..a0b52ab24 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs @@ -5,8 +5,9 @@ use brk_cohort::{ SpendableType, Term, UnderAge, UnderAmount, }; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, CentsSquaredSats, Dollars, Height, Indexes, Sats, Version}; +use brk_types::{Cents, CentsSquaredSats, Dollars, Height, Sats, Version}; use rayon::prelude::*; use vecdb::{ AnyStoredVec, AnyVec, Database, Exit, ReadOnlyClone, ReadableVec, Rw, StorageMode, WritableVec, @@ -414,7 +415,7 @@ impl UTXOCohorts { pub(crate) fn compute_overlapping_vecs( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { let Self { @@ -432,7 +433,7 @@ impl UTXOCohorts { let ar = &*age_range; let amr = &*amount_range; - let si = starting_indexes; + let si = starting_lengths; let tasks: Vec Result<()> + Send + '_>> = vec![ Box::new(|| { @@ -483,7 +484,7 @@ impl UTXOCohorts { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { // 1. Compute all metrics except net_sentiment (all cohorts via DynCohortVecs) @@ -527,16 +528,16 @@ impl UTXOCohorts { ); all.extend(self.type_.iter_mut().map(|x| x as &mut dyn DynCohortVecs)); all.into_par_iter() - .try_for_each(|v| v.compute_rest_part1(prices, starting_indexes, exit))?; + .try_for_each(|v| v.compute_rest_part1(prices, starting_lengths, exit))?; } // Compute matured cumulative + cents from sats × price self.matured .par_iter_mut() - .try_for_each(|v| v.compute_rest(starting_indexes.height, prices, exit))?; + .try_for_each(|v| v.compute_rest(starting_lengths.height, prices, exit))?; // Compute profitability supply cents and realized price - self.profitability.compute(prices, starting_indexes, exit)?; + self.profitability.compute(prices, starting_lengths, exit)?; Ok(()) } @@ -546,7 +547,7 @@ impl UTXOCohorts { &mut self, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_market_cap: &impl ReadableVec, exit: &Exit, ) -> Result<()> { @@ -574,7 +575,7 @@ impl UTXOCohorts { self.all.metrics.compute_rest_part2( blocks, prices, - starting_indexes, + starting_lengths, height_to_market_cap, &under_1h_value_created, &under_1h_value_destroyed, @@ -619,7 +620,7 @@ impl UTXOCohorts { sth.metrics.compute_rest_part2( blocks, prices, - starting_indexes, + starting_lengths, height_to_market_cap, vc, vd, @@ -632,7 +633,7 @@ impl UTXOCohorts { lth.metrics.compute_rest_part2( blocks, prices, - starting_indexes, + starting_lengths, height_to_market_cap, ss, au, @@ -642,55 +643,55 @@ impl UTXOCohorts { Box::new(|| { age_range.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { under_age.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { over_age.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { over_amount.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { epoch.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { class.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { amount_range.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { under_amount.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), Box::new(|| { type_.par_iter_mut().try_for_each(|v| { v.metrics - .compute_rest_part2(prices, starting_indexes, ss, au, exit) + .compute_rest_part2(prices, starting_lengths, ss, au, exit) }) }), ]; diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/core.rs b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/core.rs index 20bed684b..3ff8d5d3d 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/core.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/core.rs @@ -1,6 +1,7 @@ use brk_cohort::{Filter, Filtered}; use brk_error::Result; -use brk_types::{Cents, Height, Indexes, Version}; +use brk_indexer::Lengths; +use brk_types::{Cents, Height, Version}; use vecdb::{Exit, ReadableVec}; use crate::{ @@ -56,11 +57,11 @@ impl DynCohortVecs for UTXOCohortVecs { fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.metrics - .compute_rest_part1(prices, starting_indexes, exit) + .compute_rest_part1(prices, starting_lengths, exit) } fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> { diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/minimal.rs b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/minimal.rs index a4bb12e2f..1f023aba9 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/minimal.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/minimal.rs @@ -1,6 +1,7 @@ use brk_cohort::{Filter, Filtered}; use brk_error::Result; -use brk_types::{Cents, Height, Indexes, Version}; +use brk_indexer::Lengths; +use brk_types::{Cents, Height, Version}; use vecdb::{Exit, ReadableVec}; use crate::{ @@ -49,11 +50,11 @@ impl DynCohortVecs for UTXOCohortVecs { fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.metrics - .compute_rest_part1(prices, starting_indexes, exit) + .compute_rest_part1(prices, starting_lengths, exit) } fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> { diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/mod.rs b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/mod.rs index c03fbaa2c..f454d62a3 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/mod.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/mod.rs @@ -44,8 +44,9 @@ mod r#type; use brk_cohort::{Filter, Filtered}; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Height, Indexes, Version}; +use brk_types::{Cents, Height, Version}; use vecdb::{Exit, ReadableVec}; use crate::{ @@ -186,11 +187,11 @@ impl DynCohortVecs for UTXOCohortVecs { fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.metrics - .compute_rest_part1(prices, starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/type.rs b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/type.rs index 96f885a8b..df32af77f 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/vecs/type.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/vecs/type.rs @@ -1,6 +1,7 @@ use brk_cohort::{Filter, Filtered}; use brk_error::Result; -use brk_types::{Cents, Height, Indexes, Version}; +use brk_indexer::Lengths; +use brk_types::{Cents, Height, Version}; use vecdb::{Exit, ReadableVec}; use crate::{ @@ -55,11 +56,11 @@ impl DynCohortVecs for UTXOCohortVecs { fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.metrics - .compute_rest_part1(prices, starting_indexes, exit) + .compute_rest_part1(prices, starting_lengths, exit) } fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> { diff --git a/crates/brk_computer/src/distribution/metrics/activity/core.rs b/crates/brk_computer/src/distribution/metrics/activity/core.rs index 01fc2ca90..c833e0507 100644 --- a/crates/brk_computer/src/distribution/metrics/activity/core.rs +++ b/crates/brk_computer/src/distribution/metrics/activity/core.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Bitcoin, Indexes, StoredF64, Version}; +use brk_types::{Bitcoin, StoredF64, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode, WritableVec}; @@ -80,17 +81,17 @@ impl ActivityCore { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { let minimal_refs: Vec<&ActivityMinimal> = others.iter().map(|o| &o.minimal).collect(); self.minimal - .compute_from_stateful(starting_indexes, &minimal_refs, exit)?; + .compute_from_stateful(starting_lengths, &minimal_refs, exit)?; - sum_others!(self, starting_indexes, others, exit; coindays_destroyed.block); - sum_others!(self, starting_indexes, others, exit; transfer_volume_in_profit.block.sats); - sum_others!(self, starting_indexes, others, exit; transfer_volume_in_loss.block.sats); + sum_others!(self, starting_lengths, others, exit; coindays_destroyed.block); + sum_others!(self, starting_lengths, others, exit; transfer_volume_in_profit.block.sats); + sum_others!(self, starting_lengths, others, exit; transfer_volume_in_loss.block.sats); Ok(()) } @@ -98,17 +99,17 @@ impl ActivityCore { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.minimal - .compute_rest_part1(prices, starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; self.coindays_destroyed - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.transfer_volume_in_profit - .compute_rest(starting_indexes.height, prices, exit)?; + .compute_rest(starting_lengths.height, prices, exit)?; self.transfer_volume_in_loss - .compute_rest(starting_indexes.height, prices, exit)?; + .compute_rest(starting_lengths.height, prices, exit)?; Ok(()) } } diff --git a/crates/brk_computer/src/distribution/metrics/activity/full.rs b/crates/brk_computer/src/distribution/metrics/activity/full.rs index b10334746..e20d43e02 100644 --- a/crates/brk_computer/src/distribution/metrics/activity/full.rs +++ b/crates/brk_computer/src/distribution/metrics/activity/full.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Indexes, StoredF32, StoredF64, Version}; +use brk_types::{StoredF32, StoredF64, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, Exit, Rw, StorageMode}; @@ -78,22 +79,22 @@ impl ActivityFull { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&ActivityCore], exit: &Exit, ) -> Result<()> { self.inner - .compute_from_stateful(starting_indexes, others, exit) + .compute_from_stateful(starting_lengths, others, exit) } pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.inner - .compute_rest_part1(prices, starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; for ((dormancy, cdd_sum), tv_sum) in self .dormancy @@ -103,7 +104,7 @@ impl ActivityFull { .zip(self.inner.minimal.transfer_volume.sum.0.as_array()) { dormancy.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, &cdd_sum.height, &tv_sum.btc.height, |(i, rolling_cdd, rolling_btc, ..)| { diff --git a/crates/brk_computer/src/distribution/metrics/activity/minimal.rs b/crates/brk_computer/src/distribution/metrics/activity/minimal.rs index 3fa22407b..03fb86a92 100644 --- a/crates/brk_computer/src/distribution/metrics/activity/minimal.rs +++ b/crates/brk_computer/src/distribution/metrics/activity/minimal.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Indexes, Version}; +use brk_types::Version; use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode, WritableVec}; use crate::{ @@ -44,12 +45,12 @@ impl ActivityMinimal { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { self.transfer_volume.block.sats.compute_sum_of_others( - starting_indexes.height, + starting_lengths.height, &others .iter() .map(|v| &v.transfer_volume.block.sats) @@ -63,11 +64,11 @@ impl ActivityMinimal { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.transfer_volume - .compute_rest(starting_indexes.height, prices, exit)?; + .compute_rest(starting_lengths.height, prices, exit)?; Ok(()) } } diff --git a/crates/brk_computer/src/distribution/metrics/activity/mod.rs b/crates/brk_computer/src/distribution/metrics/activity/mod.rs index fee676677..567347e7b 100644 --- a/crates/brk_computer/src/distribution/metrics/activity/mod.rs +++ b/crates/brk_computer/src/distribution/metrics/activity/mod.rs @@ -7,7 +7,8 @@ pub use full::ActivityFull; pub use minimal::ActivityMinimal; use brk_error::Result; -use brk_types::{Indexes, Version}; +use brk_indexer::Lengths; +use brk_types::Version; use vecdb::Exit; use crate::{ @@ -23,14 +24,14 @@ pub trait ActivityLike: Send + Sync { fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>; fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&ActivityCore], exit: &Exit, ) -> Result<()>; fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()>; } @@ -53,19 +54,19 @@ impl ActivityLike for ActivityCore { } fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&ActivityCore], exit: &Exit, ) -> Result<()> { - self.compute_from_stateful(starting_indexes, others, exit) + self.compute_from_stateful(starting_lengths, others, exit) } fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.compute_rest_part1(prices, starting_indexes, exit) + self.compute_rest_part1(prices, starting_lengths, exit) } } @@ -87,18 +88,18 @@ impl ActivityLike for ActivityFull { } fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&ActivityCore], exit: &Exit, ) -> Result<()> { - self.compute_from_stateful(starting_indexes, others, exit) + self.compute_from_stateful(starting_lengths, others, exit) } fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.compute_rest_part1(prices, starting_indexes, exit) + self.compute_rest_part1(prices, starting_lengths, exit) } } diff --git a/crates/brk_computer/src/distribution/metrics/cohort/all.rs b/crates/brk_computer/src/distribution/metrics/cohort/all.rs index 6c7dcbb98..cd36fcbbd 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/all.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/all.rs @@ -1,7 +1,8 @@ use brk_cohort::Filter; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Dollars, Height, Indexes, Version}; +use brk_types::{Cents, Dollars, Height, Version}; use vecdb::{AnyStoredVec, Exit, ReadOnlyClone, ReadableVec, Rw, StorageMode}; use crate::{ @@ -100,7 +101,7 @@ impl AllCohortMetrics { &mut self, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_market_cap: &impl ReadableVec, under_1h_value_created: &impl ReadableVec, under_1h_value_destroyed: &impl ReadableVec, @@ -109,7 +110,7 @@ impl AllCohortMetrics { self.realized.compute_rest_part2( blocks, prices, - starting_indexes, + starting_lengths, &self.supply.total.btc.height, height_to_market_cap, &self.activity.transfer_volume, @@ -117,14 +118,14 @@ impl AllCohortMetrics { )?; self.unrealized.compute( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &self.realized.price.cents.height, exit, )?; self.asopr.compute_rest_part2( - starting_indexes, + starting_lengths, &self.activity.transfer_volume.block.cents, &self.realized.core.sopr.value_destroyed.block, under_1h_value_created, @@ -134,10 +135,10 @@ impl AllCohortMetrics { let all_utxo_count = self.outputs.unspent_count.height.read_only_clone(); self.outputs - .compute_part2(starting_indexes.height, &all_utxo_count, exit)?; + .compute_part2(starting_lengths.height, &all_utxo_count, exit)?; self.cost_basis.compute_prices( - starting_indexes, + starting_lengths, &prices.spot.cents.height, &self.unrealized.invested_capital.in_profit.cents.height, &self.unrealized.invested_capital.in_loss.cents.height, @@ -149,14 +150,14 @@ impl AllCohortMetrics { )?; self.unrealized - .compute_sentiment(starting_indexes, &prices.spot.cents.height, exit)?; + .compute_sentiment(starting_lengths, &prices.spot.cents.height, exit)?; let own_supply_sats = self.supply.total.sats.height.read_only_clone(); self.supply - .compute_dominance(starting_indexes.height, &own_supply_sats, exit)?; + .compute_dominance(starting_lengths.height, &own_supply_sats, exit)?; self.relative.compute( - starting_indexes.height, + starting_lengths.height, &self.supply, &self.unrealized, &self.realized, diff --git a/crates/brk_computer/src/distribution/metrics/cohort/basic.rs b/crates/brk_computer/src/distribution/metrics/cohort/basic.rs index 84c6bab57..948e3f430 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/basic.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/basic.rs @@ -1,7 +1,8 @@ use brk_cohort::Filter; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, StoredU64}; +use brk_types::{Height, Sats, StoredU64}; use vecdb::{AnyStoredVec, Exit, ReadableVec, Rw, StorageMode}; use crate::{ @@ -61,31 +62,31 @@ impl BasicCohortMetrics { pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.realized.compute_rest_part2( prices, - starting_indexes, + starting_lengths, &self.supply.total.btc.height, &self.activity.transfer_volume.sum._24h.cents.height, exit, )?; self.unrealized.compute( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &self.realized.price.cents.height, exit, )?; self.supply - .compute_dominance(starting_indexes.height, all_supply_sats, exit)?; + .compute_dominance(starting_lengths.height, all_supply_sats, exit)?; self.outputs - .compute_part2(starting_indexes.height, all_utxo_count, exit)?; + .compute_part2(starting_lengths.height, all_utxo_count, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/distribution/metrics/cohort/core.rs b/crates/brk_computer/src/distribution/metrics/cohort/core.rs index d215bdb88..e9ba51e35 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/core.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/core.rs @@ -1,7 +1,8 @@ use brk_cohort::Filter; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, StoredU64, Version}; +use brk_types::{Height, Sats, StoredU64, Version}; use vecdb::{AnyStoredVec, Exit, ReadableVec, Rw, StorageMode}; use crate::{ @@ -63,32 +64,32 @@ impl CoreCohortMetrics { /// Aggregate Core-tier fields from CohortMetricsBase sources (e.g. age_range -> under_age/over_age). pub(crate) fn compute_from_base_sources( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&T], exit: &Exit, ) -> Result<()> { self.supply.compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.supply()).collect::>(), exit, )?; self.outputs.compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.outputs()).collect::>(), exit, )?; self.activity.compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.activity_core()).collect::>(), exit, )?; self.realized.compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.realized_core()).collect::>(), exit, )?; self.unrealized.compute_from_stateful( - starting_indexes, + starting_lengths, &others .iter() .map(|v| v.unrealized_core()) @@ -102,19 +103,19 @@ impl CoreCohortMetrics { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.supply.compute(prices, starting_indexes.height, exit)?; + self.supply.compute(prices, starting_lengths.height, exit)?; - self.outputs.compute_rest(starting_indexes.height, exit)?; + self.outputs.compute_rest(starting_lengths.height, exit)?; self.activity - .compute_rest_part1(prices, starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; - self.realized.compute_rest_part1(starting_indexes, exit)?; + self.realized.compute_rest_part1(starting_lengths, exit)?; - self.unrealized.compute_rest(starting_indexes, exit)?; + self.unrealized.compute_rest(starting_lengths, exit)?; Ok(()) } @@ -122,31 +123,31 @@ impl CoreCohortMetrics { pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.realized.compute_rest_part2( prices, - starting_indexes, + starting_lengths, &self.supply.total.btc.height, &self.activity.transfer_volume.sum._24h.cents.height, exit, )?; self.unrealized.compute( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &self.realized.price.cents.height, exit, )?; self.supply - .compute_dominance(starting_indexes.height, all_supply_sats, exit)?; + .compute_dominance(starting_lengths.height, all_supply_sats, exit)?; self.outputs - .compute_part2(starting_indexes.height, all_utxo_count, exit)?; + .compute_part2(starting_lengths.height, all_utxo_count, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/distribution/metrics/cohort/extended.rs b/crates/brk_computer/src/distribution/metrics/cohort/extended.rs index 8f708ac7b..c74be21c1 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/extended.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/extended.rs @@ -1,7 +1,8 @@ use brk_cohort::Filter; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Dollars, Height, Indexes, Sats, StoredU64, Version}; +use brk_types::{Dollars, Height, Sats, StoredU64, Version}; use vecdb::AnyStoredVec; use vecdb::{Exit, ReadableVec, Rw, StorageMode}; @@ -90,7 +91,7 @@ impl ExtendedCohortMetrics { &mut self, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_market_cap: &impl ReadableVec, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, @@ -99,7 +100,7 @@ impl ExtendedCohortMetrics { self.realized.compute_rest_part2( blocks, prices, - starting_indexes, + starting_lengths, &self.supply.total.btc.height, height_to_market_cap, &self.activity.transfer_volume, @@ -107,14 +108,14 @@ impl ExtendedCohortMetrics { )?; self.unrealized.compute( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &self.realized.price.cents.height, exit, )?; self.cost_basis.compute_prices( - starting_indexes, + starting_lengths, &prices.spot.cents.height, &self.unrealized.invested_capital.in_profit.cents.height, &self.unrealized.invested_capital.in_loss.cents.height, @@ -126,13 +127,13 @@ impl ExtendedCohortMetrics { )?; self.unrealized - .compute_sentiment(starting_indexes, &prices.spot.cents.height, exit)?; + .compute_sentiment(starting_lengths, &prices.spot.cents.height, exit)?; self.supply - .compute_dominance(starting_indexes.height, all_supply_sats, exit)?; + .compute_dominance(starting_lengths.height, all_supply_sats, exit)?; self.relative.compute( - starting_indexes.height, + starting_lengths.height, &self.supply, &self.unrealized, &self.realized, @@ -142,7 +143,7 @@ impl ExtendedCohortMetrics { )?; self.outputs - .compute_part2(starting_indexes.height, all_utxo_count, exit)?; + .compute_part2(starting_lengths.height, all_utxo_count, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/distribution/metrics/cohort/extended_adjusted.rs b/crates/brk_computer/src/distribution/metrics/cohort/extended_adjusted.rs index e6d693a28..8f8724f38 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/extended_adjusted.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/extended_adjusted.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Dollars, Height, Indexes, Sats, StoredU64, Version}; +use brk_types::{Cents, Dollars, Height, Sats, StoredU64, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, Exit, ReadableVec, Rw, StorageMode}; @@ -62,7 +63,7 @@ impl ExtendedAdjustedCohortMetrics { &mut self, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_market_cap: &impl ReadableVec, under_1h_value_created: &impl ReadableVec, under_1h_value_destroyed: &impl ReadableVec, @@ -73,7 +74,7 @@ impl ExtendedAdjustedCohortMetrics { self.inner.compute_rest_part2( blocks, prices, - starting_indexes, + starting_lengths, height_to_market_cap, all_supply_sats, all_utxo_count, @@ -81,7 +82,7 @@ impl ExtendedAdjustedCohortMetrics { )?; self.asopr.compute_rest_part2( - starting_indexes, + starting_lengths, &self.inner.activity.transfer_volume.block.cents, &self.inner.realized.core.sopr.value_destroyed.block, under_1h_value_created, diff --git a/crates/brk_computer/src/distribution/metrics/cohort/minimal.rs b/crates/brk_computer/src/distribution/metrics/cohort/minimal.rs index c4cc9c84a..f130bda20 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/minimal.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/minimal.rs @@ -1,7 +1,8 @@ use brk_cohort::Filter; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, StoredU64}; +use brk_types::{Height, Sats, StoredU64}; use vecdb::{AnyStoredVec, Exit, ReadableVec, Rw, StorageMode}; use crate::{ @@ -58,17 +59,17 @@ impl MinimalCohortMetrics { /// Aggregate Minimal-tier metrics from other MinimalCohortMetrics sources. pub(crate) fn compute_from_sources( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&MinimalCohortMetrics], exit: &Exit, ) -> Result<()> { self.supply.compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.supply.as_ref()).collect::>(), exit, )?; self.outputs.compute_from_stateful( - starting_indexes, + starting_lengths, &others .iter() .map(|v| v.outputs.as_ref()) @@ -76,7 +77,7 @@ impl MinimalCohortMetrics { exit, )?; self.activity.compute_from_stateful( - starting_indexes, + starting_lengths, &others .iter() .map(|v| v.activity.as_ref()) @@ -84,7 +85,7 @@ impl MinimalCohortMetrics { exit, )?; self.realized.compute_from_stateful( - starting_indexes, + starting_lengths, &others .iter() .map(|v| v.realized.as_ref()) @@ -97,44 +98,44 @@ impl MinimalCohortMetrics { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.supply.compute(prices, starting_indexes.height, exit)?; - self.outputs.compute_rest(starting_indexes.height, exit)?; + self.supply.compute(prices, starting_lengths.height, exit)?; + self.outputs.compute_rest(starting_lengths.height, exit)?; self.activity - .compute_rest_part1(prices, starting_indexes, exit)?; - self.realized.compute_rest_part1(starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; + self.realized.compute_rest_part1(starting_lengths, exit)?; Ok(()) } pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.realized.compute_rest_part2( prices, - starting_indexes, + starting_lengths, &self.supply.total.btc.height, exit, )?; self.unrealized.compute( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &self.realized.price.cents.height, exit, )?; self.supply - .compute_dominance(starting_indexes.height, all_supply_sats, exit)?; + .compute_dominance(starting_lengths.height, all_supply_sats, exit)?; self.outputs - .compute_part2(starting_indexes.height, all_utxo_count, exit)?; + .compute_part2(starting_lengths.height, all_utxo_count, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/distribution/metrics/cohort/type.rs b/crates/brk_computer/src/distribution/metrics/cohort/type.rs index cbd2c67d8..ed3177a6b 100644 --- a/crates/brk_computer/src/distribution/metrics/cohort/type.rs +++ b/crates/brk_computer/src/distribution/metrics/cohort/type.rs @@ -1,7 +1,8 @@ use brk_cohort::Filter; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, StoredU64}; +use brk_types::{Height, Sats, StoredU64}; use vecdb::{AnyStoredVec, Exit, ReadableVec, Rw, StorageMode}; use crate::{ @@ -59,44 +60,44 @@ impl TypeCohortMetrics { pub(crate) fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.supply.compute(prices, starting_indexes.height, exit)?; - self.outputs.compute_rest(starting_indexes.height, exit)?; + self.supply.compute(prices, starting_lengths.height, exit)?; + self.outputs.compute_rest(starting_lengths.height, exit)?; self.activity - .compute_rest_part1(prices, starting_indexes, exit)?; - self.realized.compute_rest_part1(starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; + self.realized.compute_rest_part1(starting_lengths, exit)?; Ok(()) } pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, all_supply_sats: &impl ReadableVec, all_utxo_count: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.realized.compute_rest_part2( prices, - starting_indexes, + starting_lengths, &self.supply.total.btc.height, exit, )?; self.unrealized.compute( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &self.realized.price.cents.height, exit, )?; self.supply - .compute_dominance(starting_indexes.height, all_supply_sats, exit)?; + .compute_dominance(starting_lengths.height, all_supply_sats, exit)?; self.outputs - .compute_part2(starting_indexes.height, all_utxo_count, exit)?; + .compute_part2(starting_lengths.height, all_utxo_count, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/distribution/metrics/cost_basis/mod.rs b/crates/brk_computer/src/distribution/metrics/cost_basis/mod.rs index 44958431f..4d49110cf 100644 --- a/crates/brk_computer/src/distribution/metrics/cost_basis/mod.rs +++ b/crates/brk_computer/src/distribution/metrics/cost_basis/mod.rs @@ -1,7 +1,8 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::CentsSquaredSats; -use brk_types::{BasisPoints16, Cents, Height, Indexes, Sats, Version}; +use brk_types::{BasisPoints16, Cents, Height, Sats, Version}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVec}; use crate::internal::{PERCENTILES_LEN, PerBlock, PercentPerBlock, PercentilesVecs, Price}; @@ -147,7 +148,7 @@ impl CostBasis { #[allow(clippy::too_many_arguments)] pub(crate) fn compute_prices( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, spot: &impl ReadableVec, invested_cap_in_profit: &impl ReadableVec, invested_cap_in_loss: &impl ReadableVec, @@ -158,7 +159,7 @@ impl CostBasis { exit: &Exit, ) -> Result<()> { self.in_profit.per_coin.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, invested_cap_in_profit, supply_in_profit_sats, spot, @@ -175,7 +176,7 @@ impl CostBasis { exit, )?; self.in_loss.per_coin.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, invested_cap_in_loss, supply_in_loss_sats, spot, @@ -192,7 +193,7 @@ impl CostBasis { exit, )?; self.in_profit.per_dollar.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, capitalized_cap_in_profit_raw, invested_cap_in_profit, spot, @@ -209,7 +210,7 @@ impl CostBasis { exit, )?; self.in_loss.per_dollar.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, capitalized_cap_in_loss_raw, invested_cap_in_loss, spot, diff --git a/crates/brk_computer/src/distribution/metrics/mod.rs b/crates/brk_computer/src/distribution/metrics/mod.rs index dcbee2e70..5dfa55f25 100644 --- a/crates/brk_computer/src/distribution/metrics/mod.rs +++ b/crates/brk_computer/src/distribution/metrics/mod.rs @@ -140,7 +140,8 @@ pub use unrealized::{ use brk_cohort::Filter; use brk_error::Result; -use brk_types::{Cents, Indexes, Version}; +use brk_indexer::Lengths; +use brk_types::{Cents, Version}; use vecdb::{AnyStoredVec, Exit, StorageMode}; use crate::{ @@ -270,23 +271,23 @@ pub trait CohortMetricsBase: fn compute_rest_part1( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { self.supply_mut() - .compute(prices, starting_indexes.height, exit)?; + .compute(prices, starting_lengths.height, exit)?; self.outputs_mut() - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.activity_mut() - .compute_rest_part1(prices, starting_indexes, exit)?; + .compute_rest_part1(prices, starting_lengths, exit)?; self.realized_mut() - .compute_rest_part1(starting_indexes, exit)?; + .compute_rest_part1(starting_lengths, exit)?; let (supply, unrealized) = self.supply_and_unrealized_mut(); unrealized.compute_rest( prices, - starting_indexes, + starting_lengths, &supply.in_profit.sats.height, &supply.in_loss.sats.height, exit, @@ -298,32 +299,32 @@ pub trait CohortMetricsBase: /// Compute aggregate base metrics from source cohorts. fn compute_base_from_others( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&T], exit: &Exit, ) -> Result<()> { self.supply_mut().compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.supply()).collect::>(), exit, )?; self.outputs_mut().compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.outputs()).collect::>(), exit, )?; self.activity_mut().compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.activity_core()).collect::>(), exit, )?; self.realized_mut().compute_from_stateful( - starting_indexes, + starting_lengths, &others.iter().map(|v| v.realized_core()).collect::>(), exit, )?; self.unrealized_core_mut().compute_from_stateful( - starting_indexes, + starting_lengths, &others .iter() .map(|v| v.unrealized_core()) diff --git a/crates/brk_computer/src/distribution/metrics/outputs/base.rs b/crates/brk_computer/src/distribution/metrics/outputs/base.rs index 88f1e7cf2..bbff4bce1 100644 --- a/crates/brk_computer/src/distribution/metrics/outputs/base.rs +++ b/crates/brk_computer/src/distribution/metrics/outputs/base.rs @@ -1,8 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{ - BasisPointsSigned32, Height, Indexes, StoredF32, StoredI64, StoredU32, StoredU64, Version, -}; +use brk_types::{BasisPointsSigned32, Height, StoredF32, StoredI64, StoredU32, StoredU64, Version}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVec}; use crate::{ @@ -83,19 +82,19 @@ impl OutputsBase { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { self.unspent_count.height.compute_sum_of_others( - starting_indexes.height, + starting_lengths.height, &others .iter() .map(|v| &v.unspent_count.height) .collect::>(), exit, )?; - sum_others!(self, starting_indexes, others, exit; spent_count.block); + sum_others!(self, starting_lengths, others, exit; spent_count.block); Ok(()) } } diff --git a/crates/brk_computer/src/distribution/metrics/profitability.rs b/crates/brk_computer/src/distribution/metrics/profitability.rs index 39faa4c23..becb92d69 100644 --- a/crates/brk_computer/src/distribution/metrics/profitability.rs +++ b/crates/brk_computer/src/distribution/metrics/profitability.rs @@ -1,7 +1,8 @@ use brk_cohort::{Loss, Profit, ProfitabilityRange}; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Dollars, Indexes, Sats, Version}; +use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Dollars, Sats, Version}; use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec}; use crate::{ @@ -115,11 +116,11 @@ impl ProfitabilityBucket { pub(crate) fn compute( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, is_profit: bool, exit: &Exit, ) -> Result<()> { - let max_from = starting_indexes.height; + let max_from = starting_lengths.height; self.supply.all.compute(prices, max_from, exit)?; self.supply.sth.compute(prices, max_from, exit)?; @@ -176,12 +177,12 @@ impl ProfitabilityBucket { pub(crate) fn compute_from_ranges( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, is_profit: bool, sources: &[&ProfitabilityBucket], exit: &Exit, ) -> Result<()> { - let max_from = starting_indexes.height; + let max_from = starting_lengths.height; self.supply.all.sats.height.compute_sum_of_others( max_from, @@ -216,7 +217,7 @@ impl ProfitabilityBucket { exit, )?; - self.compute(prices, starting_indexes, is_profit, exit) + self.compute(prices, starting_lengths, is_profit, exit) } pub(crate) fn collect_all_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> { @@ -293,20 +294,20 @@ impl ProfitabilityMetrics { pub(crate) fn compute( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { for (is_profit, bucket) in self.range.iter_mut_with_is_profit() { - bucket.compute(prices, starting_indexes, is_profit, exit)?; + bucket.compute(prices, starting_lengths, is_profit, exit)?; } let range_arr = self.range.as_array(); for (threshold, sources) in self.profit.iter_mut_with_growing_prefix(&range_arr) { - threshold.compute_from_ranges(prices, starting_indexes, true, sources, exit)?; + threshold.compute_from_ranges(prices, starting_lengths, true, sources, exit)?; } for (threshold, sources) in self.loss.iter_mut_with_growing_suffix(&range_arr) { - threshold.compute_from_ranges(prices, starting_indexes, false, sources, exit)?; + threshold.compute_from_ranges(prices, starting_lengths, false, sources, exit)?; } Ok(()) diff --git a/crates/brk_computer/src/distribution/metrics/realized/adjusted.rs b/crates/brk_computer/src/distribution/metrics/realized/adjusted.rs index 456a6572a..4b33de087 100644 --- a/crates/brk_computer/src/distribution/metrics/realized/adjusted.rs +++ b/crates/brk_computer/src/distribution/metrics/realized/adjusted.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Height, Indexes, StoredF64, Version}; +use brk_types::{Cents, Height, StoredF64, Version}; use vecdb::{Exit, ReadableVec, Rw, StorageMode}; use crate::{ @@ -27,7 +28,7 @@ impl AdjustedSopr { #[allow(clippy::too_many_arguments)] pub(crate) fn compute_rest_part2( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, base_transfer_volume: &impl ReadableVec, base_value_destroyed: &impl ReadableVec, under_1h_transfer_volume: &impl ReadableVec, @@ -35,22 +36,22 @@ impl AdjustedSopr { exit: &Exit, ) -> Result<()> { self.transfer_volume.block.compute_subtract( - starting_indexes.height, + starting_lengths.height, base_transfer_volume, under_1h_transfer_volume, exit, )?; self.value_destroyed.block.compute_subtract( - starting_indexes.height, + starting_lengths.height, base_value_destroyed, under_1h_value_destroyed, exit, )?; self.transfer_volume - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.value_destroyed - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; for ((sopr, tv), vd) in self .ratio @@ -60,7 +61,7 @@ impl AdjustedSopr { .zip(self.value_destroyed.sum.as_array()) { sopr.compute_binary::( - starting_indexes.height, + starting_lengths.height, &tv.height, &vd.height, exit, diff --git a/crates/brk_computer/src/distribution/metrics/realized/core.rs b/crates/brk_computer/src/distribution/metrics/realized/core.rs index d531cec77..caae24e51 100644 --- a/crates/brk_computer/src/distribution/metrics/realized/core.rs +++ b/crates/brk_computer/src/distribution/metrics/realized/core.rs @@ -1,7 +1,8 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::{ - BasisPointsSigned32, Bitcoin, Cents, CentsSigned, Dollars, Height, Indexes, StoredF64, Version, + BasisPointsSigned32, Bitcoin, Cents, CentsSigned, Dollars, Height, StoredF64, Version, }; use derive_more::{Deref, DerefMut}; use vecdb::{ @@ -124,31 +125,31 @@ impl RealizedCore { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { let minimal_refs: Vec<&RealizedMinimal> = others.iter().map(|o| &o.minimal).collect(); self.minimal - .compute_from_stateful(starting_indexes, &minimal_refs, exit)?; + .compute_from_stateful(starting_lengths, &minimal_refs, exit)?; - sum_others!(self, starting_indexes, others, exit; sopr.value_destroyed.block); + sum_others!(self, starting_lengths, others, exit; sopr.value_destroyed.block); Ok(()) } pub(crate) fn compute_rest_part1( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.minimal.compute_rest_part1(starting_indexes, exit)?; + self.minimal.compute_rest_part1(starting_lengths, exit)?; self.sopr .value_destroyed - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.net_pnl.block.cents.compute_transform2( - starting_indexes.height, + starting_lengths.height, &self.minimal.profit.block.cents, &self.minimal.loss.block.cents, |(i, profit, loss, ..)| { @@ -166,21 +167,21 @@ impl RealizedCore { pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_supply: &impl ReadableVec, transfer_volume_sum_24h_cents: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.minimal - .compute_rest_part2(prices, starting_indexes, height_to_supply, exit)?; + .compute_rest_part2(prices, starting_lengths, height_to_supply, exit)?; - self.net_pnl.compute_rest(starting_indexes.height, exit)?; + self.net_pnl.compute_rest(starting_lengths.height, exit)?; self.sopr .ratio ._24h .compute_binary::( - starting_indexes.height, + starting_lengths.height, transfer_volume_sum_24h_cents, &self.sopr.value_destroyed.sum._24h.height, exit, diff --git a/crates/brk_computer/src/distribution/metrics/realized/full.rs b/crates/brk_computer/src/distribution/metrics/realized/full.rs index 9ad500ed9..df6345f1c 100644 --- a/crates/brk_computer/src/distribution/metrics/realized/full.rs +++ b/crates/brk_computer/src/distribution/metrics/realized/full.rs @@ -1,8 +1,9 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::{ BasisPoints32, BasisPointsSigned32, Bitcoin, Cents, CentsSats, CentsSigned, CentsSquaredSats, - Dollars, Height, Indexes, StoredF64, Version, + Dollars, Height, StoredF64, Version, }; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, AnyVec, BytesVec, Exit, ReadableVec, Rw, StorageMode, WritableVec}; @@ -194,12 +195,12 @@ impl RealizedFull { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&RealizedCore], exit: &Exit, ) -> Result<()> { self.core - .compute_from_stateful(starting_indexes, others, exit)?; + .compute_from_stateful(starting_lengths, others, exit)?; Ok(()) } @@ -224,14 +225,14 @@ impl RealizedFull { pub(crate) fn compute_rest_part1( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.core.compute_rest_part1(starting_indexes, exit)?; + self.core.compute_rest_part1(starting_lengths, exit)?; self.peak_regret .value - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; Ok(()) } @@ -240,7 +241,7 @@ impl RealizedFull { &mut self, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_supply: &impl ReadableVec, height_to_market_cap: &impl ReadableVec, activity_transfer_volume: &ValuePerBlockCumulativeRolling, @@ -248,7 +249,7 @@ impl RealizedFull { ) -> Result<()> { self.core.compute_rest_part2( prices, - starting_indexes, + starting_lengths, height_to_supply, &activity_transfer_volume.sum._24h.cents.height, exit, @@ -264,7 +265,7 @@ impl RealizedFull { .zip(self.core.sopr.value_destroyed.sum.as_array()[1..].iter()) { sopr.compute_binary::( - starting_indexes.height, + starting_lengths.height, &vc.cents.height, &vd.height, exit, @@ -273,18 +274,18 @@ impl RealizedFull { // Gross PnL self.gross_pnl.block.cents.compute_add( - starting_indexes.height, + starting_lengths.height, &self.core.minimal.profit.block.cents, &self.core.minimal.loss.block.cents, exit, )?; - self.gross_pnl.compute_rest(starting_indexes.height, exit)?; + self.gross_pnl.compute_rest(starting_lengths.height, exit)?; // Net PnL 1m change relative to rcap and mcap self.net_pnl .change_1m_to_rcap .compute_binary::( - starting_indexes.height, + starting_lengths.height, &self.core.net_pnl.delta.absolute._1m.cents.height, &self.core.minimal.cap.cents.height, exit, @@ -292,7 +293,7 @@ impl RealizedFull { self.net_pnl .change_1m_to_mcap .compute_binary::( - starting_indexes.height, + starting_lengths.height, &self.core.net_pnl.delta.absolute._1m.cents.height, height_to_market_cap, exit, @@ -301,7 +302,7 @@ impl RealizedFull { // Capitalized price ratio, percentiles and bands self.capitalized .price - .compute_rest(prices, starting_indexes, exit)?; + .compute_rest(prices, starting_lengths, exit)?; // Sell-side risk ratios for (ssrr, rv) in self @@ -311,7 +312,7 @@ impl RealizedFull { .zip(self.gross_pnl.sum.as_array()) { ssrr.compute_binary::( - starting_indexes.height, + starting_lengths.height, &rv.cents.height, &self.core.minimal.cap.cents.height, exit, @@ -321,7 +322,7 @@ impl RealizedFull { // Realized cap relative to own market cap self.cap_to_own_mcap .compute_binary::( - starting_indexes.height, + starting_lengths.height, &self.core.minimal.cap.usd.height, height_to_market_cap, exit, @@ -336,7 +337,7 @@ impl RealizedFull { .zip(self.core.minimal.loss.sum.as_array()) { ratio.compute_binary::( - starting_indexes.height, + starting_lengths.height, &profit.cents.height, &loss.cents.height, exit, @@ -345,7 +346,7 @@ impl RealizedFull { // Price ratio: percentiles, sma and std dev bands self.price_ratio_percentiles.compute( - starting_indexes, + starting_lengths, exit, &self.core.minimal.price.ratio.height, &self.core.minimal.price.cents.height, @@ -353,14 +354,14 @@ impl RealizedFull { self.price_ratio_sma.compute( blocks, - starting_indexes, + starting_lengths, exit, &self.core.minimal.price.ratio.height, )?; self.price_ratio_std_dev.compute( blocks, - starting_indexes, + starting_lengths, exit, &self.core.minimal.price.ratio.height, &self.core.minimal.price.cents.height, diff --git a/crates/brk_computer/src/distribution/metrics/realized/minimal.rs b/crates/brk_computer/src/distribution/metrics/realized/minimal.rs index 2a54eca8a..7980806fb 100644 --- a/crates/brk_computer/src/distribution/metrics/realized/minimal.rs +++ b/crates/brk_computer/src/distribution/metrics/realized/minimal.rs @@ -1,8 +1,9 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::{ - BasisPoints32, BasisPointsSigned32, Bitcoin, Cents, CentsSigned, Height, Indexes, Sats, - StoredF32, Version, + BasisPoints32, BasisPointsSigned32, Bitcoin, Cents, CentsSigned, Height, Sats, StoredF32, + Version, }; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVec}; @@ -81,38 +82,38 @@ impl RealizedMinimal { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { - sum_others!(self, starting_indexes, others, exit; cap.cents.height); - sum_others!(self, starting_indexes, others, exit; profit.block.cents); - sum_others!(self, starting_indexes, others, exit; loss.block.cents); + sum_others!(self, starting_lengths, others, exit; cap.cents.height); + sum_others!(self, starting_lengths, others, exit; profit.block.cents); + sum_others!(self, starting_lengths, others, exit; loss.block.cents); Ok(()) } pub(crate) fn compute_rest_part1( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - self.profit.compute_rest(starting_indexes.height, exit)?; - self.loss.compute_rest(starting_indexes.height, exit)?; + self.profit.compute_rest(starting_lengths.height, exit)?; + self.loss.compute_rest(starting_lengths.height, exit)?; Ok(()) } pub(crate) fn compute_rest_part2( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_to_supply: &impl ReadableVec, exit: &Exit, ) -> Result<()> { let cap = &self.cap.cents.height; self.price - .compute_all(prices, starting_indexes, exit, |v| { + .compute_all(prices, starting_lengths, exit, |v| { Ok(v.compute_transform2( - starting_indexes.height, + starting_lengths.height, cap, height_to_supply, |(i, cap_cents, supply, ..)| { diff --git a/crates/brk_computer/src/distribution/metrics/realized/mod.rs b/crates/brk_computer/src/distribution/metrics/realized/mod.rs index 67971773e..05f0e3a31 100644 --- a/crates/brk_computer/src/distribution/metrics/realized/mod.rs +++ b/crates/brk_computer/src/distribution/metrics/realized/mod.rs @@ -9,7 +9,7 @@ pub use full::{RealizedFull, RealizedFullAccum}; pub use minimal::RealizedMinimal; use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Lengths; use vecdb::Exit; use crate::distribution::state::{CohortState, CostBasisData, RealizedState, WithCapital}; @@ -19,10 +19,10 @@ pub trait RealizedLike: Send + Sync { fn as_core_mut(&mut self) -> &mut RealizedCore; fn min_stateful_len(&self) -> usize; fn push_state(&mut self, state: &CohortState>); - fn compute_rest_part1(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>; + fn compute_rest_part1(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()>; fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&RealizedCore], exit: &Exit, ) -> Result<()>; @@ -42,16 +42,16 @@ impl RealizedLike for RealizedCore { fn push_state(&mut self, state: &CohortState>) { self.push_state(state) } - fn compute_rest_part1(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> { - self.compute_rest_part1(starting_indexes, exit) + fn compute_rest_part1(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()> { + self.compute_rest_part1(starting_lengths, exit) } fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&RealizedCore], exit: &Exit, ) -> Result<()> { - self.compute_from_stateful(starting_indexes, others, exit) + self.compute_from_stateful(starting_lengths, others, exit) } } @@ -69,15 +69,15 @@ impl RealizedLike for RealizedFull { fn push_state(&mut self, state: &CohortState>) { self.push_state(state) } - fn compute_rest_part1(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> { - self.compute_rest_part1(starting_indexes, exit) + fn compute_rest_part1(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()> { + self.compute_rest_part1(starting_lengths, exit) } fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&RealizedCore], exit: &Exit, ) -> Result<()> { - self.compute_from_stateful(starting_indexes, others, exit) + self.compute_from_stateful(starting_lengths, others, exit) } } diff --git a/crates/brk_computer/src/distribution/metrics/supply/base.rs b/crates/brk_computer/src/distribution/metrics/supply/base.rs index f617aed62..7a1519b60 100644 --- a/crates/brk_computer/src/distribution/metrics/supply/base.rs +++ b/crates/brk_computer/src/distribution/metrics/supply/base.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPoints16, BasisPointsSigned32, Height, Indexes, Sats, SatsSigned, Version}; +use brk_types::{BasisPoints16, BasisPointsSigned32, Height, Sats, SatsSigned, Version}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVec}; use crate::{ @@ -86,12 +87,12 @@ impl SupplyBase { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { self.total.sats.height.compute_sum_of_others( - starting_indexes.height, + starting_lengths.height, &others .iter() .map(|v| &v.total.sats.height) diff --git a/crates/brk_computer/src/distribution/metrics/supply/core.rs b/crates/brk_computer/src/distribution/metrics/supply/core.rs index bac4fbc39..1de602ce3 100644 --- a/crates/brk_computer/src/distribution/metrics/supply/core.rs +++ b/crates/brk_computer/src/distribution/metrics/supply/core.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Version}; +use brk_types::{Height, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode, WritableVec}; @@ -87,15 +88,15 @@ impl SupplyCore { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { let base_refs: Vec<&SupplyBase> = others.iter().map(|o| &o.base).collect(); self.base - .compute_from_stateful(starting_indexes, &base_refs, exit)?; - sum_others!(self, starting_indexes, others, exit; in_profit.sats.height); - sum_others!(self, starting_indexes, others, exit; in_loss.sats.height); + .compute_from_stateful(starting_lengths, &base_refs, exit)?; + sum_others!(self, starting_lengths, others, exit; in_profit.sats.height); + sum_others!(self, starting_lengths, others, exit; in_loss.sats.height); Ok(()) } } diff --git a/crates/brk_computer/src/distribution/metrics/unrealized/basic.rs b/crates/brk_computer/src/distribution/metrics/unrealized/basic.rs index 3c3d43a48..ed2769f65 100644 --- a/crates/brk_computer/src/distribution/metrics/unrealized/basic.rs +++ b/crates/brk_computer/src/distribution/metrics/unrealized/basic.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Dollars, Indexes, Version}; +use brk_types::{Cents, Dollars, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableCloneableVec, Rw, StorageMode, WritableVec}; @@ -67,12 +68,12 @@ impl UnrealizedBasic { pub(crate) fn compute_from_sources( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { - sum_others!(self, starting_indexes, others, exit; profit.cents.height); - sum_others!(self, starting_indexes, others, exit; loss.cents.height); + sum_others!(self, starting_lengths, others, exit; profit.cents.height); + sum_others!(self, starting_lengths, others, exit; loss.cents.height); Ok(()) } } diff --git a/crates/brk_computer/src/distribution/metrics/unrealized/core.rs b/crates/brk_computer/src/distribution/metrics/unrealized/core.rs index dc66b4c4b..808392535 100644 --- a/crates/brk_computer/src/distribution/metrics/unrealized/core.rs +++ b/crates/brk_computer/src/distribution/metrics/unrealized/core.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, CentsSigned, Indexes, Version}; +use brk_types::{Cents, CentsSigned, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, Exit, Rw, StorageMode}; @@ -47,22 +48,22 @@ impl UnrealizedCore { pub(crate) fn compute_from_stateful( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, others: &[&Self], exit: &Exit, ) -> Result<()> { let basic_refs: Vec<&UnrealizedBasic> = others.iter().map(|o| &o.basic).collect(); self.basic - .compute_from_sources(starting_indexes, &basic_refs, exit)?; + .compute_from_sources(starting_lengths, &basic_refs, exit)?; Ok(()) } - pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> { + pub(crate) fn compute_rest(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()> { self.net_pnl .cents .height .compute_binary::( - starting_indexes.height, + starting_lengths.height, &self.basic.profit.cents.height, &self.basic.loss.cents.height, exit, diff --git a/crates/brk_computer/src/distribution/metrics/unrealized/full.rs b/crates/brk_computer/src/distribution/metrics/unrealized/full.rs index 5d46dd078..a9af8bc71 100644 --- a/crates/brk_computer/src/distribution/metrics/unrealized/full.rs +++ b/crates/brk_computer/src/distribution/metrics/unrealized/full.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, CentsSigned, CentsSquaredSats, Height, Indexes, Sats, Version}; +use brk_types::{Cents, CentsSigned, CentsSquaredSats, Height, Sats, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{AnyStoredVec, AnyVec, BytesVec, Exit, ReadableVec, Rw, StorageMode, WritableVec}; @@ -99,16 +100,16 @@ impl UnrealizedFull { pub(crate) fn compute_rest_all( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, supply_in_profit_sats: &(impl ReadableVec + Sync), supply_in_loss_sats: &(impl ReadableVec + Sync), exit: &Exit, ) -> Result<()> { - self.inner.compute_rest(starting_indexes, exit)?; + self.inner.compute_rest(starting_lengths, exit)?; // gross_pnl = profit + loss self.gross_pnl.cents.height.compute_add( - starting_indexes.height, + starting_lengths.height, &self.inner.basic.profit.cents.height, &self.inner.basic.loss.cents.height, exit, @@ -120,7 +121,7 @@ impl UnrealizedFull { .cents .height .compute_transform3( - starting_indexes.height, + starting_lengths.height, supply_in_profit_sats, &prices.spot.cents.height, &self.inner.basic.profit.cents.height, @@ -140,7 +141,7 @@ impl UnrealizedFull { .cents .height .compute_transform3( - starting_indexes.height, + starting_lengths.height, supply_in_loss_sats, &prices.spot.cents.height, &self.inner.basic.loss.cents.height, @@ -158,7 +159,7 @@ impl UnrealizedFull { /// Called after cost_basis.in_profit/loss are computed at the cohort level. pub(crate) fn compute_sentiment( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, spot: &impl ReadableVec, exit: &Exit, ) -> Result<()> { @@ -166,7 +167,7 @@ impl UnrealizedFull { // capitalized_price = capitalized_cap / invested_cap // invested_cap is in Cents (already / ONE_BTC), multiply back for CentsSats scale self.sentiment.greed_index.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, &self.capitalized_cap_in_profit_raw, &self.invested_capital.in_profit.cents.height, spot, @@ -187,7 +188,7 @@ impl UnrealizedFull { // pain = capitalized_price_losers - spot self.sentiment.pain_index.cents.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, &self.capitalized_cap_in_loss_raw, &self.invested_capital.in_loss.cents.height, spot, @@ -212,7 +213,7 @@ impl UnrealizedFull { .cents .height .compute_binary::( - starting_indexes.height, + starting_lengths.height, &self.sentiment.greed_index.cents.height, &self.sentiment.pain_index.cents.height, exit, diff --git a/crates/brk_computer/src/distribution/metrics/unrealized/mod.rs b/crates/brk_computer/src/distribution/metrics/unrealized/mod.rs index 183445f2b..073642b5f 100644 --- a/crates/brk_computer/src/distribution/metrics/unrealized/mod.rs +++ b/crates/brk_computer/src/distribution/metrics/unrealized/mod.rs @@ -9,7 +9,8 @@ pub use full::UnrealizedFull; pub use minimal::UnrealizedMinimal; use brk_error::Result; -use brk_types::{Height, Indexes, Sats}; +use brk_indexer::Lengths; +use brk_types::{Height, Sats}; use vecdb::{Exit, ReadableVec}; use crate::{distribution::state::UnrealizedState, prices}; @@ -22,7 +23,7 @@ pub trait UnrealizedLike: Send + Sync { fn compute_rest( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, supply_in_profit_sats: &(impl ReadableVec + Sync), supply_in_loss_sats: &(impl ReadableVec + Sync), exit: &Exit, @@ -46,12 +47,12 @@ impl UnrealizedLike for UnrealizedCore { fn compute_rest( &mut self, _prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, _supply_in_profit_sats: &(impl ReadableVec + Sync), _supply_in_loss_sats: &(impl ReadableVec + Sync), exit: &Exit, ) -> Result<()> { - self.compute_rest(starting_indexes, exit) + self.compute_rest(starting_lengths, exit) } } @@ -72,14 +73,14 @@ impl UnrealizedLike for UnrealizedFull { fn compute_rest( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, supply_in_profit_sats: &(impl ReadableVec + Sync), supply_in_loss_sats: &(impl ReadableVec + Sync), exit: &Exit, ) -> Result<()> { self.compute_rest_all( prices, - starting_indexes, + starting_lengths, supply_in_profit_sats, supply_in_loss_sats, exit, diff --git a/crates/brk_computer/src/distribution/vecs.rs b/crates/brk_computer/src/distribution/vecs.rs index 139104550..bb5a4f45f 100644 --- a/crates/brk_computer/src/distribution/vecs.rs +++ b/crates/brk_computer/src/distribution/vecs.rs @@ -5,8 +5,8 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{ - Cents, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, Height, Indexes, - StoredF64, SupplyState, Timestamp, TxIndex, Version, + Cents, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, Height, StoredF64, + SupplyState, Timestamp, TxIndex, Version, }; use rayon::prelude::*; use tracing::{debug, info}; @@ -317,18 +317,19 @@ impl Vecs { transactions: &transactions::Vecs, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &mut Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; + let starting_lengths = indexer.safe_lengths(); + // 1. Find minimum height we have data for across stateful vecs let current_height = Height::from(self.supply_state.len()); let min_stateful = self.min_stateful_len(); // 2. Determine start mode and recover/reset state - // Clamp to starting_indexes.height to handle reorg (indexer may require earlier start) - let resume_target = current_height.min(starting_indexes.height); + // Clamp to starting_lengths.height to handle reorg (indexer may require earlier start) + let resume_target = current_height.min(starting_lengths.height); if resume_target < current_height { info!( "Reorg detected: rolling back from {} to {}", @@ -451,11 +452,6 @@ impl Vecs { recovered_height }; - // Update starting_indexes if we need to recompute from an earlier point - if starting_height < starting_indexes.height { - starting_indexes.height = starting_height; - } - // 2c. Validate computed versions debug!("validating computed versions"); let base_version = VERSION; @@ -510,11 +506,11 @@ impl Vecs { let (r1, r2) = rayon::join( || { self.utxo_cohorts - .compute_overlapping_vecs(starting_indexes, exit) + .compute_overlapping_vecs(&starting_lengths, exit) }, || { self.addr_cohorts - .compute_overlapping_vecs(starting_indexes, exit) + .compute_overlapping_vecs(&starting_lengths, exit) }, ); r1?; @@ -523,7 +519,7 @@ impl Vecs { // 5b. Compute coinblocks_destroyed cumulative from raw self.coinblocks_destroyed - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; // 6. Compute rest part1 (day1 mappings) info!("Computing rest part 1..."); @@ -531,11 +527,11 @@ impl Vecs { let (r1, r2) = rayon::join( || { self.utxo_cohorts - .compute_rest_part1(prices, starting_indexes, exit) + .compute_rest_part1(prices, &starting_lengths, exit) }, || { self.addr_cohorts - .compute_rest_part1(prices, starting_indexes, exit) + .compute_rest_part1(prices, &starting_lengths, exit) }, ); r1?; @@ -543,8 +539,8 @@ impl Vecs { } // 6b. Compute address count sum (by addr_type -> all) - self.addrs.funded.compute_rest(starting_indexes, exit)?; - self.addrs.empty.compute_rest(starting_indexes, exit)?; + self.addrs.funded.compute_rest(&starting_lengths, exit)?; + self.addrs.empty.compute_rest(&starting_lengths, exit)?; let t = &self.utxo_cohorts.type_; let type_supply_sats = ByAddrType::new(|filter| { let Filter::Type(ot) = filter else { @@ -554,7 +550,7 @@ impl Vecs { }); let all_supply_sats = &self.utxo_cohorts.all.metrics.supply.total.sats.height; self.addrs.reused.compute_rest( - starting_indexes, + &starting_lengths, &outputs.by_type, &inputs.by_type, prices, @@ -563,7 +559,7 @@ impl Vecs { exit, )?; self.addrs.respent.compute_rest( - starting_indexes, + &starting_lengths, &outputs.by_type, &inputs.by_type, prices, @@ -572,7 +568,7 @@ impl Vecs { exit, )?; self.addrs.exposed.compute_rest( - starting_indexes, + &starting_lengths, prices, all_supply_sats, &type_supply_sats, @@ -586,7 +582,7 @@ impl Vecs { &all_m.supply.total.sats.height, &all_m.outputs.unspent_count.height, &self.addrs.funded.all.height, - starting_indexes.height, + starting_lengths.height, exit, )?; for ((ot, avg), (_, funded)) in self @@ -602,14 +598,14 @@ impl Vecs { &type_m.supply.total.sats.height, &type_m.outputs.unspent_count.height, &funded.height, - starting_indexes.height, + starting_lengths.height, exit, )?; } // 6c. Compute total_addr_count = addr_count + empty_addr_count self.addrs.total.compute( - starting_indexes.height, + starting_lengths.height, &self.addrs.funded, &self.addrs.empty, exit, @@ -617,10 +613,10 @@ impl Vecs { self.addrs .activity - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.addrs .new - .compute(starting_indexes.height, &self.addrs.total, exit)?; + .compute(starting_lengths.height, &self.addrs.total, exit)?; // 7. Compute rest part2 (relative metrics) let height_to_market_cap = self @@ -637,7 +633,7 @@ impl Vecs { self.utxo_cohorts.compute_rest_part2( blocks, prices, - starting_indexes, + &starting_lengths, &height_to_market_cap, exit, )?; @@ -661,7 +657,7 @@ impl Vecs { .read_only_clone(); self.addr_cohorts.compute_rest_part2( prices, - starting_indexes, + &starting_lengths, &all_supply_sats, &all_utxo_count, exit, diff --git a/crates/brk_computer/src/indexes/mod.rs b/crates/brk_computer/src/indexes/mod.rs index a1ce10bad..4c7b412f1 100644 --- a/crates/brk_computer/src/indexes/mod.rs +++ b/crates/brk_computer/src/indexes/mod.rs @@ -13,8 +13,8 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_traversable::Traversable; use brk_types::{ - Date, Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Indexes, Minute10, Minute30, - Month1, Month3, Month6, Version, Week1, Year1, Year10, + Date, Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Minute10, Minute30, Month1, + Month3, Month6, Version, Week1, Year1, Year10, }; use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode}; @@ -141,38 +141,34 @@ impl Vecs { Ok(this) } - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: Indexes, - exit: &Exit, - ) -> Result { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { self.db.sync_bg_tasks()?; - self.tx_heights.update(indexer, starting_indexes.height); + let starting_height = indexer.safe_lengths().height; + + self.tx_heights.update(indexer, starting_height); // timestamp_monotonic must be computed first — other mappings read it self.timestamp - .compute_monotonic(indexer, starting_indexes.height, exit)?; + .compute_monotonic(indexer, starting_height, exit)?; - self.compute_tx_indexes(indexer, &starting_indexes, exit)?; - self.compute_height_indexes(indexer, &starting_indexes, exit)?; + self.compute_tx_indexes(indexer, exit)?; + self.compute_height_indexes(indexer, exit)?; - let prev_height = starting_indexes.height.decremented().unwrap_or_default(); + let prev_height = starting_height.decremented().unwrap_or_default(); - self.compute_timestamp_mappings(&starting_indexes, exit)?; + self.compute_timestamp_mappings(indexer, exit)?; - let starting_day1 = - self.compute_calendar_mappings(indexer, &starting_indexes, prev_height, exit)?; + let starting_day1 = self.compute_calendar_mappings(indexer, prev_height, exit)?; - self.compute_period_vecs(&starting_indexes, prev_height, starting_day1, exit)?; + self.compute_period_vecs(indexer, prev_height, starting_day1, exit)?; self.timestamp.compute_per_resolution( indexer, &self.height, &self.halving, &self.epoch, - &starting_indexes, + &indexer.safe_lengths(), exit, )?; @@ -181,19 +177,15 @@ impl Vecs { let _lock = exit.lock(); db.compact_deferred_default() }); - Ok(starting_indexes) + Ok(()) } - fn compute_tx_indexes( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + fn compute_tx_indexes(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); let (r1, r2) = rayon::join( || { self.tx_index.input_count.compute_count_from_indexes( - starting_indexes.tx_index, + starting_lengths.tx_index, &indexer.vecs.transactions.first_txin_index, &indexer.vecs.inputs.outpoint, exit, @@ -201,7 +193,7 @@ impl Vecs { }, || { self.tx_index.output_count.compute_count_from_indexes( - starting_indexes.tx_index, + starting_lengths.tx_index, &indexer.vecs.transactions.first_txout_index, &indexer.vecs.outputs.value, exit, @@ -213,14 +205,10 @@ impl Vecs { Ok(()) } - fn compute_height_indexes( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + fn compute_height_indexes(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; self.height.tx_index_count.compute_count_from_indexes( - starting_indexes.height, + starting_height, &indexer.vecs.transactions.first_tx_index, &indexer.vecs.transactions.txid, exit, @@ -228,15 +216,13 @@ impl Vecs { Ok(()) } - fn compute_timestamp_mappings( - &mut self, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + fn compute_timestamp_mappings(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + macro_rules! from_timestamp { ($field:ident, $period:ty) => { self.height.$field.compute_transform( - starting_indexes.height, + starting_height, &self.timestamp.monotonic, |(h, ts, _)| (h, <$period>::from_timestamp(ts)), exit, @@ -257,10 +243,11 @@ impl Vecs { fn compute_calendar_mappings( &mut self, indexer: &Indexer, - starting_indexes: &Indexes, prev_height: Height, exit: &Exit, ) -> Result { + let starting_height = indexer.safe_lengths().height; + let starting_day1 = self .height .day1 @@ -268,7 +255,7 @@ impl Vecs { .unwrap_or_default(); self.height.day1.compute_transform( - starting_indexes.height, + starting_height, &self.timestamp.monotonic, |(h, ts, ..)| (h, Day1::try_from(Date::from(ts)).unwrap()), exit, @@ -280,40 +267,40 @@ impl Vecs { starting_day1 }; - self.compute_epoch(indexer, starting_indexes, exit)?; + self.compute_epoch(indexer, exit)?; self.height.week1.compute_transform( - starting_indexes.height, + starting_height, &self.height.day1, |(h, di, _)| (h, Week1::from(di)), exit, )?; self.height.month1.compute_transform( - starting_indexes.height, + starting_height, &self.height.day1, |(h, di, _)| (h, Month1::from(di)), exit, )?; self.height.month3.compute_transform( - starting_indexes.height, + starting_height, &self.height.month1, |(h, mi, _)| (h, Month3::from(mi)), exit, )?; self.height.month6.compute_transform( - starting_indexes.height, + starting_height, &self.height.month1, |(h, mi, _)| (h, Month6::from(mi)), exit, )?; self.height.year1.compute_transform( - starting_indexes.height, + starting_height, &self.height.month1, |(h, mi, _)| (h, Year1::from(mi)), exit, )?; self.height.year10.compute_transform( - starting_indexes.height, + starting_height, &self.height.year1, |(h, yi, _)| (h, Year10::from(yi)), exit, @@ -322,30 +309,25 @@ impl Vecs { Ok(starting_day1) } - fn compute_epoch( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { - self.height.epoch.compute_from_index( - starting_indexes.height, - &indexer.vecs.blocks.weight, - exit, - )?; + fn compute_epoch(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + + self.height + .epoch + .compute_from_index(starting_height, &indexer.vecs.blocks.weight, exit)?; self.epoch.first_height.inner.compute_first_per_index( - starting_indexes.height, + starting_height, &self.height.epoch, exit, )?; self.height.halving.compute_from_index( - starting_indexes.height, + starting_height, &indexer.vecs.blocks.weight, exit, )?; self.halving.first_height.inner.compute_first_per_index( - starting_indexes.height, + starting_height, &self.height.halving, exit, )?; @@ -354,15 +336,17 @@ impl Vecs { fn compute_period_vecs( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, prev_height: Height, starting_day1: Day1, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + macro_rules! basic_period { ($period:ident) => { self.$period.first_height.inner.compute_first_per_index( - starting_indexes.height, + starting_height, &self.height.$period, exit, )?; @@ -376,7 +360,7 @@ impl Vecs { basic_period!(hour12); self.day1.first_height.inner.compute_first_per_index( - starting_indexes.height, + starting_height, &self.height.day1, exit, )?; @@ -391,7 +375,7 @@ impl Vecs { macro_rules! dated_period { ($period:ident) => {{ self.$period.first_height.inner.compute_first_per_index( - starting_indexes.height, + starting_height, &self.height.$period, exit, )?; diff --git a/crates/brk_computer/src/indexes/timestamp.rs b/crates/brk_computer/src/indexes/timestamp.rs index e1c434a53..4d0233016 100644 --- a/crates/brk_computer/src/indexes/timestamp.rs +++ b/crates/brk_computer/src/indexes/timestamp.rs @@ -1,8 +1,9 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::{ - Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Indexes, Minute10, Minute30, Month1, - Month3, Month6, Timestamp, Week1, Year1, Year10, + Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Minute10, Minute30, Month1, Month3, + Month6, Timestamp, Week1, Year1, Year10, }; use derive_more::{Deref, DerefMut}; use vecdb::{ @@ -129,10 +130,10 @@ impl Timestamps { height: &super::HeightVecs, halving_vecs: &super::HalvingVecs, epoch_vecs: &super::EpochVecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - let prev_height = starting_indexes.height.decremented().unwrap_or_default(); + let prev_height = starting_lengths.height.decremented().unwrap_or_default(); self.resolutions.halving.compute_indirect_sequential( height.halving.collect_one(prev_height).unwrap_or_default(), &halving_vecs.first_height, diff --git a/crates/brk_computer/src/indicators/compute.rs b/crates/brk_computer/src/indicators/compute.rs index ab71b3a68..6814f4aee 100644 --- a/crates/brk_computer/src/indicators/compute.rs +++ b/crates/brk_computer/src/indicators/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Bitcoin, Dollars, Indexes, StoredF32}; +use brk_indexer::Indexer; +use brk_types::{Bitcoin, Dollars, StoredF32}; use vecdb::Exit; use super::{Vecs, gini}; @@ -9,33 +10,35 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, + indexer: &Indexer, mining: &mining::Vecs, distribution: &distribution::Vecs, transactions: &transactions::Vecs, market: &market::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; + let starting_lengths = indexer.safe_lengths(); + // Puell Multiple: daily_subsidy_usd / sma_365d_subsidy_usd self.puell_multiple .bps .compute_binary::( - starting_indexes.height, + starting_lengths.height, &mining.rewards.subsidy.block.usd, &mining.rewards.subsidy.average._1y.usd.height, exit, )?; // Gini coefficient (UTXO distribution inequality) - gini::compute(&mut self.gini, distribution, starting_indexes, exit)?; + gini::compute(&mut self.gini, distribution, indexer, exit)?; // RHODL Ratio: 1d-1w realized cap / 1y-2y realized cap self.rhodl_ratio .bps .compute_binary::( - starting_indexes.height, + starting_lengths.height, &distribution .utxo_cohorts .age_range @@ -69,7 +72,7 @@ impl Vecs { self.nvt .bps .compute_binary::( - starting_indexes.height, + starting_lengths.height, market_cap, &transactions.volume.transfer_volume.sum._24h.usd.height, exit, @@ -79,7 +82,7 @@ impl Vecs { self.thermo_cap_multiple .bps .compute_binary::( - starting_indexes.height, + starting_lengths.height, market_cap, &mining.rewards.subsidy.cumulative.usd.height, exit, @@ -93,7 +96,7 @@ impl Vecs { self.coindays_destroyed_supply_adj .height .compute_transform2( - starting_indexes.height, + starting_lengths.height, &all_activity.coindays_destroyed.sum._24h.height, supply_total_sats, |(i, cdd_24h, supply_sats, ..)| { @@ -111,7 +114,7 @@ impl Vecs { self.coinyears_destroyed_supply_adj .height .compute_transform2( - starting_indexes.height, + starting_lengths.height, &all_activity.coinyears_destroyed.height, supply_total_sats, |(i, cyd, supply_sats, ..)| { @@ -127,7 +130,7 @@ impl Vecs { // Supply-Adjusted Dormancy = dormancy / circulating_supply_btc self.dormancy.supply_adj.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, &all_activity.dormancy._24h.height, supply_total_sats, |(i, dormancy, supply_sats, ..)| { @@ -144,7 +147,7 @@ impl Vecs { // Stock-to-Flow: supply / annual_issuance // annual_issuance ≈ subsidy_per_block × 52560 (blocks/year) self.stock_to_flow.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, supply_total_sats, &mining.rewards.subsidy.block.sats, |(i, supply_sats, subsidy_sats, ..)| { @@ -163,7 +166,7 @@ impl Vecs { // Dormancy Flow: supply_btc / dormancy self.dormancy.flow.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, supply_total_sats, &all_activity.dormancy._24h.height, |(i, supply_sats, dormancy, ..)| { @@ -180,7 +183,7 @@ impl Vecs { // Seller Exhaustion Constant: % supply_in_profit × 30d_volatility self.seller_exhaustion.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, &all_metrics.supply.in_profit.sats.height, &market.volatility._1m.height, supply_total_sats, diff --git a/crates/brk_computer/src/indicators/gini.rs b/crates/brk_computer/src/indicators/gini.rs index 712810250..96ebab26b 100644 --- a/crates/brk_computer/src/indicators/gini.rs +++ b/crates/brk_computer/src/indicators/gini.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{BasisPoints16, Indexes, Sats, StoredU64, Version}; +use brk_indexer::Indexer; +use brk_types::{BasisPoints16, Sats, StoredU64, Version}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use crate::{distribution, internal::PercentPerBlock}; @@ -7,9 +8,10 @@ use crate::{distribution, internal::PercentPerBlock}; pub(super) fn compute( gini: &mut PercentPerBlock, distribution: &distribution::Vecs, - starting_indexes: &Indexes, + indexer: &Indexer, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let amount_range = &distribution.utxo_cohorts.amount_range; let supply_vecs: Vec<&_> = amount_range @@ -36,11 +38,7 @@ pub(super) fn compute( .height .validate_computed_version_or_reset(source_version)?; - let min_len = gini - .bps - .height - .len() - .min(starting_indexes.height.to_usize()); + let min_len = gini.bps.height.len().min(starting_height.to_usize()); gini.bps.height.truncate_if_needed_at(min_len)?; diff --git a/crates/brk_computer/src/indicators/rarity_meter/inner.rs b/crates/brk_computer/src/indicators/rarity_meter/inner.rs index 561d31665..948c872f4 100644 --- a/crates/brk_computer/src/indicators/rarity_meter/inner.rs +++ b/crates/brk_computer/src/indicators/rarity_meter/inner.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{Cents, Height, Indexes, StoredI8, Version}; +use brk_types::{Cents, Height, StoredI8, Version}; use vecdb::{AnyVec, Database, Exit, ReadableVec, Rw, StorageMode, WritableVec}; use crate::{ @@ -47,60 +48,61 @@ impl RarityMeterInner { &mut self, models: &[&RatioPerBlockPercentiles], spot: &impl ReadableVec, - starting_indexes: &Indexes, + indexer: &Indexer, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let gather = |f: fn(&RatioPerBlockPercentiles) -> &_| -> Vec<_> { models.iter().map(|m| f(m)).collect() }; // Lower percentiles: max across all models (tightest lower bound) self.pct0_5.cents.height.compute_max_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct0_5.price.cents.height), exit, )?; self.pct1.cents.height.compute_max_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct1.price.cents.height), exit, )?; self.pct2.cents.height.compute_max_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct2.price.cents.height), exit, )?; self.pct5.cents.height.compute_max_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct5.price.cents.height), exit, )?; // Upper percentiles: min across all models (tightest upper bound) self.pct95.cents.height.compute_min_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct95.price.cents.height), exit, )?; self.pct98.cents.height.compute_min_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct98.price.cents.height), exit, )?; self.pct99.cents.height.compute_min_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct99.price.cents.height), exit, )?; self.pct99_5.cents.height.compute_min_of_others( - starting_indexes.height, + starting_height, &gather(|m| &m.pct99_5.price.cents.height), exit, )?; - self.compute_index(spot, starting_indexes, exit)?; + self.compute_index(spot, indexer, exit)?; - self.compute_score(models, spot, starting_indexes, exit)?; + self.compute_score(models, spot, indexer, exit)?; Ok(()) } @@ -108,9 +110,10 @@ impl RarityMeterInner { fn compute_index( &mut self, spot: &impl ReadableVec, - starting_indexes: &Indexes, + indexer: &Indexer, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let bands = [ &self.pct0_5.cents.height, &self.pct1.cents.height, @@ -128,9 +131,7 @@ impl RarityMeterInner { self.index .height .validate_computed_version_or_reset(dep_version)?; - self.index - .height - .truncate_if_needed(starting_indexes.height)?; + self.index.height.truncate_if_needed(starting_height)?; self.index.height.repeat_until_complete(exit, |vec| { let skip = vec.len(); @@ -186,9 +187,10 @@ impl RarityMeterInner { &mut self, models: &[&RatioPerBlockPercentiles], spot: &impl ReadableVec, - starting_indexes: &Indexes, + indexer: &Indexer, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let dep_version: Version = models .iter() .map(|p| { @@ -207,9 +209,7 @@ impl RarityMeterInner { self.score .height .validate_computed_version_or_reset(dep_version)?; - self.score - .height - .truncate_if_needed(starting_indexes.height)?; + self.score.height.truncate_if_needed(starting_height)?; self.score.height.repeat_until_complete(exit, |vec| { let skip = vec.len(); diff --git a/crates/brk_computer/src/indicators/rarity_meter/mod.rs b/crates/brk_computer/src/indicators/rarity_meter/mod.rs index a6ae7b716..dfa2d412e 100644 --- a/crates/brk_computer/src/indicators/rarity_meter/mod.rs +++ b/crates/brk_computer/src/indicators/rarity_meter/mod.rs @@ -1,8 +1,9 @@ mod inner; use brk_error::Result; +use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{Indexes, Version}; +use brk_types::Version; use vecdb::{Database, Exit, Rw, StorageMode}; use crate::{distribution, indexes, prices}; @@ -34,9 +35,9 @@ impl RarityMeter { pub(crate) fn compute( &mut self, + indexer: &Indexer, distribution: &distribution::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { let realized = &distribution.utxo_cohorts.all.metrics.realized; @@ -55,7 +56,7 @@ impl RarityMeter { <h_realized.capitalized.price.percentiles, ], spot, - starting_indexes, + indexer, exit, )?; @@ -66,7 +67,7 @@ impl RarityMeter { &sth_realized.capitalized.price.percentiles, ], spot, - starting_indexes, + indexer, exit, )?; @@ -79,7 +80,7 @@ impl RarityMeter { <h_realized.capitalized.price.percentiles, ], spot, - starting_indexes, + indexer, exit, )?; diff --git a/crates/brk_computer/src/inputs/by_type/compute.rs b/crates/brk_computer/src/inputs/by_type/compute.rs index 3b3c95036..621865dd5 100644 --- a/crates/brk_computer/src/inputs/by_type/compute.rs +++ b/crates/brk_computer/src/inputs/by_type/compute.rs @@ -1,27 +1,24 @@ use brk_error::{OptionData, Result}; use brk_indexer::Indexer; -use brk_types::{Indexes, StoredU64}; +use brk_types::StoredU64; use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::{Vecs, WithInputTypes}; use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks}; impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); + let dep_version = indexer.vecs.inputs.output_type.version() + indexer.vecs.transactions.first_tx_index.version() + indexer.vecs.transactions.first_txin_index.version() + indexer.vecs.transactions.txid.version(); self.input_count - .validate_and_truncate(dep_version, starting_indexes.height)?; + .validate_and_truncate(dep_version, starting_lengths.height)?; self.tx_count - .validate_and_truncate(dep_version, starting_indexes.height)?; + .validate_and_truncate(dep_version, starting_lengths.height)?; let skip = self .input_count @@ -84,15 +81,15 @@ impl Vecs { } self.input_count - .compute_rest(starting_indexes.height, exit)?; - self.tx_count.compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; + self.tx_count.compute_rest(starting_lengths.height, exit)?; } for (otype, source) in self.input_count.by_type.iter_typed() { self.input_share.get_mut(otype).compute_count_ratio( source, &self.input_count.all, - starting_indexes.height, + starting_lengths.height, exit, )?; } @@ -101,7 +98,7 @@ impl Vecs { self.tx_share.get_mut(otype).compute_count_ratio( source, &self.tx_count.all, - starting_indexes.height, + starting_lengths.height, exit, )?; } diff --git a/crates/brk_computer/src/inputs/compute.rs b/crates/brk_computer/src/inputs/compute.rs index 2aec3b755..d5388e114 100644 --- a/crates/brk_computer/src/inputs/compute.rs +++ b/crates/brk_computer/src/inputs/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -12,16 +11,16 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, blocks: &blocks::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; - self.spent.compute(indexer, starting_indexes, exit)?; - self.count - .compute(indexer, indexes, blocks, starting_indexes, exit)?; - self.per_sec.compute(&self.count, starting_indexes, exit)?; - self.by_type.compute(indexer, starting_indexes, exit)?; + let starting_lengths = indexer.safe_lengths(); + + self.spent.compute(indexer, exit)?; + self.count.compute(indexer, indexes, blocks, exit)?; + self.per_sec.compute(&self.count, &starting_lengths, exit)?; + self.by_type.compute(indexer, exit)?; let exit = exit.clone(); self.db.run_bg(move |db| { diff --git a/crates/brk_computer/src/inputs/count/compute.rs b/crates/brk_computer/src/inputs/count/compute.rs index dd2349bd3..50c78cee3 100644 --- a/crates/brk_computer/src/inputs/count/compute.rs +++ b/crates/brk_computer/src/inputs/count/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -12,12 +11,12 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, blocks: &blocks::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let window_starts = blocks.lookback.window_starts(); self.0.compute( - starting_indexes.height, + starting_height, &indexes.tx_index.input_count, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, diff --git a/crates/brk_computer/src/inputs/per_sec/compute.rs b/crates/brk_computer/src/inputs/per_sec/compute.rs index 0f9b6adf5..0a617e93f 100644 --- a/crates/brk_computer/src/inputs/per_sec/compute.rs +++ b/crates/brk_computer/src/inputs/per_sec/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, StoredF32}; +use brk_indexer::Lengths; +use brk_types::StoredF32; use vecdb::Exit; use super::Vecs; @@ -9,10 +10,10 @@ impl Vecs { pub(crate) fn compute( &mut self, count: &CountVecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - let h = starting_indexes.height; + let h = starting_lengths.height; let sums = count.rolling.sum.0.as_array(); let per_sec = self.0.as_mut_array(); for (i, &secs) in Windows::<()>::SECS.iter().enumerate() { diff --git a/crates/brk_computer/src/inputs/spent/compute.rs b/crates/brk_computer/src/inputs/spent/compute.rs index 2e3ba2413..ad427703b 100644 --- a/crates/brk_computer/src/inputs/spent/compute.rs +++ b/crates/brk_computer/src/inputs/spent/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Indexes, Sats, TxIndex, TxOutIndex, Vout}; +use brk_types::{Sats, TxIndex, TxOutIndex, Vout}; use rayon::prelude::*; use tracing::info; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; @@ -10,12 +10,8 @@ use super::Vecs; const BATCH_SIZE: usize = 2 * 1024 * 1024 * 1024 / size_of::(); impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); // Validate computed versions against dependencies let dep_version = indexer.vecs.inputs.outpoint.version() + indexer.vecs.transactions.first_txout_index.version() @@ -31,7 +27,7 @@ impl Vecs { let len1 = self.txout_index.len(); let len2 = self.value.len(); - let starting = starting_indexes.txin_index.to_usize(); + let starting = starting_lengths.txin_index.to_usize(); let min = len1.min(len2).min(starting); if min >= target { diff --git a/crates/brk_computer/src/internal/indexes/eager.rs b/crates/brk_computer/src/internal/indexes/eager.rs index eb449faa8..db69dfb6f 100644 --- a/crates/brk_computer/src/internal/indexes/eager.rs +++ b/crates/brk_computer/src/internal/indexes/eager.rs @@ -1,9 +1,10 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::{ - Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Indexes, Minute10, Minute30, Month1, - Month3, Month6, Version, Week1, Year1, Year10, + Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Minute10, Minute30, Month1, Month3, + Month6, Version, Week1, Year1, Year10, }; use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; @@ -74,12 +75,12 @@ where pub(crate) fn compute_first( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_source: &impl ReadableVec, indexes: &indexes::Vecs, exit: &Exit, ) -> Result<()> { - let prev_height = starting_indexes.height.decremented().unwrap_or_default(); + let prev_height = starting_lengths.height.decremented().unwrap_or_default(); macro_rules! period { ($field:ident) => { @@ -117,13 +118,13 @@ where pub(crate) fn compute_max( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_source: &impl ReadableVec, indexes: &indexes::Vecs, exit: &Exit, ) -> Result<()> { let src_len = height_source.len(); - let prev_height = starting_indexes.height.decremented().unwrap_or_default(); + let prev_height = starting_lengths.height.decremented().unwrap_or_default(); macro_rules! period { ($field:ident) => { @@ -164,13 +165,13 @@ where pub(crate) fn compute_min( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, height_source: &impl ReadableVec, indexes: &indexes::Vecs, exit: &Exit, ) -> Result<()> { let src_len = height_source.len(); - let prev_height = starting_indexes.height.decremented().unwrap_or_default(); + let prev_height = starting_lengths.height.decremented().unwrap_or_default(); macro_rules! period { ($field:ident) => { diff --git a/crates/brk_computer/src/internal/per_block/ratio/base.rs b/crates/brk_computer/src/internal/per_block/ratio/base.rs index b1f1fe61f..e888ef7f6 100644 --- a/crates/brk_computer/src/internal/per_block/ratio/base.rs +++ b/crates/brk_computer/src/internal/per_block/ratio/base.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPoints32, Cents, Height, Indexes, StoredF32, Version}; +use brk_types::{BasisPoints32, Cents, Height, StoredF32, Version}; use vecdb::{Database, Exit, ReadableCloneableVec, ReadableVec, Rw, StorageMode}; use crate::{ @@ -50,13 +51,13 @@ impl RatioPerBlock { impl RatioPerBlock { pub(crate) fn compute_ratio( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, close_price: &impl ReadableVec, series_price: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.bps.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, close_price, series_price, |(i, close, price, ..)| { diff --git a/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs b/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs index cd1ca1145..3e02d81b0 100644 --- a/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs +++ b/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPoints32, Cents, Height, Indexes, StoredF32, Version}; +use brk_types::{BasisPoints32, Cents, Height, StoredF32, Version}; use vecdb::{ AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode, VecIndex, WritableVec, @@ -86,7 +87,7 @@ impl RatioPerBlockPercentiles { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ratio_source: &impl ReadableVec, series_price: &impl ReadableVec, @@ -102,7 +103,7 @@ impl RatioPerBlockPercentiles { .map(|v| Height::from(v.len())) .min() .unwrap() - .min(starting_indexes.height); + .min(starting_lengths.height); let start = starting_height.to_usize(); let ratio_len = ratio_source.len(); @@ -159,7 +160,7 @@ impl RatioPerBlockPercentiles { .price .cents .compute_binary::( - starting_indexes.height, + starting_lengths.height, series_price, &self.$band.ratio.bps.height, exit, diff --git a/crates/brk_computer/src/internal/per_block/ratio/price_extended.rs b/crates/brk_computer/src/internal/per_block/ratio/price_extended.rs index 13fe4a799..358d827d8 100644 --- a/crates/brk_computer/src/internal/per_block/ratio/price_extended.rs +++ b/crates/brk_computer/src/internal/per_block/ratio/price_extended.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPoints32, Cents, Dollars, Height, Indexes, SatsFract, StoredF32, Version}; +use brk_types::{BasisPoints32, Cents, Dollars, Height, SatsFract, StoredF32, Version}; use derive_more::{Deref, DerefMut}; use vecdb::{Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode}; @@ -39,12 +40,12 @@ impl PriceWithRatioPerBlock { /// Compute ratio from close price and this metric's price. pub(crate) fn compute_ratio( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, close_price: &impl ReadableVec, exit: &Exit, ) -> Result<()> { self.bps.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, close_price, &self.cents.height, |(i, close, price, ..)| { @@ -63,7 +64,7 @@ impl PriceWithRatioPerBlock { pub(crate) fn compute_all( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, mut compute_price: F, ) -> Result<()> @@ -71,7 +72,7 @@ impl PriceWithRatioPerBlock { F: FnMut(&mut EagerVec>) -> Result<()>, { compute_price(&mut self.cents.height)?; - self.compute_ratio(starting_indexes, &prices.spot.cents.height, exit) + self.compute_ratio(starting_lengths, &prices.spot.cents.height, exit) } } @@ -101,14 +102,14 @@ impl PriceWithRatioExtendedPerBlock { pub(crate) fn compute_rest( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { let close_price = &prices.spot.cents.height; self.base - .compute_ratio(starting_indexes, close_price, exit)?; + .compute_ratio(starting_lengths, close_price, exit)?; self.percentiles.compute( - starting_indexes, + starting_lengths, exit, &self.base.ratio.height, &self.base.cents.height, @@ -120,7 +121,7 @@ impl PriceWithRatioExtendedPerBlock { pub(crate) fn compute_all( &mut self, prices: &prices::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, mut compute_price: F, ) -> Result<()> @@ -128,6 +129,6 @@ impl PriceWithRatioExtendedPerBlock { F: FnMut(&mut EagerVec>) -> Result<()>, { compute_price(&mut self.base.cents.height)?; - self.compute_rest(prices, starting_indexes, exit) + self.compute_rest(prices, starting_lengths, exit) } } diff --git a/crates/brk_computer/src/internal/per_block/ratio/sma.rs b/crates/brk_computer/src/internal/per_block/ratio/sma.rs index f9097acfe..287885a5e 100644 --- a/crates/brk_computer/src/internal/per_block/ratio/sma.rs +++ b/crates/brk_computer/src/internal/per_block/ratio/sma.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{BasisPoints32, Height, Indexes, StoredF32, Version}; +use brk_types::{BasisPoints32, Height, StoredF32, Version}; use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode}; use crate::{blocks, indexes}; @@ -52,13 +53,13 @@ impl RatioSma { pub(crate) fn compute( &mut self, blocks: &blocks::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ratio_source: &impl ReadableVec, ) -> Result<()> { // Expanding SMA (all history) self.all.bps.height.compute_sma_( - starting_indexes.height, + starting_lengths.height, ratio_source, usize::MAX, exit, @@ -74,7 +75,7 @@ impl RatioSma { (&mut self._4y, &blocks.lookback._4y), ] { sma.bps.height.compute_rolling_average( - starting_indexes.height, + starting_lengths.height, lookback, ratio_source, exit, diff --git a/crates/brk_computer/src/internal/per_block/ratio/std_dev_bands.rs b/crates/brk_computer/src/internal/per_block/ratio/std_dev_bands.rs index cc340fc88..265ce6ffa 100644 --- a/crates/brk_computer/src/internal/per_block/ratio/std_dev_bands.rs +++ b/crates/brk_computer/src/internal/per_block/ratio/std_dev_bands.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Height, Indexes, StoredF32, Version}; +use brk_types::{Cents, Height, StoredF32, Version}; use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode}; use crate::{blocks, indexes, internal::StdDevPerBlockExtended}; @@ -43,7 +44,7 @@ impl RatioPerBlockStdDevBands { pub(crate) fn compute( &mut self, blocks: &blocks::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ratio_source: &impl ReadableVec, series_price: &impl ReadableVec, @@ -55,8 +56,8 @@ impl RatioPerBlockStdDevBands { (&mut self._2y, &sma._2y.ratio.height), (&mut self._1y, &sma._1y.ratio.height), ] { - sd.compute_all(blocks, starting_indexes, exit, ratio_source, sma_ratio)?; - sd.compute_cents_bands(starting_indexes, series_price, sma_ratio, exit)?; + sd.compute_all(blocks, starting_lengths, exit, ratio_source, sma_ratio)?; + sd.compute_cents_bands(starting_lengths, series_price, sma_ratio, exit)?; } Ok(()) diff --git a/crates/brk_computer/src/internal/per_block/stddev/base.rs b/crates/brk_computer/src/internal/per_block/stddev/base.rs index 1b4f9d24b..9176681d5 100644 --- a/crates/brk_computer/src/internal/per_block/stddev/base.rs +++ b/crates/brk_computer/src/internal/per_block/stddev/base.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, StoredF32, Version}; +use brk_types::{Height, StoredF32, Version}; use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode}; use crate::{blocks, indexes, internal::PerBlock}; @@ -35,20 +36,20 @@ impl StdDevPerBlock { pub(crate) fn compute_all( &mut self, blocks: &blocks::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, source: &impl ReadableVec, ) -> Result<()> { if self.days == usize::MAX { self.sma.height.compute_sma_( - starting_indexes.height, + starting_lengths.height, source, usize::MAX, exit, None, )?; self.sd.height.compute_expanding_sd( - starting_indexes.height, + starting_lengths.height, source, &self.sma.height, exit, @@ -59,14 +60,14 @@ impl StdDevPerBlock { let window_starts = blocks.lookback.start_vec(self.days); self.sma.height.compute_rolling_average( - starting_indexes.height, + starting_lengths.height, window_starts, source, exit, )?; self.sd.height.compute_rolling_sd( - starting_indexes.height, + starting_lengths.height, window_starts, source, &self.sma.height, diff --git a/crates/brk_computer/src/internal/per_block/stddev/extended.rs b/crates/brk_computer/src/internal/per_block/stddev/extended.rs index 797be803e..0b3f327da 100644 --- a/crates/brk_computer/src/internal/per_block/stddev/extended.rs +++ b/crates/brk_computer/src/internal/per_block/stddev/extended.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Cents, Height, Indexes, StoredF32, Version}; +use brk_types::{Cents, Height, StoredF32, Version}; use vecdb::{ AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode, VecIndex, WritableVec, @@ -95,7 +96,7 @@ impl StdDevPerBlockExtended { pub(crate) fn compute_all( &mut self, blocks: &blocks::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, source: &impl ReadableVec, sma: &impl ReadableVec, @@ -103,11 +104,11 @@ impl StdDevPerBlockExtended { if self.days == usize::MAX { self.sd .height - .compute_expanding_sd(starting_indexes.height, source, sma, exit)?; + .compute_expanding_sd(starting_lengths.height, source, sma, exit)?; } else { let window_starts = blocks.lookback.start_vec(self.days); self.sd.height.compute_rolling_sd( - starting_indexes.height, + starting_lengths.height, window_starts, source, sma, @@ -115,12 +116,12 @@ impl StdDevPerBlockExtended { )?; } - self.compute_bands(starting_indexes, exit, sma, source) + self.compute_bands(starting_lengths, exit, sma, source) } fn compute_bands( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, sma: &impl ReadableVec, source: &impl ReadableVec, @@ -138,7 +139,7 @@ impl StdDevPerBlockExtended { .map(|v| Height::from(v.len())) .min() .unwrap() - .min(starting_indexes.height); + .min(starting_lengths.height); let start = starting_height.to_usize(); @@ -167,7 +168,7 @@ impl StdDevPerBlockExtended { } self.zscore.height.compute_zscore( - starting_indexes.height, + starting_lengths.height, source, sma, &self.sd.height, @@ -179,7 +180,7 @@ impl StdDevPerBlockExtended { pub(crate) fn compute_cents_bands( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, series_price: &impl ReadableVec, sma: &impl ReadableVec, exit: &Exit, @@ -189,7 +190,7 @@ impl StdDevPerBlockExtended { $price .cents .compute_binary::( - starting_indexes.height, + starting_lengths.height, series_price, $band_source, exit, diff --git a/crates/brk_computer/src/internal/per_tx/derived.rs b/crates/brk_computer/src/internal/per_tx/derived.rs index 5a08ac615..448d1a98a 100644 --- a/crates/brk_computer/src/internal/per_tx/derived.rs +++ b/crates/brk_computer/src/internal/per_tx/derived.rs @@ -1,8 +1,8 @@ use brk_error::Result; -use brk_indexer::Indexer; +use brk_indexer::{Indexer, Lengths}; use brk_traversable::Traversable; -use brk_types::{Indexes, TxIndex, VSize}; +use brk_types::{TxIndex, VSize}; use schemars::JsonSchema; use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode, Version}; @@ -68,7 +68,7 @@ where &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, tx_index_source: &impl ReadableVec, exit: &Exit, ) -> Result<()> @@ -76,7 +76,7 @@ where T: Copy + Ord + From + Default, f64: From, { - self.derive_from_with_skip(indexer, indexes, starting_indexes, tx_index_source, exit, 0) + self.derive_from_with_skip(indexer, indexes, starting_lengths, tx_index_source, exit, 0) } #[allow(clippy::too_many_arguments)] @@ -84,7 +84,7 @@ where &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, tx_index_source: &impl ReadableVec, exit: &Exit, skip_count: usize, @@ -94,7 +94,7 @@ where f64: From, { self.block.compute_with_skip( - starting_indexes.height, + starting_lengths.height, tx_index_source, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, @@ -103,7 +103,7 @@ where )?; self.distribution._6b.compute_from_nblocks( - starting_indexes.height, + starting_lengths.height, tx_index_source, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, @@ -121,7 +121,7 @@ where &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, tx_index_source: &impl ReadableVec, vsize_source: &impl ReadableVec, exit: &Exit, @@ -132,7 +132,7 @@ where f64: From, { self.block.compute_with_skip_weighted( - starting_indexes.height, + starting_lengths.height, tx_index_source, vsize_source, &indexer.vecs.transactions.first_tx_index, @@ -142,7 +142,7 @@ where )?; self.distribution._6b.compute_from_nblocks( - starting_indexes.height, + starting_lengths.height, tx_index_source, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, diff --git a/crates/brk_computer/src/internal/per_tx/distribution.rs b/crates/brk_computer/src/internal/per_tx/distribution.rs index a56133fa7..ed8484184 100644 --- a/crates/brk_computer/src/internal/per_tx/distribution.rs +++ b/crates/brk_computer/src/internal/per_tx/distribution.rs @@ -4,9 +4,9 @@ //! and stored rather than lazily derived. use brk_error::Result; -use brk_indexer::Indexer; +use brk_indexer::{Indexer, Lengths}; use brk_traversable::Traversable; -use brk_types::{Indexes, TxIndex, VSize}; +use brk_types::{TxIndex, VSize}; use schemars::JsonSchema; use vecdb::{ Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw, StorageMode, Version, @@ -50,7 +50,7 @@ where &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, skip_count: usize, ) -> Result<()> @@ -61,7 +61,7 @@ where self.distribution.derive_from_with_skip( indexer, indexes, - starting_indexes, + starting_lengths, &self.tx_index, exit, skip_count, @@ -73,7 +73,7 @@ where &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, vsize_source: &impl ReadableVec, exit: &Exit, skip_count: usize, @@ -85,7 +85,7 @@ where self.distribution.derive_from_with_skip_weighted( indexer, indexes, - starting_indexes, + starting_lengths, &self.tx_index, vsize_source, exit, diff --git a/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs b/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs index fa724170c..fb55bdd79 100644 --- a/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs +++ b/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs @@ -1,7 +1,7 @@ use brk_error::Result; -use brk_indexer::Indexer; +use brk_indexer::{Indexer, Lengths}; use brk_traversable::Traversable; -use brk_types::{Indexes, TxIndex}; +use brk_types::TxIndex; use schemars::JsonSchema; use vecdb::{Database, Exit, LazyVecFrom2, ReadableVec, Rw, StorageMode, Version}; @@ -46,7 +46,7 @@ where &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> where @@ -55,6 +55,6 @@ where LazyVecFrom2: ReadableVec, { self.distribution - .derive_from(indexer, indexes, starting_indexes, &self.tx_index, exit) + .derive_from(indexer, indexes, starting_lengths, &self.tx_index, exit) } } diff --git a/crates/brk_computer/src/internal/with_addr_types.rs b/crates/brk_computer/src/internal/with_addr_types.rs index 3aea4d54a..f68427e5f 100644 --- a/crates/brk_computer/src/internal/with_addr_types.rs +++ b/crates/brk_computer/src/internal/with_addr_types.rs @@ -4,8 +4,9 @@ use brk_cohort::ByAddrType; use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; -use brk_types::{Height, Indexes, Sats, Version}; +use brk_types::{Height, Sats, Version}; use rayon::prelude::*; use schemars::JsonSchema; use vecdb::{AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, WritableVec}; @@ -83,12 +84,12 @@ where } /// Compute `all.height` as the per-block sum of the per-type vecs. - pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> { + pub(crate) fn compute_rest(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()> { let sources: Vec<&EagerVec>> = self.by_addr_type.values().map(|v| &v.height).collect(); self.all .height - .compute_sum_of_others(starting_indexes.height, &sources, exit)?; + .compute_sum_of_others(starting_lengths.height, &sources, exit)?; Ok(()) } } diff --git a/crates/brk_computer/src/investing/compute.rs b/crates/brk_computer/src/investing/compute.rs index c89456bd5..9a3d2332b 100644 --- a/crates/brk_computer/src/investing/compute.rs +++ b/crates/brk_computer/src/investing/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Date, Day1, Dollars, Indexes, Sats}; +use brk_indexer::Indexer; +use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Date, Day1, Dollars, Sats}; use vecdb::{AnyVec, Exit, ReadableOptionVec, ReadableVec, VecIndex}; use super::{ByDcaPeriod, Vecs}; @@ -10,15 +11,16 @@ const DCA_AMOUNT: Dollars = Dollars::mint(100.0); impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, indexes: &indexes::Vecs, prices: &prices::Vecs, blocks: &blocks::Vecs, lookback: &market::lookback::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; + let starting_lengths = indexer.safe_lengths(); let h2d = &indexes.height.day1; let close = &prices.split.close.usd.day1; @@ -29,7 +31,7 @@ impl Vecs { { let mut last_di: Option = None; self.sats_per_day.compute_transform( - starting_indexes.height, + starting_lengths.height, h2d, |(h, di, _)| { if last_di.is_none() && h.to_usize() > 0 { @@ -55,7 +57,7 @@ impl Vecs { for (stack, days) in self.period.dca_stack.iter_mut_with_days() { let window_starts = blocks.lookback.start_vec(days as usize); stack.sats.height.compute_rolling_sum( - starting_indexes.height, + starting_lengths.height, window_starts, &self.sats_per_day, exit, @@ -64,11 +66,11 @@ impl Vecs { // DCA by period - stack cents (sats × price) for stack in self.period.dca_stack.iter_mut() { - stack.compute(prices, starting_indexes.height, exit)?; + stack.compute(prices, starting_lengths.height, exit)?; } // DCA by period - average price (derived from stack) - let starting_height = starting_indexes.height.to_usize(); + let starting_height_usize = starting_lengths.height.to_usize(); for (average_price, stack, days) in self .period .dca_cost_basis @@ -76,7 +78,7 @@ impl Vecs { { let days = days as usize; average_price.cents.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, h2d, &stack.sats.height, |(h, di, stack_sats, ..)| { @@ -101,7 +103,7 @@ impl Vecs { .zip(self.period.dca_cost_basis.iter_with_days()) { returns.compute_binary::( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &average_price.cents.height, exit, @@ -116,7 +118,7 @@ impl Vecs { { let years = days as f64 / 365.0; cagr.bps.height.compute_transform( - starting_indexes.height, + starting_lengths.height, &returns.bps.height, |(h, r, ..)| { let ratio = f64::from(r); @@ -134,7 +136,7 @@ impl Vecs { { let total_invested = DCA_AMOUNT * days as usize; stack.sats.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, h2d, &lookback_price.cents.height, |(h, _di, lp, ..)| { @@ -151,7 +153,7 @@ impl Vecs { // Lump sum by period - stack cents (sats × price) for stack in self.period.lump_sum_stack.iter_mut() { - stack.compute(prices, starting_indexes.height, exit)?; + stack.compute(prices, starting_lengths.height, exit)?; } // Lump sum by period - returns (compute from lookback price) @@ -162,7 +164,7 @@ impl Vecs { .zip(lookback_dca.iter_with_days()) { returns.compute_binary::( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &lookback_price.cents.height, exit, @@ -173,7 +175,7 @@ impl Vecs { let start_days = super::ByDcaClass::<()>::start_days(); for (stack, day1) in self.class.dca_stack.iter_mut().zip(start_days) { let mut last_di: Option = None; - let cls_start = stack.sats.height.len().min(starting_height); + let cls_start = stack.sats.height.len().min(starting_height_usize); let mut prev_value = if cls_start > 0 { stack .sats @@ -185,7 +187,7 @@ impl Vecs { }; stack.sats.height.compute_transform( - starting_indexes.height, + starting_lengths.height, h2d, |(h, di, _)| { let hi = h.to_usize(); @@ -227,7 +229,7 @@ impl Vecs { // DCA by year class - stack cents (sats × price) for stack in self.class.dca_stack.iter_mut() { - stack.compute(prices, starting_indexes.height, exit)?; + stack.compute(prices, starting_lengths.height, exit)?; } // DCA by year class - average price (derived from stack) @@ -241,7 +243,7 @@ impl Vecs { { let from_usize = from.to_usize(); average_price.cents.height.compute_transform2( - starting_indexes.height, + starting_lengths.height, h2d, &stack.sats.height, |(h, di, stack_sats, ..)| { @@ -265,7 +267,7 @@ impl Vecs { .zip(self.class.dca_cost_basis.iter()) { returns.compute_binary::( - starting_indexes.height, + starting_lengths.height, &prices.spot.cents.height, &average_price.cents.height, exit, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index f35119219..c7e51c2d8 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -277,42 +277,28 @@ impl Computer { Ok(()) } - pub fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: brk_indexer::Indexes, - exit: &Exit, - ) -> Result<()> { + pub fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { internal::cache_clear_all(); let compute_start = Instant::now(); - let mut starting_indexes = timed("Computed indexes", || { - self.indexes.compute(indexer, starting_indexes, exit) - })?; + timed("Computed indexes", || self.indexes.compute(indexer, exit))?; thread::scope(|scope| -> Result<()> { timed("Computed blocks", || { - self.blocks - .compute(indexer, &self.indexes, &starting_indexes, exit) + self.blocks.compute(indexer, &self.indexes, exit) })?; let (inputs_result, prices_result) = rayon::join( || { timed("Computed inputs", || { - self.inputs.compute( - indexer, - &self.indexes, - &self.blocks, - &starting_indexes, - exit, - ) + self.inputs + .compute(indexer, &self.indexes, &self.blocks, exit) }) }, || { timed("Computed prices", || { - self.prices - .compute(indexer, &self.indexes, &starting_indexes, exit) + self.prices.compute(indexer, &self.indexes, exit) }) }, ); @@ -323,13 +309,8 @@ impl Computer { // independent. Run all three in parallel. let market = scope.spawn(|| { timed("Computed market", || { - self.market.compute( - &self.prices, - &self.indexes, - &self.blocks, - &starting_indexes, - exit, - ) + self.market + .compute(indexer, &self.prices, &self.indexes, &self.blocks, exit) }) }); @@ -341,7 +322,6 @@ impl Computer { &self.blocks, &self.inputs, &self.prices, - &starting_indexes, exit, ) })?; @@ -352,7 +332,6 @@ impl Computer { &self.blocks, &self.transactions, &self.prices, - &starting_indexes, exit, ) }) @@ -365,7 +344,6 @@ impl Computer { &self.inputs, &self.blocks, &self.prices, - &starting_indexes, exit, ) })?; @@ -375,7 +353,6 @@ impl Computer { Ok(()) })?; - let starting_indexes_clone = starting_indexes.clone(); thread::scope(|scope| -> Result<()> { let pools = scope.spawn(|| { timed("Computed pools", || { @@ -385,7 +362,6 @@ impl Computer { &self.blocks, &self.prices, &self.mining, - &starting_indexes_clone, exit, ) }) @@ -394,11 +370,11 @@ impl Computer { let investing = scope.spawn(|| { timed("Computed investing", || { self.investing.compute( + indexer, &self.indexes, &self.prices, &self.blocks, &self.market.lookback, - &starting_indexes_clone, exit, ) }) @@ -413,7 +389,6 @@ impl Computer { &self.transactions, &self.blocks, &self.prices, - &mut starting_indexes, exit, ) })?; @@ -429,11 +404,11 @@ impl Computer { let indicators = scope.spawn(|| { timed("Computed indicators", || { self.indicators.compute( + indexer, &self.mining, &self.distribution, &self.transactions, &self.market, - &starting_indexes, exit, ) }) @@ -441,20 +416,20 @@ impl Computer { timed("Computed supply", || { self.supply.compute( + indexer, &self.outputs, &self.blocks, &self.mining, &self.transactions, &self.prices, &self.distribution, - &starting_indexes, exit, ) })?; timed("Computed cointime", || { self.cointime.compute( - &starting_indexes, + indexer, &self.prices, &self.blocks, &self.mining, @@ -468,12 +443,9 @@ impl Computer { Ok(()) })?; - self.indicators.rarity_meter.compute( - &self.distribution, - &self.prices, - &starting_indexes, - exit, - )?; + self.indicators + .rarity_meter + .compute(indexer, &self.distribution, &self.prices, exit)?; info!("Total compute time: {:?}", compute_start.elapsed()); Ok(()) @@ -481,8 +453,9 @@ impl Computer { } impl Computer { - /// Last height whose computed-side state is durably stamped, derived - /// from `distribution.supply_state`'s stamp. + /// Live computer stamp for diagnostics. Derived from + /// `distribution.supply_state`'s stamp. For data reads use + /// `Query::height` (clamped against the safe-lengths snapshot). pub fn computed_height(&self) -> Height { Height::from(self.distribution.supply_state.stamp()) } diff --git a/crates/brk_computer/src/market/ath/compute.rs b/crates/brk_computer/src/market/ath/compute.rs index 6b03d2152..6a3a45aae 100644 --- a/crates/brk_computer/src/market/ath/compute.rs +++ b/crates/brk_computer/src/market/ath/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, StoredF32, Timestamp}; +use brk_indexer::Indexer; +use brk_types::{StoredF32, Timestamp}; use vecdb::{Exit, ReadableVec, VecIndex}; use super::Vecs; @@ -8,20 +9,22 @@ use crate::{indexes, prices}; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, prices: &prices::Vecs, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + self.high.cents.height.compute_all_time_high( - starting_indexes.height, + starting_height, &prices.spot.cents.height, exit, )?; let mut ath_ts: Option = None; self.days_since.height.compute_transform3( - starting_indexes.height, + starting_height, &self.high.cents.height, &prices.spot.cents.height, &indexes.timestamp.monotonic, @@ -48,7 +51,7 @@ impl Vecs { let mut prev = None; self.max_days_between.height.compute_transform( - starting_indexes.height, + starting_height, &self.days_since.height, |(i, days, slf)| { if prev.is_none() { @@ -67,7 +70,7 @@ impl Vecs { )?; self.drawdown.compute_drawdown( - starting_indexes.height, + starting_height, &prices.spot.cents.height, &self.high.cents.height, exit, diff --git a/crates/brk_computer/src/market/compute.rs b/crates/brk_computer/src/market/compute.rs index 8aeba53da..3df1ffd84 100644 --- a/crates/brk_computer/src/market/compute.rs +++ b/crates/brk_computer/src/market/compute.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use crate::{blocks, indexes, prices}; @@ -9,10 +9,10 @@ use super::Vecs; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, prices: &prices::Vecs, indexes: &indexes::Vecs, blocks: &blocks::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; @@ -21,20 +21,14 @@ impl Vecs { let (r1, r2) = rayon::join( || { rayon::join( - || self.ath.compute(prices, indexes, starting_indexes, exit), - || { - self.lookback - .compute(blocks, prices, starting_indexes, exit) - }, + || self.ath.compute(indexer, prices, indexes, exit), + || self.lookback.compute(indexer, blocks, prices, exit), ) }, || { rayon::join( - || self.range.compute(prices, blocks, starting_indexes, exit), - || { - self.moving_average - .compute(blocks, prices, starting_indexes, exit) - }, + || self.range.compute(indexer, prices, blocks, exit), + || self.moving_average.compute(indexer, blocks, prices, exit), ) }, ); @@ -45,15 +39,15 @@ impl Vecs { // Phase 2: Depend on lookback self.returns - .compute(prices, blocks, &self.lookback, starting_indexes, exit)?; + .compute(indexer, prices, blocks, &self.lookback, exit)?; // Phase 3: Depends on returns, moving_average self.technical.compute( + indexer, &self.returns, prices, blocks, &self.moving_average, - starting_indexes, exit, )?; diff --git a/crates/brk_computer/src/market/lookback/compute.rs b/crates/brk_computer/src/market/lookback/compute.rs index 394aff667..c356e8105 100644 --- a/crates/brk_computer/src/market/lookback/compute.rs +++ b/crates/brk_computer/src/market/lookback/compute.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; @@ -8,17 +8,18 @@ use crate::{blocks, prices}; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let price = &prices.spot.cents.height; for (price_past, days) in self.price_past.iter_mut_with_days() { let window_starts = blocks.lookback.start_vec(days as usize); price_past.cents.height.compute_lookback( - starting_indexes.height, + starting_height, window_starts, price, exit, diff --git a/crates/brk_computer/src/market/moving_average/compute.rs b/crates/brk_computer/src/market/moving_average/compute.rs index 69eeaa10d..7f4c8c498 100644 --- a/crates/brk_computer/src/market/moving_average/compute.rs +++ b/crates/brk_computer/src/market/moving_average/compute.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; @@ -8,11 +8,12 @@ use crate::{blocks, prices}; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); let close = &prices.spot.cents.height; for (sma, period) in [ @@ -34,8 +35,8 @@ impl Vecs { (&mut self.sma._4y, 4 * 365), ] { let window_starts = blocks.lookback.start_vec(period); - sma.compute_all(prices, starting_indexes, exit, |v| { - v.compute_rolling_average(starting_indexes.height, window_starts, close, exit)?; + sma.compute_all(prices, &starting_lengths, exit, |v| { + v.compute_rolling_average(starting_lengths.height, window_starts, close, exit)?; Ok(()) })?; } @@ -59,8 +60,8 @@ impl Vecs { (&mut self.ema._4y, 4 * 365), ] { let window_starts = blocks.lookback.start_vec(period); - ema.compute_all(prices, starting_indexes, exit, |v| { - v.compute_rolling_ema(starting_indexes.height, window_starts, close, exit)?; + ema.compute_all(prices, &starting_lengths, exit, |v| { + v.compute_rolling_ema(starting_lengths.height, window_starts, close, exit)?; Ok(()) })?; } diff --git a/crates/brk_computer/src/market/range/compute.rs b/crates/brk_computer/src/market/range/compute.rs index 2d333dbb0..8fb436137 100644 --- a/crates/brk_computer/src/market/range/compute.rs +++ b/crates/brk_computer/src/market/range/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{BasisPoints16, Indexes, StoredF32}; +use brk_indexer::Indexer; +use brk_types::{BasisPoints16, StoredF32}; use vecdb::{Exit, ReadableVec, VecIndex}; use super::Vecs; @@ -8,11 +9,12 @@ use crate::{blocks, prices}; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, prices: &prices::Vecs, blocks: &blocks::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let price = &prices.spot.cents.height; for (min_vec, max_vec, starts) in [ @@ -37,24 +39,14 @@ impl Vecs { &blocks.lookback._1y.inner, ), ] { - min_vec.compute_rolling_min_from_starts( - starting_indexes.height, - starts, - price, - exit, - )?; - max_vec.compute_rolling_max_from_starts( - starting_indexes.height, - starts, - price, - exit, - )?; + min_vec.compute_rolling_min_from_starts(starting_height, starts, price, exit)?; + max_vec.compute_rolling_max_from_starts(starting_height, starts, price, exit)?; } // True range at block level: |price[h] - price[h-1]| let mut prev_price = None; self.true_range.height.compute_transform( - starting_indexes.height, + starting_height, price, |(h, current, ..)| { let prev = prev_price.unwrap_or_else(|| { @@ -74,14 +66,14 @@ impl Vecs { // 2w rolling sum of true range self.true_range_sum_2w.height.compute_rolling_sum( - starting_indexes.height, + starting_height, &blocks.lookback._2w, &self.true_range.height, exit, )?; self.choppiness_index_2w.bps.height.compute_transform4( - starting_indexes.height, + starting_height, &self.true_range_sum_2w.height, &self.max._2w.cents.height, &self.min._2w.cents.height, diff --git a/crates/brk_computer/src/market/returns/compute.rs b/crates/brk_computer/src/market/returns/compute.rs index 4f46c022d..6defdf674 100644 --- a/crates/brk_computer/src/market/returns/compute.rs +++ b/crates/brk_computer/src/market/returns/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{BasisPointsSigned32, Dollars, Indexes}; +use brk_indexer::Indexer; +use brk_types::{BasisPointsSigned32, Dollars}; use vecdb::Exit; use super::Vecs; @@ -10,12 +11,14 @@ use crate::{ impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, prices: &prices::Vecs, blocks: &blocks::Vecs, lookback: &lookback::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); + // Compute price returns at height level for ((returns, _), (lookback_price, _)) in self .periods @@ -23,7 +26,7 @@ impl Vecs { .zip(lookback.price_past.iter_with_days()) { returns.compute_binary::( - starting_indexes.height, + starting_lengths.height, &prices.spot.usd.height, &lookback_price.usd.height, exit, @@ -35,7 +38,7 @@ impl Vecs { for (cagr, returns, days) in self.cagr.zip_mut_with_period(&price_return_dca) { let years = days as f64 / 365.0; cagr.bps.height.compute_transform( - starting_indexes.height, + starting_lengths.height, &returns.bps.height, |(h, r, ..)| { let ratio = f64::from(r); @@ -49,7 +52,7 @@ impl Vecs { let _24h_price_return_ratio = &self.periods._24h.ratio.height; for sd in self.sd_24h.as_mut_array() { - sd.compute_all(blocks, starting_indexes, exit, _24h_price_return_ratio)?; + sd.compute_all(blocks, &starting_lengths, exit, _24h_price_return_ratio)?; } Ok(()) diff --git a/crates/brk_computer/src/market/technical/compute.rs b/crates/brk_computer/src/market/technical/compute.rs index a787bf6a4..1d25e7456 100644 --- a/crates/brk_computer/src/market/technical/compute.rs +++ b/crates/brk_computer/src/market/technical/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Dollars, Indexes}; +use brk_indexer::Indexer; +use brk_types::Dollars; use vecdb::Exit; use super::{ @@ -16,13 +17,14 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, + indexer: &Indexer, returns: &returns::Vecs, prices: &prices::Vecs, blocks: &blocks::Vecs, moving_average: &moving_average::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let daily_returns = &returns.periods._24h.ratio.height; for (rsi_chain, &m) in self .rsi @@ -32,11 +34,11 @@ impl Vecs { { rsi::compute( rsi_chain, + indexer, blocks, daily_returns, 14 * m, 3 * m, - starting_indexes, exit, )?; } @@ -49,12 +51,12 @@ impl Vecs { { macd::compute( macd_chain, + indexer, blocks, prices, 12 * m, 26 * m, 9 * m, - starting_indexes, exit, )?; } @@ -62,7 +64,7 @@ impl Vecs { self.pi_cycle .bps .compute_binary::( - starting_indexes.height, + starting_height, &moving_average.sma._111d.usd.height, &moving_average.sma._350d_x2.usd.height, exit, diff --git a/crates/brk_computer/src/market/technical/macd.rs b/crates/brk_computer/src/market/technical/macd.rs index 8af7b3fd5..522c8a2c8 100644 --- a/crates/brk_computer/src/market/technical/macd.rs +++ b/crates/brk_computer/src/market/technical/macd.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use super::MacdChain; @@ -8,14 +8,15 @@ use crate::{blocks, prices}; #[allow(clippy::too_many_arguments)] pub(super) fn compute( chain: &mut MacdChain, + indexer: &Indexer, blocks: &blocks::Vecs, prices: &prices::Vecs, fast_days: usize, slow_days: usize, signal_days: usize, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let close = &prices.spot.usd.height; let ws_fast = blocks.lookback.start_vec(fast_days); let ws_slow = blocks.lookback.start_vec(slow_days); @@ -24,29 +25,29 @@ pub(super) fn compute( chain .ema_fast .height - .compute_rolling_ema(starting_indexes.height, ws_fast, close, exit)?; + .compute_rolling_ema(starting_height, ws_fast, close, exit)?; chain .ema_slow .height - .compute_rolling_ema(starting_indexes.height, ws_slow, close, exit)?; + .compute_rolling_ema(starting_height, ws_slow, close, exit)?; chain.line.height.compute_subtract( - starting_indexes.height, + starting_height, &chain.ema_fast.height, &chain.ema_slow.height, exit, )?; chain.signal.height.compute_rolling_ema( - starting_indexes.height, + starting_height, ws_signal, &chain.line.height, exit, )?; chain.histogram.height.compute_subtract( - starting_indexes.height, + starting_height, &chain.line.height, &chain.signal.height, exit, diff --git a/crates/brk_computer/src/market/technical/rsi.rs b/crates/brk_computer/src/market/technical/rsi.rs index f17e25659..9ce45586e 100644 --- a/crates/brk_computer/src/market/technical/rsi.rs +++ b/crates/brk_computer/src/market/technical/rsi.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{BasisPoints16, Height, Indexes, StoredF32}; +use brk_indexer::Indexer; +use brk_types::{BasisPoints16, Height, StoredF32}; use vecdb::{Exit, ReadableVec}; use super::RsiChain; @@ -7,46 +8,47 @@ use crate::blocks; pub(super) fn compute( chain: &mut RsiChain, + indexer: &Indexer, blocks: &blocks::Vecs, returns_source: &impl ReadableVec, rma_days: usize, stoch_sma_days: usize, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let ws_rma = blocks.lookback.start_vec(rma_days); let ws_sma = blocks.lookback.start_vec(stoch_sma_days); chain.gains.height.compute_transform( - starting_indexes.height, + starting_height, returns_source, |(h, r, ..)| (h, StoredF32::from((*r).max(0.0))), exit, )?; chain.losses.height.compute_transform( - starting_indexes.height, + starting_height, returns_source, |(h, r, ..)| (h, StoredF32::from((-*r).max(0.0))), exit, )?; chain.average_gain.height.compute_rolling_rma( - starting_indexes.height, + starting_height, ws_rma, &chain.gains.height, exit, )?; chain.average_loss.height.compute_rolling_rma( - starting_indexes.height, + starting_height, ws_rma, &chain.losses.height, exit, )?; chain.rsi.bps.height.compute_transform2( - starting_indexes.height, + starting_height, &chain.average_gain.height, &chain.average_loss.height, |(h, g, l, ..)| { @@ -58,21 +60,21 @@ pub(super) fn compute( )?; chain.rsi_min.bps.height.compute_rolling_min_from_starts( - starting_indexes.height, + starting_height, ws_rma, &chain.rsi.bps.height, exit, )?; chain.rsi_max.bps.height.compute_rolling_max_from_starts( - starting_indexes.height, + starting_height, ws_rma, &chain.rsi.bps.height, exit, )?; chain.stoch_rsi.bps.height.compute_transform3( - starting_indexes.height, + starting_height, &chain.rsi.bps.height, &chain.rsi_min.bps.height, &chain.rsi_max.bps.height, @@ -89,14 +91,14 @@ pub(super) fn compute( )?; chain.stoch_rsi_k.bps.height.compute_rolling_average( - starting_indexes.height, + starting_height, ws_sma, &chain.stoch_rsi.bps.height, exit, )?; chain.stoch_rsi_d.bps.height.compute_rolling_average( - starting_indexes.height, + starting_height, ws_sma, &chain.stoch_rsi_k.bps.height, exit, diff --git a/crates/brk_computer/src/mining/compute.rs b/crates/brk_computer/src/mining/compute.rs index 71302e133..8dd36d4f6 100644 --- a/crates/brk_computer/src/mining/compute.rs +++ b/crates/brk_computer/src/mining/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -15,7 +14,6 @@ impl Vecs { blocks: &blocks::Vecs, transactions: &transactions::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; @@ -27,17 +25,16 @@ impl Vecs { &blocks.lookback, transactions, prices, - starting_indexes, exit, )?; self.hashrate.compute( + indexer, &blocks.count, &blocks.lookback, &blocks.difficulty, &self.rewards.coinbase.sum._24h.sats.height, &self.rewards.coinbase.sum._24h.usd.height, - starting_indexes, exit, )?; diff --git a/crates/brk_computer/src/mining/hashrate/compute.rs b/crates/brk_computer/src/mining/hashrate/compute.rs index d49f5bf03..457ec91fa 100644 --- a/crates/brk_computer/src/mining/hashrate/compute.rs +++ b/crates/brk_computer/src/mining/hashrate/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Dollars, Height, Indexes, Sats, StoredF32, StoredF64}; +use brk_indexer::Indexer; +use brk_types::{Dollars, Height, Sats, StoredF32, StoredF64}; use vecdb::{Exit, ReadableVec}; use super::Vecs; @@ -12,16 +13,18 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, + indexer: &Indexer, count_vecs: &blocks::CountVecs, lookback: &blocks::LookbackVecs, difficulty_vecs: &blocks::DifficultyVecs, coinbase_sats_24h_sum: &impl ReadableVec, coinbase_usd_24h_sum: &impl ReadableVec, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + self.rate.base.height.compute_transform2( - starting_indexes.height, + starting_height, &count_vecs.total.sum._24h.height, &difficulty_vecs.hashrate.height, |(i, block_count_sum, difficulty_as_hash, ..)| { @@ -43,24 +46,24 @@ impl Vecs { (&mut self.rate.sma._2m.height, &lookback._2m), (&mut self.rate.sma._1y.height, &lookback._1y.inner), ] { - sma.compute_rolling_average(starting_indexes.height, window, hash_rate, exit)?; + sma.compute_rolling_average(starting_height, window, hash_rate, exit)?; } self.rate.ath.height.compute_all_time_high( - starting_indexes.height, + starting_height, &self.rate.base.height, exit, )?; self.rate.drawdown.compute_drawdown( - starting_indexes.height, + starting_height, &self.rate.base.height, &self.rate.ath.height, exit, )?; self.price.ths.height.compute_transform2( - starting_indexes.height, + starting_height, coinbase_usd_24h_sum, &self.rate.base.height, |(i, coinbase_sum, hashrate, ..)| { @@ -76,7 +79,7 @@ impl Vecs { )?; self.value.ths.height.compute_transform2( - starting_indexes.height, + starting_height, coinbase_sats_24h_sum, &self.rate.base.height, |(i, coinbase_sum, hashrate, ..)| { @@ -95,13 +98,13 @@ impl Vecs { (&mut self.price.ths_min.height, &self.price.ths.height), (&mut self.value.ths_min.height, &self.value.ths.height), ] { - min_vec.compute_all_time_low_(starting_indexes.height, src_vec, exit, true)?; + min_vec.compute_all_time_low_(starting_height, src_vec, exit, true)?; } self.price .rebound .compute_binary::( - starting_indexes.height, + starting_height, &self.price.phs.height, &self.price.phs_min.height, exit, @@ -110,7 +113,7 @@ impl Vecs { self.value .rebound .compute_binary::( - starting_indexes.height, + starting_height, &self.value.phs.height, &self.value.phs_min.height, exit, diff --git a/crates/brk_computer/src/mining/rewards/compute.rs b/crates/brk_computer/src/mining/rewards/compute.rs index de70a297b..10cfc5623 100644 --- a/crates/brk_computer/src/mining/rewards/compute.rs +++ b/crates/brk_computer/src/mining/rewards/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{CheckedSub, Dollars, Halving, Indexes, Sats}; +use brk_types::{CheckedSub, Dollars, Halving, Sats}; use vecdb::{Exit, ReadableVec, VecIndex}; use super::Vecs; @@ -19,67 +19,62 @@ impl Vecs { lookback: &blocks::LookbackVecs, transactions: &transactions::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + // coinbase and fees are independent — parallelize let window_starts = lookback.window_starts(); let (r_coinbase, r_fees) = rayon::join( || { - self.coinbase - .compute(starting_indexes.height, prices, exit, |vec| { - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); - let mut count_cursor = indexes.tx_index.output_count.cursor(); + self.coinbase.compute(starting_height, prices, exit, |vec| { + let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut count_cursor = indexes.tx_index.output_count.cursor(); - vec.compute_transform( - starting_indexes.height, - &indexer.vecs.transactions.first_tx_index, - |(height, tx_index, ..)| { - let ti = tx_index.to_usize(); + vec.compute_transform( + starting_height, + &indexer.vecs.transactions.first_tx_index, + |(height, tx_index, ..)| { + let ti = tx_index.to_usize(); - txout_cursor.advance(ti - txout_cursor.position()); - let first_txout_index = txout_cursor.next().unwrap().to_usize(); + txout_cursor.advance(ti - txout_cursor.position()); + let first_txout_index = txout_cursor.next().unwrap().to_usize(); - count_cursor.advance(ti - count_cursor.position()); - let output_count: usize = count_cursor.next().unwrap().into(); + count_cursor.advance(ti - count_cursor.position()); + let output_count: usize = count_cursor.next().unwrap().into(); - let sats = indexer.vecs.outputs.value.fold_range_at( - first_txout_index, - first_txout_index + output_count, - Sats::ZERO, - |acc, v| acc + v, - ); - (height, sats) - }, - exit, - )?; - Ok(()) - }) + let sats = indexer.vecs.outputs.value.fold_range_at( + first_txout_index, + first_txout_index + output_count, + Sats::ZERO, + |acc, v| acc + v, + ); + (height, sats) + }, + exit, + )?; + Ok(()) + }) }, || { - self.fees.compute( - starting_indexes.height, - &window_starts, - prices, - exit, - |vec| { + self.fees + .compute(starting_height, &window_starts, prices, exit, |vec| { vec.compute_sum_from_indexes( - starting_indexes.height, + starting_height, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, &transactions.fees.fee.tx_index, exit, )?; Ok(()) - }, - ) + }) }, ); r_coinbase?; r_fees?; self.subsidy.block.sats.compute_transform2( - starting_indexes.height, + starting_height, &self.coinbase.block.sats, &self.fees.block.sats, |(height, coinbase, fees, ..)| { @@ -92,18 +87,17 @@ impl Vecs { }, exit, )?; - self.subsidy - .compute_rest(starting_indexes.height, prices, exit)?; + self.subsidy.compute_rest(starting_height, prices, exit)?; self.output_volume.compute_subtract( - starting_indexes.height, + starting_height, &transactions.volume.transfer_volume.block.sats, &self.fees.block.sats, exit, )?; self.unclaimed.block.sats.compute_transform( - starting_indexes.height, + starting_height, &self.subsidy.block.sats, |(height, subsidy, ..)| { let halving = Halving::from(height); @@ -112,12 +106,11 @@ impl Vecs { }, exit, )?; - self.unclaimed - .compute(prices, starting_indexes.height, exit)?; + self.unclaimed.compute(prices, starting_height, exit)?; self.fee_dominance .compute_binary::( - starting_indexes.height, + starting_height, &self.fees.cumulative.sats.height, &self.coinbase.cumulative.sats.height, self.fees.sum.as_array().map(|w| &w.sats.height), @@ -127,7 +120,7 @@ impl Vecs { self.fee_to_subsidy_ratio .compute_binary::( - starting_indexes.height, + starting_height, self.coinbase.sum.as_array().map(|w| &w.usd.height), self.fees.sum.as_array().map(|w| &w.usd.height), exit, diff --git a/crates/brk_computer/src/outputs/by_type/compute.rs b/crates/brk_computer/src/outputs/by_type/compute.rs index 744d0312a..55692336c 100644 --- a/crates/brk_computer/src/outputs/by_type/compute.rs +++ b/crates/brk_computer/src/outputs/by_type/compute.rs @@ -1,30 +1,27 @@ use brk_error::{OptionData, Result}; use brk_indexer::Indexer; -use brk_types::{Indexes, OutputType, StoredU64}; +use brk_types::{OutputType, StoredU64}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::{Vecs, WithOutputTypes}; use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks}; impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); + let dep_version = indexer.vecs.outputs.output_type.version() + indexer.vecs.transactions.first_tx_index.version() + indexer.vecs.transactions.first_txout_index.version() + indexer.vecs.transactions.txid.version(); self.output_count - .validate_and_truncate(dep_version, starting_indexes.height)?; + .validate_and_truncate(dep_version, starting_lengths.height)?; self.spendable_output_count .block - .validate_and_truncate(dep_version, starting_indexes.height)?; + .validate_and_truncate(dep_version, starting_lengths.height)?; self.tx_count - .validate_and_truncate(dep_version, starting_indexes.height)?; + .validate_and_truncate(dep_version, starting_lengths.height)?; let skip = self .output_count @@ -98,17 +95,17 @@ impl Vecs { } self.output_count - .compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; self.spendable_output_count - .compute_rest(starting_indexes.height, exit)?; - self.tx_count.compute_rest(starting_indexes.height, exit)?; + .compute_rest(starting_lengths.height, exit)?; + self.tx_count.compute_rest(starting_lengths.height, exit)?; } for (otype, source) in self.output_count.by_type.iter_typed() { self.output_share.get_mut(otype).compute_count_ratio( source, &self.output_count.all, - starting_indexes.height, + starting_lengths.height, exit, )?; } @@ -117,7 +114,7 @@ impl Vecs { self.tx_share.get_mut(otype).compute_count_ratio( source, &self.tx_count.all, - starting_indexes.height, + starting_lengths.height, exit, )?; } diff --git a/crates/brk_computer/src/outputs/compute.rs b/crates/brk_computer/src/outputs/compute.rs index 7461c06a7..47d429a74 100644 --- a/crates/brk_computer/src/outputs/compute.rs +++ b/crates/brk_computer/src/outputs/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -15,27 +14,24 @@ impl Vecs { inputs: &inputs::Vecs, blocks: &blocks::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; - self.count - .compute(indexer, indexes, blocks, starting_indexes, exit)?; - self.per_sec.compute(&self.count, starting_indexes, exit)?; - self.value - .compute(indexer, prices, starting_indexes, exit)?; - self.by_type.compute(indexer, starting_indexes, exit)?; + let starting_lengths = indexer.safe_lengths(); + + self.count.compute(indexer, indexes, blocks, exit)?; + self.per_sec.compute(&self.count, &starting_lengths, exit)?; + self.value.compute(indexer, prices, exit)?; + self.by_type.compute(indexer, exit)?; self.unspent.compute( &self.count, &inputs.count, &self.by_type, - starting_indexes, + &starting_lengths, exit, )?; - let lock = self - .spent - .compute(indexer, inputs, starting_indexes, exit)?; + let lock = self.spent.compute(indexer, inputs, exit)?; self.db.run_bg(move |db| { let _lock = lock; db.compact_deferred_default() diff --git a/crates/brk_computer/src/outputs/count/compute.rs b/crates/brk_computer/src/outputs/count/compute.rs index 1775a3b4c..35adbf9d5 100644 --- a/crates/brk_computer/src/outputs/count/compute.rs +++ b/crates/brk_computer/src/outputs/count/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -12,12 +11,12 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, blocks: &blocks::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; let window_starts = blocks.lookback.window_starts(); self.total.compute( - starting_indexes.height, + starting_height, &indexes.tx_index.output_count, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, diff --git a/crates/brk_computer/src/outputs/per_sec/compute.rs b/crates/brk_computer/src/outputs/per_sec/compute.rs index f1753ae51..7da14b644 100644 --- a/crates/brk_computer/src/outputs/per_sec/compute.rs +++ b/crates/brk_computer/src/outputs/per_sec/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, StoredF32}; +use brk_indexer::Lengths; +use brk_types::StoredF32; use vecdb::Exit; use super::Vecs; @@ -9,10 +10,10 @@ impl Vecs { pub(crate) fn compute( &mut self, count: &CountVecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { - let h = starting_indexes.height; + let h = starting_lengths.height; let sums = count.total.rolling.sum.0.as_array(); let per_sec = self.0.as_mut_array(); for (i, &secs) in Windows::<()>::SECS.iter().enumerate() { diff --git a/crates/brk_computer/src/outputs/spent/compute.rs b/crates/brk_computer/src/outputs/spent/compute.rs index 44ad24950..82d26b9e8 100644 --- a/crates/brk_computer/src/outputs/spent/compute.rs +++ b/crates/brk_computer/src/outputs/spent/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Height, Indexes, TxInIndex, TxOutIndex}; +use brk_types::{Height, TxInIndex, TxOutIndex}; use tracing::info; use vecdb::{AnyStoredVec, AnyVec, Exit, ExitGuard, ReadableVec, Stamp, VecIndex, WritableVec}; @@ -14,9 +14,10 @@ impl Vecs { &mut self, indexer: &Indexer, inputs: &inputs::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result { + let starting_lengths = indexer.safe_lengths(); + let target_height = indexer.vecs.blocks.blockhash.len(); if target_height == 0 { return Ok(exit.lock()); @@ -25,9 +26,9 @@ impl Vecs { // Find min_height from current vec length let current_txout_index = self.txin_index.len(); - let min_txout_index = current_txout_index.min(starting_indexes.txout_index.to_usize()); + let min_txout_index = current_txout_index.min(starting_lengths.txout_index.to_usize()); - let starting_stamp = Stamp::from(starting_indexes.height); + let starting_stamp = Stamp::from(starting_lengths.height); let _ = self.txin_index.rollback_before(starting_stamp); self.txin_index @@ -39,11 +40,11 @@ impl Vecs { let first_txout_index_vec = &indexer.vecs.outputs.first_txout_index; let min_height = if min_txout_index == 0 { Height::ZERO - } else if min_txout_index >= starting_indexes.txout_index.to_usize() { - starting_indexes.height + } else if min_txout_index >= starting_lengths.txout_index.to_usize() { + starting_lengths.height } else { let mut lo = 0usize; - let mut hi = starting_indexes.height.to_usize() + 1; + let mut hi = starting_lengths.height.to_usize() + 1; while lo < hi { let mid = lo + (hi - lo) / 2; if first_txout_index_vec @@ -72,10 +73,10 @@ impl Vecs { // Validate: computed height must not exceed starting height assert!( - min_height <= starting_indexes.height, - "txouts min_height ({}) exceeds starting_indexes.height ({})", + min_height <= starting_lengths.height, + "txouts min_height ({}) exceeds starting_lengths.height ({})", min_height, - starting_indexes.height + starting_lengths.height ); let mut pairs: Vec<(TxOutIndex, TxInIndex)> = Vec::new(); diff --git a/crates/brk_computer/src/outputs/unspent/compute.rs b/crates/brk_computer/src/outputs/unspent/compute.rs index be677e2bf..c28b46f68 100644 --- a/crates/brk_computer/src/outputs/unspent/compute.rs +++ b/crates/brk_computer/src/outputs/unspent/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Height, Indexes, StoredU64}; +use brk_indexer::Lengths; +use brk_types::{Height, StoredU64}; use vecdb::Exit; use super::Vecs; @@ -15,14 +16,14 @@ impl Vecs { count: &CountVecs, inputs_count: &inputs::CountVecs, by_type: &ByTypeVecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, exit: &Exit, ) -> Result<()> { let op_return: &PerBlockCumulativeRolling = &by_type.output_count.by_type.unspendable.op_return; self.count.height.compute_transform3( - starting_indexes.height, + starting_lengths.height, &count.total.cumulative.height, &inputs_count.cumulative.height, &op_return.cumulative.height, diff --git a/crates/brk_computer/src/outputs/value/compute.rs b/crates/brk_computer/src/outputs/value/compute.rs index 22d80324a..d601b5ace 100644 --- a/crates/brk_computer/src/outputs/value/compute.rs +++ b/crates/brk_computer/src/outputs/value/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Height, Indexes, OutputType, Sats, TxOutIndex}; +use brk_types::{Height, OutputType, Sats, TxOutIndex}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::Vecs; @@ -11,11 +11,11 @@ impl Vecs { &mut self, indexer: &Indexer, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); self.op_return - .compute_with(starting_indexes.height, prices, exit, |height_vec| { + .compute_with(starting_lengths.height, prices, exit, |height_vec| { // Validate computed versions against dependencies let dep_version = indexer.vecs.outputs.first_txout_index.version() + indexer.vecs.outputs.output_type.version() @@ -32,7 +32,7 @@ impl Vecs { // Find starting height for this vec let current_len = height_vec.len(); let starting_height = - Height::from(current_len.min(starting_indexes.height.to_usize())); + Height::from(current_len.min(starting_lengths.height.to_usize())); if starting_height > target_height { return Ok(()); diff --git a/crates/brk_computer/src/pools/major.rs b/crates/brk_computer/src/pools/major.rs index bc12d2b62..d50c53e2a 100644 --- a/crates/brk_computer/src/pools/major.rs +++ b/crates/brk_computer/src/pools/major.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{BasisPoints16, Height, Indexes, PoolSlug, StoredU64}; +use brk_types::{BasisPoints16, Height, PoolSlug, StoredU64}; use derive_more::{Deref, DerefMut}; use vecdb::{BinaryTransform, Database, Exit, ReadableVec, Rw, StorageMode, Version}; @@ -59,14 +60,16 @@ impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, pool: &impl ReadableVec, blocks: &blocks::Vecs, prices: &prices::Vecs, mining: &mining::Vecs, exit: &Exit, ) -> Result<()> { - self.base.compute(starting_indexes, pool, blocks, exit)?; + let starting_height = indexer.safe_lengths().height; + + self.base.compute(indexer, pool, blocks, exit)?; for (dom, (mined, total)) in self.dominance_rolling.as_mut_array().into_iter().zip( self.base @@ -77,23 +80,22 @@ impl Vecs { .zip(blocks.count.total.sum.as_array()), ) { dom.compute_binary::( - starting_indexes.height, + starting_height, &mined.height, &total.height, exit, )?; } - self.rewards - .compute(starting_indexes.height, prices, exit, |vec| { - Ok(vec.compute_transform2( - starting_indexes.height, - &self.base.blocks_mined.block, - &mining.rewards.coinbase.block.sats, - |(h, mask, val, ..)| (h, MaskSats::apply(mask, val)), - exit, - )?) - })?; + self.rewards.compute(starting_height, prices, exit, |vec| { + Ok(vec.compute_transform2( + starting_height, + &self.base.blocks_mined.block, + &mining.rewards.coinbase.block.sats, + |(h, mask, val, ..)| (h, MaskSats::apply(mask, val)), + exit, + )?) + })?; Ok(()) } diff --git a/crates/brk_computer/src/pools/minor.rs b/crates/brk_computer/src/pools/minor.rs index 5028c05b2..531b6ad49 100644 --- a/crates/brk_computer/src/pools/minor.rs +++ b/crates/brk_computer/src/pools/minor.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{BasisPoints16, Height, Indexes, PoolSlug, StoredU32, StoredU64}; +use brk_types::{BasisPoints16, Height, PoolSlug, StoredU32, StoredU64}; use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode, Version}; use crate::{ @@ -46,34 +47,35 @@ impl Vecs { pub(crate) fn compute( &mut self, - starting_indexes: &Indexes, + indexer: &Indexer, pool: &impl ReadableVec, blocks: &blocks::Vecs, exit: &Exit, ) -> Result<()> { - self.blocks_mined - .compute(starting_indexes.height, exit, |vec| { - vec.compute_transform( - starting_indexes.height, - pool, - |(h, id, ..)| { - ( - h, - if id == self.slug { - StoredU32::ONE - } else { - StoredU32::ZERO - }, - ) - }, - exit, - )?; - Ok(()) - })?; + let starting_height = indexer.safe_lengths().height; + + self.blocks_mined.compute(starting_height, exit, |vec| { + vec.compute_transform( + starting_height, + pool, + |(h, id, ..)| { + ( + h, + if id == self.slug { + StoredU32::ONE + } else { + StoredU32::ZERO + }, + ) + }, + exit, + )?; + Ok(()) + })?; self.dominance .compute_binary::( - starting_indexes.height, + starting_height, &self.blocks_mined.cumulative.height, &blocks.count.total.cumulative.height, exit, diff --git a/crates/brk_computer/src/pools/mod.rs b/crates/brk_computer/src/pools/mod.rs index 87546e2e0..be86e8eca 100644 --- a/crates/brk_computer/src/pools/mod.rs +++ b/crates/brk_computer/src/pools/mod.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, path::Path}; use brk_error::Result; use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{Addr, AddrBytes, Height, Indexes, OutputType, PoolSlug, Pools, TxOutIndex, pools}; +use brk_types::{Addr, AddrBytes, Height, OutputType, PoolSlug, Pools, TxOutIndex, pools}; use rayon::prelude::*; use vecdb::{ AnyStoredVec, AnyVec, BytesVec, Database, Exit, ImportableVec, ReadableVec, Rw, StorageMode, @@ -92,20 +92,19 @@ impl Vecs { blocks: &blocks::Vecs, prices: &prices::Vecs, mining: &mining::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; - self.compute_pool(indexer, indexes, starting_indexes, exit)?; + self.compute_pool(indexer, indexes, exit)?; self.major.par_iter_mut().try_for_each(|(_, vecs)| { - vecs.compute(starting_indexes, &self.pool, blocks, prices, mining, exit) + vecs.compute(indexer, &self.pool, blocks, prices, mining, exit) })?; self.minor .par_iter_mut() - .try_for_each(|(_, vecs)| vecs.compute(starting_indexes, &self.pool, blocks, exit))?; + .try_for_each(|(_, vecs)| vecs.compute(indexer, &self.pool, blocks, exit))?; let exit = exit.clone(); self.db.run_bg(move |db| { @@ -119,9 +118,10 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let dep_version = indexer.vecs.blocks.coinbase_tag.version(); let pool_vec_version = self.pool.header().vec_version(); let pool_computed = self.pool.header().computed_version(); @@ -148,7 +148,7 @@ impl Vecs { let unknown = self.pools.get_unknown(); - let min = starting_indexes.height.to_usize().min(self.pool.len()); + let min = starting_height.to_usize().min(self.pool.len()); // Cursors avoid per-height PcoVec page decompression. // Heights are sequential, tx_index values derived from them are monotonically diff --git a/crates/brk_computer/src/prices/compute.rs b/crates/brk_computer/src/prices/compute.rs index d91ae988f..01ca8672c 100644 --- a/crates/brk_computer/src/prices/compute.rs +++ b/crates/brk_computer/src/prices/compute.rs @@ -1,9 +1,9 @@ use std::ops::Range; -use brk_error::Result; -use brk_indexer::Indexer; +use brk_error::{Error, Result}; +use brk_indexer::{Indexer, Lengths}; use brk_oracle::{Config, NUM_BINS, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin}; -use brk_types::{Cents, Indexes, OutputType, Sats, TxIndex, TxOutIndex}; +use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex}; use tracing::info; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, StorageMode, VecIndex, WritableVec}; @@ -15,32 +15,33 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; - self.compute_prices(indexer, starting_indexes, exit)?; + let starting_lengths = indexer.safe_lengths(); + + self.compute_prices(indexer, exit)?; self.split.open.cents.compute_first( - starting_indexes, + &starting_lengths, &self.spot.cents.height, indexes, exit, )?; self.split.high.cents.compute_max( - starting_indexes, + &starting_lengths, &self.spot.cents.height, indexes, exit, )?; self.split.low.cents.compute_min( - starting_indexes, + &starting_lengths, &self.spot.cents.height, indexes, exit, )?; self.ohlc.cents.compute_from_split( - starting_indexes, + &starting_lengths, indexes, &self.split.open.cents, &self.split.high.cents, @@ -57,12 +58,9 @@ impl Vecs { Ok(()) } - fn compute_prices( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + fn compute_prices(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let source_version = indexer.vecs.outputs.value.version() + indexer.vecs.outputs.output_type.version(); self.spot @@ -77,13 +75,8 @@ impl Vecs { return Ok(()); } - // Reorg: truncate to starting_indexes - let truncate_to = self - .spot - .cents - .height - .len() - .min(starting_indexes.height.to_usize()); + // Reorg: truncate to starting_lengths + let truncate_to = self.spot.cents.height.len().min(starting_height.to_usize()); self.spot .cents .height @@ -157,17 +150,50 @@ impl Vecs { } /// Feed a range of blocks from the indexer into an Oracle (skipping coinbase), - /// returning per-block ref_bin values. + /// returning per-block ref_bin values. Uncapped: derives boundaries from + /// raw indexer vec lengths. Use during compute, when the indexer is + /// quiescent and `safe_lengths` is still pinned at the pre-pass value. fn feed_blocks( oracle: &mut Oracle, indexer: &Indexer, range: Range, ) -> Vec { - let total_txs = indexer.vecs.transactions.txid.len(); - let total_outputs = indexer.vecs.outputs.value.len(); + Self::feed_blocks_inner(oracle, indexer, range, None) + } + + /// Capped variant: derives boundaries from `cap` instead of raw vec + /// lengths, so concurrent writer pushes past `cap` are invisible. + /// Reader paths (live_oracle) use this with the current `safe_lengths`. + fn feed_blocks_capped( + oracle: &mut Oracle, + indexer: &Indexer, + range: Range, + cap: &Lengths, + ) -> Vec { + Self::feed_blocks_inner(oracle, indexer, range, Some(cap)) + } + + fn feed_blocks_inner( + oracle: &mut Oracle, + indexer: &Indexer, + range: Range, + cap: Option<&Lengths>, + ) -> Vec { + let (total_txs, total_outputs, height_len) = match cap { + Some(c) => ( + c.tx_index.to_usize(), + c.txout_index.to_usize(), + c.height.to_usize(), + ), + None => ( + indexer.vecs.transactions.txid.len(), + indexer.vecs.outputs.value.len(), + indexer.vecs.transactions.first_tx_index.len(), + ), + }; // Pre-collect height-indexed data for the range (plus one extra for next-block lookups) - let collect_end = (range.end + 1).min(indexer.vecs.transactions.first_tx_index.len()); + let collect_end = (range.end + 1).min(height_len); let first_tx_indexes: Vec = indexer .vecs .transactions @@ -243,17 +269,34 @@ impl Vecs { /// window_size blocks already processed. Ready for additional blocks (e.g. mempool). pub fn live_oracle(&self, indexer: &Indexer) -> Result { let config = Config::default(); - let height = indexer.vecs.blocks.timestamp.len(); + let safe_lengths = indexer.safe_lengths(); + let height = safe_lengths.height.to_usize(); + let last_idx = self + .spot + .cents + .height + .len() + .checked_sub(1) + .ok_or(Error::NotFound( + "oracle prices not yet computed".to_string(), + ))?; let last_cents = self .spot .cents .height - .collect_one_at(self.spot.cents.height.len() - 1) - .unwrap(); + .collect_one_at(last_idx) + .ok_or(Error::NotFound( + "oracle prices not yet computed".to_string(), + ))?; let seed_bin = cents_to_bin(last_cents.inner() as f64); let window_size = config.window_size; let oracle = Oracle::from_checkpoint(seed_bin, config, |o| { - Vecs::feed_blocks(o, indexer, height.saturating_sub(window_size)..height); + Vecs::feed_blocks_capped( + o, + indexer, + height.saturating_sub(window_size)..height, + &safe_lengths, + ); }); Ok(oracle) diff --git a/crates/brk_computer/src/prices/ohlcs.rs b/crates/brk_computer/src/prices/ohlcs.rs index 357267c1f..57c9fc819 100644 --- a/crates/brk_computer/src/prices/ohlcs.rs +++ b/crates/brk_computer/src/prices/ohlcs.rs @@ -1,8 +1,9 @@ use brk_error::Result; +use brk_indexer::Lengths; use brk_traversable::Traversable; use brk_types::{ - Cents, Close, Day1, Day3, Epoch, Halving, High, Hour1, Hour4, Hour12, Indexes, Low, Minute10, - Minute30, Month1, Month3, Month6, OHLCCents, Open, Version, Week1, Year1, Year10, + Cents, Close, Day1, Day3, Epoch, Halving, High, Hour1, Hour4, Hour12, Low, Minute10, Minute30, + Month1, Month3, Month6, OHLCCents, Open, Version, Week1, Year1, Year10, }; use derive_more::{Deref, DerefMut}; use schemars::JsonSchema; @@ -81,7 +82,7 @@ impl OhlcVecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute_from_split( &mut self, - starting_indexes: &Indexes, + starting_lengths: &Lengths, indexes: &indexes::Vecs, open: &EagerIndexes, high: &EagerIndexes, @@ -89,7 +90,7 @@ impl OhlcVecs { close: &Resolutions, exit: &Exit, ) -> Result<()> { - let prev_height = starting_indexes.height.decremented().unwrap_or_default(); + let prev_height = starting_lengths.height.decremented().unwrap_or_default(); macro_rules! period { ($field:ident) => { diff --git a/crates/brk_computer/src/supply/burned/compute.rs b/crates/brk_computer/src/supply/burned/compute.rs index ab9c36032..b544ff142 100644 --- a/crates/brk_computer/src/supply/burned/compute.rs +++ b/crates/brk_computer/src/supply/burned/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, Sats}; +use brk_indexer::Indexer; +use brk_types::Sats; use vecdb::{Exit, VecIndex}; use super::Vecs; @@ -8,16 +9,18 @@ use crate::{mining, outputs, prices}; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, outputs: &outputs::Vecs, mining: &mining::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + self.total - .compute_with(starting_indexes.height, prices, exit, |sats| { + .compute_with(starting_height, prices, exit, |sats| { Ok(sats.compute_transform2( - starting_indexes.height, + starting_height, &outputs.value.op_return.block.sats, &mining.rewards.unclaimed.block.sats, |(h, op_return, unclaimed, ..)| { diff --git a/crates/brk_computer/src/supply/compute.rs b/crates/brk_computer/src/supply/compute.rs index b115d7511..ca717a4c0 100644 --- a/crates/brk_computer/src/supply/compute.rs +++ b/crates/brk_computer/src/supply/compute.rs @@ -1,5 +1,6 @@ use brk_error::Result; -use brk_types::{Indexes, Sats}; +use brk_indexer::Indexer; +use brk_types::Sats; use vecdb::Exit; /// Initial block subsidy (50 BTC) in sats, as f64 for floating-point comparisons. @@ -12,20 +13,22 @@ impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, + indexer: &Indexer, outputs: &outputs::Vecs, blocks: &blocks::Vecs, mining: &mining::Vecs, transactions: &transactions::Vecs, prices: &prices::Vecs, distribution: &distribution::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; + let starting_height = indexer.safe_lengths().height; + // 1. Compute burned/unspendable supply self.burned - .compute(outputs, mining, prices, starting_indexes, exit)?; + .compute(indexer, outputs, mining, prices, exit)?; // 2. Compute inflation rate: (supply[h] / supply[1y_ago]) - 1 // Skip when lookback supply <= first block (50 BTC = 5B sats), @@ -35,7 +38,7 @@ impl Vecs { .bps .height .compute_rolling_from_window_starts( - starting_indexes.height, + starting_height, &blocks.lookback._1y, &circulating_supply.height, exit, @@ -50,7 +53,7 @@ impl Vecs { // 3. Compute velocity at height level self.velocity - .compute(blocks, transactions, distribution, starting_indexes, exit)?; + .compute(indexer, blocks, transactions, distribution, exit)?; // 4. market_cap_rate - realized_cap_rate per window let all_realized = &distribution.utxo_cohorts.all.metrics.realized; @@ -66,7 +69,7 @@ impl Vecs { for i in 0..4 { diff_arr[i].height.compute_subtract( - starting_indexes.height, + starting_height, &mcr_arr[i].bps.height, rcr_rates[i], exit, diff --git a/crates/brk_computer/src/supply/velocity/compute.rs b/crates/brk_computer/src/supply/velocity/compute.rs index daf3dd4ad..567f5cfc7 100644 --- a/crates/brk_computer/src/supply/velocity/compute.rs +++ b/crates/brk_computer/src/supply/velocity/compute.rs @@ -1,5 +1,5 @@ use brk_error::Result; -use brk_types::Indexes; +use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; @@ -8,18 +8,20 @@ use crate::{blocks, distribution, transactions}; impl Vecs { pub(crate) fn compute( &mut self, + indexer: &Indexer, blocks: &blocks::Vecs, transactions: &transactions::Vecs, distribution: &distribution::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + // velocity = rolling_1y_sum(volume) / circulating_supply let circulating_supply = &distribution.utxo_cohorts.all.metrics.supply.total; // Native velocity at height level self.native.height.compute_rolling_ratio( - starting_indexes.height, + starting_height, &blocks.lookback._1y, &transactions.volume.transfer_volume.block.sats, &circulating_supply.sats.height, @@ -28,7 +30,7 @@ impl Vecs { // Fiat velocity at height level self.fiat.height.compute_rolling_ratio( - starting_indexes.height, + starting_height, &blocks.lookback._1y, &transactions.volume.transfer_volume.block.usd, &circulating_supply.usd.height, diff --git a/crates/brk_computer/src/transactions/compute.rs b/crates/brk_computer/src/transactions/compute.rs index 96e1064d2..1656257fe 100644 --- a/crates/brk_computer/src/transactions/compute.rs +++ b/crates/brk_computer/src/transactions/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -15,20 +14,16 @@ impl Vecs { blocks: &blocks::Vecs, inputs: &inputs::Vecs, prices: &prices::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { self.db.sync_bg_tasks()?; let (r1, (r2, r3)) = rayon::join( - || { - self.count - .compute(indexer, &blocks.lookback, starting_indexes, exit) - }, + || self.count.compute(indexer, &blocks.lookback, exit), || { rayon::join( - || self.versions.compute(indexer, starting_indexes, exit), - || self.size.compute(indexer, indexes, starting_indexes, exit), + || self.versions.compute(indexer, exit), + || self.size.compute(indexer, indexes, exit), ) }, ); @@ -36,24 +31,11 @@ impl Vecs { r2?; r3?; - self.fees.compute( - indexer, - indexes, - &inputs.spent, - &self.size, - starting_indexes, - exit, - )?; + self.fees + .compute(indexer, indexes, &inputs.spent, &self.size, exit)?; - self.volume.compute( - indexer, - indexes, - prices, - &self.count, - &self.fees, - starting_indexes, - exit, - )?; + self.volume + .compute(indexer, indexes, prices, &self.count, &self.fees, exit)?; let exit = exit.clone(); self.db.run_bg(move |db| { diff --git a/crates/brk_computer/src/transactions/count/compute.rs b/crates/brk_computer/src/transactions/count/compute.rs index 79d706252..491d26a97 100644 --- a/crates/brk_computer/src/transactions/count/compute.rs +++ b/crates/brk_computer/src/transactions/count/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -11,14 +10,15 @@ impl Vecs { &mut self, indexer: &Indexer, lookback: &blocks::LookbackVecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let window_starts = lookback.window_starts(); self.total - .compute(starting_indexes.height, &window_starts, exit, |height| { + .compute(starting_height, &window_starts, exit, |height| { Ok(height.compute_count_from_indexes( - starting_indexes.height, + starting_height, &indexer.vecs.transactions.first_tx_index, &indexer.vecs.transactions.txid, exit, diff --git a/crates/brk_computer/src/transactions/fees/compute.rs b/crates/brk_computer/src/transactions/fees/compute.rs index 02e1460f7..7c15f23f0 100644 --- a/crates/brk_computer/src/transactions/fees/compute.rs +++ b/crates/brk_computer/src/transactions/fees/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{FeeRate, Indexes, OutPoint, Sats, TxInIndex, TxIndex, VSize}; +use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely}; use super::super::size; @@ -15,37 +15,38 @@ impl Vecs { indexes: &indexes::Vecs, spent: &inputs::SpentVecs, size_vecs: &size::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); + self.input_value.compute_sum_from_indexes( - starting_indexes.tx_index, + starting_lengths.tx_index, &indexer.vecs.transactions.first_txin_index, &indexes.tx_index.input_count, &spent.value, exit, )?; self.output_value.compute_sum_from_indexes( - starting_indexes.tx_index, + starting_lengths.tx_index, &indexer.vecs.transactions.first_txout_index, &indexes.tx_index.output_count, &indexer.vecs.outputs.value, exit, )?; - self.compute_fees(indexer, indexes, size_vecs, starting_indexes, exit)?; + self.compute_fees(indexer, indexes, size_vecs, exit)?; let vsize_source = &size_vecs.vsize.tx_index; let (r1, r2) = rayon::join( || { self.fee - .derive_from_with_skip(indexer, indexes, starting_indexes, exit, 1) + .derive_from_with_skip(indexer, indexes, &starting_lengths, exit, 1) }, || { self.effective_fee_rate.derive_from_with_skip_weighted( indexer, indexes, - starting_indexes, + &starting_lengths, vsize_source, exit, 1, @@ -63,9 +64,10 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, size_vecs: &size::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); + let dep_version = self.input_value.version() + self.output_value.version() + size_vecs.vsize.tx_index.version(); @@ -90,7 +92,7 @@ impl Vecs { .len() .min(self.fee_rate.len()) .min(self.effective_fee_rate.tx_index.len()) - .min(starting_indexes.tx_index.to_usize()); + .min(starting_lengths.tx_index.to_usize()); if min >= target { return Ok(()); @@ -98,12 +100,12 @@ impl Vecs { self.fee .tx_index - .truncate_if_needed(starting_indexes.tx_index)?; + .truncate_if_needed(starting_lengths.tx_index)?; self.fee_rate - .truncate_if_needed(starting_indexes.tx_index)?; + .truncate_if_needed(starting_lengths.tx_index)?; self.effective_fee_rate .tx_index - .truncate_if_needed(starting_indexes.tx_index)?; + .truncate_if_needed(starting_lengths.tx_index)?; let start_tx = self.fee.tx_index.len(); let max_height = indexer.vecs.transactions.first_tx_index.len(); diff --git a/crates/brk_computer/src/transactions/size/compute.rs b/crates/brk_computer/src/transactions/size/compute.rs index 1e06930ba..a4bc55b45 100644 --- a/crates/brk_computer/src/transactions/size/compute.rs +++ b/crates/brk_computer/src/transactions/size/compute.rs @@ -1,6 +1,5 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::Indexes; use vecdb::Exit; use super::Vecs; @@ -11,11 +10,12 @@ impl Vecs { &mut self, indexer: &Indexer, indexes: &indexes::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_lengths = indexer.safe_lengths(); + self.vsize - .derive_from(indexer, indexes, starting_indexes, exit)?; + .derive_from(indexer, indexes, &starting_lengths, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/transactions/versions/compute.rs b/crates/brk_computer/src/transactions/versions/compute.rs index 223c6e8f6..885e879c5 100644 --- a/crates/brk_computer/src/transactions/versions/compute.rs +++ b/crates/brk_computer/src/transactions/versions/compute.rs @@ -1,23 +1,20 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Indexes, StoredU64, TxVersion}; +use brk_types::{StoredU64, TxVersion}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::Vecs; impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let dep_version = indexer.vecs.transactions.tx_version.version() + indexer.vecs.transactions.first_tx_index.version() + indexer.vecs.transactions.txid.version(); for vec in [&mut self.v1.block, &mut self.v2.block, &mut self.v3.block] { - vec.validate_and_truncate(dep_version, starting_indexes.height)?; + vec.validate_and_truncate(dep_version, starting_height)?; } let skip = self @@ -84,9 +81,9 @@ impl Vecs { } // Derive cumulative + sums from base - self.v1.compute_rest(starting_indexes.height, exit)?; - self.v2.compute_rest(starting_indexes.height, exit)?; - self.v3.compute_rest(starting_indexes.height, exit)?; + self.v1.compute_rest(starting_height, exit)?; + self.v2.compute_rest(starting_height, exit)?; + self.v3.compute_rest(starting_height, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/transactions/volume/compute.rs b/crates/brk_computer/src/transactions/volume/compute.rs index c138a929b..3b5d944dc 100644 --- a/crates/brk_computer/src/transactions/volume/compute.rs +++ b/crates/brk_computer/src/transactions/volume/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{Indexes, StoredF32}; +use brk_types::StoredF32; use vecdb::Exit; use super::Vecs; @@ -16,13 +16,14 @@ impl Vecs { prices: &prices::Vecs, count_vecs: &count::Vecs, fees_vecs: &fees::Vecs, - starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + self.transfer_volume - .compute(starting_indexes.height, prices, exit, |sats_vec| { + .compute(starting_height, prices, exit, |sats_vec| { Ok(sats_vec.compute_filtered_sum_from_indexes( - starting_indexes.height, + starting_height, &indexer.vecs.transactions.first_tx_index, &indexes.height.tx_index_count, &fees_vecs.input_value, @@ -31,12 +32,11 @@ impl Vecs { )?) })?; - let h = starting_indexes.height; let tx_sums = count_vecs.total.rolling.sum.0.as_array(); let tx_per_sec = self.tx_per_sec.as_mut_array(); for (i, &secs) in Windows::<()>::SECS.iter().enumerate() { tx_per_sec[i].height.compute_transform( - h, + starting_height, &tx_sums[i].height, |(h, sum, ..)| (h, StoredF32::from(*sum as f64 / secs)), exit, diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index c8b77e44e..87d5646aa 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -25,7 +25,6 @@ serde = { workspace = true } tracing = { workspace = true } rayon = { workspace = true } rustc-hash = { workspace = true } -smallvec = { workspace = true } vecdb = { workspace = true } [dev-dependencies] diff --git a/crates/brk_indexer/examples/indexer_bench.rs b/crates/brk_indexer/examples/indexer_bench.rs index 9b1f8e1b3..406a45b9e 100644 --- a/crates/brk_indexer/examples/indexer_bench.rs +++ b/crates/brk_indexer/examples/indexer_bench.rs @@ -50,14 +50,16 @@ fn main() -> Result<()> { indexer.index(&reader, &client, &exit)?; info!("Done in {:?}", i.elapsed()); + sleep(Duration::from_secs(60)); + // We want to benchmark the drop too drop(indexer); - sleep(Duration::from_secs(10)); + sleep(Duration::from_secs(60)); Mimalloc::collect(); - sleep(Duration::from_secs(10)); + sleep(Duration::from_secs(60)); Ok(()) } diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs deleted file mode 100644 index feb76d006..000000000 --- a/crates/brk_indexer/src/indexes.rs +++ /dev/null @@ -1,240 +0,0 @@ -use brk_error::Result; -use brk_types::{Height, Indexes}; -use tracing::{debug, info}; -use vecdb::{AnyStoredVec, PcoVec, PcoVecValue, ReadableVec, VecIndex, VecValue, WritableVec}; - -use crate::{Stores, Vecs}; - -/// Extension trait for Indexes with brk_indexer-specific functionality. -pub trait IndexesExt { - fn checked_push(&self, vecs: &mut Vecs) -> Result<()>; - fn from_vecs_and_stores( - required_height: Height, - vecs: &mut Vecs, - stores: &Stores, - ) -> Option - where - Self: Sized; -} - -impl IndexesExt for Indexes { - fn checked_push(&self, vecs: &mut Vecs) -> Result<()> { - let height = self.height; - vecs.transactions - .first_tx_index - .checked_push(height, self.tx_index)?; - vecs.inputs - .first_txin_index - .checked_push(height, self.txin_index)?; - vecs.outputs - .first_txout_index - .checked_push(height, self.txout_index)?; - vecs.scripts - .empty - .first_index - .checked_push(height, self.empty_output_index)?; - vecs.scripts - .p2ms - .first_index - .checked_push(height, self.p2ms_output_index)?; - vecs.scripts - .op_return - .first_index - .checked_push(height, self.op_return_index)?; - vecs.addrs - .p2a - .first_index - .checked_push(height, self.p2a_addr_index)?; - vecs.scripts - .unknown - .first_index - .checked_push(height, self.unknown_output_index)?; - vecs.addrs - .p2pk33 - .first_index - .checked_push(height, self.p2pk33_addr_index)?; - vecs.addrs - .p2pk65 - .first_index - .checked_push(height, self.p2pk65_addr_index)?; - vecs.addrs - .p2pkh - .first_index - .checked_push(height, self.p2pkh_addr_index)?; - vecs.addrs - .p2sh - .first_index - .checked_push(height, self.p2sh_addr_index)?; - vecs.addrs - .p2tr - .first_index - .checked_push(height, self.p2tr_addr_index)?; - vecs.addrs - .p2wpkh - .first_index - .checked_push(height, self.p2wpkh_addr_index)?; - vecs.addrs - .p2wsh - .first_index - .checked_push(height, self.p2wsh_addr_index)?; - - Ok(()) - } - - fn from_vecs_and_stores( - required_height: Height, - vecs: &mut Vecs, - stores: &Stores, - ) -> Option { - debug!("Creating indexes from vecs and stores..."); - - // Local data height: minimum of vecs and stores - let vecs_height = vecs.starting_height(); - let stores_height = stores.starting_height(); - let local_height = vecs_height.min(stores_height); - - // Data inconsistency: local data behind required height - if local_height < required_height { - return None; - } - - // Handle reorg: local data ahead of required height - let starting_height = if local_height > required_height { - info!( - "Reorg detected: rolling back from {} to {}", - local_height, required_height - ); - required_height - } else { - local_height - }; - - let empty_output_index = starting_index( - &vecs.scripts.empty.first_index, - &vecs.scripts.empty.to_tx_index, - starting_height, - )?; - - let p2ms_output_index = starting_index( - &vecs.scripts.p2ms.first_index, - &vecs.scripts.p2ms.to_tx_index, - starting_height, - )?; - - let op_return_index = starting_index( - &vecs.scripts.op_return.first_index, - &vecs.scripts.op_return.to_tx_index, - starting_height, - )?; - - let p2pk33_addr_index = starting_index( - &vecs.addrs.p2pk33.first_index, - &vecs.addrs.p2pk33.bytes, - starting_height, - )?; - - let p2pk65_addr_index = starting_index( - &vecs.addrs.p2pk65.first_index, - &vecs.addrs.p2pk65.bytes, - starting_height, - )?; - - let p2pkh_addr_index = starting_index( - &vecs.addrs.p2pkh.first_index, - &vecs.addrs.p2pkh.bytes, - starting_height, - )?; - - let p2sh_addr_index = starting_index( - &vecs.addrs.p2sh.first_index, - &vecs.addrs.p2sh.bytes, - starting_height, - )?; - - let p2tr_addr_index = starting_index( - &vecs.addrs.p2tr.first_index, - &vecs.addrs.p2tr.bytes, - starting_height, - )?; - - let p2wpkh_addr_index = starting_index( - &vecs.addrs.p2wpkh.first_index, - &vecs.addrs.p2wpkh.bytes, - starting_height, - )?; - - let p2wsh_addr_index = starting_index( - &vecs.addrs.p2wsh.first_index, - &vecs.addrs.p2wsh.bytes, - starting_height, - )?; - - let p2a_addr_index = starting_index( - &vecs.addrs.p2a.first_index, - &vecs.addrs.p2a.bytes, - starting_height, - )?; - - let tx_index = starting_index( - &vecs.transactions.first_tx_index, - &vecs.transactions.txid, - starting_height, - )?; - - let txin_index = starting_index( - &vecs.inputs.first_txin_index, - &vecs.inputs.outpoint, - starting_height, - )?; - - let txout_index = starting_index( - &vecs.outputs.first_txout_index, - &vecs.outputs.value, - starting_height, - )?; - - let unknown_output_index = starting_index( - &vecs.scripts.unknown.first_index, - &vecs.scripts.unknown.to_tx_index, - starting_height, - )?; - - Some(Indexes { - empty_output_index, - height: starting_height, - p2ms_output_index, - op_return_index, - p2pk33_addr_index, - p2pk65_addr_index, - p2pkh_addr_index, - p2sh_addr_index, - p2tr_addr_index, - p2wpkh_addr_index, - p2wsh_addr_index, - p2a_addr_index, - tx_index, - txin_index, - txout_index, - unknown_output_index, - }) - } -} - -pub fn starting_index( - height_to_index: &PcoVec, - index_to_else: &impl ReadableVec, - starting_height: Height, -) -> Option -where - I: VecIndex + PcoVecValue + From, - T: VecValue, -{ - let h = Height::from(height_to_index.stamp()); - if h.is_zero() { - None - } else if h + 1_u32 == starting_height { - Some(I::from(index_to_else.len())) - } else { - height_to_index.collect_one(starting_height) - } -} diff --git a/crates/brk_indexer/src/lengths.rs b/crates/brk_indexer/src/lengths.rs new file mode 100644 index 000000000..5b094b1ad --- /dev/null +++ b/crates/brk_indexer/src/lengths.rs @@ -0,0 +1,254 @@ +use brk_error::Result; +use brk_types::{ + EmptyOutputIndex, Height, OpReturnIndex, OutputType, P2AAddrIndex, P2MSOutputIndex, + P2PK33AddrIndex, P2PK65AddrIndex, P2PKHAddrIndex, P2SHAddrIndex, P2TRAddrIndex, + P2WPKHAddrIndex, P2WSHAddrIndex, TxInIndex, TxIndex, TxOutIndex, TypeIndex, UnknownOutputIndex, +}; +use tracing::info; +use vecdb::{AnyStoredVec, PcoVec, PcoVecValue, ReadableVec, VecIndex, VecValue, WritableVec}; + +use crate::{Stores, Vecs}; + +/// Pipeline-wide length/count snapshot. Lengths semantics: +/// `bound.f = N` means positions `0..N` are fully written; readers +/// reject `pos >= bound.f`. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Lengths { + pub empty_output_index: EmptyOutputIndex, + pub height: Height, + pub op_return_index: OpReturnIndex, + pub p2ms_output_index: P2MSOutputIndex, + pub p2pk33_addr_index: P2PK33AddrIndex, + pub p2pk65_addr_index: P2PK65AddrIndex, + pub p2pkh_addr_index: P2PKHAddrIndex, + pub p2sh_addr_index: P2SHAddrIndex, + pub p2tr_addr_index: P2TRAddrIndex, + pub p2wpkh_addr_index: P2WPKHAddrIndex, + pub p2wsh_addr_index: P2WSHAddrIndex, + pub p2a_addr_index: P2AAddrIndex, + pub tx_index: TxIndex, + pub txin_index: TxInIndex, + pub txout_index: TxOutIndex, + pub unknown_output_index: UnknownOutputIndex, +} + +impl Lengths { + pub fn to_type_index(&self, output_type: OutputType) -> TypeIndex { + match output_type { + OutputType::Empty => *self.empty_output_index, + OutputType::OpReturn => *self.op_return_index, + OutputType::P2A => *self.p2a_addr_index, + OutputType::P2MS => *self.p2ms_output_index, + OutputType::P2PK33 => *self.p2pk33_addr_index, + OutputType::P2PK65 => *self.p2pk65_addr_index, + OutputType::P2PKH => *self.p2pkh_addr_index, + OutputType::P2SH => *self.p2sh_addr_index, + OutputType::P2TR => *self.p2tr_addr_index, + OutputType::P2WPKH => *self.p2wpkh_addr_index, + OutputType::P2WSH => *self.p2wsh_addr_index, + OutputType::Unknown => *self.unknown_output_index, + } + } + + /// Bump per-block totals after processing a block. + pub fn add_block(&mut self, tx_count: usize, input_count: usize, output_count: usize) { + self.tx_index += TxIndex::from(tx_count); + self.txin_index += TxInIndex::from(input_count); + self.txout_index += TxOutIndex::from(output_count); + } + + /// Increments the address index for the given address type and returns the previous value. + /// Only call this for address types (P2PK65, P2PK33, P2PKH, P2SH, P2WPKH, P2WSH, P2TR, P2A). + #[inline] + pub fn increment_addr_index(&mut self, addr_type: OutputType) -> TypeIndex { + match addr_type { + OutputType::P2PK65 => self.p2pk65_addr_index.copy_then_increment(), + OutputType::P2PK33 => self.p2pk33_addr_index.copy_then_increment(), + OutputType::P2PKH => self.p2pkh_addr_index.copy_then_increment(), + OutputType::P2SH => self.p2sh_addr_index.copy_then_increment(), + OutputType::P2WPKH => self.p2wpkh_addr_index.copy_then_increment(), + OutputType::P2WSH => self.p2wsh_addr_index.copy_then_increment(), + OutputType::P2TR => self.p2tr_addr_index.copy_then_increment(), + OutputType::P2A => self.p2a_addr_index.copy_then_increment(), + _ => unreachable!(), + } + } + + pub fn checked_push(&self, vecs: &mut Vecs) -> Result<()> { + let height = self.height; + vecs.transactions + .first_tx_index + .checked_push(height, self.tx_index)?; + vecs.inputs + .first_txin_index + .checked_push(height, self.txin_index)?; + vecs.outputs + .first_txout_index + .checked_push(height, self.txout_index)?; + vecs.scripts + .empty + .first_index + .checked_push(height, self.empty_output_index)?; + vecs.scripts + .p2ms + .first_index + .checked_push(height, self.p2ms_output_index)?; + vecs.scripts + .op_return + .first_index + .checked_push(height, self.op_return_index)?; + vecs.addrs + .p2a + .first_index + .checked_push(height, self.p2a_addr_index)?; + vecs.scripts + .unknown + .first_index + .checked_push(height, self.unknown_output_index)?; + vecs.addrs + .p2pk33 + .first_index + .checked_push(height, self.p2pk33_addr_index)?; + vecs.addrs + .p2pk65 + .first_index + .checked_push(height, self.p2pk65_addr_index)?; + vecs.addrs + .p2pkh + .first_index + .checked_push(height, self.p2pkh_addr_index)?; + vecs.addrs + .p2sh + .first_index + .checked_push(height, self.p2sh_addr_index)?; + vecs.addrs + .p2tr + .first_index + .checked_push(height, self.p2tr_addr_index)?; + vecs.addrs + .p2wpkh + .first_index + .checked_push(height, self.p2wpkh_addr_index)?; + vecs.addrs + .p2wsh + .first_index + .checked_push(height, self.p2wsh_addr_index)?; + + Ok(()) + } + + /// Read current local lengths. `None` pre-genesis. + pub fn from_local(vecs: &mut Vecs, stores: &Stores) -> Option { + let height = vecs.next_height().min(stores.next_height()); + Self::collect_at(height, vecs) + } + + /// Read lengths to resume at `required_height`. Reorg-aware: + /// - if local is ahead, clamp down to `required_height`; + /// - if local is behind, return `None` (caller must full-reset). + pub fn resume_at(required_height: Height, vecs: &mut Vecs, stores: &Stores) -> Option { + let local = vecs.next_height().min(stores.next_height()); + if local < required_height { + return None; + } + let height = if local > required_height { + info!( + "Reorg detected: rolling back from {} to {}", + local, required_height + ); + required_height + } else { + local + }; + Self::collect_at(height, vecs) + } + + fn collect_at(height: Height, vecs: &mut Vecs) -> Option { + Some(Self { + empty_output_index: next_index( + &vecs.scripts.empty.first_index, + &vecs.scripts.empty.to_tx_index, + height, + )?, + height, + p2ms_output_index: next_index( + &vecs.scripts.p2ms.first_index, + &vecs.scripts.p2ms.to_tx_index, + height, + )?, + op_return_index: next_index( + &vecs.scripts.op_return.first_index, + &vecs.scripts.op_return.to_tx_index, + height, + )?, + p2pk33_addr_index: next_index( + &vecs.addrs.p2pk33.first_index, + &vecs.addrs.p2pk33.bytes, + height, + )?, + p2pk65_addr_index: next_index( + &vecs.addrs.p2pk65.first_index, + &vecs.addrs.p2pk65.bytes, + height, + )?, + p2pkh_addr_index: next_index( + &vecs.addrs.p2pkh.first_index, + &vecs.addrs.p2pkh.bytes, + height, + )?, + p2sh_addr_index: next_index( + &vecs.addrs.p2sh.first_index, + &vecs.addrs.p2sh.bytes, + height, + )?, + p2tr_addr_index: next_index( + &vecs.addrs.p2tr.first_index, + &vecs.addrs.p2tr.bytes, + height, + )?, + p2wpkh_addr_index: next_index( + &vecs.addrs.p2wpkh.first_index, + &vecs.addrs.p2wpkh.bytes, + height, + )?, + p2wsh_addr_index: next_index( + &vecs.addrs.p2wsh.first_index, + &vecs.addrs.p2wsh.bytes, + height, + )?, + p2a_addr_index: next_index(&vecs.addrs.p2a.first_index, &vecs.addrs.p2a.bytes, height)?, + tx_index: next_index( + &vecs.transactions.first_tx_index, + &vecs.transactions.txid, + height, + )?, + txin_index: next_index(&vecs.inputs.first_txin_index, &vecs.inputs.outpoint, height)?, + txout_index: next_index(&vecs.outputs.first_txout_index, &vecs.outputs.value, height)?, + unknown_output_index: next_index( + &vecs.scripts.unknown.first_index, + &vecs.scripts.unknown.to_tx_index, + height, + )?, + }) + } +} + +/// Per-type next-to-write counter at `next_height`. `None` pre-genesis. +fn next_index( + height_to_index: &PcoVec, + index_to_else: &impl ReadableVec, + next_height: Height, +) -> Option +where + I: VecIndex + PcoVecValue + From, + T: VecValue, +{ + let h = Height::from(height_to_index.stamp()); + if h.is_zero() { + None + } else if h + 1_u32 == next_height { + Some(I::from(index_to_else.len())) + } else { + height_to_index.collect_one(next_height) + } +} diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index d58997e63..0919ddc8d 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -4,7 +4,7 @@ use std::{ fs, path::{Path, PathBuf}, sync::Arc, - thread::{self, sleep}, + thread, time::{Duration, Instant}, }; @@ -19,18 +19,19 @@ use vecdb::{ Exit, RawDBError, ReadOnlyClone, ReadableVec, Ro, Rw, StorageMode, WritableVec, unlikely, }; mod constants; -mod indexes; +mod lengths; mod processor; mod readers; +mod safe_lengths; mod stores; mod vecs; use constants::*; -use indexes::IndexesExt; use processor::{BlockBuffers, BlockProcessor}; use readers::Readers; -pub use brk_types::Indexes; +pub use lengths::Lengths; +pub use safe_lengths::SafeLengths; pub use stores::Stores; pub use vecs::*; @@ -39,32 +40,27 @@ pub struct Indexer { pub vecs: Vecs, pub stores: Stores, tip_blockhash: Arc>, + safe_lengths: SafeLengths, } impl Indexer { pub fn tip_blockhash(&self) -> BlockHash { *self.tip_blockhash.read() } -} -impl Indexer { - /// Last height whose data is durably indexed, derived from the - /// `blockhash` vec's stamp. - pub fn indexed_height(&self) -> Height { - Height::from(self.vecs.blocks.blockhash.inner.stamp()) + /// Pipeline-safe `Lengths` snapshot shared with `Query`. Writers + /// advance and lower this internally; readers clamp non-series + /// answers against this loaded snapshot. + pub fn safe_lengths(&self) -> Lengths { + self.safe_lengths.load() } } -impl ReadOnlyClone for Indexer { - type ReadOnly = Indexer; - - fn read_only_clone(&self) -> Indexer { - Indexer { - path: self.path.clone(), - vecs: self.vecs.read_only_clone(), - stores: self.stores.clone(), - tip_blockhash: self.tip_blockhash.clone(), - } +impl Indexer { + /// Live indexer stamp for diagnostics. For data reads use + /// [`crate::SafeLengths::load`] (via `Query::height`). + pub fn indexed_height(&self) -> Height { + Height::from(self.vecs.blocks.blockhash.inner.stamp()) } } @@ -94,6 +90,7 @@ impl Indexer { vecs, stores, tip_blockhash: Arc::new(RwLock::new(tip_blockhash)), + safe_lengths: SafeLengths::new(), }) }; @@ -119,6 +116,8 @@ impl Indexer { /// record that gets replayed on every recovery), this cleanly recreates. fn full_reset(&mut self) -> Result<()> { info!("Full reset..."); + self.safe_lengths.reset(); + *self.tip_blockhash.write() = BlockHash::default(); self.vecs.reset()?; let stores_path = self.path.join("stores"); fs::remove_dir_all(&stores_path).ok(); @@ -126,41 +125,21 @@ impl Indexer { Ok(()) } - pub fn index(&mut self, reader: &Reader, client: &Client, exit: &Exit) -> Result { + pub fn index(&mut self, reader: &Reader, client: &Client, exit: &Exit) -> Result<()> { self.index_(reader, client, exit, false) } - pub fn checked_index( - &mut self, - reader: &Reader, - client: &Client, - exit: &Exit, - ) -> Result { + pub fn checked_index(&mut self, reader: &Reader, client: &Client, exit: &Exit) -> Result<()> { self.index_(reader, client, exit, true) } - fn check_xor_bytes(&mut self, reader: &Reader) -> Result<()> { - let current = reader.xor_bytes(); - let cached = XORBytes::from(self.path.as_path()); - - if cached == current { - return Ok(()); - } - - self.full_reset()?; - - fs::write(self.path.join("xor.dat"), *current)?; - - Ok(()) - } - fn index_( &mut self, reader: &Reader, client: &Client, exit: &Exit, check_collisions: bool, - ) -> Result { + ) -> Result<()> { self.vecs.db.sync_bg_tasks()?; self.check_xor_bytes(reader)?; @@ -176,42 +155,40 @@ impl Indexer { // .collect_one_at(self.vecs.blocks.blockhash.len() - 2); debug!("Last block hash found."); - let (starting_indexes, prev_hash) = if let Some(hash) = last_blockhash { + let (starting_lengths, prev_hash) = if let Some(hash) = last_blockhash { let (height, hash) = client.get_closest_valid_height(hash)?; - match Indexes::from_vecs_and_stores(height.incremented(), &mut self.vecs, &self.stores) - { - Some(starting_indexes) => { - if starting_indexes.height > client.get_last_height()? { + match Lengths::resume_at(height.incremented(), &mut self.vecs, &self.stores) { + Some(starting_lengths) => { + if starting_lengths.height > client.get_last_height()? { info!("Up to date, nothing to index."); - return Ok(starting_indexes); + return Ok(()); } - (starting_indexes, Some(hash)) + (starting_lengths, Some(hash)) } None => { info!("Data inconsistency detected, resetting indexer..."); self.full_reset()?; - (Indexes::default(), None) + (Lengths::default(), None) } } } else { - (Indexes::default(), None) + (Lengths::default(), None) }; - debug!("Starting indexes set."); + debug!("Starting lengths set."); let lock = exit.lock(); + self.safe_lengths.lower_before(&starting_lengths); self.stores - .rollback_if_needed(&mut self.vecs, &starting_indexes)?; + .rollback_if_needed(&mut self.vecs, &starting_lengths)?; debug!("Rollback stores done."); - self.vecs.rollback_if_needed(&starting_indexes)?; + self.vecs.rollback_if_needed(&starting_lengths)?; debug!("Rollback vecs done."); if let Some(hash) = prev_hash.as_ref() { *self.tip_blockhash.write() = *hash; } drop(lock); - // Cloned because we want to return starting indexes for the computer - let mut indexes = starting_indexes.clone(); - debug!("Indexes cloned."); + let mut lengths = starting_lengths; let is_export_height = |height: Height| -> bool { height != 0 && height % SNAPSHOT_BLOCK_RANGE == 0 }; @@ -268,7 +245,7 @@ impl Indexer { debug!("Indexing block {height}..."); } - indexes.height = height; + lengths.height = height; vecs.blocks.position.push(block.metadata().position()); block.tx_metadata().iter().for_each(|m| { @@ -279,7 +256,7 @@ impl Indexer { block: &block, height, check_collisions, - indexes: &mut indexes, + lengths: &mut lengths, vecs, stores, readers: &readers, @@ -309,7 +286,7 @@ impl Indexer { processor.check_txid_collisions(&txs)?; - let sigops = processor.compute_sigops(&txins); + let sigops = processor.compute_sigops(&txins, &txouts); processor.finalize_and_store_metadata( txs, @@ -321,16 +298,14 @@ impl Indexer { &mut buffers.same_block_output_info, )?; - processor.update_indexes(tx_count, input_count, output_count); + processor + .lengths + .add_block(tx_count, input_count, output_count); if is_export_height(height) { drop(readers); export(stores, vecs, height)?; readers = Readers::new(vecs); - - if height == Height::new(500_000) { - break; - } } *self.tip_blockhash.write() = block.block_hash().into(); @@ -339,14 +314,14 @@ impl Indexer { drop(readers); let lock = exit.lock(); - let tasks = self.stores.take_all_pending_ingests(indexes.height)?; - self.vecs.stamped_write(indexes.height)?; + let tasks = self.stores.take_all_pending_ingests(lengths.height)?; + self.vecs.stamped_write(lengths.height)?; let fjall_db = self.stores.db.clone(); self.vecs.db.run_bg(move |db| { let _lock = lock; - sleep(Duration::from_secs(5)); + db.bg_sleep(Duration::from_secs(3)); info!("Exporting..."); let i = Instant::now(); @@ -371,6 +346,45 @@ impl Indexer { Ok(()) }); - Ok(starting_indexes) + Ok(()) + } + + fn check_xor_bytes(&mut self, reader: &Reader) -> Result<()> { + let current = reader.xor_bytes(); + let cached = XORBytes::from(self.path.as_path()); + + if cached == current { + return Ok(()); + } + + self.full_reset()?; + + fs::write(self.path.join("xor.dat"), *current)?; + + Ok(()) + } + + /// Publish disk state as the new safe-lengths snapshot. Drains pending + /// bg ingest first so stores are queryable at the new bound. + pub fn advance_safe_lengths(&mut self) -> Result<()> { + self.vecs.db.sync_bg_tasks()?; + if let Some(lengths) = Lengths::from_local(&mut self.vecs, &self.stores) { + self.safe_lengths.advance(lengths); + } + Ok(()) + } +} + +impl ReadOnlyClone for Indexer { + type ReadOnly = Indexer; + + fn read_only_clone(&self) -> Indexer { + Indexer { + path: self.path.clone(), + vecs: self.vecs.read_only_clone(), + stores: self.stores.clone(), + tip_blockhash: self.tip_blockhash.clone(), + safe_lengths: self.safe_lengths.clone(), + } } } diff --git a/crates/brk_indexer/src/processor/metadata.rs b/crates/brk_indexer/src/processor/metadata.rs index 26b67ca87..41146b41b 100644 --- a/crates/brk_indexer/src/processor/metadata.rs +++ b/crates/brk_indexer/src/processor/metadata.rs @@ -4,7 +4,6 @@ use tracing::error; use vecdb::WritableVec; use super::{BlockProcessor, ComputedTx}; -use crate::IndexesExt; impl BlockProcessor<'_> { pub fn process_block_metadata(&mut self) -> Result<()> { @@ -22,7 +21,7 @@ impl BlockProcessor<'_> { return Err(Error::Internal("BlockHash prefix collision")); } - self.indexes.checked_push(self.vecs)?; + self.lengths.checked_push(self.vecs)?; self.stores .blockhash_prefix_to_height diff --git a/crates/brk_indexer/src/processor/mod.rs b/crates/brk_indexer/src/processor/mod.rs index d63bab969..bf9013f22 100644 --- a/crates/brk_indexer/src/processor/mod.rs +++ b/crates/brk_indexer/src/processor/mod.rs @@ -9,32 +9,23 @@ pub use types::*; use brk_cohort::ByAddrType; use brk_error::Result; -use brk_types::{ - AddrHash, Block, Height, OutPoint, SigOps, TxInIndex, TxIndex, TxOutIndex, TypeIndex, -}; +use brk_types::{AddrHash, Block, Height, OutPoint, SigOps, TxInIndex, TypeIndex}; use rustc_hash::{FxHashMap, FxHashSet}; -use crate::{Indexes, Readers, Stores, Vecs}; +use crate::{Lengths, Readers, Stores, Vecs}; /// Processes a single block, extracting and storing all indexed data. pub struct BlockProcessor<'a> { pub block: &'a Block, pub height: Height, pub check_collisions: bool, - pub indexes: &'a mut Indexes, + pub lengths: &'a mut Lengths, pub vecs: &'a mut Vecs, pub stores: &'a mut Stores, pub readers: &'a Readers, } impl BlockProcessor<'_> { - /// Update global indexes after processing a block. - pub fn update_indexes(&mut self, tx_count: usize, input_count: usize, output_count: usize) { - self.indexes.tx_index += TxIndex::from(tx_count); - self.indexes.txin_index += TxInIndex::from(input_count); - self.indexes.txout_index += TxOutIndex::from(output_count); - } - /// Finalizes outputs/inputs in parallel with storing tx metadata. #[allow(clippy::too_many_arguments)] pub fn finalize_and_store_metadata( @@ -47,7 +38,7 @@ impl BlockProcessor<'_> { already_added: &mut ByAddrType>, same_block_info: &mut FxHashMap, ) -> Result<()> { - let indexes = &mut *self.indexes; + let lengths = &mut *self.lengths; // Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest let (first_txout_index, first_txin_index, mut tx_metadata) = @@ -66,7 +57,7 @@ impl BlockProcessor<'_> { let (finalize_result, metadata_result) = rayon::join( || -> Result<()> { txout::finalize_outputs( - indexes, + lengths, first_txout_index, outputs, addrs, diff --git a/crates/brk_indexer/src/processor/sigops.rs b/crates/brk_indexer/src/processor/sigops.rs index ab352b184..993b96e55 100644 --- a/crates/brk_indexer/src/processor/sigops.rs +++ b/crates/brk_indexer/src/processor/sigops.rs @@ -1,24 +1,37 @@ +use bitcoin::{Script, script::Instruction}; use brk_types::{OutputType, SigOps, TxInIndex}; use rayon::prelude::*; -use smallvec::SmallVec; -use super::{BlockProcessor, InputSource}; +use super::{BlockProcessor, InputSource, ProcessedOutput}; impl BlockProcessor<'_> { - /// BIP-141 sigop cost per tx in the block. Uses each input's prevout - /// `OutputType` (already resolved by `process_inputs` for the - /// previous-block case, looked up from `block.txdata` for the - /// same-block case) to feed canonical-shaped synthetic prevouts into - /// `bitcoin::Transaction::total_sigop_cost`. - pub fn compute_sigops(&self, txins: &[(TxInIndex, InputSource)]) -> Vec { + /// BIP-141 sigop cost per tx in the block. Mirrors + /// `bitcoin::Transaction::total_sigop_cost` but dispatches on each + /// input's prevout `OutputType` and each output's `OutputType` + /// (already resolved by `process_inputs`/`process_outputs`) instead + /// of round-tripping through bitcoin's closure API with + /// synthetic-prevout `ScriptBuf` allocations. The legacy-sigop walk + /// is short-circuited by `OutputType` for every script with a + /// canonical shape (~99% of outputs and ~95% of inputs on mainnet); + /// only `OpReturn`/`Unknown` outputs and non-segwit/non-P2SH inputs + /// fall back to a real script walk. + pub fn compute_sigops( + &self, + txins: &[(TxInIndex, InputSource)], + txouts: &[ProcessedOutput<'_>], + ) -> Vec { let txdata = &self.block.txdata; - let base_tx_index = u32::from(self.indexes.tx_index); + let base_tx_index = u32::from(self.lengths.tx_index); let mut tx_input_offsets = Vec::with_capacity(txdata.len()); - let mut offset = 0usize; + let mut tx_output_offsets = Vec::with_capacity(txdata.len()); + let mut input_offset = 0usize; + let mut output_offset = 0usize; for tx in txdata { - tx_input_offsets.push(offset); - offset += tx.input.len(); + tx_input_offsets.push(input_offset); + input_offset += tx.input.len(); + tx_output_offsets.push(output_offset); + output_offset += tx.output.len(); } txdata @@ -28,31 +41,144 @@ impl BlockProcessor<'_> { if tx.is_coinbase() { return SigOps::ZERO; } - let start = tx_input_offsets[i]; - let tx_inputs = &txins[start..start + tx.input.len()]; + let in_start = tx_input_offsets[i]; + let tx_inputs = &txins[in_start..in_start + tx.input.len()]; + let out_start = tx_output_offsets[i]; + let tx_outputs = &txouts[out_start..out_start + tx.output.len()]; - let kinds: SmallVec<[(bitcoin::OutPoint, OutputType); 4]> = tx - .input - .iter() - .zip(tx_inputs.iter()) - .map(|(txin, (_, source))| { - let kind = match source { - InputSource::PreviousBlock { output_type, .. } => *output_type, - InputSource::SameBlock { outpoint, .. } => { - let local = - (u32::from(outpoint.tx_index()) - base_tx_index) as usize; - let vout = u32::from(outpoint.vout()) as usize; - OutputType::from(&txdata[local].output[vout].script_pubkey) + let mut legacy: usize = 0; + let mut redeem: usize = 0; + let mut witness: usize = 0; + + for (input, (_, source)) in tx.input.iter().zip(tx_inputs.iter()) { + let prev_kind = match source { + InputSource::PreviousBlock { output_type, .. } => *output_type, + InputSource::SameBlock { outpoint, .. } => { + let local = + (u32::from(outpoint.tx_index()) - base_tx_index) as usize; + let vout = u32::from(outpoint.vout()) as usize; + txouts[tx_output_offsets[local] + vout].output_type + } + }; + + // Single match per input: legacy script_sig sigops AND the + // redeem/witness contribution. Consensus enforces a + // push-only or empty script_sig in the four cases below + // (BIP-16 for P2SH from block 173805 onwards; BIP-141 / + // BIP-341 for segwit/taproot from activation), so legacy + // sigops are guaranteed 0 there. Everything else falls + // through to a real `count_sigops_legacy` walk. + match prev_kind { + OutputType::P2SH => { + // Faithful to bitcoin's count_p2sh_sigops + the + // nested-segwit branch of count_witness_sigops in + // a single script walk: redeem sigops use + // last_pushdata (no push-only check), wrapped + // witness sigops require both push-only and + // last_pushdata. + let (last_push, is_push_only) = + last_push_and_push_only(&input.script_sig); + let Some(redeem_bytes) = last_push else { + continue; + }; + let rs = Script::from_bytes(redeem_bytes); + redeem = redeem.saturating_add(rs.count_sigops()); + if !is_push_only { + continue; } - }; - (txin.previous_output, kind) - }) - .collect(); + if rs.is_p2wpkh() { + witness = witness.saturating_add(1); + } else if rs.is_p2wsh() + && let Some(last) = input.witness.last() + { + witness = witness + .saturating_add(Script::from_bytes(last).count_sigops()); + } + } + OutputType::P2WPKH => { + witness = witness.saturating_add(1); + } + OutputType::P2WSH => { + if let Some(last) = input.witness.last() { + witness = witness + .saturating_add(Script::from_bytes(last).count_sigops()); + } + } + OutputType::P2TR => {} + _ => { + legacy = legacy + .saturating_add(input.script_sig.count_sigops_legacy()); + } + } + } - SigOps::of_bitcoin_tx_with_kinds(tx, |op| { - kinds.iter().find(|(o, _)| o == op).map(|(_, k)| *k) - }) + for processed in tx_outputs { + legacy = legacy.saturating_add(legacy_sigops_for_output( + processed.output_type, + &processed.txout.script_pubkey, + )); + } + + SigOps::from( + legacy + .saturating_mul(4) + .saturating_add(redeem.saturating_mul(4)) + .saturating_add(witness), + ) }) .collect() } } + +/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`. +/// Every variant except `OpReturn` and `Unknown` has a canonical shape +/// recognised by `OutputType::from`'s exact byte-pattern matchers, so +/// the legacy sigop count is fixed: P2PKH and P2PK both end in a +/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted +/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain +/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn` +/// payloads can include 0xac/0xae bytes outside a push, and `Unknown` +/// can be anything, so both fall back to a real script walk. +#[inline] +fn legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> usize { + match output_type { + OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1, + OutputType::P2MS => 20, + OutputType::P2SH + | OutputType::P2WPKH + | OutputType::P2WSH + | OutputType::P2TR + | OutputType::P2A + | OutputType::Empty => 0, + OutputType::OpReturn | OutputType::Unknown => script_pubkey.count_sigops_legacy(), + } +} + +/// Single-pass equivalent of bitcoin's private `last_pushdata()` plus the +/// public `is_push_only()`: returns the bytes of the script's last +/// `Instruction::PushBytes` (only when it is the *last* instruction) +/// alongside whether every instruction was a push (per Core, +/// `OP_RESERVED` and `OP_PUSHNUM_1..16` count as pushes too). +fn last_push_and_push_only(script: &Script) -> (Option<&[u8]>, bool) { + let mut last: Option<&[u8]> = None; + let mut push_only = true; + for inst in script.instructions() { + match inst { + Ok(Instruction::PushBytes(b)) => { + last = Some(b.as_bytes()); + } + Ok(Instruction::Op(op)) => { + last = None; + if op.to_u8() > 0x60 { + push_only = false; + } + } + Err(_) => { + last = None; + push_only = false; + break; + } + } + } + (last, push_only) +} diff --git a/crates/brk_indexer/src/processor/tx.rs b/crates/brk_indexer/src/processor/tx.rs index e98f20421..c3174b635 100644 --- a/crates/brk_indexer/src/processor/tx.rs +++ b/crates/brk_indexer/src/processor/tx.rs @@ -13,7 +13,7 @@ use super::{BlockProcessor, ComputedTx}; impl<'a> BlockProcessor<'a> { pub fn compute_txids(&self) -> Result>> { let will_check_collisions = self.check_collisions; - let base_tx_index = self.indexes.tx_index; + let base_tx_index = self.lengths.tx_index; self.block .txdata diff --git a/crates/brk_indexer/src/processor/txin.rs b/crates/brk_indexer/src/processor/txin.rs index 3389a98c9..c1eb39504 100644 --- a/crates/brk_indexer/src/processor/txin.rs +++ b/crates/brk_indexer/src/processor/txin.rs @@ -22,8 +22,8 @@ impl<'a> BlockProcessor<'a> { txid_prefix_to_tx_index.clear(); txid_prefix_to_tx_index.extend(txs.iter().map(|ct| (ct.txid_prefix, ct.tx_index))); - let base_tx_index = self.indexes.tx_index; - let base_txin_index = self.indexes.txin_index; + let base_tx_index = self.lengths.tx_index; + let base_txin_index = self.lengths.txin_index; let total_inputs: usize = self.block.txdata.iter().map(|tx| tx.input.len()).sum(); let mut items = Vec::with_capacity(total_inputs); @@ -79,11 +79,11 @@ impl<'a> BlockProcessor<'a> { .map(|v| *v); let prev_tx_index = match store_result { - Some(tx_index) if tx_index < self.indexes.tx_index => tx_index, + Some(tx_index) if tx_index < self.lengths.tx_index => tx_index, _ => { error!( "UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}", - txid, txid_prefix, store_result, self.indexes.tx_index + txid, txid_prefix, store_result, self.lengths.tx_index ); return Err(Error::UnknownTxid); } diff --git a/crates/brk_indexer/src/processor/txout.rs b/crates/brk_indexer/src/processor/txout.rs index c88051bcc..335d06826 100644 --- a/crates/brk_indexer/src/processor/txout.rs +++ b/crates/brk_indexer/src/processor/txout.rs @@ -11,15 +11,15 @@ use tracing::error; use vecdb::{BytesVec, WritableVec}; use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo}; -use crate::{AddrsVecs, Indexes, OutputsVecs, ScriptsVecs}; +use crate::{AddrsVecs, Lengths, OutputsVecs, ScriptsVecs}; impl<'a> BlockProcessor<'a> { pub fn process_outputs(&self) -> Result>> { let height = self.height; let check_collisions = self.check_collisions; - let base_tx_index = self.indexes.tx_index; - let base_txout_index = self.indexes.txout_index; + let base_tx_index = self.lengths.tx_index; + let base_txout_index = self.lengths.txout_index; let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum(); let mut items = Vec::with_capacity(total_outputs); @@ -63,7 +63,7 @@ impl<'a> BlockProcessor<'a> { .get(&addr_hash)? .map(|v| *v) .and_then(|type_index_local| { - (type_index_local < self.indexes.to_type_index(addr_type)) + (type_index_local < self.lengths.to_type_index(addr_type)) .then_some(type_index_local) }); @@ -106,7 +106,7 @@ impl<'a> BlockProcessor<'a> { #[allow(clippy::too_many_arguments)] pub(super) fn finalize_outputs( - indexes: &mut Indexes, + lengths: &mut Lengths, first_txout_index: &mut BytesVec, outputs: &mut OutputsVecs, addrs: &mut AddrsVecs, @@ -150,7 +150,7 @@ pub(super) fn finalize_outputs( { ti } else { - let ti = indexes.increment_addr_index(addr_type); + let ti = lengths.increment_addr_index(addr_type); already_added_addr_hash .get_mut_unwrap(addr_type) @@ -168,29 +168,29 @@ pub(super) fn finalize_outputs( scripts .p2ms .to_tx_index - .checked_push(indexes.p2ms_output_index, tx_index)?; - indexes.p2ms_output_index.copy_then_increment() + .checked_push(lengths.p2ms_output_index, tx_index)?; + lengths.p2ms_output_index.copy_then_increment() } OutputType::OpReturn => { scripts .op_return .to_tx_index - .checked_push(indexes.op_return_index, tx_index)?; - indexes.op_return_index.copy_then_increment() + .checked_push(lengths.op_return_index, tx_index)?; + lengths.op_return_index.copy_then_increment() } OutputType::Empty => { scripts .empty .to_tx_index - .checked_push(indexes.empty_output_index, tx_index)?; - indexes.empty_output_index.copy_then_increment() + .checked_push(lengths.empty_output_index, tx_index)?; + lengths.empty_output_index.copy_then_increment() } OutputType::Unknown => { scripts .unknown .to_tx_index - .checked_push(indexes.unknown_output_index, tx_index)?; - indexes.unknown_output_index.copy_then_increment() + .checked_push(lengths.unknown_output_index, tx_index)?; + lengths.unknown_output_index.copy_then_increment() } _ => unreachable!(), } diff --git a/crates/brk_indexer/src/safe_lengths.rs b/crates/brk_indexer/src/safe_lengths.rs new file mode 100644 index 000000000..80ddb8c44 --- /dev/null +++ b/crates/brk_indexer/src/safe_lengths.rs @@ -0,0 +1,112 @@ +use std::sync::Arc; + +use parking_lot::RwLock; + +use crate::lengths::Lengths; + +/// Pipeline-wide safe-read snapshot. All fields are lengths/counts +/// (next-to-write totals): `bound.f = N` means positions `0..N` are +/// fully written; readers reject `pos >= bound.f`. Covers vecs only: +/// reorg store rewrites can briefly tear in-flight reads. +#[derive(Clone, Default)] +pub struct SafeLengths(Arc>); + +impl SafeLengths { + pub fn new() -> Self { + Self::default() + } + + pub fn load(&self) -> Lengths { + self.0.read().clone() + } + + pub fn reset(&self) { + *self.0.write() = Lengths::default(); + } + + pub fn advance(&self, next: Lengths) { + let mut g = self.0.write(); + debug_assert!( + next.height >= g.height + && next.tx_index >= g.tx_index + && next.txin_index >= g.txin_index + && next.txout_index >= g.txout_index + && next.empty_output_index >= g.empty_output_index + && next.op_return_index >= g.op_return_index + && next.p2ms_output_index >= g.p2ms_output_index + && next.p2pk33_addr_index >= g.p2pk33_addr_index + && next.p2pk65_addr_index >= g.p2pk65_addr_index + && next.p2pkh_addr_index >= g.p2pkh_addr_index + && next.p2sh_addr_index >= g.p2sh_addr_index + && next.p2tr_addr_index >= g.p2tr_addr_index + && next.p2wpkh_addr_index >= g.p2wpkh_addr_index + && next.p2wsh_addr_index >= g.p2wsh_addr_index + && next.p2a_addr_index >= g.p2a_addr_index + && next.unknown_output_index >= g.unknown_output_index, + "advance: per-field regression" + ); + *g = next; + } + + /// Drop each field to at most `starting`. Must be called BEFORE + /// any rewrite at positions `>= starting`. + pub fn lower_before(&self, starting: &Lengths) { + let mut g = self.0.write(); + g.height = g.height.min(starting.height); + g.tx_index = g.tx_index.min(starting.tx_index); + g.txin_index = g.txin_index.min(starting.txin_index); + g.txout_index = g.txout_index.min(starting.txout_index); + g.empty_output_index = g.empty_output_index.min(starting.empty_output_index); + g.op_return_index = g.op_return_index.min(starting.op_return_index); + g.p2ms_output_index = g.p2ms_output_index.min(starting.p2ms_output_index); + g.p2pk33_addr_index = g.p2pk33_addr_index.min(starting.p2pk33_addr_index); + g.p2pk65_addr_index = g.p2pk65_addr_index.min(starting.p2pk65_addr_index); + g.p2pkh_addr_index = g.p2pkh_addr_index.min(starting.p2pkh_addr_index); + g.p2sh_addr_index = g.p2sh_addr_index.min(starting.p2sh_addr_index); + g.p2tr_addr_index = g.p2tr_addr_index.min(starting.p2tr_addr_index); + g.p2wpkh_addr_index = g.p2wpkh_addr_index.min(starting.p2wpkh_addr_index); + g.p2wsh_addr_index = g.p2wsh_addr_index.min(starting.p2wsh_addr_index); + g.p2a_addr_index = g.p2a_addr_index.min(starting.p2a_addr_index); + g.unknown_output_index = g.unknown_output_index.min(starting.unknown_output_index); + } +} + +#[cfg(test)] +mod tests { + use brk_types::{ + EmptyOutputIndex, Height, OpReturnIndex, P2AAddrIndex, P2MSOutputIndex, P2PK33AddrIndex, + P2PK65AddrIndex, P2PKHAddrIndex, P2SHAddrIndex, P2TRAddrIndex, P2WPKHAddrIndex, + P2WSHAddrIndex, TxInIndex, TxIndex, TxOutIndex, UnknownOutputIndex, + }; + + use super::*; + + #[test] + fn lower_before_clamps_every_field() { + let sentinel = u32::MAX as usize; + let max = Lengths { + empty_output_index: EmptyOutputIndex::from(sentinel), + height: Height::from(sentinel), + op_return_index: OpReturnIndex::from(sentinel), + p2ms_output_index: P2MSOutputIndex::from(sentinel), + p2pk33_addr_index: P2PK33AddrIndex::from(sentinel), + p2pk65_addr_index: P2PK65AddrIndex::from(sentinel), + p2pkh_addr_index: P2PKHAddrIndex::from(sentinel), + p2sh_addr_index: P2SHAddrIndex::from(sentinel), + p2tr_addr_index: P2TRAddrIndex::from(sentinel), + p2wpkh_addr_index: P2WPKHAddrIndex::from(sentinel), + p2wsh_addr_index: P2WSHAddrIndex::from(sentinel), + p2a_addr_index: P2AAddrIndex::from(sentinel), + tx_index: TxIndex::from(sentinel), + txin_index: TxInIndex::from(sentinel), + txout_index: TxOutIndex::from(sentinel), + unknown_output_index: UnknownOutputIndex::from(sentinel), + }; + + let safe = SafeLengths::new(); + safe.advance(max); + safe.lower_before(&Lengths::default()); + + assert_eq!(safe.load(), Lengths::default()); + } +} diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index 868dee05f..a6d1f6b09 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -14,7 +14,7 @@ use rayon::prelude::*; use tracing::{debug, info}; use vecdb::{AnyVec, ReadableVec, VecIndex}; -use crate::{Indexes, constants::DUPLICATE_TXID_PREFIXES}; +use crate::{Lengths, constants::DUPLICATE_TXID_PREFIXES}; use super::Vecs; @@ -119,7 +119,7 @@ impl Stores { Ok(stores) } - pub fn starting_height(&self) -> Height { + pub fn next_height(&self) -> Height { self.iter_any() .map(|store| store.height().map(Height::incremented).unwrap_or_default()) .min() @@ -220,24 +220,26 @@ impl Stores { Ok(tasks) } + /// Rewrites reverse-key entries below the lowered bound. In-flight + /// readers may briefly see torn state. pub fn rollback_if_needed( &mut self, vecs: &mut Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, ) -> Result<()> { if self.is_empty()? { return Ok(()); } - debug_assert!(starting_indexes.height != Height::ZERO); - debug_assert!(starting_indexes.tx_index != TxIndex::ZERO); - debug_assert!(starting_indexes.txout_index != TxOutIndex::ZERO); + debug_assert!(starting_lengths.height != Height::ZERO); + debug_assert!(starting_lengths.tx_index != TxIndex::ZERO); + debug_assert!(starting_lengths.txout_index != TxOutIndex::ZERO); - self.rollback_block_metadata(vecs, starting_indexes)?; - self.rollback_txids(vecs, starting_indexes); - self.rollback_outputs_and_inputs(vecs, starting_indexes); + self.rollback_block_metadata(vecs, starting_lengths)?; + self.rollback_txids(vecs, starting_lengths); + self.rollback_outputs_and_inputs(vecs, starting_lengths); - let rollback_height = starting_indexes.height.decremented().unwrap_or_default(); + let rollback_height = starting_lengths.height.decremented().unwrap_or_default(); self.par_iter_any_mut() .try_for_each(|store| store.export_meta(rollback_height))?; self.commit(rollback_height)?; @@ -265,10 +267,10 @@ impl Stores { fn rollback_block_metadata( &mut self, vecs: &mut Vecs, - starting_indexes: &Indexes, + starting_lengths: &Lengths, ) -> Result<()> { vecs.blocks.blockhash.for_each_range_at( - starting_indexes.height.to_usize(), + starting_lengths.height.to_usize(), vecs.blocks.blockhash.len(), |blockhash| { self.blockhash_prefix_to_height @@ -277,7 +279,7 @@ impl Stores { ); for addr_type in OutputType::ADDR_TYPES { - for hash in vecs.iter_addr_hashes_from(addr_type, starting_indexes.height)? { + for hash in vecs.iter_addr_hashes_from(addr_type, starting_lengths.height)? { self.addr_type_to_addr_hash_to_addr_index .get_mut_unwrap(addr_type) .remove(hash); @@ -287,8 +289,8 @@ impl Stores { Ok(()) } - fn rollback_txids(&mut self, vecs: &mut Vecs, starting_indexes: &Indexes) { - let start = starting_indexes.tx_index.to_usize(); + fn rollback_txids(&mut self, vecs: &mut Vecs, starting_lengths: &Lengths) { + let start = starting_lengths.tx_index.to_usize(); let end = vecs.transactions.txid.len(); let mut current_index = start; vecs.transactions @@ -313,7 +315,7 @@ impl Stores { self.txid_prefix_to_tx_index.clear_caches(); } - fn rollback_outputs_and_inputs(&mut self, vecs: &mut Vecs, starting_indexes: &Indexes) { + fn rollback_outputs_and_inputs(&mut self, vecs: &mut Vecs, starting_lengths: &Lengths) { let tx_index_to_first_txout_index_reader = vecs.transactions.first_txout_index.reader(); let txout_index_to_output_type_reader = vecs.outputs.output_type.reader(); let txout_index_to_type_index_reader = vecs.outputs.type_index.reader(); @@ -321,7 +323,7 @@ impl Stores { let mut addr_index_tx_index_to_remove: FxHashSet<(OutputType, TypeIndex, TxIndex)> = FxHashSet::default(); - let rollback_start = starting_indexes.txout_index.to_usize(); + let rollback_start = starting_lengths.txout_index.to_usize(); let rollback_end = vecs.outputs.output_type.len(); let tx_indexes: Vec = vecs @@ -354,7 +356,7 @@ impl Stores { .remove(AddrIndexOutPoint::from((addr_index, outpoint))); } - let start = starting_indexes.txin_index.to_usize(); + let start = starting_lengths.txin_index.to_usize(); let end = vecs.inputs.outpoint.len(); let outpoints: Vec = vecs.inputs.outpoint.collect_range_at(start, end); let spending_tx_indexes: Vec = vecs.inputs.tx_index.collect_range_at(start, end); @@ -372,7 +374,7 @@ impl Stores { let txout_index = tx_index_to_first_txout_index_reader.get(output_tx_index.to_usize()) + vout; - if txout_index < starting_indexes.txout_index { + if txout_index < starting_lengths.txout_index { let output_type = txout_index_to_output_type_reader.get(txout_index.to_usize()); let type_index = txout_index_to_type_index_reader.get(txout_index.to_usize()); Some((outpoint, output_type, type_index, spending_tx_index)) diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 7a25ee373..52134a787 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -25,7 +25,7 @@ pub use outputs::*; pub use scripts::*; pub use transactions::*; -use crate::Indexes; +use crate::Lengths; #[derive(Traversable)] pub struct Vecs { @@ -80,40 +80,40 @@ impl Vecs { Ok(this) } - pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> Result<()> { - let saved_height = starting_indexes.height.decremented().unwrap_or_default(); + pub fn rollback_if_needed(&mut self, starting_lengths: &Lengths) -> Result<()> { + let saved_height = starting_lengths.height.decremented().unwrap_or_default(); let stamp = Stamp::from(u64::from(saved_height)); - self.blocks.truncate(starting_indexes.height, stamp)?; + self.blocks.truncate(starting_lengths.height, stamp)?; self.transactions - .truncate(starting_indexes.height, starting_indexes.tx_index, stamp)?; + .truncate(starting_lengths.height, starting_lengths.tx_index, stamp)?; self.inputs - .truncate(starting_indexes.height, starting_indexes.txin_index, stamp)?; + .truncate(starting_lengths.height, starting_lengths.txin_index, stamp)?; self.outputs - .truncate(starting_indexes.height, starting_indexes.txout_index, stamp)?; + .truncate(starting_lengths.height, starting_lengths.txout_index, stamp)?; self.addrs.truncate( - starting_indexes.height, - starting_indexes.p2pk65_addr_index, - starting_indexes.p2pk33_addr_index, - starting_indexes.p2pkh_addr_index, - starting_indexes.p2sh_addr_index, - starting_indexes.p2wpkh_addr_index, - starting_indexes.p2wsh_addr_index, - starting_indexes.p2tr_addr_index, - starting_indexes.p2a_addr_index, + starting_lengths.height, + starting_lengths.p2pk65_addr_index, + starting_lengths.p2pk33_addr_index, + starting_lengths.p2pkh_addr_index, + starting_lengths.p2sh_addr_index, + starting_lengths.p2wpkh_addr_index, + starting_lengths.p2wsh_addr_index, + starting_lengths.p2tr_addr_index, + starting_lengths.p2a_addr_index, stamp, )?; self.scripts.truncate( - starting_indexes.height, - starting_indexes.empty_output_index, - starting_indexes.op_return_index, - starting_indexes.p2ms_output_index, - starting_indexes.unknown_output_index, + starting_lengths.height, + starting_lengths.empty_output_index, + starting_lengths.op_return_index, + starting_lengths.p2ms_output_index, + starting_lengths.unknown_output_index, stamp, )?; @@ -126,7 +126,7 @@ impl Vecs { Ok(()) } - pub fn starting_height(&mut self) -> Height { + pub fn next_height(&mut self) -> Height { self.par_iter_mut_any_stored_vec() .map(|vec| { let h = Height::from(vec.stamp()); diff --git a/crates/brk_iterator/examples/sigops_bench.rs b/crates/brk_iterator/examples/sigops_bench.rs index 5ac7576c3..face932ee 100644 --- a/crates/brk_iterator/examples/sigops_bench.rs +++ b/crates/brk_iterator/examples/sigops_bench.rs @@ -105,9 +105,7 @@ fn main() -> Result<()> { let t2 = Instant::now(); let mut sum_hi: u64 = 0; for tx in &all_txs { - sum_hi += tx - .total_sigop_cost(|_op: &OutPoint| Some(synthetic_txout.clone())) - as u64; + sum_hi += tx.total_sigop_cost(|_op: &OutPoint| Some(synthetic_txout.clone())) as u64; } let elapsed_hi = t2.elapsed(); println!( diff --git a/crates/brk_logger/README.md b/crates/brk_logger/README.md index b5ec35763..2f0b4042a 100644 --- a/crates/brk_logger/README.md +++ b/crates/brk_logger/README.md @@ -8,7 +8,10 @@ Drop-in logging initialization that silences noisy dependencies (bitcoin, fjall, ## Key Features -- **Dual output**: Console (colorized) + optional file logging with size-based rotation (42MB, 2 files) +- **Dual output**: Console (colorized) + optional file logging with daily rotation +- **Per-level files**: One combined log plus one file per tracing level (error/warn/info/debug/trace) +- **Per-level rate limit**: 100 writes/sec per level so a chatty level can't starve the others; combined file mirrors what the per-level files accept +- **Auto-cleanup**: `*.txt` files older than 7 days are pruned on startup - **Log hooks**: Register callbacks to intercept log messages programmatically - **Sensible defaults**: Pre-configured filters silence common verbose libraries - **Timestamp formatting**: Uses system timezone via jiff @@ -21,8 +24,8 @@ Drop-in logging initialization that silences noisy dependencies (bitcoin, fjall, ## Core API ```rust,ignore -brk_logger::init(Some(Path::new("app.log")))?; // Console + file -brk_logger::init(None)?; // Console only +brk_logger::init(Some(Path::new("logs")))?; // Console + files in logs/ +brk_logger::init(None)?; // Console only brk_logger::register_hook(|msg| { // React to log messages diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index 70b32015b..428863fcc 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -10,12 +10,18 @@ use tracing_subscriber::{filter::Targets, fmt, layer::SubscriberExt, util::Subsc use format::Formatter; use hook::{HookLayer, LOG_HOOK}; -use rate_limit::RateLimitedFile; +use rate_limit::{RateLimitedFile, is_log_file}; /// Days to keep log files before cleanup const MAX_LOG_AGE_DAYS: u64 = 7; -pub fn init(path: Option<&Path>) -> io::Result<()> { +/// Initialize the global tracing subscriber with a colorized console layer. +/// +/// If `dir` is `Some`, also writes daily log files to that directory: +/// `YYYY-MM-DD.txt` for the combined log and `YYYY-MM-DD_.txt` for each +/// tracing level. The directory is created if it does not exist, and any +/// `*.txt` file older than 7 days is pruned on startup. +pub fn init(dir: Option<&Path>) -> io::Result<()> { tracing_log::LogTracer::init().ok(); #[cfg(debug_assertions)] @@ -40,16 +46,10 @@ pub fn init(path: Option<&Path>) -> io::Result<()> { .with(fmt::layer().event_format(Formatter::)) .with(HookLayer); - if let Some(path) = path { - let dir = path.parent().unwrap_or(Path::new(".")); - let prefix = path - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or("app.log"); + if let Some(dir) = dir { + let writer = RateLimitedFile::new(dir)?; - cleanup_old_logs(dir, prefix); - - let writer = RateLimitedFile::new(dir, prefix); + cleanup_old_logs(dir); registry .with( @@ -75,7 +75,7 @@ where .map_err(|_| "Hook already registered") } -fn cleanup_old_logs(dir: &Path, prefix: &str) { +fn cleanup_old_logs(dir: &Path) { let max_age = Duration::from_secs(MAX_LOG_AGE_DAYS * 24 * 60 * 60); let Ok(entries) = std::fs::read_dir(dir) else { return; @@ -86,8 +86,7 @@ fn cleanup_old_logs(dir: &Path, prefix: &str) { let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue; }; - - if !name.starts_with(prefix) || name == prefix { + if !is_log_file(name) { continue; } diff --git a/crates/brk_logger/src/rate_limit.rs b/crates/brk_logger/src/rate_limit.rs index 02af3a198..a5a96a4b1 100644 --- a/crates/brk_logger/src/rate_limit.rs +++ b/crates/brk_logger/src/rate_limit.rs @@ -1,27 +1,68 @@ use std::{ - fs::OpenOptions, + fs::{File, OpenOptions, create_dir_all}, io::{self, Write}, - path::PathBuf, + path::{Path, PathBuf}, sync::{ - Arc, + Arc, Mutex, atomic::{AtomicU64, Ordering}, }, time::{SystemTime, UNIX_EPOCH}, }; use jiff::{Timestamp, tz}; +use tracing::{Level, Metadata}; use tracing_subscriber::fmt::MakeWriter; const MAX_WRITES_PER_SEC: u64 = 100; +const LEVELS: usize = 5; +const LEVEL_SUFFIX: [&str; LEVELS] = ["error", "warn", "info", "debug", "trace"]; -struct Inner { - dir: PathBuf, - prefix: String, +const fn level_index(level: Level) -> usize { + match level { + Level::ERROR => 0, + Level::WARN => 1, + Level::INFO => 2, + Level::DEBUG => 3, + Level::TRACE => 4, + } +} + +/// Returns true if `name` matches a file produced by this writer: +/// `YYYY-MM-DD.txt` or `YYYY-MM-DD_.txt`. +pub(crate) fn is_log_file(name: &str) -> bool { + let Some(stem) = name.strip_suffix(".txt") else { + return false; + }; + if stem.len() < 10 { + return false; + } + let (date, rest) = stem.split_at(10); + if !is_date_yyyymmdd(date) { + return false; + } + rest.is_empty() + || rest + .strip_prefix('_') + .is_some_and(|s| LEVEL_SUFFIX.contains(&s)) +} + +fn is_date_yyyymmdd(s: &str) -> bool { + let b = s.as_bytes(); + b.len() == 10 + && b[0..4].iter().all(u8::is_ascii_digit) + && b[4] == b'-' + && b[5..7].iter().all(u8::is_ascii_digit) + && b[7] == b'-' + && b[8..10].iter().all(u8::is_ascii_digit) +} + +#[derive(Default)] +struct RateLimit { count: AtomicU64, last_second: AtomicU64, } -impl Inner { +impl RateLimit { fn can_write(&self) -> bool { let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -37,43 +78,112 @@ impl Inner { self.count.fetch_add(1, Ordering::Relaxed) < MAX_WRITES_PER_SEC } } +} - fn path(&self) -> PathBuf { - let date = Timestamp::now() - .to_zoned(tz::TimeZone::system()) - .strftime("%Y-%m-%d") - .to_string(); - self.dir.join(format!("{}_{}.txt", self.prefix, date)) +struct Cached { + date: String, + file: File, +} + +#[derive(Default)] +struct FileSlot(Mutex>); + +fn open_append(path: &Path) -> io::Result { + OpenOptions::new().create(true).append(true).open(path) +} + +impl FileSlot { + fn write( + &self, + dir: &Path, + date: &str, + buf: &[u8], + path: impl FnOnce() -> PathBuf, + ) -> io::Result<()> { + let mut guard = self.0.lock().unwrap(); + let cached = match guard.as_mut() { + Some(c) if c.date == date => c, + _ => { + let p = path(); + let file = match open_append(&p) { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + create_dir_all(dir)?; + open_append(&p)? + } + Err(e) => return Err(e), + }; + guard.insert(Cached { + date: date.to_string(), + file, + }) + } + }; + cached.file.write_all(buf) } } -#[derive(Clone)] +fn today() -> String { + Timestamp::now() + .to_zoned(tz::TimeZone::system()) + .strftime("%Y-%m-%d") + .to_string() +} + +struct Inner { + dir: PathBuf, + level_limits: [RateLimit; LEVELS], + combined_slot: FileSlot, + level_slots: [FileSlot; LEVELS], +} + +/// Rate-limited daily log files: one combined file plus one per tracing level. +/// Each level has its own 100/sec limiter so a chatty level cannot starve the +/// others; the combined file is a true superset, written iff a per-level write +/// is permitted (or unconditionally for events with no associated level). pub struct RateLimitedFile(Arc); impl RateLimitedFile { - pub fn new(dir: &std::path::Path, prefix: &str) -> Self { - Self(Arc::new(Inner { + pub fn new(dir: &Path) -> io::Result { + create_dir_all(dir)?; + Ok(Self(Arc::new(Inner { dir: dir.to_path_buf(), - prefix: prefix.to_string(), - count: AtomicU64::new(0), - last_second: AtomicU64::new(0), - })) + level_limits: Default::default(), + combined_slot: FileSlot::default(), + level_slots: Default::default(), + }))) } } -pub struct FileWriter(Arc); +pub struct FileWriter { + inner: Arc, + level: Option, +} impl Write for FileWriter { fn write(&mut self, buf: &[u8]) -> io::Result { - if !self.0.can_write() { + let level_idx = self.level.map(level_index); + + if let Some(i) = level_idx + && !self.inner.level_limits[i].can_write() + { return Ok(buf.len()); } - OpenOptions::new() - .create(true) - .append(true) - .open(self.0.path())? - .write(buf) + let date = today(); + let dir = &self.inner.dir; + + self.inner + .combined_slot + .write(dir, &date, buf, || dir.join(format!("{date}.txt")))?; + + if let Some(i) = level_idx { + self.inner.level_slots[i].write(dir, &date, buf, || { + dir.join(format!("{date}_{}.txt", LEVEL_SUFFIX[i])) + })?; + } + + Ok(buf.len()) } fn flush(&mut self) -> io::Result<()> { @@ -84,7 +194,56 @@ impl Write for FileWriter { impl<'a> MakeWriter<'a> for RateLimitedFile { type Writer = FileWriter; + /// Fallback used only by callers that bypass `make_writer_for`. The fmt + /// layer always provides metadata, so this path is unused in practice; it + /// writes to the combined file only and skips per-level routing. fn make_writer(&'a self) -> Self::Writer { - FileWriter(Arc::clone(&self.0)) + FileWriter { + inner: Arc::clone(&self.0), + level: None, + } + } + + fn make_writer_for(&'a self, meta: &Metadata<'_>) -> Self::Writer { + FileWriter { + inner: Arc::clone(&self.0), + level: Some(*meta.level()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_log_file_accepts_combined() { + assert!(is_log_file("2026-05-06.txt")); + } + + #[test] + fn is_log_file_accepts_each_level() { + for suffix in LEVEL_SUFFIX { + assert!(is_log_file(&format!("2026-05-06_{suffix}.txt"))); + } + } + + #[test] + fn is_log_file_rejects_unknown_level() { + assert!(!is_log_file("2026-05-06_notice.txt")); + } + + #[test] + fn is_log_file_rejects_bad_date() { + assert!(!is_log_file("2026-5-06.txt")); + assert!(!is_log_file("abcd-ef-gh.txt")); + assert!(!is_log_file("2026/05/06.txt")); + } + + #[test] + fn is_log_file_rejects_user_files() { + assert!(!is_log_file("notes.txt")); + assert!(!is_log_file("README")); + assert!(!is_log_file("2026-05-06.log")); } } diff --git a/crates/brk_mempool/src/cluster/mod.rs b/crates/brk_mempool/src/cluster/mod.rs index 42a53639e..75dac089e 100644 --- a/crates/brk_mempool/src/cluster/mod.rs +++ b/crates/brk_mempool/src/cluster/mod.rs @@ -71,8 +71,7 @@ impl Cluster { /// strictly less than `i`. fn permute_to_topo_order(mut nodes: Vec>) -> Vec> { let n = nodes.len(); - let mut children: Vec> = - (0..n).map(|_| SmallVec::new()).collect(); + let mut children: Vec> = (0..n).map(|_| SmallVec::new()).collect(); let mut indegree: Vec = vec![0; n]; for (i, node) in nodes.iter().enumerate() { indegree[i] = node.parents.len() as u32; @@ -118,10 +117,7 @@ impl Cluster { /// `trailing_zeros` visits each chunk's bits in ascending order, and /// nodes are stored in topo order (`LocalIdx == position`), so each /// pushed `LocalIdx` lands parents-first in `chunk.txs`. - fn materialize_chunks( - chunk_masks: &[sfl::ChunkMask], - n: usize, - ) -> (Vec, Vec) { + fn materialize_chunks(chunk_masks: &[sfl::ChunkMask], n: usize) -> (Vec, Vec) { let mut chunks: Vec = Vec::with_capacity(chunk_masks.len()); let mut node_to_chunk = vec![ChunkId::ZERO; n]; for (cid, cm) in chunk_masks.iter().enumerate() { diff --git a/crates/brk_mempool/src/cluster/sfl.rs b/crates/brk_mempool/src/cluster/sfl.rs index f150a65e0..380bbf398 100644 --- a/crates/brk_mempool/src/cluster/sfl.rs +++ b/crates/brk_mempool/src/cluster/sfl.rs @@ -84,7 +84,10 @@ fn extract_chunks(t: &Tables) -> Vec { /// `0..n`; since the cluster is stored in topological order, that *is* /// a topological sweep. fn best_subset(t: &Tables, remaining: u128) -> (u128, Sats, VSize) { - let ctx = Ctx { tables: t, remaining }; + let ctx = Ctx { + tables: t, + remaining, + }; let mut best = (0u128, Sats::ZERO, VSize::default()); recurse(&ctx, 0, 0, Sats::ZERO, VSize::default(), &mut best); best @@ -99,8 +102,7 @@ fn recurse( best: &mut (u128, Sats, VSize), ) { if idx == ctx.tables.n { - if included != 0 - && (best.0 == 0 || FeeRate::from((f, v)) > FeeRate::from((best.1, best.2))) + if included != 0 && (best.0 == 0 || FeeRate::from((f, v)) > FeeRate::from((best.1, best.2))) { *best = (included, f, v); } @@ -110,8 +112,7 @@ fn recurse( // Not in remaining, or a parent (within remaining) is excluded: // this node is forced-excluded, no branching. - if (bit & ctx.remaining) == 0 - || (ctx.tables.parents_mask[idx] & ctx.remaining & !included) != 0 + if (bit & ctx.remaining) == 0 || (ctx.tables.parents_mask[idx] & ctx.remaining & !included) != 0 { recurse(ctx, idx + 1, included, f, v, best); return; diff --git a/crates/brk_mempool/src/cpfp.rs b/crates/brk_mempool/src/cpfp.rs index 65197ffde..5c105ad7b 100644 --- a/crates/brk_mempool/src/cpfp.rs +++ b/crates/brk_mempool/src/cpfp.rs @@ -22,7 +22,6 @@ use brk_types::{ VSize, }; - use crate::Mempool; use crate::cluster::{Cluster, ClusterRef, LocalIdx}; @@ -81,7 +80,11 @@ impl Cluster { let mut reachable = 1u128 << seed.inner(); let mut out: Vec = Vec::new(); for (i, node) in self.nodes.iter().enumerate().skip(seed_pos + 1) { - if node.parents.iter().any(|&p| reachable & (1u128 << p.inner()) != 0) { + if node + .parents + .iter() + .any(|&p| reachable & (1u128 << p.inner()) != 0) + { reachable |= 1u128 << i; out.push(CpfpEntry::from(node)); } @@ -108,7 +111,10 @@ impl Mempool { pub fn cpfp_info(&self, prefix: &TxidPrefix) -> Option { let snapshot = self.snapshot(); let seed_idx = self.entries().idx_of(prefix)?; - let ClusterRef { cluster_id, local: seed_local } = snapshot.cluster_of(seed_idx)?; + let ClusterRef { + cluster_id, + local: seed_local, + } = snapshot.cluster_of(seed_idx)?; let cluster = &snapshot.clusters[cluster_id.as_usize()]; let seed_txid = &cluster.nodes[seed_local.as_usize()].txid; diff --git a/crates/brk_mempool/src/steps/preparer/mod.rs b/crates/brk_mempool/src/steps/preparer/mod.rs index a13ff06a0..127b37507 100644 --- a/crates/brk_mempool/src/steps/preparer/mod.rs +++ b/crates/brk_mempool/src/steps/preparer/mod.rs @@ -43,7 +43,10 @@ impl Preparer { } fn live_set(entries_info: &[MempoolEntryInfo]) -> FxHashSet { - entries_info.iter().map(|info| TxidPrefix::from(&info.txid)).collect() + entries_info + .iter() + .map(|info| TxidPrefix::from(&info.txid)) + .collect() } fn classify_additions( @@ -59,9 +62,7 @@ impl Preparer { entries_info .iter() - .filter_map(|info| { - Self::classify(info, known, graveyard, &mut new_raws, &parent_raws) - }) + .filter_map(|info| Self::classify(info, known, graveyard, &mut new_raws, &parent_raws)) .collect() } diff --git a/crates/brk_mempool/src/steps/rebuilder/clusters.rs b/crates/brk_mempool/src/steps/rebuilder/clusters.rs index fff709c33..bbe8dcd1e 100644 --- a/crates/brk_mempool/src/steps/rebuilder/clusters.rs +++ b/crates/brk_mempool/src/steps/rebuilder/clusters.rs @@ -80,7 +80,10 @@ fn flood_component( while let Some(pos) = stack.pop() { members.push(pos); - for &n in parents[pos as usize].iter().chain(children[pos as usize].iter()) { + for &n in parents[pos as usize] + .iter() + .chain(children[pos as usize].iter()) + { if !seen[n as usize] { seen[n as usize] = true; stack.push(n); diff --git a/crates/brk_mempool/src/steps/rebuilder/mod.rs b/crates/brk_mempool/src/steps/rebuilder/mod.rs index 00a483d0f..eda989dc0 100644 --- a/crates/brk_mempool/src/steps/rebuilder/mod.rs +++ b/crates/brk_mempool/src/steps/rebuilder/mod.rs @@ -11,11 +11,11 @@ use brk_types::FeeRate; use parking_lot::{Mutex, RwLock}; use tracing::warn; +use crate::stores::MempoolState; use clusters::build_clusters; use partition::Partitioner; #[cfg(debug_assertions)] use verify::Verifier; -use crate::stores::MempoolState; pub(crate) mod clusters; mod partition; diff --git a/crates/brk_mempool/src/steps/rebuilder/partition.rs b/crates/brk_mempool/src/steps/rebuilder/partition.rs index a6c930671..37686cf26 100644 --- a/crates/brk_mempool/src/steps/rebuilder/partition.rs +++ b/crates/brk_mempool/src/steps/rebuilder/partition.rs @@ -114,7 +114,9 @@ impl<'a> Partitioner<'a> { fn try_fill_with_smaller(&mut self, start: usize, remaining_space: VSize) -> bool { let end = (start + LOOK_AHEAD_COUNT).min(self.slots.len()); for idx in (start + 1)..end { - let Some(cand) = self.slots[idx] else { continue }; + let Some(cand) = self.slots[idx] else { + continue; + }; if cand.vsize > remaining_space { continue; } diff --git a/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs b/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs index 49d47acaa..ee65e9b4a 100644 --- a/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs +++ b/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs @@ -88,6 +88,10 @@ impl Snapshot { /// the full `CpfpInfo`. pub fn chunk_rate_of(&self, idx: TxIndex) -> Option { let ClusterRef { cluster_id, local } = self.cluster_of(idx)?; - Some(self.clusters[cluster_id.as_usize()].chunk_of(local).fee_rate()) + Some( + self.clusters[cluster_id.as_usize()] + .chunk_of(local) + .fee_rate(), + ) } } diff --git a/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs b/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs index c6923f2bb..44f31d74a 100644 --- a/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs +++ b/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs @@ -44,7 +44,11 @@ impl BlockStats { total_fee += entry.fee; total_vsize += entry.vsize; total_size += entry.size; - fee_rates.push(clusters[cref.cluster_id.as_usize()].chunk_of(cref.local).fee_rate()); + fee_rates.push( + clusters[cref.cluster_id.as_usize()] + .chunk_of(cref.local) + .fee_rate(), + ); } let tx_count = fee_rates.len() as u32; diff --git a/crates/brk_mempool/src/tests/linearize/oracle.rs b/crates/brk_mempool/src/tests/linearize/oracle.rs index b8506cfca..2f857cec8 100644 --- a/crates/brk_mempool/src/tests/linearize/oracle.rs +++ b/crates/brk_mempool/src/tests/linearize/oracle.rs @@ -59,9 +59,7 @@ fn all_topo_orders(parents: &[Vec]) -> Vec> { out.push(current.clone()); return; } - let ready: Vec = (0..n as u32) - .filter(|&i| indeg[i as usize] == 0) - .collect(); + let ready: Vec = (0..n as u32).filter(|&i| indeg[i as usize] == 0).collect(); for v in ready { indeg[v as usize] = u32::MAX; current.push(v); diff --git a/crates/brk_mempool/src/tests/linearize/stress.rs b/crates/brk_mempool/src/tests/linearize/stress.rs index 4bebd3571..ff10113d2 100644 --- a/crates/brk_mempool/src/tests/linearize/stress.rs +++ b/crates/brk_mempool/src/tests/linearize/stress.rs @@ -72,16 +72,8 @@ fn check_invariants(fees_vsizes: &[(u64, u64)], edges: &[(u32, u32)], cluster: & } for chunk in chunks { - let fee: u64 = chunk - .txs - .iter() - .map(|&l| fees_vsizes[input_of(l)].0) - .sum(); - let vsize: u64 = chunk - .txs - .iter() - .map(|&l| fees_vsizes[input_of(l)].1) - .sum(); + let fee: u64 = chunk.txs.iter().map(|&l| fees_vsizes[input_of(l)].0).sum(); + let vsize: u64 = chunk.txs.iter().map(|&l| fees_vsizes[input_of(l)].1).sum(); assert_eq!(chunk.fee, Sats::from(fee), "chunk fee mismatch"); assert_eq!(chunk.vsize, VSize::from(vsize), "chunk vsize mismatch"); } diff --git a/crates/brk_query/src/impl/addr.rs b/crates/brk_query/src/impl/addr.rs index 8e43ce0db..bde9caa54 100644 --- a/crates/brk_query/src/impl/addr.rs +++ b/crates/brk_query/src/impl/addr.rs @@ -34,6 +34,10 @@ impl Query { let hash = AddrHash::from(&bytes); let type_index = self.type_index_for(output_type, &hash)?; + if type_index >= self.safe_lengths().to_type_index(output_type) { + return Err(Error::UnknownAddr); + } + let any_addr_index = computer .distribution .any_addr_indexes @@ -139,22 +143,26 @@ impl Query { .get(output_type) .data()?; + let tx_index_len = self.safe_lengths().tx_index; + if let Some(after_txid) = after_txid { let after_tx_index = self.resolve_tx_index(&after_txid)?; let min = AddrIndexTxIndex::min_for_addr(type_index); - let bound = AddrIndexTxIndex::from((type_index, after_tx_index)); + let cursor = AddrIndexTxIndex::from((type_index, after_tx_index)); Ok(store - .range(min..bound) + .range(min..cursor) .rev() - .take(limit) .map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index()) + .filter(|tx_index| *tx_index < tx_index_len) + .take(limit) .collect()) } else { Ok(store .prefix(type_index) .rev() - .take(limit) .map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index()) + .filter(|tx_index| *tx_index < tx_index_len) + .take(limit) .collect()) } } @@ -171,9 +179,11 @@ impl Query { .get(output_type) .data()?; + let tx_index_len = self.safe_lengths().tx_index; let outpoints: Vec<(TxIndex, Vout)> = store .prefix(type_index) .map(|(key, _): (AddrIndexOutPoint, Unit)| (key.tx_index(), key.vout())) + .filter(|(tx_index, _)| *tx_index < tx_index_len) .take(max_utxos + 1) .collect(); if outpoints.len() > max_utxos { @@ -257,10 +267,12 @@ impl Query { .addr_type_to_addr_index_and_tx_index .get(output_type) .data()?; + let tx_index_len = self.safe_lengths().tx_index; let last_tx_index = store .prefix(type_index) - .next_back() + .rev() .map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index()) + .find(|tx_index| *tx_index < tx_index_len) .ok_or(Error::UnknownAddr)?; self.confirmed_status_height(last_tx_index) } diff --git a/crates/brk_query/src/impl/block/info.rs b/crates/brk_query/src/impl/block/info.rs index 9f081dde4..fb18ea4cf 100644 --- a/crates/brk_query/src/impl/block/info.rs +++ b/crates/brk_query/src/impl/block/info.rs @@ -7,7 +7,7 @@ use brk_types::{ BlockExtras, BlockHash, BlockHashPrefix, BlockHeader, BlockInfo, BlockInfoV1, BlockPool, FeeRate, Height, PoolSlug, Sats, Timestamp, TxIndex, VSize, pools, }; -use vecdb::{AnyVec, ReadableVec, VecIndex}; +use vecdb::{ReadableVec, VecIndex}; use crate::Query; @@ -43,9 +43,9 @@ impl Query { self.block_by_height(height) } - /// Block by height. Height > tip → `OutOfRange`. + /// Block by height. Height past tip (or pre-genesis) → `OutOfRange`. pub fn block_by_height(&self, height: Height) -> Result { - if height > self.tip_height() { + if height >= self.safe_lengths().height { return Err(Error::OutOfRange("Block height out of range".into())); } let h = height.to_usize(); @@ -58,7 +58,7 @@ impl Query { /// `blocks_v1_range` reads computer-stamped series (pools, fees, /// supply state). Anything past `computed_height` would short-read. pub fn block_by_height_v1(&self, height: Height) -> Result { - if height > self.height() { + if height >= self.safe_lengths().height { return Err(Error::OutOfRange("Block height out of range".into())); } let h = height.to_usize(); @@ -71,6 +71,9 @@ impl Query { /// doubles as a corruption check on the on-disk bytes. pub fn block_header_hex(&self, hash: &BlockHash) -> Result { let height = self.height_by_hash(hash)?; + if height >= self.safe_lengths().height { + return Err(Error::OutOfRange("Block height out of range".into())); + } let header = self.read_block_header(height)?; Ok(bitcoin::consensus::encode::serialize_hex(&header)) } @@ -79,7 +82,7 @@ impl Query { /// bounds gate (`OutOfRange` for past-tip, `Internal` if the data /// is unexpectedly missing inside the gate). pub fn block_hash_by_height(&self, height: Height) -> Result { - if height > self.tip_height() { + if height >= self.safe_lengths().height { return Err(Error::OutOfRange("Block height out of range".into())); } self.indexer().vecs.blocks.blockhash.get(height).data() @@ -88,7 +91,7 @@ impl Query { /// Most recent `count` blocks ending at `start_height` (default tip), /// returned in descending-height order. pub fn blocks(&self, start_height: Option, count: u32) -> Result> { - let (begin, end) = self.resolve_block_range(start_height, count, self.tip_height()); + let (begin, end) = self.resolve_block_range(start_height, count, self.height()); self.blocks_range(begin, end) } @@ -102,51 +105,45 @@ impl Query { // === Range queries (bulk reads) === /// Build `BlockInfo` rows for `[begin, end)` in descending-height order. - /// Caller must bounds-check `end <= tip + 1`. Returns `Internal` if any - /// bulk read short-returns under per-vec stamp races. + /// `end` is re-clamped to `safe.height` (single snapshot) so two-snapshot + /// tearing under a concurrent reorg cannot short-read past the loop guards. fn blocks_range(&self, begin: usize, end: usize) -> Result> { + let safe = self.safe_lengths(); + let height_len = safe.height.to_usize(); + let tx_index_len = safe.tx_index.to_usize(); + let end = end.min(height_len); if begin >= end { return Ok(Vec::new()); } let indexer = self.indexer(); - let computer = self.computer(); let reader = self.reader(); let count = end - begin; - // Bulk read all indexed data + // Bulk read all indexed data. `end <= safe.height` ⇒ these per-block + // vecs are populated for `[begin, end)`, so short reads are impossible. let blockhashes = indexer.vecs.blocks.blockhash.collect_range_at(begin, end); let difficulties = indexer.vecs.blocks.difficulty.collect_range_at(begin, end); let timestamps = indexer.vecs.blocks.timestamp.collect_range_at(begin, end); let sizes = indexer.vecs.blocks.total.collect_range_at(begin, end); let weights = indexer.vecs.blocks.weight.collect_range_at(begin, end); let positions = indexer.vecs.blocks.position.collect_range_at(begin, end); - if blockhashes.len() != count - || difficulties.len() != count - || timestamps.len() != count - || sizes.len() != count - || weights.len() != count - || positions.len() != count - { - return Err(Error::Internal("blocks_range: short read on per-block vecs")); - } + debug_assert_eq!(blockhashes.len(), count); + debug_assert_eq!(difficulties.len(), count); + debug_assert_eq!(timestamps.len(), count); + debug_assert_eq!(sizes.len(), count); + debug_assert_eq!(weights.len(), count); + debug_assert_eq!(positions.len(), count); - // Bulk read tx indexes for tx_count - let max_height = self.indexed_height(); - let tx_index_end = if end <= max_height.to_usize() { - end + 1 - } else { - end - }; + // Read one past the last block for its tx-count, capped by the snapshot's + // exclusive height bound. Tip block falls back to `tx_index_len` in the loop. + let tx_index_end = (end + 1).min(height_len); let first_tx_indexes: Vec = indexer .vecs .transactions .first_tx_index .collect_range_at(begin, tx_index_end); - if first_tx_indexes.len() < count { - return Err(Error::Internal("blocks_range: short read on first_tx_index")); - } - let total_txs = computer.indexes.tx_index.identity.len(); + debug_assert!(first_tx_indexes.len() >= count); // Bulk read median time window let median_start = begin.saturating_sub(10); @@ -155,9 +152,7 @@ impl Query { .blocks .timestamp .collect_range_at(median_start, end); - if median_timestamps.len() != end - median_start { - return Err(Error::Internal("blocks_range: short read on median window")); - } + debug_assert_eq!(median_timestamps.len(), end - median_start); let mut blocks = Vec::with_capacity(count); @@ -168,7 +163,7 @@ impl Query { let tx_count = if i + 1 < first_tx_indexes.len() { (first_tx_indexes[i + 1].to_usize() - first_tx_indexes[i].to_usize()) as u32 } else { - (total_txs - first_tx_indexes[i].to_usize()) as u32 + (tx_index_len - first_tx_indexes[i].to_usize()) as u32 }; let median_time = @@ -195,9 +190,15 @@ impl Query { } /// Build `BlockInfoV1` rows for `[begin, end)` in descending-height order. - /// Caller must bounds-check `end <= min(indexed, computed) + 1`. Returns - /// `Internal` on bulk-read short returns or per-block header read failures. + /// `end` is re-clamped to `bound.height` (single snapshot covering both + /// indexer-stamped and computer-stamped vecs, since `safe_lengths` only + /// advances after compute). Returns `Internal` on per-block header read + /// failures. pub(crate) fn blocks_v1_range(&self, begin: usize, end: usize) -> Result> { + let safe = self.safe_lengths(); + let height_len = safe.height.to_usize(); + let tx_index_len = safe.tx_index.to_usize(); + let end = end.min(height_len); if begin >= end { return Ok(Vec::new()); } @@ -217,19 +218,14 @@ impl Query { let positions = indexer.vecs.blocks.position.collect_range_at(begin, end); let pool_slugs = computer.pools.pool.collect_range_at(begin, end); - // Bulk read tx indexes - let max_height = self.indexed_height(); - let tx_index_end = if end <= max_height.to_usize() { - end + 1 - } else { - end - }; + // Read one past the last block for its tx-count, capped by the snapshot's + // exclusive height bound. Tip block falls back to `tx_index_len` in the loop. + let tx_index_end = (end + 1).min(height_len); let first_tx_indexes: Vec = indexer .vecs .transactions .first_tx_index .collect_range_at(begin, tx_index_end); - let total_txs = computer.indexes.tx_index.identity.len(); // Bulk read segwit stats let segwit_txs = indexer.vecs.blocks.segwit_txs.collect_range_at(begin, end); @@ -315,49 +311,49 @@ impl Query { .timestamp .collect_range_at(median_start, end); - let per_block_lens = [ - blockhashes.len(), - difficulties.len(), - timestamps.len(), - sizes.len(), - weights.len(), - positions.len(), - pool_slugs.len(), - segwit_txs.len(), - segwit_sizes.len(), - segwit_weights.len(), - fee_sats.len(), - subsidy_sats.len(), - input_counts.len(), - output_counts.len(), - utxo_set_sizes.len(), - input_volumes.len(), - prices.len(), - output_volumes.len(), - fr_min.len(), - fr_pct10.len(), - fr_pct25.len(), - fr_median.len(), - fr_pct75.len(), - fr_pct90.len(), - fr_max.len(), - fa_min.len(), - fa_pct10.len(), - fa_pct25.len(), - fa_median.len(), - fa_pct75.len(), - fa_pct90.len(), - fa_max.len(), - ]; - if per_block_lens.iter().any(|&l| l != count) { - return Err(Error::Internal("blocks_v1_range: short read on per-block vecs")); - } - if first_tx_indexes.len() < count { - return Err(Error::Internal("blocks_v1_range: short read on first_tx_index")); - } - if median_timestamps.len() != end - median_start { - return Err(Error::Internal("blocks_v1_range: short read on median window")); - } + // All bulk reads above span `[begin, end)` (or `[median_start, end)`). + // Caller's `end <= bound.height + 1` precondition guarantees populated + // slots, so short reads are impossible. + debug_assert!( + [ + blockhashes.len(), + difficulties.len(), + timestamps.len(), + sizes.len(), + weights.len(), + positions.len(), + pool_slugs.len(), + segwit_txs.len(), + segwit_sizes.len(), + segwit_weights.len(), + fee_sats.len(), + subsidy_sats.len(), + input_counts.len(), + output_counts.len(), + utxo_set_sizes.len(), + input_volumes.len(), + prices.len(), + output_volumes.len(), + fr_min.len(), + fr_pct10.len(), + fr_pct25.len(), + fr_median.len(), + fr_pct75.len(), + fr_pct90.len(), + fr_max.len(), + fa_min.len(), + fa_pct10.len(), + fa_pct25.len(), + fa_median.len(), + fa_pct75.len(), + fa_pct90.len(), + fa_max.len(), + ] + .iter() + .all(|&l| l == count) + ); + debug_assert!(first_tx_indexes.len() >= count); + debug_assert_eq!(median_timestamps.len(), end - median_start); let mut blocks = Vec::with_capacity(count); @@ -365,7 +361,7 @@ impl Query { let tx_count = if i + 1 < first_tx_indexes.len() { (first_tx_indexes[i + 1].to_usize() - first_tx_indexes[i].to_usize()) as u32 } else { - (total_txs - first_tx_indexes[i].to_usize()) as u32 + (tx_index_len - first_tx_indexes[i].to_usize()) as u32 }; // Single reader for header + coinbase (adjacent in blk file). @@ -503,10 +499,6 @@ impl Query { // === Helper methods === - pub fn tip_height(&self) -> Height { - Height::from(self.indexer().vecs.blocks.blockhash.len().saturating_sub(1)) - } - /// Hash to height. The prefix store keys on the first 8 bytes of /// the hash, so the resolved height is verified against the full /// `blockhash[height]` before being returned. Prefix collisions @@ -545,9 +537,8 @@ impl Query { /// `(begin, end)` half-open window of up to `count` blocks ending /// at `start_height` (default `cap`), clamped to `[0, cap]`. Caller - /// supplies `cap`: `tip_height()` when reading indexer-only series, - /// `height() = min(indexed, computed)` when reading computer-stamped - /// series too. + /// supplies `cap`: typically [`Query::height`] (the highest fully-written + /// height per the safe-lengths snapshot). fn resolve_block_range( &self, start_height: Option, @@ -668,10 +659,11 @@ impl Query { /// but accepts coinbase parse failures (they manifest as missing /// `extras` rather than a 5xx). fn parse_coinbase_from_read(reader: impl Read) -> Coinbase { - let tx = match bitcoin::Transaction::consensus_decode(&mut bitcoin::io::FromStd::new(reader)) { - Ok(tx) => tx, - Err(_) => return Coinbase::default(), - }; + let tx = + match bitcoin::Transaction::consensus_decode(&mut bitcoin::io::FromStd::new(reader)) { + Ok(tx) => tx, + Err(_) => return Coinbase::default(), + }; let total_size = tx.total_size(); diff --git a/crates/brk_query/src/impl/block/raw.rs b/crates/brk_query/src/impl/block/raw.rs index cd3c199d2..7a3e26d99 100644 --- a/crates/brk_query/src/impl/block/raw.rs +++ b/crates/brk_query/src/impl/block/raw.rs @@ -11,10 +11,10 @@ impl Query { } fn block_raw_by_height(&self, height: Height) -> Result> { - let max_height = self.tip_height(); - if height > max_height { + let bound = self.safe_lengths().height; + if height >= bound { return Err(Error::OutOfRange( - format!("Block height {height} out of range (tip {max_height})").into(), + format!("Block height {height} out of range (tip {})", self.height()).into(), )); } diff --git a/crates/brk_query/src/impl/block/status.rs b/crates/brk_query/src/impl/block/status.rs index 6699c589c..d1924e4b2 100644 --- a/crates/brk_query/src/impl/block/status.rs +++ b/crates/brk_query/src/impl/block/status.rs @@ -10,13 +10,14 @@ impl Query { } fn block_status_by_height(&self, height: Height) -> Result { - let max_height = self.tip_height(); + let bound = self.safe_lengths().height; - if height > max_height { + if height >= bound { return Ok(BlockStatus::not_in_best_chain()); } - let next_best = if height < max_height { + let tip = self.height(); + let next_best = if height < tip { Some( self.indexer() .vecs diff --git a/crates/brk_query/src/impl/block/timestamp.rs b/crates/brk_query/src/impl/block/timestamp.rs index 8eb4d2452..ed1cda422 100644 --- a/crates/brk_query/src/impl/block/timestamp.rs +++ b/crates/brk_query/src/impl/block/timestamp.rs @@ -1,7 +1,7 @@ use brk_error::{Error, OptionData, Result}; use brk_types::{BlockTimestamp, Date, Day1, Height, Timestamp}; use jiff::Timestamp as JiffTimestamp; -use vecdb::{AnyVec, ReadableVec}; +use vecdb::ReadableVec; use crate::Query; @@ -23,10 +23,10 @@ impl Query { let indexer = self.indexer(); let computer = self.computer(); - if indexer.vecs.blocks.blockhash.len() == 0 { + if self.safe_lengths().height == Height::ZERO { return Err(Error::NotFound("No blocks indexed".into())); } - let tip: usize = self.tip_height().into(); + let tip: usize = self.height().into(); let target = timestamp; let date = Date::from(target); diff --git a/crates/brk_query/src/impl/block/txs.rs b/crates/brk_query/src/impl/block/txs.rs index 5418f88de..234f00cc4 100644 --- a/crates/brk_query/src/impl/block/txs.rs +++ b/crates/brk_query/src/impl/block/txs.rs @@ -8,7 +8,7 @@ use brk_types::{ Weight, }; use rustc_hash::FxHashMap; -use vecdb::{AnyVec, ReadableVec, VecIndex}; +use vecdb::{ReadableVec, VecIndex}; use crate::Query; @@ -71,9 +71,7 @@ impl Query { .txid .collect_range_at(first, first + tx_count); if txids.len() != tx_count { - return Err(Error::Internal( - "block_txids_by_height: short txid read", - )); + return Err(Error::Internal("block_txids_by_height: short txid read")); } Ok(txids) } @@ -310,28 +308,22 @@ impl Query { /// /// `OutOfRange` when `height` is past the indexed-tip stamp. /// `Internal` when `first_tx_index[height]` is missing under the - /// stamp-before-data race. For the tip block (where - /// `first_tx_index[height+1]` is not yet written), `next` falls back - /// to `txid.len()`. + /// stamp-before-data race. The tip-of-safe block falls back to + /// `safe.tx_index` (not live `txid.len()`, which can be ahead of the + /// writer's stamped boundary mid-block). fn block_tx_range(&self, height: Height) -> Result<(usize, usize)> { - let indexer = self.indexer(); - if height > self.indexed_height() { + let safe = self.safe_lengths(); + if height >= safe.height { return Err(Error::OutOfRange("Block height out of range".into())); } - let first: usize = indexer - .vecs - .transactions - .first_tx_index - .collect_one(height) - .data()? - .into(); - let next: usize = indexer - .vecs - .transactions - .first_tx_index - .collect_one(height.incremented()) - .unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len())) - .into(); + let first_tx_index_vec = &self.indexer().vecs.transactions.first_tx_index; + let first: usize = first_tx_index_vec.collect_one(height).data()?.into(); + let next_height = height.incremented(); + let next: usize = if next_height < safe.height { + first_tx_index_vec.collect_one(next_height).data()?.into() + } else { + safe.tx_index.to_usize() + }; Ok((first, next - first)) } } diff --git a/crates/brk_query/src/impl/cpfp.rs b/crates/brk_query/src/impl/cpfp.rs index cd1f69165..7c66e2810 100644 --- a/crates/brk_query/src/impl/cpfp.rs +++ b/crates/brk_query/src/impl/cpfp.rs @@ -11,12 +11,10 @@ use brk_error::{Error, OptionData, Result}; use brk_mempool::cluster::{Cluster, ClusterNode, LocalIdx}; -use brk_types::{ - CpfpInfo, FeeRate, Height, TxIndex, TxInIndex, Txid, TxidPrefix, VSize, Weight, -}; +use brk_types::{CpfpInfo, FeeRate, Height, TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight}; use rustc_hash::{FxBuildHasher, FxHashMap}; use smallvec::SmallVec; -use vecdb::{AnyVec, ReadableVec, VecIndex}; +use vecdb::{ReadableVec, VecIndex}; use crate::Query; @@ -128,18 +126,15 @@ impl Query { ) -> Result<(Cluster, LocalIdx)> { let indexer = self.indexer(); let computer = self.computer(); - let block_first = indexer - .vecs - .transactions - .first_tx_index - .collect_one(height) - .data()?; - let block_end = indexer - .vecs - .transactions - .first_tx_index - .collect_one(height.incremented()) - .unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len())); + let safe = self.safe_lengths(); + let first_tx_index_vec = &indexer.vecs.transactions.first_tx_index; + let block_first = first_tx_index_vec.collect_one(height).data()?; + let next_height = height.incremented(); + let block_end = if next_height < safe.height { + first_tx_index_vec.collect_one(next_height).data()? + } else { + safe.tx_index + }; let same_block = |idx: TxIndex| idx >= block_first && idx < block_end; let WalkResult { nodes, seed_local } = self.walk_same_block_edges(seed, same_block); @@ -153,7 +148,8 @@ impl Query { .into_iter() .map(|(tx_index, parents)| { let i = tx_index.to_usize(); - let weight = Weight::from_sizes(*base_size.get(i).data()?, *total_size.get(i).data()?); + let weight = + Weight::from_sizes(*base_size.get(i).data()?, *total_size.get(i).data()?); Ok(ClusterNode { id: tx_index, txid: txid_reader.get(i), @@ -189,10 +185,16 @@ impl Query { let mut walk_inputs = |tx: TxIndex| -> SmallVec<[TxIndex; 2]> { let mut out: SmallVec<[TxIndex; 2]> = SmallVec::new(); - let Ok(start) = first_txin.get(tx.to_usize()).data() else { return out }; - let Ok(end) = first_txin.get(tx.to_usize() + 1).data() else { return out }; + let Ok(start) = first_txin.get(tx.to_usize()).data() else { + return out; + }; + let Ok(end) = first_txin.get(tx.to_usize() + 1).data() else { + return out; + }; for i in usize::from(start)..usize::from(end) { - let Ok(op) = outpoint.get(i).data() else { continue }; + let Ok(op) = outpoint.get(i).data() else { + continue; + }; if op.is_coinbase() { continue; } @@ -237,14 +239,22 @@ impl Query { let mut stack: Vec = vec![seed]; let mut descendant_count = 0; 'd: while let Some(cur) = stack.pop() { - let Ok(start) = first_txout.get(cur.to_usize()).data() else { continue }; - let Ok(end) = first_txout.get(cur.to_usize() + 1).data() else { continue }; + let Ok(start) = first_txout.get(cur.to_usize()).data() else { + continue; + }; + let Ok(end) = first_txout.get(cur.to_usize() + 1).data() else { + continue; + }; for i in usize::from(start)..usize::from(end) { - let Ok(txin_idx) = spent.get(i).data() else { continue }; + let Ok(txin_idx) = spent.get(i).data() else { + continue; + }; if txin_idx == TxInIndex::UNSPENT { continue; } - let Ok(child) = spending_tx.get(usize::from(txin_idx)).data() else { continue }; + let Ok(child) = spending_tx.get(usize::from(txin_idx)).data() else { + continue; + }; if local_of.contains_key(&child) || !same_block(child) { continue; } diff --git a/crates/brk_query/src/impl/mempool.rs b/crates/brk_query/src/impl/mempool.rs index 7084355bb..2d439e2c2 100644 --- a/crates/brk_query/src/impl/mempool.rs +++ b/crates/brk_query/src/impl/mempool.rs @@ -67,6 +67,9 @@ impl Query { let indexer = self.indexer(); let stores = &indexer.stores; + let safe = self.safe_lengths(); + let tx_index_len = safe.tx_index; + let txout_index_len = safe.txout_index; let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader(); let output_type_reader = indexer.vecs.outputs.output_type.reader(); let type_index_reader = indexer.vecs.outputs.type_index.reader(); @@ -79,8 +82,15 @@ impl Query { .get(&TxidPrefix::from(prev_txid)) .ok()?? .into_owned(); + if prev_tx_index >= tx_index_len { + return None; + } let first_txout: TxOutIndex = first_txout_index_reader.get(prev_tx_index.to_usize()); - let txout_index = usize::from(first_txout + vout); + let txout = first_txout + vout; + if txout >= txout_index_len { + return None; + } + let txout_index = usize::from(txout); let output_type: OutputType = output_type_reader.get(txout_index); let type_index: TypeIndex = type_index_reader.get(txout_index); let value: Sats = value_reader.get(txout_index); @@ -106,10 +116,7 @@ impl Query { let root_txid = Self::walk_to_replacement_root(&graveyard, *txid); - let replaces_vec: Vec = graveyard - .predecessors_of(txid) - .map(|(p, _)| *p) - .collect(); + let replaces_vec: Vec = graveyard.predecessors_of(txid).map(|(p, _)| *p).collect(); let replaces = (!replaces_vec.is_empty()).then_some(replaces_vec); let replacements = self.build_rbf_node(&root_txid, None, &txs, &entries, &graveyard); @@ -124,9 +131,7 @@ impl Query { /// replacer of an RBF chain. Returns `txid` itself if it's already /// the root. fn walk_to_replacement_root(graveyard: &TxGraveyard, mut root: Txid) -> Txid { - while let Some(TxRemoval::Replaced { by }) = - graveyard.get(&root).map(TxTombstone::reason) - { + while let Some(TxRemoval::Replaced { by }) = graveyard.get(&root).map(TxTombstone::reason) { root = *by; } root diff --git a/crates/brk_query/src/impl/mining/difficulty.rs b/crates/brk_query/src/impl/mining/difficulty.rs index 40b667efd..ae5e22332 100644 --- a/crates/brk_query/src/impl/mining/difficulty.rs +++ b/crates/brk_query/src/impl/mining/difficulty.rs @@ -62,8 +62,7 @@ impl Query { // Bitcoin block timestamps can step backward within MTP rules, so // saturate the subtraction to avoid u32 underflow on a backwards-going // first block of an epoch. - let elapsed_time = - u64::from((*current_timestamp).saturating_sub(*epoch_start_timestamp)); + let elapsed_time = u64::from((*current_timestamp).saturating_sub(*epoch_start_timestamp)); let time_avg = if blocks_into_epoch > 0 { elapsed_time / blocks_into_epoch as u64 } else { diff --git a/crates/brk_query/src/impl/mining/epochs.rs b/crates/brk_query/src/impl/mining/epochs.rs index c9f0e2716..5b61f163d 100644 --- a/crates/brk_query/src/impl/mining/epochs.rs +++ b/crates/brk_query/src/impl/mining/epochs.rs @@ -49,9 +49,9 @@ pub(super) fn iter_difficulty_epochs( // Epochs that start before the window are skipped; we still record // their difficulty so the next in-window entry can compute its ratio. if epoch_height.to_usize() < start_height { - prev_difficulty = Some(*difficulty_cursor.get(epoch_usize).ok_or( - Error::Internal("iter_difficulty_epochs: missing pre-window epoch difficulty"), - )?); + prev_difficulty = Some(*difficulty_cursor.get(epoch_usize).ok_or(Error::Internal( + "iter_difficulty_epochs: missing pre-window epoch difficulty", + ))?); continue; } diff --git a/crates/brk_query/src/impl/mining/pools.rs b/crates/brk_query/src/impl/mining/pools.rs index 9d4a462c4..4777fbe86 100644 --- a/crates/brk_query/src/impl/mining/pools.rs +++ b/crates/brk_query/src/impl/mining/pools.rs @@ -194,7 +194,15 @@ impl Query { let estimated_hashrate = (share_24h * network_hr as f64) as u128; let total_reward = if let Some(major) = computer.pools.major.get(&slug) { - Some(major.rewards.cumulative.sats.height.collect_one(current_height).data()?) + Some( + major + .rewards + .cumulative + .sats + .height + .collect_one(current_height) + .data()?, + ) } else { None }; @@ -232,7 +240,7 @@ impl Query { let computer = self.computer(); let tip = self.height().to_usize(); let upper = before_height.map(|h| h.to_usize()).unwrap_or(tip); - let end = upper.min(computer.pools.pool.len().saturating_sub(1)); + let end = upper.min(tip); let heights: Vec = computer .pools diff --git a/crates/brk_query/src/impl/series.rs b/crates/brk_query/src/impl/series.rs index 5ebe87167..642d6404f 100644 --- a/crates/brk_query/src/impl/series.rs +++ b/crates/brk_query/src/impl/series.rs @@ -102,11 +102,7 @@ impl Query { Ok(csv) } - fn get_vec( - &self, - series: &SeriesName, - index: Index, - ) -> Result<&'static dyn AnyExportableVec> { + fn get_vec(&self, series: &SeriesName, index: Index) -> Result<&'static dyn AnyExportableVec> { self.vecs() .get(series, index) .ok_or_else(|| self.series_not_found_error(series)) @@ -158,7 +154,11 @@ impl Query { let resolve_bound = |ri: RangeIndex, fallback: usize| -> Result { let i = self.range_index_to_i64(ri, index)?; - Ok(vecs.iter().map(|v| v.i64_to_usize(i)).min().unwrap_or(fallback)) + Ok(vecs + .iter() + .map(|v| v.i64_to_usize(i)) + .min() + .unwrap_or(fallback)) }; let start = match params.start() { @@ -186,7 +186,7 @@ impl Query { // Snapshot tip-derived state together so the historical-branch ETag stays // self-consistent: stable_count is computed from tip_height, hash_prefix // is the live tip. - let tip_height = self.indexed_height(); + let tip_height = self.height(); let hash_prefix = self.tip_hash_prefix(); let stable_count = self.stable_count(params.index, total, tip_height); @@ -213,12 +213,7 @@ impl Query { /// its live tail as stable. /// - Mutable (Funded/Empty addr): `None`. No immutable region exists, so /// the caller must use the tip-bound ETag for every range. - pub fn stable_count( - &self, - index: Index, - total: usize, - tip_height: Height, - ) -> Option { + pub fn stable_count(&self, index: Index, total: usize, tip_height: Height) -> Option { match index.cache_class() { CacheClass::Bucket { margin } => Some(total.saturating_sub(margin)), CacheClass::Entity => { @@ -232,13 +227,27 @@ impl Query { fn entity_index_at(&self, index: Index, h: Height) -> Option { let v = &self.indexer().vecs; match index { - Index::TxIndex => v.transactions.first_tx_index.collect_one(h).map(usize::from), + Index::TxIndex => v + .transactions + .first_tx_index + .collect_one(h) + .map(usize::from), Index::TxInIndex => v.inputs.first_txin_index.collect_one(h).map(usize::from), Index::TxOutIndex => v.outputs.first_txout_index.collect_one(h).map(usize::from), Index::EmptyOutputIndex => v.scripts.empty.first_index.collect_one(h).map(usize::from), - Index::OpReturnIndex => v.scripts.op_return.first_index.collect_one(h).map(usize::from), + Index::OpReturnIndex => v + .scripts + .op_return + .first_index + .collect_one(h) + .map(usize::from), Index::P2MSOutputIndex => v.scripts.p2ms.first_index.collect_one(h).map(usize::from), - Index::UnknownOutputIndex => v.scripts.unknown.first_index.collect_one(h).map(usize::from), + Index::UnknownOutputIndex => v + .scripts + .unknown + .first_index + .collect_one(h) + .map(usize::from), Index::P2AAddrIndex => v.addrs.p2a.first_index.collect_one(h).map(usize::from), Index::P2PK33AddrIndex => v.addrs.p2pk33.first_index.collect_one(h).map(usize::from), Index::P2PK65AddrIndex => v.addrs.p2pk65.first_index.collect_one(h).map(usize::from), diff --git a/crates/brk_query/src/impl/tx.rs b/crates/brk_query/src/impl/tx.rs index 0903b7580..049be6828 100644 --- a/crates/brk_query/src/impl/tx.rs +++ b/crates/brk_query/src/impl/tx.rs @@ -7,7 +7,7 @@ use brk_types::{ BlockHash, Height, MerkleProof, Timestamp, Transaction, TxInIndex, TxIndex, TxOutIndex, TxOutspend, TxStatus, Txid, TxidPrefix, Vin, Vout, }; -use vecdb::{AnyVec, ReadableVec, VecIndex}; +use vecdb::{ReadableVec, VecIndex}; use crate::Query; @@ -15,6 +15,10 @@ impl Query { // ── Txid → TxIndex resolution (single source of truth) ───────── /// Resolve a txid to its internal TxIndex via prefix lookup. + /// Raw store hit — caller should prefer [`Self::resolve_tx_index_bounded`] + /// when subsequent reads dereference indexer/computer vecs by `tx_index`. + /// Use this raw form only for "is this mined?" probes that don't deref + /// derived data (mempool merge, cpfp fee-rate fall-through). #[inline] pub(crate) fn resolve_tx_index(&self, txid: &Txid) -> Result { self.indexer() @@ -25,7 +29,23 @@ impl Query { .ok_or(Error::UnknownTxid) } + /// `resolve_tx_index` clamped against the safe-lengths snapshot. + /// Returns `UnknownTxid` for tx_indices the store knows but the snapshot + /// has not yet covered. Use this from any path that will subsequently + /// dereference indexer/computer vecs by `tx_index`. + #[inline] + pub(crate) fn resolve_tx_index_bounded(&self, txid: &Txid) -> Result { + let tx_index = self.resolve_tx_index(txid)?; + if tx_index >= self.safe_lengths().tx_index { + return Err(Error::UnknownTxid); + } + Ok(tx_index) + } + pub fn txid_by_index(&self, index: TxIndex) -> Result { + if index >= self.safe_lengths().tx_index { + return Err(Error::OutOfRange("Transaction index out of range".into())); + } self.indexer() .vecs .transactions @@ -36,7 +56,7 @@ impl Query { /// Resolve a txid to (TxIndex, Height). pub fn resolve_tx(&self, txid: &Txid) -> Result<(TxIndex, Height)> { - let tx_index = self.resolve_tx_index(txid)?; + let tx_index = self.resolve_tx_index_bounded(txid)?; let height = self.confirmed_status_height(tx_index)?; Ok((tx_index, height)) } @@ -44,8 +64,14 @@ impl Query { // ── TxStatus construction (single source of truth) ───────────── /// Height for a confirmed tx_index via in-memory TxHeights lookup. + /// Bounded against the safe-lengths snapshot so rejected tx_indices + /// never dereference slots a concurrent writer might be populating. #[inline] pub(crate) fn confirmed_status_height(&self, tx_index: TxIndex) -> Result { + let bound = self.safe_lengths(); + if tx_index >= bound.tx_index { + return Err(Error::UnknownTxid); + } self.computer() .indexes .tx_heights @@ -81,7 +107,7 @@ impl Query { if let Some(tx) = self.map_mempool_tx(txid, Transaction::clone) { return Ok(tx); } - self.transaction_by_index(self.resolve_tx_index(txid)?) + self.transaction_by_index(self.resolve_tx_index_bounded(txid)?) } pub fn transaction_status(&self, txid: &Txid) -> Result { @@ -96,14 +122,14 @@ impl Query { if let Some(bytes) = self.map_mempool_tx(txid, Transaction::encode_bytes) { return Ok(bytes); } - self.transaction_raw_by_index(self.resolve_tx_index(txid)?) + self.transaction_raw_by_index(self.resolve_tx_index_bounded(txid)?) } pub fn transaction_hex(&self, txid: &Txid) -> Result { if let Some(hex) = self.map_mempool_tx(txid, |tx| tx.encode_bytes().to_lower_hex_string()) { return Ok(hex); } - self.transaction_hex_by_index(self.resolve_tx_index(txid)?) + self.transaction_hex_by_index(self.resolve_tx_index_bounded(txid)?) } // ── Outspend queries ─────────────────────────────────────────── @@ -142,8 +168,7 @@ impl Query { } fn mempool_outspend(&self, txid: &Txid, vout: Vout) -> TxOutspend { - let Some((spender_txid, vin)) = - self.mempool().and_then(|m| m.lookup_spender(txid, vout)) + let Some((spender_txid, vin)) = self.mempool().and_then(|m| m.lookup_spender(txid, vout)) else { return TxOutspend::UNSPENT; }; @@ -187,6 +212,8 @@ impl Query { let mut input_tx_cursor = indexer.vecs.inputs.tx_index.cursor(); let mut first_txin_cursor = indexer.vecs.transactions.first_txin_index.cursor(); + let bound = self.safe_lengths(); + let mut cached_status: Option<(Height, BlockHash, Timestamp)> = None; let mut outspends = Vec::with_capacity(output_count); for i in 0..output_count { @@ -198,6 +225,10 @@ impl Query { } let spending_tx_index = input_tx_cursor.get(usize::from(txin_index)).data()?; + if spending_tx_index >= bound.tx_index { + outspends.push(TxOutspend::UNSPENT); + continue; + } let spending_first_txin = first_txin_cursor.get(spending_tx_index.to_usize()).data()?; let vin = Vin::from(usize::from(txin_index) - usize::from(spending_first_txin)); let spending_txid = txid_reader.get(spending_tx_index.to_usize()); @@ -258,16 +289,23 @@ impl Query { } /// Resolve txid to (tx_index, first_txout_index, output_count). + /// Snapshots `safe_lengths` once and uses `safe.txout_index` as the + /// upper bound for the tip-of-safe tx, so the fallback never reads past + /// the writer's stamped boundary (`vecs.outputs.value.len()` can be + /// ahead of `safe.txout_index` when the writer is mid-block). fn resolve_tx_outputs(&self, txid: &Txid) -> Result<(TxIndex, TxOutIndex, usize)> { + let safe = self.safe_lengths(); let tx_index = self.resolve_tx_index(txid)?; - let indexer = self.indexer(); - let first_txout_vec = &indexer.vecs.transactions.first_txout_index; + if tx_index >= safe.tx_index { + return Err(Error::UnknownTxid); + } + let first_txout_vec = &self.indexer().vecs.transactions.first_txout_index; let first = first_txout_vec.read_once(tx_index)?; let next_tx = tx_index.incremented(); - let next = if next_tx.to_usize() < first_txout_vec.len() { + let next = if next_tx < safe.tx_index { first_txout_vec.read_once(next_tx)? } else { - TxOutIndex::from(indexer.vecs.outputs.value.len()) + safe.txout_index }; Ok((tx_index, first, usize::from(next) - usize::from(first))) } diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index 2136af468..10742b0db 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -5,7 +5,7 @@ use std::{path::Path, sync::Arc}; use brk_computer::Computer; use brk_error::{OptionData, Result}; -use brk_indexer::Indexer; +use brk_indexer::{Indexer, Lengths}; use brk_mempool::Mempool; use brk_reader::Reader; use brk_rpc::Client; @@ -57,19 +57,24 @@ impl Query { })) } - /// Current indexed height - pub fn indexed_height(&self) -> Height { - self.indexer().indexed_height() - } - - /// Current computed height (series) - pub fn computed_height(&self) -> Height { - self.computer().computed_height() - } - - /// Minimum of indexed and computed heights + /// Pipeline-safe ceiling: the highest height for which both the + /// indexer and computer have committed durable data. Backed by + /// `Indexer::safe_lengths()`, advanced by `main.rs` after each + /// compute pass and lowered before any rollback. + /// + /// Returns a height (the last fully-written block), not a length. + /// `safe_lengths().height` is a count: `N` means heights `0..N` are + /// committed, so the highest is `N-1`. Pre-genesis (`N == 0`) falls + /// back to `Height::default()` and clients treat it as "nothing + /// indexed yet". pub fn height(&self) -> Height { - self.indexed_height().min(self.computed_height()) + self.safe_lengths().height.decremented().unwrap_or_default() + } + + /// Snapshot of the pipeline-safe `Lengths`. Hot paths that need + /// multiple bound fields should call this once at entry and reuse. + pub(crate) fn safe_lengths(&self) -> Lengths { + self.indexer().safe_lengths() } /// Tip block hash, cached in the indexer. @@ -84,17 +89,20 @@ impl Query { BlockHashPrefix::from(&self.tip_blockhash()) } - /// Build sync status with the given tip height + /// Build sync status with the given tip height. `indexed_height` and + /// `computed_height` reflect live per-vec stamps (diagnostic) and may be + /// briefly ahead of fully-flushed data; the timestamp data read uses the + /// safe-lengths-derived height so it never outruns committed bytes. pub fn sync_status(&self, tip_height: Height) -> Result { - let indexed_height = self.indexed_height(); - let computed_height = self.computed_height(); + let indexed_height = self.indexer().indexed_height(); + let computed_height = self.computer().computed_height(); let blocks_behind = Height::from(tip_height.saturating_sub(*indexed_height)); let last_indexed_at_unix = self .indexer() .vecs .blocks .timestamp - .collect_one(indexed_height) + .collect_one(self.height()) .data()?; Ok(SyncStatus { diff --git a/crates/brk_query/src/vecs.rs b/crates/brk_query/src/vecs.rs index f90211587..3f9225f4b 100644 --- a/crates/brk_query/src/vecs.rs +++ b/crates/brk_query/src/vecs.rs @@ -135,7 +135,9 @@ impl<'a> Vecs<'a> { } pub fn series_info(&self, series: &SeriesName) -> Option { - let index_to_vec = self.series_to_index_to_vec.get(series.normalize().as_ref())?; + let index_to_vec = self + .series_to_index_to_vec + .get(series.normalize().as_ref())?; let value_type = index_to_vec.values().next()?.value_type_to_string(); let indexes = index_to_vec.keys().copied().collect(); Some(SeriesInfo { @@ -191,7 +193,10 @@ impl<'a> Builder<'a> { .entry(name) .or_default() .insert(index, vec); - assert!(prev.is_none(), "Duplicate series: {name} for index {index:?}"); + assert!( + prev.is_none(), + "Duplicate series: {name} for index {index:?}" + ); self.index_to_series_to_vec .entry(index) diff --git a/crates/brk_rpc/src/methods.rs b/crates/brk_rpc/src/methods.rs index 1475dbcc6..11103b993 100644 --- a/crates/brk_rpc/src/methods.rs +++ b/crates/brk_rpc/src/methods.rs @@ -297,8 +297,7 @@ impl Client { Ok(raw) => { out.insert(*txid, raw); } - Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc))) - if rpc.code == RPC_NOT_FOUND => {} + Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc))) if rpc.code == RPC_NOT_FOUND => {} Err(e) => { debug!(txid = %txid, error = %e, "getrawtransaction batch: item failed") } diff --git a/crates/brk_server/src/api/blocks.rs b/crates/brk_server/src/api/blocks.rs index 10f24ccc4..43421caa1 100644 --- a/crates/brk_server/src/api/blocks.rs +++ b/crates/brk_server/src/api/blocks.rs @@ -191,7 +191,7 @@ impl BlockRoutes for ApiRouter { "/api/blocks/tip/height", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.respond_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.indexed_height().to_string())).await + state.respond_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.height().to_string())).await }, |op| { op.id("get_block_tip_height") diff --git a/crates/brk_server/src/api/series_legacy.rs b/crates/brk_server/src/api/series_legacy.rs index 61c10416c..4ef3166b3 100644 --- a/crates/brk_server/src/api/series_legacy.rs +++ b/crates/brk_server/src/api/series_legacy.rs @@ -42,8 +42,7 @@ pub async fn handler( Query(params): Query, State(state): State, ) -> Result { - let mut response = - super::series::serve(state, uri, headers, params, legacy_bytes).await?; + let mut response = super::series::serve(state, uri, headers, params, legacy_bytes).await?; if response.status() == StatusCode::OK { response.headers_mut().insert_deprecation(SUNSET); } diff --git a/crates/brk_server/src/api/server.rs b/crates/brk_server/src/api/server.rs index 9ddf50586..f5941adaf 100644 --- a/crates/brk_server/src/api/server.rs +++ b/crates/brk_server/src/api/server.rs @@ -33,7 +33,7 @@ impl ServerRoutes for ApiRouter { let tip_height = q .client() .get_last_height() - .unwrap_or(q.indexed_height()); + .unwrap_or(q.height()); q.sync_status(tip_height) }) .await diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index e6684d641..cf18becc3 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -67,8 +67,7 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); /// like `; charset=utf-8`. Used to skip JSON-error rewriting for already-JSON bodies. fn is_json_content_type(s: &str) -> bool { let mime = s.split(';').next().unwrap_or("").trim(); - mime == "application/json" - || (mime.starts_with("application/") && mime.ends_with("+json")) + mime == "application/json" || (mime.starts_with("application/") && mime.ends_with("+json")) } pub struct Server(AppState); @@ -300,7 +299,11 @@ impl Server { // NormalizePath must wrap the router (not be a layer) to run before route matching let app = NormalizePathLayer::trim_trailing_slash().layer(router); - serve(listener, ServiceExt::>::into_make_service(app)).await?; + serve( + listener, + ServiceExt::>::into_make_service(app), + ) + .await?; Ok(()) } @@ -336,7 +339,9 @@ mod tests { assert!(is_json_content_type("application/json; charset=utf-8")); assert!(is_json_content_type(" application/json ")); assert!(is_json_content_type("application/problem+json")); - assert!(is_json_content_type("application/vnd.api+json; charset=utf-8")); + assert!(is_json_content_type( + "application/vnd.api+json; charset=utf-8" + )); } #[test] diff --git a/crates/brk_server/src/params/txids_param.rs b/crates/brk_server/src/params/txids_param.rs index c51532ce2..b46826198 100644 --- a/crates/brk_server/src/params/txids_param.rs +++ b/crates/brk_server/src/params/txids_param.rs @@ -85,7 +85,13 @@ mod tests { #[test] fn parses_single_and_multi() { - assert_eq!(TxidsParam::from_query(&format!("txId[]={T1}")).unwrap().txids.len(), 1); + assert_eq!( + TxidsParam::from_query(&format!("txId[]={T1}")) + .unwrap() + .txids + .len(), + 1 + ); assert_eq!( TxidsParam::from_query(&format!("txId%5B%5D={T1}&txId[]={T2}")) .unwrap() diff --git a/crates/brk_server/src/state.rs b/crates/brk_server/src/state.rs index 7edf51026..befc84598 100644 --- a/crates/brk_server/src/state.rs +++ b/crates/brk_server/src/state.rs @@ -53,7 +53,7 @@ impl AppState { /// date's day, then settle once the tip crosses the day boundary. pub fn date_strategy(&self, version: Version, date: Date) -> CacheStrategy { self.sync(|q| { - let height = q.indexed_height(); + let height = q.height(); q.indexer() .vecs .blocks @@ -76,9 +76,7 @@ impl AppState { /// - Unknown address → `Tip` pub fn addr_strategy(&self, version: Version, addr: &Addr, chain_only: bool) -> CacheStrategy { self.sync(|q| { - if !chain_only - && let Some(mempool_hash) = q.addr_mempool_hash(addr) - { + if !chain_only && let Some(mempool_hash) = q.addr_mempool_hash(addr) { return CacheStrategy::MempoolHash(mempool_hash); } q.addr_last_activity_height(addr) diff --git a/crates/brk_types/src/cohort.rs b/crates/brk_types/src/cohort.rs index 8b007140b..115381dd7 100644 --- a/crates/brk_types/src/cohort.rs +++ b/crates/brk_types/src/cohort.rs @@ -24,7 +24,11 @@ impl Cohort { /// Returns `Some(Cohort)` iff `s` is non-empty ASCII `[a-z0-9_]+`. pub fn new(s: impl Into) -> Option { let s = s.into(); - if s.is_empty() || !s.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') { + if s.is_empty() + || !s + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + { return None; } Some(Self(s)) diff --git a/crates/brk_types/src/indexes.rs b/crates/brk_types/src/indexes.rs deleted file mode 100644 index 04614747e..000000000 --- a/crates/brk_types/src/indexes.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::{ - EmptyOutputIndex, Height, OpReturnIndex, OutputType, P2AAddrIndex, P2MSOutputIndex, - P2PK33AddrIndex, P2PK65AddrIndex, P2PKHAddrIndex, P2SHAddrIndex, P2TRAddrIndex, - P2WPKHAddrIndex, P2WSHAddrIndex, TxInIndex, TxIndex, TxOutIndex, TypeIndex, UnknownOutputIndex, -}; - -/// Blockchain-level indexes tracking current positions for various entity types. -/// Used by brk_indexer during block processing. -#[derive(Debug, Default, Clone)] -pub struct Indexes { - pub empty_output_index: EmptyOutputIndex, - pub height: Height, - pub op_return_index: OpReturnIndex, - pub p2ms_output_index: P2MSOutputIndex, - pub p2pk33_addr_index: P2PK33AddrIndex, - pub p2pk65_addr_index: P2PK65AddrIndex, - pub p2pkh_addr_index: P2PKHAddrIndex, - pub p2sh_addr_index: P2SHAddrIndex, - pub p2tr_addr_index: P2TRAddrIndex, - pub p2wpkh_addr_index: P2WPKHAddrIndex, - pub p2wsh_addr_index: P2WSHAddrIndex, - pub p2a_addr_index: P2AAddrIndex, - pub tx_index: TxIndex, - pub txin_index: TxInIndex, - pub txout_index: TxOutIndex, - pub unknown_output_index: UnknownOutputIndex, -} - -impl Indexes { - pub fn to_type_index(&self, output_type: OutputType) -> TypeIndex { - match output_type { - OutputType::Empty => *self.empty_output_index, - OutputType::OpReturn => *self.op_return_index, - OutputType::P2A => *self.p2a_addr_index, - OutputType::P2MS => *self.p2ms_output_index, - OutputType::P2PK33 => *self.p2pk33_addr_index, - OutputType::P2PK65 => *self.p2pk65_addr_index, - OutputType::P2PKH => *self.p2pkh_addr_index, - OutputType::P2SH => *self.p2sh_addr_index, - OutputType::P2TR => *self.p2tr_addr_index, - OutputType::P2WPKH => *self.p2wpkh_addr_index, - OutputType::P2WSH => *self.p2wsh_addr_index, - OutputType::Unknown => *self.unknown_output_index, - } - } - - /// Increments the address index for the given address type and returns the previous value. - /// Only call this for address types (P2PK65, P2PK33, P2PKH, P2SH, P2WPKH, P2WSH, P2TR, P2A). - #[inline] - pub fn increment_addr_index(&mut self, addr_type: OutputType) -> TypeIndex { - match addr_type { - OutputType::P2PK65 => self.p2pk65_addr_index.copy_then_increment(), - OutputType::P2PK33 => self.p2pk33_addr_index.copy_then_increment(), - OutputType::P2PKH => self.p2pkh_addr_index.copy_then_increment(), - OutputType::P2SH => self.p2sh_addr_index.copy_then_increment(), - OutputType::P2WPKH => self.p2wpkh_addr_index.copy_then_increment(), - OutputType::P2WSH => self.p2wsh_addr_index.copy_then_increment(), - OutputType::P2TR => self.p2tr_addr_index.copy_then_increment(), - OutputType::P2A => self.p2a_addr_index.copy_then_increment(), - _ => unreachable!(), - } - } -} diff --git a/crates/brk_types/src/lib.rs b/crates/brk_types/src/lib.rs index fb5d2ce95..2cc13d60b 100644 --- a/crates/brk_types/src/lib.rs +++ b/crates/brk_types/src/lib.rs @@ -79,7 +79,6 @@ mod hour12; mod hour4; mod index; mod index_info; -mod indexes; mod limit; mod mempool_block; mod mempool_entry_info; @@ -136,7 +135,6 @@ mod reward_stats; mod sats; mod sats_fract; mod sats_signed; -mod sigops; mod search_query; mod series_count; mod series_data; @@ -148,6 +146,7 @@ mod series_output; mod series_paginated; mod series_selection; mod series_selection_legacy; +mod sigops; mod stored_bool; mod stored_f32; mod stored_f64; @@ -273,7 +272,6 @@ pub use hour4::*; pub use hour12::*; pub use index::*; pub use index_info::*; -pub use indexes::*; pub use limit::*; pub use mempool_block::*; pub use mempool_entry_info::*; @@ -330,7 +328,6 @@ pub use reward_stats::*; pub use sats::*; pub use sats_fract::*; pub use sats_signed::*; -pub use sigops::*; pub use search_query::*; pub use series_count::*; pub use series_data::*; @@ -342,6 +339,7 @@ pub use series_output::*; pub use series_paginated::*; pub use series_selection::*; pub use series_selection_legacy::*; +pub use sigops::*; pub use stored_bool::*; pub use stored_f32::*; pub use stored_f64::*; diff --git a/crates/brk_types/src/sigops.rs b/crates/brk_types/src/sigops.rs index 5fe17cc92..c9521a31c 100644 --- a/crates/brk_types/src/sigops.rs +++ b/crates/brk_types/src/sigops.rs @@ -1,9 +1,8 @@ -use bitcoin::{Amount, OutPoint, ScriptBuf, TxOut}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use vecdb::{Formattable, Pco}; -use crate::{OutputType, VSize}; +use crate::VSize; /// BIP-141 sigop cost. The block-level budget is 80,000, so a `u32` /// fits a single tx's count with room to spare. @@ -53,58 +52,6 @@ impl SigOps { pub fn adjust_vsize(self, vsize: VSize) -> VSize { vsize.max(self.vsize_cost()) } - - /// BIP-141 sigop cost using only each input's prevout `OutputType` as - /// hint. Avoids reading the real `script_pubkey`: bitcoin-rs's sigop - /// walk only inspects script *structure* (`is_p2sh` / `is_p2wpkh` / - /// `is_p2wsh`), so a canonical empty-hash script of the matching shape - /// produces the same count as the real one. Other output types - /// contribute nothing on the input side, so we return `None` for them. - pub fn of_bitcoin_tx_with_kinds(tx: &bitcoin::Transaction, mut kind_at: F) -> Self - where - F: FnMut(&OutPoint) -> Option, - { - Self::from(tx.total_sigop_cost(|outpoint| { - let script_pubkey = match kind_at(outpoint)? { - OutputType::P2SH => synthetic_p2sh_spk(), - OutputType::P2WPKH => synthetic_p2wpkh_spk(), - OutputType::P2WSH => synthetic_p2wsh_spk(), - _ => return None, - }; - Some(TxOut { - value: Amount::ZERO, - script_pubkey, - }) - })) - } -} - -fn synthetic_p2sh_spk() -> ScriptBuf { - // OP_HASH160 PUSH20 <20 zero bytes> OP_EQUAL - let mut bytes = Vec::with_capacity(23); - bytes.push(0xa9); - bytes.push(0x14); - bytes.extend_from_slice(&[0u8; 20]); - bytes.push(0x87); - ScriptBuf::from_bytes(bytes) -} - -fn synthetic_p2wpkh_spk() -> ScriptBuf { - // OP_0 PUSH20 <20 zero bytes> - let mut bytes = Vec::with_capacity(22); - bytes.push(0x00); - bytes.push(0x14); - bytes.extend_from_slice(&[0u8; 20]); - ScriptBuf::from_bytes(bytes) -} - -fn synthetic_p2wsh_spk() -> ScriptBuf { - // OP_0 PUSH32 <32 zero bytes> - let mut bytes = Vec::with_capacity(34); - bytes.push(0x00); - bytes.push(0x20); - bytes.extend_from_slice(&[0u8; 32]); - ScriptBuf::from_bytes(bytes) } impl From for SigOps {