global: big snapshot part 2

crates/brk_computer/src/internal/block_walker.rs

@@ -0,0 +1,84 @@
+//! Shared per-block-per-type cursor walker used by `outputs/by_type/` and
+//! `inputs/by_type/`. The walker iterates blocks and aggregates the
+//! per-tx output-type counts; pushing into a particular wrapper is left
+//! to the caller.
+
+use brk_error::Result;
+use brk_types::TxIndex;
+use vecdb::VecIndex;
+
+/// Aggregated per-block counters produced by [`walk_blocks`].
+pub(crate) struct BlockAggregate {
+    pub entries_all: u64,
+    pub entries_per_type: [u64; 12],
+    pub txs_all: u64,
+    pub txs_per_type: [u64; 12],
+}
+
+/// Whether to include the coinbase tx (first tx in each block) in the walk.
+#[derive(Clone, Copy)]
+pub(crate) enum CoinbasePolicy {
+    Include,
+    Skip,
+}
+
+/// Walk every block in `fi_batch`, calling `scan_tx` once per tx (which
+/// fills a `[u32; 12]` with the per-output-type count for that tx),
+/// aggregating into a [`BlockAggregate`] and handing it to `store`.
+///
+/// `entries_all` and `txs_all` aggregate over the 12 output types
+/// indistinguishably; downstream consumers can cap to the 11 spendable
+/// types if op_return is non-applicable.
+#[inline]
+pub(crate) fn walk_blocks(
+    fi_batch: &[TxIndex],
+    txid_len: usize,
+    coinbase: CoinbasePolicy,
+    mut scan_tx: impl FnMut(usize, &mut [u32; 12]) -> Result<()>,
+    mut store: impl FnMut(BlockAggregate) -> Result<()>,
+) -> Result<()> {
+    for (j, first_tx) in fi_batch.iter().enumerate() {
+        let fi = first_tx.to_usize();
+        let next_fi = fi_batch
+            .get(j + 1)
+            .map(|v| v.to_usize())
+            .unwrap_or(txid_len);
+
+        let start_tx = match coinbase {
+            CoinbasePolicy::Include => fi,
+            CoinbasePolicy::Skip => fi + 1,
+        };
+
+        let mut entries_per_type = [0u64; 12];
+        let mut txs_per_type = [0u64; 12];
+        let mut entries_all = 0u64;
+        let mut txs_all = 0u64;
+
+        for tx_pos in start_tx..next_fi {
+            let mut per_tx = [0u32; 12];
+            scan_tx(tx_pos, &mut per_tx)?;
+            let mut tx_has_any = false;
+            for (i, &n) in per_tx.iter().enumerate() {
+                if n > 0 {
+                    entries_per_type[i] += u64::from(n);
+                    txs_per_type[i] += 1;
+                    entries_all += u64::from(n);
+                    tx_has_any = true;
+                }
+            }
+            if tx_has_any {
+                txs_all += 1;
+            }
+        }
+
+        store(BlockAggregate {
+            entries_all,
+            entries_per_type,
+            txs_all,
+            txs_per_type,
+        })?;
+    }
+
+    Ok(())
+}
+

crates/brk_computer/src/internal/by_type_counts.rs

@@ -1,125 +0,0 @@
-//! Shared per-block per-address-type counters.
-//!
-//! Used by `outputs/by_type/` (counts outputs per type) and `inputs/by_type/`
-//! (counts inputs per type). Walks each block's tx range, calls a scanner
-//! callback that fills a `[u32; 12]` per-tx counter, and produces two
-//! per-block aggregates in a single pass:
-//!
-//! - `entry_count` — total number of items (outputs / inputs) per type
-//! - `tx_count`    — number of txs that contain at least one item of each type
-
-use brk_cohort::ByAddrType;
-use brk_error::Result;
-use brk_types::{BasisPoints16, Height, Indexes, OutputType, StoredU64, TxIndex};
-use vecdb::{AnyStoredVec, Exit, VecIndex, WritableVec};
-
-use crate::internal::{
-    PerBlockCumulativeRolling, PerBlockFull, PercentCumulativeRolling, RatioU64Bp16,
-};
-
-/// Per-block scan that simultaneously computes:
-/// - `entry_count[type] += per_tx[type]` (sum of items)
-/// - `tx_count[type] += 1 if per_tx[type] > 0` (presence flag)
-///
-/// `scan_tx` is called once per tx with a zeroed `[u32; 12]` buffer that
-/// it must fill with the per-type item count for that tx.
-#[allow(clippy::too_many_arguments)]
-pub(crate) fn compute_by_addr_type_block_counts(
-    entry_count: &mut ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64>>,
-    tx_count: &mut ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64>>,
-    fi_batch: &[TxIndex],
-    txid_len: usize,
-    skip_first_tx: bool,
-    starting_height: Height,
-    exit: &Exit,
-    mut scan_tx: impl FnMut(usize, &mut [u32; 12]) -> Result<()>,
-) -> Result<()> {
-    for (j, first_tx) in fi_batch.iter().enumerate() {
-        let fi = first_tx.to_usize();
-        let next_fi = fi_batch
-            .get(j + 1)
-            .map(|v| v.to_usize())
-            .unwrap_or(txid_len);
-
-        let start_tx = if skip_first_tx { fi + 1 } else { fi };
-
-        let mut entries_per_block = [0u64; 12];
-        let mut txs_per_block = [0u64; 12];
-
-        for tx_pos in start_tx..next_fi {
-            let mut per_tx = [0u32; 12];
-            scan_tx(tx_pos, &mut per_tx)?;
-            for (i, &n) in per_tx.iter().enumerate() {
-                if n > 0 {
-                    entries_per_block[i] += u64::from(n);
-                    txs_per_block[i] += 1;
-                }
-            }
-        }
-
-        for otype in OutputType::ADDR_TYPES {
-            let idx = otype as usize;
-            entry_count
-                .get_mut_unwrap(otype)
-                .block
-                .push(StoredU64::from(entries_per_block[idx]));
-            tx_count
-                .get_mut_unwrap(otype)
-                .block
-                .push(StoredU64::from(txs_per_block[idx]));
-        }
-
-        if entry_count.p2pkh.block.batch_limit_reached() {
-            let _lock = exit.lock();
-            for (_, v) in entry_count.iter_mut() {
-                v.block.write()?;
-            }
-            for (_, v) in tx_count.iter_mut() {
-                v.block.write()?;
-            }
-        }
-    }
-
-    {
-        let _lock = exit.lock();
-        for (_, v) in entry_count.iter_mut() {
-            v.block.write()?;
-        }
-        for (_, v) in tx_count.iter_mut() {
-            v.block.write()?;
-        }
-    }
-
-    for (_, v) in entry_count.iter_mut() {
-        v.compute_rest(starting_height, exit)?;
-    }
-    for (_, v) in tx_count.iter_mut() {
-        v.compute_rest(starting_height, exit)?;
-    }
-
-    Ok(())
-}
-
-/// Compute per-type tx-count percent over total tx count, for all 8 address types.
-pub(crate) fn compute_by_addr_type_tx_percents(
-    tx_count: &ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64>>,
-    tx_percent: &mut ByAddrType<PercentCumulativeRolling<BasisPoints16>>,
-    count_total: &PerBlockFull<StoredU64>,
-    starting_indexes: &Indexes,
-    exit: &Exit,
-) -> Result<()> {
-    for otype in OutputType::ADDR_TYPES {
-        let source = tx_count.get_unwrap(otype);
-        tx_percent
-            .get_mut_unwrap(otype)
-            .compute_binary::<StoredU64, StoredU64, RatioU64Bp16, _, _, _, _>(
-                starting_indexes.height,
-                &source.cumulative.height,
-                &count_total.cumulative.height,
-                source.sum.as_array().map(|w| &w.height),
-                count_total.rolling.sum.as_array().map(|w| &w.height),
-                exit,
-            )?;
-    }
-    Ok(())
-}

crates/brk_computer/src/internal/containers/windows.rs

@@ -11,6 +11,12 @@ pub struct Windows<A> {
 impl<A> Windows<A> {
     pub const SUFFIXES: [&'static str; 4] = ["24h", "1w", "1m", "1y"];
     pub const DAYS: [usize; 4] = [1, 7, 30, 365];
+    pub const SECS: [f64; 4] = [
+        Self::DAYS[0] as f64 * 86400.0,
+        Self::DAYS[1] as f64 * 86400.0,
+        Self::DAYS[2] as f64 * 86400.0,
+        Self::DAYS[3] as f64 * 86400.0,
+    ];
 
     pub fn try_from_fn<E>(
         mut f: impl FnMut(&str) -> std::result::Result<A, E>,

crates/brk_computer/src/internal/mod.rs

@@ -1,6 +1,6 @@
 pub(crate) mod algo;
 mod amount;
-mod by_type_counts;
+mod block_walker;
 mod cache_budget;
 mod containers;
 pub(crate) mod db_utils;
@@ -9,9 +9,10 @@ mod per_block;
 mod per_tx;
 mod traits;
 mod transform;
+mod with_addr_types;
 
 pub(crate) use amount::*;
-pub(crate) use by_type_counts::*;
+pub(crate) use block_walker::*;
 pub(crate) use cache_budget::*;
 pub(crate) use containers::*;
 pub(crate) use indexes::*;
@@ -19,3 +20,4 @@ pub(crate) use per_block::*;
 pub(crate) use per_tx::*;
 pub(crate) use traits::*;
 pub use transform::*;
+pub(crate) use with_addr_types::*;

crates/brk_computer/src/internal/per_block/percent/cumulative_rolling.rs

@@ -5,16 +5,17 @@
 
 use brk_error::Result;
 use brk_traversable::Traversable;
-use brk_types::{Height, Version};
+use brk_types::{BasisPoints16, Height, StoredU64, Version};
 use vecdb::{BinaryTransform, Database, Exit, ReadableVec, Rw, StorageMode, VecValue};
 
 use crate::{
     indexes,
-    internal::{BpsType, PercentPerBlock, PercentRollingWindows},
+    internal::{BpsType, PerBlockCumulativeRolling, PercentPerBlock, PercentRollingWindows, RatioU64Bp16},
 };
 
 #[derive(Traversable)]
 pub struct PercentCumulativeRolling<B: BpsType, M: StorageMode = Rw> {
+    #[traversable(flatten)]
     pub cumulative: PercentPerBlock<B, M>,
     #[traversable(flatten)]
     pub rolling: PercentRollingWindows<B, M>,
@@ -26,26 +27,6 @@ impl<B: BpsType> PercentCumulativeRolling<B> {
         name: &str,
         version: Version,
         indexes: &indexes::Vecs,
-    ) -> Result<Self> {
-        let cumulative =
-            PercentPerBlock::forced_import(db, &format!("{name}_cumulative"), version, indexes)?;
-        let rolling =
-            PercentRollingWindows::forced_import(db, &format!("{name}_sum"), version, indexes)?;
-        Ok(Self {
-            cumulative,
-            rolling,
-        })
-    }
-
-    /// Alternate constructor that uses the same base name for both the
-    /// cumulative `PercentPerBlock` and the `PercentRollingWindows`, relying on
-    /// the window suffix to disambiguate. Useful for preserving legacy disk
-    /// names where the two variants historically shared a prefix.
-    pub(crate) fn forced_import_flat(
-        db: &Database,
-        name: &str,
-        version: Version,
-        indexes: &indexes::Vecs,
     ) -> Result<Self> {
         let cumulative = PercentPerBlock::forced_import(db, name, version, indexes)?;
         let rolling = PercentRollingWindows::forced_import(db, name, version, indexes)?;
@@ -89,3 +70,26 @@ impl<B: BpsType> PercentCumulativeRolling<B> {
         Ok(())
     }
 }
+
+impl PercentCumulativeRolling<BasisPoints16> {
+    /// Derive a percent from two `PerBlockCumulativeRolling<StoredU64>`
+    /// sources (numerator and denominator). Both sources must already have
+    /// their cumulative and rolling sums computed.
+    #[inline]
+    pub(crate) fn compute_count_ratio(
+        &mut self,
+        numerator: &PerBlockCumulativeRolling<StoredU64, StoredU64>,
+        denominator: &PerBlockCumulativeRolling<StoredU64, StoredU64>,
+        starting_height: Height,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.compute_binary::<StoredU64, StoredU64, RatioU64Bp16, _, _, _, _>(
+            starting_height,
+            &numerator.cumulative.height,
+            &denominator.cumulative.height,
+            numerator.sum.as_array().map(|w| &w.height),
+            denominator.sum.as_array().map(|w| &w.height),
+            exit,
+        )
+    }
+}

crates/brk_computer/src/internal/per_block/percent/lazy_cumulative_rolling.rs

@@ -0,0 +1,38 @@
+use brk_traversable::Traversable;
+use brk_types::Version;
+use vecdb::UnaryTransform;
+
+use crate::internal::{
+    BpsType, LazyPercentPerBlock, LazyPercentRollingWindows, PercentCumulativeRolling,
+};
+
+/// Fully lazy variant of `PercentCumulativeRolling` — no stored vecs.
+///
+/// Mirrors the flat shape of `PercentCumulativeRolling`: cumulative and
+/// rolling window fields are both flattened to the same tree level, so
+/// consumers see `{ bps, percent, ratio, _24h, _1w, _1m, _1y }`.
+#[derive(Clone, Traversable)]
+pub struct LazyPercentCumulativeRolling<B: BpsType> {
+    #[traversable(flatten)]
+    pub cumulative: LazyPercentPerBlock<B>,
+    #[traversable(flatten)]
+    pub rolling: LazyPercentRollingWindows<B>,
+}
+
+impl<B: BpsType> LazyPercentCumulativeRolling<B> {
+    /// Derive from a stored `PercentCumulativeRolling` source via a
+    /// BPS-to-BPS unary transform applied to both cumulative and rolling.
+    pub(crate) fn from_source<F: UnaryTransform<B, B>>(
+        name: &str,
+        version: Version,
+        source: &PercentCumulativeRolling<B>,
+    ) -> Self {
+        let cumulative =
+            LazyPercentPerBlock::from_percent::<F>(name, version, &source.cumulative);
+        let rolling = LazyPercentRollingWindows::from_rolling::<F>(name, version, &source.rolling);
+        Self {
+            cumulative,
+            rolling,
+        }
+    }
+}

crates/brk_computer/src/internal/per_block/percent/mod.rs

@@ -1,6 +1,7 @@
 mod base;
 mod cumulative_rolling;
 mod lazy;
+mod lazy_cumulative_rolling;
 mod lazy_windows;
 mod vec;
 mod windows;
@@ -8,6 +9,7 @@ mod windows;
 pub use base::*;
 pub use cumulative_rolling::*;
 pub use lazy::*;
+pub use lazy_cumulative_rolling::*;
 pub use lazy_windows::*;
 pub use vec::*;
 pub use windows::*;

crates/brk_computer/src/internal/with_addr_types.rs

@@ -0,0 +1,246 @@
+//! Generic `all` + per-`AddrType` container, mirrors the `WithSth` pattern
+//! along the address-type axis. Used by every metric that tracks one
+//! aggregate value alongside a per-address-type breakdown.
+
+use brk_cohort::ByAddrType;
+use brk_error::Result;
+use brk_traversable::Traversable;
+use brk_types::{Height, Indexes, Sats, Version};
+use rayon::prelude::*;
+use schemars::JsonSchema;
+use vecdb::{AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, WritableVec};
+
+use crate::{indexes, prices};
+
+use super::{
+    AmountPerBlock, NumericValue, PerBlock, PerBlockCumulativeRolling, WindowStartVec, Windows,
+};
+
+/// `all` aggregate plus per-`AddrType` breakdown.
+#[derive(Clone, Traversable)]
+pub struct WithAddrTypes<T> {
+    pub all: T,
+    #[traversable(flatten)]
+    pub by_addr_type: ByAddrType<T>,
+}
+
+impl<T> WithAddrTypes<PerBlock<T>>
+where
+    T: NumericValue + JsonSchema,
+{
+    pub(crate) fn forced_import(
+        db: &Database,
+        name: &str,
+        version: Version,
+        indexes: &indexes::Vecs,
+    ) -> Result<Self> {
+        let all = PerBlock::forced_import(db, name, version, indexes)?;
+        let by_addr_type = ByAddrType::new_with_name(|type_name| {
+            PerBlock::forced_import(db, &format!("{type_name}_{name}"), version, indexes)
+        })?;
+        Ok(Self { all, by_addr_type })
+    }
+
+    pub(crate) fn min_stateful_len(&self) -> usize {
+        self.by_addr_type
+            .values()
+            .map(|v| v.height.len())
+            .min()
+            .unwrap()
+            .min(self.all.height.len())
+    }
+
+    pub(crate) fn par_iter_height_mut(
+        &mut self,
+    ) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
+        rayon::iter::once(&mut self.all.height as &mut dyn AnyStoredVec).chain(
+            self.by_addr_type
+                .par_values_mut()
+                .map(|v| &mut v.height as &mut dyn AnyStoredVec),
+        )
+    }
+
+    pub(crate) fn reset_height(&mut self) -> Result<()> {
+        self.all.height.reset()?;
+        for v in self.by_addr_type.values_mut() {
+            v.height.reset()?;
+        }
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub(crate) fn push_height<U>(&mut self, total: U, per_type: impl IntoIterator<Item = U>)
+    where
+        U: Into<T>,
+    {
+        self.all.height.push(total.into());
+        for (v, value) in self.by_addr_type.values_mut().zip(per_type) {
+            v.height.push(value.into());
+        }
+    }
+
+    /// Compute `all.height` as the per-block sum of the per-type vecs.
+    pub(crate) fn compute_rest(
+        &mut self,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        let sources: Vec<&EagerVec<PcoVec<Height, T>>> =
+            self.by_addr_type.values().map(|v| &v.height).collect();
+        self.all
+            .height
+            .compute_sum_of_others(starting_indexes.height, &sources, exit)?;
+        Ok(())
+    }
+}
+
+impl<T, C> WithAddrTypes<PerBlockCumulativeRolling<T, C>>
+where
+    T: NumericValue + JsonSchema + Into<C>,
+    C: NumericValue + JsonSchema,
+{
+    pub(crate) fn forced_import(
+        db: &Database,
+        name: &str,
+        version: Version,
+        indexes: &indexes::Vecs,
+        cached_starts: &Windows<&WindowStartVec>,
+    ) -> Result<Self> {
+        let all = PerBlockCumulativeRolling::forced_import(
+            db,
+            name,
+            version,
+            indexes,
+            cached_starts,
+        )?;
+        let by_addr_type = ByAddrType::new_with_name(|type_name| {
+            PerBlockCumulativeRolling::forced_import(
+                db,
+                &format!("{type_name}_{name}"),
+                version,
+                indexes,
+                cached_starts,
+            )
+        })?;
+        Ok(Self { all, by_addr_type })
+    }
+
+    pub(crate) fn min_stateful_len(&self) -> usize {
+        self.by_addr_type
+            .values()
+            .map(|v| v.block.len())
+            .min()
+            .unwrap()
+            .min(self.all.block.len())
+    }
+
+    pub(crate) fn par_iter_height_mut(
+        &mut self,
+    ) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
+        rayon::iter::once(&mut self.all.block as &mut dyn AnyStoredVec).chain(
+            self.by_addr_type
+                .par_values_mut()
+                .map(|v| &mut v.block as &mut dyn AnyStoredVec),
+        )
+    }
+
+    pub(crate) fn reset_height(&mut self) -> Result<()> {
+        self.all.block.reset()?;
+        for v in self.by_addr_type.values_mut() {
+            v.block.reset()?;
+        }
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub(crate) fn push_height<U>(&mut self, total: U, per_type: impl IntoIterator<Item = U>)
+    where
+        U: Into<T>,
+    {
+        self.all.block.push(total.into());
+        for (v, value) in self.by_addr_type.values_mut().zip(per_type) {
+            v.block.push(value.into());
+        }
+    }
+
+    /// Finalize `cumulative` / `sum` / `average` for `all` and every per-type vec.
+    pub(crate) fn compute_rest(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
+        self.all.compute_rest(max_from, exit)?;
+        for v in self.by_addr_type.values_mut() {
+            v.compute_rest(max_from, exit)?;
+        }
+        Ok(())
+    }
+}
+
+impl WithAddrTypes<AmountPerBlock> {
+    pub(crate) fn forced_import(
+        db: &Database,
+        name: &str,
+        version: Version,
+        indexes: &indexes::Vecs,
+    ) -> Result<Self> {
+        let all = AmountPerBlock::forced_import(db, name, version, indexes)?;
+        let by_addr_type = ByAddrType::new_with_name(|type_name| {
+            AmountPerBlock::forced_import(db, &format!("{type_name}_{name}"), version, indexes)
+        })?;
+        Ok(Self { all, by_addr_type })
+    }
+
+    pub(crate) fn min_stateful_len(&self) -> usize {
+        self.by_addr_type
+            .values()
+            .map(|v| v.sats.height.len())
+            .min()
+            .unwrap()
+            .min(self.all.sats.height.len())
+    }
+
+    pub(crate) fn par_iter_height_mut(
+        &mut self,
+    ) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
+        rayon::iter::once(&mut self.all.sats.height as &mut dyn AnyStoredVec).chain(
+            self.by_addr_type
+                .par_values_mut()
+                .map(|v| &mut v.sats.height as &mut dyn AnyStoredVec),
+        )
+    }
+
+    pub(crate) fn reset_height(&mut self) -> Result<()> {
+        self.all.sats.height.reset()?;
+        self.all.cents.height.reset()?;
+        for v in self.by_addr_type.values_mut() {
+            v.sats.height.reset()?;
+            v.cents.height.reset()?;
+        }
+        Ok(())
+    }
+
+    /// Push the stateful sats value for `all` and each per-type. Cents are
+    /// derived post-hoc from sats × price in [`Self::compute_rest`].
+    #[inline(always)]
+    pub(crate) fn push_height<U>(&mut self, total: U, per_type: impl IntoIterator<Item = U>)
+    where
+        U: Into<Sats>,
+    {
+        self.all.sats.height.push(total.into());
+        for (v, value) in self.by_addr_type.values_mut().zip(per_type) {
+            v.sats.height.push(value.into());
+        }
+    }
+
+    /// Derive cents (and thus lazy btc/usd) for `all` and every per-type vec
+    /// from the stateful sats values × spot price.
+    pub(crate) fn compute_rest(
+        &mut self,
+        max_from: Height,
+        prices: &prices::Vecs,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.all.compute(prices, max_from, exit)?;
+        for v in self.by_addr_type.values_mut() {
+            v.compute(prices, max_from, exit)?;
+        }
+        Ok(())
+    }
+}