global: snapshot

This commit is contained in:
nym21
2026-01-31 17:39:48 +01:00
parent 8dd350264a
commit ff5bb770d7
116 changed files with 13312 additions and 9530 deletions
@@ -3,7 +3,7 @@ use std::thread;
use brk_cohort::ByAddressType;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{DateIndex, Height, OutputType, Sats, TxIndex, TypeIndex};
use brk_types::{CentsUnsigned, DateIndex, Dollars, Height, OutputType, Sats, TxIndex, TypeIndex};
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use tracing::info;
@@ -75,9 +75,9 @@ pub fn process_blocks(
let txindex_to_output_count = &indexes.txindex.output_count;
let txindex_to_input_count = &indexes.txindex.input_count;
// From price (optional):
let height_to_price = price.map(|p| &p.usd.split.close.height);
let dateindex_to_price = price.map(|p| &p.usd.split.close.dateindex);
// From price (optional) - use cents for computation:
let height_to_price = price.map(|p| &p.cents.split.height.close);
let dateindex_to_price = price.map(|p| &p.cents.split.dateindex.close);
// Access pre-computed vectors from context for thread-safe access
let height_to_price_vec = &ctx.height_to_price;
@@ -329,6 +329,7 @@ pub fn process_blocks(
&mut vecs.address_cohorts,
&mut lookup,
block_price,
ctx.price_range_max.as_ref(),
&mut addr_counts,
&mut empty_addr_counts,
&mut activity_counts,
@@ -344,7 +345,8 @@ pub fn process_blocks(
// Main thread: Update UTXO cohorts
vecs.utxo_cohorts
.receive(transacted, height, timestamp, block_price);
vecs.utxo_cohorts.send(height_to_sent, chain_state);
vecs.utxo_cohorts
.send(height_to_sent, chain_state, ctx.price_range_max.as_ref());
});
// Push to height-indexed vectors
@@ -382,8 +384,12 @@ pub fn process_blocks(
// Compute and push percentiles for aggregate cohorts (all, sth, lth)
if let Some(dateindex) = dateindex_opt {
let spot = date_price
.flatten()
.map(|c| c.to_dollars())
.unwrap_or(Dollars::NAN);
vecs.utxo_cohorts
.truncate_push_aggregate_percentiles(dateindex)?;
.truncate_push_aggregate_percentiles(dateindex, spot)?;
}
// Periodic checkpoint flush
@@ -456,9 +462,9 @@ fn push_cohort_states(
utxo_cohorts: &mut UTXOCohorts,
address_cohorts: &mut AddressCohorts,
height: Height,
height_price: Option<brk_types::Dollars>,
height_price: Option<CentsUnsigned>,
dateindex: Option<DateIndex>,
date_price: Option<Option<brk_types::Dollars>>,
date_price: Option<Option<CentsUnsigned>>,
) -> Result<()> {
// utxo_cohorts.iter_separate_mut().try_for_each(|v| {
utxo_cohorts.par_iter_separate_mut().try_for_each(|v| {
@@ -1,8 +1,99 @@
use brk_types::{Dollars, Height, Timestamp};
use std::time::Instant;
use brk_types::{CentsUnsigned, Height, Timestamp};
use tracing::debug;
use vecdb::VecIndex;
use crate::{blocks, price};
/// Sparse table for O(1) range maximum queries on prices.
/// Uses O(n log n) space (~140MB for 880k blocks).
pub struct PriceRangeMax {
/// Flattened table: table[k * n + i] = max of 2^k elements starting at index i
/// Using flat layout for better cache locality.
table: Vec<CentsUnsigned>,
/// Number of elements
n: usize,
}
impl PriceRangeMax {
/// Build sparse table from high prices. O(n log n) time and space.
pub fn build(prices: &[CentsUnsigned]) -> Self {
let start = Instant::now();
let n = prices.len();
if n == 0 {
return Self {
table: vec![],
n: 0,
};
}
// levels = floor(log2(n)) + 1
let levels = (usize::BITS - n.leading_zeros()) as usize;
// Allocate flat table: levels * n elements
let mut table = vec![CentsUnsigned::ZERO; levels * n];
// Base case: level 0 = original prices
table[..n].copy_from_slice(prices);
// Build each level from the previous
// table[k][i] = max(table[k-1][i], table[k-1][i + 2^(k-1)])
for k in 1..levels {
let prev_offset = (k - 1) * n;
let curr_offset = k * n;
let half = 1 << (k - 1);
let end = n.saturating_sub(1 << k) + 1;
// Use split_at_mut to avoid bounds checks in the loop
let (prev_level, rest) = table.split_at_mut(curr_offset);
let prev = &prev_level[prev_offset..prev_offset + n];
let curr = &mut rest[..n];
for i in 0..end {
curr[i] = prev[i].max(prev[i + half]);
}
}
let elapsed = start.elapsed();
debug!(
"PriceRangeMax built: {} heights, {} levels, {:.2}MB, {:.2}ms",
n,
levels,
(levels * n * std::mem::size_of::<CentsUnsigned>()) as f64 / 1_000_000.0,
elapsed.as_secs_f64() * 1000.0
);
Self { table, n }
}
/// Query maximum value in range [l, r] (inclusive). O(1) time.
#[inline]
pub fn range_max(&self, l: usize, r: usize) -> CentsUnsigned {
debug_assert!(l <= r && r < self.n);
let len = r - l + 1;
// k = floor(log2(len))
let k = (usize::BITS - len.leading_zeros() - 1) as usize;
let half = 1 << k;
// max of [l, l + 2^k) and [r - 2^k + 1, r + 1)
let offset = k * self.n;
unsafe {
let a = *self.table.get_unchecked(offset + l);
let b = *self.table.get_unchecked(offset + r + 1 - half);
a.max(b)
}
}
/// Query maximum value in height range. O(1) time.
#[inline]
pub fn max_between(&self, from: Height, to: Height) -> CentsUnsigned {
self.range_max(from.to_usize(), to.to_usize())
}
}
/// Context shared across block processing.
pub struct ComputeContext {
/// Starting height for this computation run
@@ -15,7 +106,11 @@ pub struct ComputeContext {
pub height_to_timestamp: Vec<Timestamp>,
/// Pre-computed height -> price mapping (if available)
pub height_to_price: Option<Vec<Dollars>>,
pub height_to_price: Option<Vec<CentsUnsigned>>,
/// Sparse table for O(1) range max queries on high prices.
/// Used for computing max price during UTXO holding periods (ATH regret).
pub price_range_max: Option<PriceRangeMax>,
}
impl ComputeContext {
@@ -29,20 +124,28 @@ impl ComputeContext {
let height_to_timestamp: Vec<Timestamp> =
blocks.time.timestamp_monotonic.into_iter().collect();
let height_to_price: Option<Vec<Dollars>> = price
.map(|p| &p.usd.split.close.height)
.map(|v| v.into_iter().map(|d| *d).collect());
let height_to_price: Option<Vec<CentsUnsigned>> = price
.map(|p| &p.cents.split.height.close)
.map(|v| v.into_iter().map(|c| *c).collect());
// Build sparse table for O(1) range max queries on HIGH prices
// Used for computing peak price during UTXO holding periods (ATH regret)
let price_range_max = price
.map(|p| &p.cents.split.height.high)
.map(|v| v.into_iter().map(|c| *c).collect::<Vec<_>>())
.map(|prices| PriceRangeMax::build(&prices));
Self {
starting_height,
last_height,
height_to_timestamp,
height_to_price,
price_range_max,
}
}
/// Get price at height (None if no price data or height out of range).
pub fn price_at(&self, height: Height) -> Option<Dollars> {
pub fn price_at(&self, height: Height) -> Option<CentsUnsigned> {
self.height_to_price
.as_ref()?
.get(height.to_usize())
@@ -6,7 +6,7 @@ mod recover;
mod write;
pub use block_loop::process_blocks;
pub use context::ComputeContext;
pub use context::{ComputeContext, PriceRangeMax};
pub use readers::{
TxInIterators, TxOutData, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
build_txoutindex_to_txindex,
@@ -108,9 +108,9 @@ pub fn reset_state(
utxo_cohorts.reset_separate_state_heights();
address_cohorts.reset_separate_state_heights();
// Reset price_to_amount for all cohorts
utxo_cohorts.reset_separate_price_to_amount()?;
address_cohorts.reset_separate_price_to_amount()?;
// Reset cost_basis_data for all cohorts
utxo_cohorts.reset_separate_cost_basis_data()?;
address_cohorts.reset_separate_cost_basis_data()?;
Ok(RecoveredState {
starting_height: Height::ZERO,