mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-07-31 04:33:05 -07:00
global: snapshot
This commit is contained in:
@@ -3,7 +3,9 @@ use std::thread;
|
|||||||
use brk_cohort::ByAddressType;
|
use brk_cohort::ByAddressType;
|
||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_indexer::Indexer;
|
use brk_indexer::Indexer;
|
||||||
use brk_types::{Cents, Date, Day1, Height, OutputType, Sats, StoredU64, TxIndex, TypeIndex};
|
use brk_types::{
|
||||||
|
Cents, Date, Day1, Height, OutputType, Sats, StoredU64, Timestamp, TxIndex, TypeIndex,
|
||||||
|
};
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use rustc_hash::FxHashSet;
|
use rustc_hash::FxHashSet;
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info};
|
||||||
@@ -20,7 +22,7 @@ use crate::{
|
|||||||
compute::write::{process_address_updates, write},
|
compute::write::{process_address_updates, write},
|
||||||
state::{BlockState, Transacted},
|
state::{BlockState, Transacted},
|
||||||
},
|
},
|
||||||
indexes, inputs, outputs, prices, transactions,
|
indexes, inputs, outputs, transactions,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
@@ -30,8 +32,8 @@ use super::{
|
|||||||
vecs::Vecs,
|
vecs::Vecs,
|
||||||
},
|
},
|
||||||
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
|
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
|
||||||
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexToTxIndexBuf, TxInReaders,
|
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexToTxIndexBuf, PriceRangeMax,
|
||||||
TxOutReaders, VecsReaders,
|
TxInReaders, TxOutReaders, VecsReaders,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Process all blocks from starting_height to last_height.
|
/// Process all blocks from starting_height to last_height.
|
||||||
@@ -44,52 +46,45 @@ pub(crate) fn process_blocks(
|
|||||||
outputs: &outputs::Vecs,
|
outputs: &outputs::Vecs,
|
||||||
transactions: &transactions::Vecs,
|
transactions: &transactions::Vecs,
|
||||||
blocks: &blocks::Vecs,
|
blocks: &blocks::Vecs,
|
||||||
prices: &prices::Vecs,
|
|
||||||
starting_height: Height,
|
starting_height: Height,
|
||||||
last_height: Height,
|
last_height: Height,
|
||||||
chain_state: &mut Vec<BlockState>,
|
chain_state: &mut Vec<BlockState>,
|
||||||
txindex_to_height: &mut RangeMap<TxIndex, Height>,
|
txindex_to_height: &mut RangeMap<TxIndex, Height>,
|
||||||
|
cached_prices: &[Cents],
|
||||||
|
cached_timestamps: &[Timestamp],
|
||||||
|
cached_price_range_max: &PriceRangeMax,
|
||||||
exit: &Exit,
|
exit: &Exit,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Create computation context with pre-computed vectors for thread-safe access
|
let ctx = ComputeContext {
|
||||||
debug!("creating ComputeContext");
|
starting_height,
|
||||||
let ctx = ComputeContext::new(starting_height, last_height, blocks, prices);
|
last_height,
|
||||||
debug!("ComputeContext created");
|
height_to_timestamp: cached_timestamps,
|
||||||
|
height_to_price: cached_prices,
|
||||||
|
price_range_max: cached_price_range_max,
|
||||||
|
};
|
||||||
|
|
||||||
if ctx.starting_height > ctx.last_height {
|
if ctx.starting_height > ctx.last_height {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// References to vectors using correct field paths
|
|
||||||
// From indexer.vecs:
|
|
||||||
let height_to_first_txindex = &indexer.vecs.transactions.first_txindex;
|
let height_to_first_txindex = &indexer.vecs.transactions.first_txindex;
|
||||||
let height_to_first_txoutindex = &indexer.vecs.outputs.first_txoutindex;
|
let height_to_first_txoutindex = &indexer.vecs.outputs.first_txoutindex;
|
||||||
let height_to_first_txinindex = &indexer.vecs.inputs.first_txinindex;
|
let height_to_first_txinindex = &indexer.vecs.inputs.first_txinindex;
|
||||||
|
|
||||||
// From transactions and inputs/outputs (via .height or .height.sum_cumulative.sum patterns):
|
|
||||||
let height_to_tx_count = &transactions.count.tx_count.height;
|
let height_to_tx_count = &transactions.count.tx_count.height;
|
||||||
let height_to_output_count = &outputs.count.total_count.full.sum_cumulative.sum.0;
|
let height_to_output_count = &outputs.count.total_count.full.sum_cumulative.sum.0;
|
||||||
let height_to_input_count = &inputs.count.full.sum_cumulative.sum.0;
|
let height_to_input_count = &inputs.count.full.sum_cumulative.sum.0;
|
||||||
// From blocks:
|
|
||||||
let height_to_timestamp = &blocks.time.timestamp_monotonic;
|
|
||||||
let height_to_date = &blocks.time.date;
|
let height_to_date = &blocks.time.date;
|
||||||
let day1_to_first_height = &indexes.day1.first_height;
|
let day1_to_first_height = &indexes.day1.first_height;
|
||||||
let day1_to_height_count = &indexes.day1.height_count;
|
let day1_to_height_count = &indexes.day1.height_count;
|
||||||
let txindex_to_output_count = &indexes.txindex.output_count;
|
let txindex_to_output_count = &indexes.txindex.output_count;
|
||||||
let txindex_to_input_count = &indexes.txindex.input_count;
|
let txindex_to_input_count = &indexes.txindex.input_count;
|
||||||
|
|
||||||
// From price - use cents for computation:
|
let height_to_price_vec = cached_prices;
|
||||||
let height_to_price = &prices.price.cents.height;
|
let height_to_timestamp_vec = cached_timestamps;
|
||||||
|
|
||||||
// Access pre-computed vectors from context for thread-safe access
|
|
||||||
let height_to_price_vec = &ctx.height_to_price;
|
|
||||||
let height_to_timestamp_vec = &ctx.height_to_timestamp;
|
|
||||||
|
|
||||||
// Range for pre-collecting height-indexed vecs
|
|
||||||
let start_usize = starting_height.to_usize();
|
let start_usize = starting_height.to_usize();
|
||||||
let end_usize = last_height.to_usize() + 1;
|
let end_usize = last_height.to_usize() + 1;
|
||||||
|
|
||||||
// Pre-collect height-indexed vecs for the block range (bulk read before hot loop)
|
|
||||||
let height_to_first_txindex_vec: Vec<TxIndex> =
|
let height_to_first_txindex_vec: Vec<TxIndex> =
|
||||||
height_to_first_txindex.collect_range_at(start_usize, end_usize);
|
height_to_first_txindex.collect_range_at(start_usize, end_usize);
|
||||||
let height_to_first_txoutindex_vec: Vec<_> =
|
let height_to_first_txoutindex_vec: Vec<_> =
|
||||||
@@ -102,10 +97,8 @@ pub(crate) fn process_blocks(
|
|||||||
height_to_output_count.collect_range_at(start_usize, end_usize);
|
height_to_output_count.collect_range_at(start_usize, end_usize);
|
||||||
let height_to_input_count_vec: Vec<_> =
|
let height_to_input_count_vec: Vec<_> =
|
||||||
height_to_input_count.collect_range_at(start_usize, end_usize);
|
height_to_input_count.collect_range_at(start_usize, end_usize);
|
||||||
let height_to_timestamp_collected: Vec<_> =
|
let height_to_timestamp_collected = &cached_timestamps[start_usize..end_usize];
|
||||||
height_to_timestamp.collect_range_at(start_usize, end_usize);
|
let height_to_price_collected = &cached_prices[start_usize..end_usize];
|
||||||
let height_to_price_collected: Vec<_> =
|
|
||||||
height_to_price.collect_range_at(start_usize, end_usize);
|
|
||||||
|
|
||||||
debug!("creating VecsReaders");
|
debug!("creating VecsReaders");
|
||||||
let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
|
let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
|
||||||
@@ -115,7 +108,10 @@ pub(crate) fn process_blocks(
|
|||||||
let target_len = indexer.vecs.transactions.first_txindex.len();
|
let target_len = indexer.vecs.transactions.first_txindex.len();
|
||||||
let current_len = txindex_to_height.len();
|
let current_len = txindex_to_height.len();
|
||||||
if current_len < target_len {
|
if current_len < target_len {
|
||||||
debug!("extending txindex_to_height RangeMap from {} to {}", current_len, target_len);
|
debug!(
|
||||||
|
"extending txindex_to_height RangeMap from {} to {}",
|
||||||
|
current_len, target_len
|
||||||
|
);
|
||||||
let new_entries: Vec<TxIndex> = indexer
|
let new_entries: Vec<TxIndex> = indexer
|
||||||
.vecs
|
.vecs
|
||||||
.transactions
|
.transactions
|
||||||
@@ -125,10 +121,16 @@ pub(crate) fn process_blocks(
|
|||||||
txindex_to_height.push(first_txindex);
|
txindex_to_height.push(first_txindex);
|
||||||
}
|
}
|
||||||
} else if current_len > target_len {
|
} else if current_len > target_len {
|
||||||
debug!("truncating txindex_to_height RangeMap from {} to {}", current_len, target_len);
|
debug!(
|
||||||
|
"truncating txindex_to_height RangeMap from {} to {}",
|
||||||
|
current_len, target_len
|
||||||
|
);
|
||||||
txindex_to_height.truncate(target_len);
|
txindex_to_height.truncate(target_len);
|
||||||
}
|
}
|
||||||
debug!("txindex_to_height RangeMap ready ({} entries)", txindex_to_height.len());
|
debug!(
|
||||||
|
"txindex_to_height RangeMap ready ({} entries)",
|
||||||
|
txindex_to_height.len()
|
||||||
|
);
|
||||||
|
|
||||||
// Create reusable iterators and buffers for per-block reads
|
// Create reusable iterators and buffers for per-block reads
|
||||||
let mut txout_iters = TxOutReaders::new(indexer);
|
let mut txout_iters = TxOutReaders::new(indexer);
|
||||||
@@ -395,7 +397,7 @@ pub(crate) fn process_blocks(
|
|||||||
&mut vecs.address_cohorts,
|
&mut vecs.address_cohorts,
|
||||||
&mut lookup,
|
&mut lookup,
|
||||||
block_price,
|
block_price,
|
||||||
&ctx.price_range_max,
|
ctx.price_range_max,
|
||||||
&mut addr_counts,
|
&mut addr_counts,
|
||||||
&mut empty_addr_counts,
|
&mut empty_addr_counts,
|
||||||
&mut activity_counts,
|
&mut activity_counts,
|
||||||
@@ -412,7 +414,7 @@ pub(crate) fn process_blocks(
|
|||||||
vecs.utxo_cohorts
|
vecs.utxo_cohorts
|
||||||
.receive(transacted, height, timestamp, block_price);
|
.receive(transacted, height, timestamp, block_price);
|
||||||
vecs.utxo_cohorts
|
vecs.utxo_cohorts
|
||||||
.send(height_to_sent, chain_state, &ctx.price_range_max);
|
.send(height_to_sent, chain_state, ctx.price_range_max);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Push to height-indexed vectors
|
// Push to height-indexed vectors
|
||||||
@@ -468,7 +470,7 @@ pub(crate) fn process_blocks(
|
|||||||
height,
|
height,
|
||||||
timestamp,
|
timestamp,
|
||||||
block_price,
|
block_price,
|
||||||
&ctx.price_range_max,
|
ctx.price_range_max,
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,150 +2,123 @@ use std::time::Instant;
|
|||||||
|
|
||||||
use brk_types::{Cents, Height, Timestamp};
|
use brk_types::{Cents, Height, Timestamp};
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
use vecdb::{ReadableVec, VecIndex};
|
use vecdb::VecIndex;
|
||||||
|
|
||||||
use crate::{blocks, prices};
|
|
||||||
|
|
||||||
/// Sparse table for O(1) range maximum queries on prices.
|
/// Sparse table for O(1) range maximum queries on prices.
|
||||||
/// Uses O(n log n) space (~140MB for 880k blocks).
|
/// Vec<Vec> per level for incremental O(new_blocks * log n) extension.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct PriceRangeMax {
|
pub struct PriceRangeMax {
|
||||||
/// Flattened table: table[k * n + i] = max of 2^k elements starting at index i
|
levels: Vec<Vec<Cents>>,
|
||||||
/// Using flat layout for better cache locality.
|
|
||||||
table: Vec<Cents>,
|
|
||||||
/// Number of elements
|
|
||||||
n: usize,
|
n: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PriceRangeMax {
|
impl PriceRangeMax {
|
||||||
/// Build sparse table from high prices. O(n log n) time and space.
|
pub(crate) fn extend(&mut self, prices: &[Cents]) {
|
||||||
pub(crate) fn build(prices: &[Cents]) -> Self {
|
let new_n = prices.len();
|
||||||
let start = Instant::now();
|
if new_n <= self.n || new_n == 0 {
|
||||||
|
return;
|
||||||
let n = prices.len();
|
|
||||||
if n == 0 {
|
|
||||||
return Self {
|
|
||||||
table: vec![],
|
|
||||||
n: 0,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// levels = floor(log2(n)) + 1
|
let start = Instant::now();
|
||||||
let levels = (usize::BITS - n.leading_zeros()) as usize;
|
let old_n = self.n;
|
||||||
|
let new_levels_count = (usize::BITS - new_n.leading_zeros()) as usize;
|
||||||
|
|
||||||
// Allocate flat table: levels * n elements
|
while self.levels.len() < new_levels_count {
|
||||||
let mut table = vec![Cents::ZERO; levels * n];
|
self.levels.push(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
// Base case: level 0 = original prices
|
self.levels[0].extend_from_slice(&prices[old_n..new_n]);
|
||||||
table[..n].copy_from_slice(prices);
|
|
||||||
|
|
||||||
// Build each level from the previous
|
for k in 1..new_levels_count {
|
||||||
// table[k][i] = max(table[k-1][i], table[k-1][i + 2^(k-1)])
|
|
||||||
for k in 1..levels {
|
|
||||||
let prev_offset = (k - 1) * n;
|
|
||||||
let curr_offset = k * n;
|
|
||||||
let half = 1 << (k - 1);
|
let half = 1 << (k - 1);
|
||||||
let end = n.saturating_sub(1 << k) + 1;
|
let new_end = if new_n >= (1 << k) {
|
||||||
|
new_n + 1 - (1 << k)
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
|
||||||
// Use split_at_mut to avoid bounds checks in the loop
|
let old_end = self.levels[k].len();
|
||||||
let (prev_level, rest) = table.split_at_mut(curr_offset);
|
if new_end > old_end {
|
||||||
let prev = &prev_level[prev_offset..prev_offset + n];
|
let (prev_levels, curr_levels) = self.levels.split_at_mut(k);
|
||||||
let curr = &mut rest[..n];
|
let prev = &prev_levels[k - 1];
|
||||||
|
let curr = &mut curr_levels[0];
|
||||||
for i in 0..end {
|
curr.reserve(new_end - old_end);
|
||||||
curr[i] = prev[i].max(prev[i + half]);
|
for i in old_end..new_end {
|
||||||
|
curr.push(prev[i].max(prev[i + half]));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.n = new_n;
|
||||||
|
|
||||||
let elapsed = start.elapsed();
|
let elapsed = start.elapsed();
|
||||||
|
let total_entries: usize = self.levels.iter().map(|l| l.len()).sum();
|
||||||
debug!(
|
debug!(
|
||||||
"PriceRangeMax built: {} heights, {} levels, {:.2}MB, {:.2}ms",
|
"PriceRangeMax extended: {} -> {} heights ({} new), {} levels, {:.2}MB, {:.2}ms",
|
||||||
n,
|
old_n,
|
||||||
levels,
|
new_n,
|
||||||
(levels * n * std::mem::size_of::<Cents>()) as f64 / 1_000_000.0,
|
new_n - old_n,
|
||||||
|
new_levels_count,
|
||||||
|
(total_entries * std::mem::size_of::<Cents>()) as f64 / 1_000_000.0,
|
||||||
elapsed.as_secs_f64() * 1000.0
|
elapsed.as_secs_f64() * 1000.0
|
||||||
);
|
);
|
||||||
|
|
||||||
Self { table, n }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Query maximum value in range [l, r] (inclusive). O(1) time.
|
pub(crate) fn truncate(&mut self, new_n: usize) {
|
||||||
|
if new_n >= self.n {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if new_n == 0 {
|
||||||
|
self.levels.clear();
|
||||||
|
self.n = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let new_levels_count = (usize::BITS - new_n.leading_zeros()) as usize;
|
||||||
|
self.levels.truncate(new_levels_count);
|
||||||
|
for k in 0..new_levels_count {
|
||||||
|
let valid = if new_n >= (1 << k) {
|
||||||
|
new_n + 1 - (1 << k)
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
self.levels[k].truncate(valid);
|
||||||
|
}
|
||||||
|
self.n = new_n;
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn range_max(&self, l: usize, r: usize) -> Cents {
|
pub(crate) fn range_max(&self, l: usize, r: usize) -> Cents {
|
||||||
debug_assert!(l <= r && r < self.n);
|
debug_assert!(l <= r && r < self.n);
|
||||||
|
|
||||||
let len = r - l + 1;
|
let len = r - l + 1;
|
||||||
// k = floor(log2(len))
|
|
||||||
let k = (usize::BITS - len.leading_zeros() - 1) as usize;
|
let k = (usize::BITS - len.leading_zeros() - 1) as usize;
|
||||||
let half = 1 << k;
|
let half = 1 << k;
|
||||||
|
let level = &self.levels[k];
|
||||||
// max of [l, l + 2^k) and [r - 2^k + 1, r + 1)
|
|
||||||
let offset = k * self.n;
|
|
||||||
unsafe {
|
unsafe {
|
||||||
let a = *self.table.get_unchecked(offset + l);
|
let a = *level.get_unchecked(l);
|
||||||
let b = *self.table.get_unchecked(offset + r + 1 - half);
|
let b = *level.get_unchecked(r + 1 - half);
|
||||||
a.max(b)
|
a.max(b)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Query maximum value in height range. O(1) time.
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn max_between(&self, from: Height, to: Height) -> Cents {
|
pub(crate) fn max_between(&self, from: Height, to: Height) -> Cents {
|
||||||
self.range_max(from.to_usize(), to.to_usize())
|
self.range_max(from.to_usize(), to.to_usize())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Context shared across block processing.
|
pub struct ComputeContext<'a> {
|
||||||
pub struct ComputeContext {
|
|
||||||
/// Starting height for this computation run
|
|
||||||
pub starting_height: Height,
|
pub starting_height: Height,
|
||||||
|
|
||||||
/// Last height to process
|
|
||||||
pub last_height: Height,
|
pub last_height: Height,
|
||||||
|
pub height_to_timestamp: &'a [Timestamp],
|
||||||
/// Pre-computed height -> timestamp mapping
|
pub height_to_price: &'a [Cents],
|
||||||
pub height_to_timestamp: Vec<Timestamp>,
|
pub price_range_max: &'a PriceRangeMax,
|
||||||
|
|
||||||
/// Pre-computed height -> price mapping
|
|
||||||
pub height_to_price: Vec<Cents>,
|
|
||||||
|
|
||||||
/// Sparse table for O(1) range max queries on high prices.
|
|
||||||
/// Used for computing max price during UTXO holding periods (peak regret).
|
|
||||||
pub price_range_max: PriceRangeMax,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ComputeContext {
|
impl<'a> ComputeContext<'a> {
|
||||||
/// Create a new computation context.
|
|
||||||
pub(crate) fn new(
|
|
||||||
starting_height: Height,
|
|
||||||
last_height: Height,
|
|
||||||
blocks: &blocks::Vecs,
|
|
||||||
prices: &prices::Vecs,
|
|
||||||
) -> Self {
|
|
||||||
let height_to_timestamp: Vec<Timestamp> =
|
|
||||||
blocks.time.timestamp_monotonic.collect();
|
|
||||||
|
|
||||||
let height_to_price: Vec<Cents> =
|
|
||||||
prices.price.cents.height.collect();
|
|
||||||
|
|
||||||
// Build sparse table for O(1) range max queries on prices
|
|
||||||
// Used for computing peak price during UTXO holding periods (peak regret)
|
|
||||||
let price_range_max = PriceRangeMax::build(&height_to_price);
|
|
||||||
|
|
||||||
Self {
|
|
||||||
starting_height,
|
|
||||||
last_height,
|
|
||||||
height_to_timestamp,
|
|
||||||
height_to_price,
|
|
||||||
price_range_max,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get price at height.
|
|
||||||
pub(crate) fn price_at(&self, height: Height) -> Cents {
|
pub(crate) fn price_at(&self, height: Height) -> Cents {
|
||||||
self.height_to_price[height.to_usize()]
|
self.height_to_price[height.to_usize()]
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get timestamp at height.
|
|
||||||
pub(crate) fn timestamp_at(&self, height: Height) -> Timestamp {
|
pub(crate) fn timestamp_at(&self, height: Height) -> Timestamp {
|
||||||
self.height_to_timestamp[height.to_usize()]
|
self.height_to_timestamp[height.to_usize()]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,16 +34,6 @@ impl<I: Default + Copy, V: Default + Copy> Default for RangeMap<I, V> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<I: Ord + Copy + Default, V: From<usize> + Copy + Default> RangeMap<I, V> {
|
impl<I: Ord + Copy + Default, V: From<usize> + Copy + Default> RangeMap<I, V> {
|
||||||
/// Create with pre-allocated capacity.
|
|
||||||
pub(crate) fn with_capacity(capacity: usize) -> Self {
|
|
||||||
Self {
|
|
||||||
first_indexes: Vec::with_capacity(capacity),
|
|
||||||
cache: [(I::default(), I::default(), V::default(), 0); CACHE_SIZE],
|
|
||||||
cache_len: 0,
|
|
||||||
_phantom: PhantomData,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Number of ranges stored.
|
/// Number of ranges stored.
|
||||||
pub(crate) fn len(&self) -> usize {
|
pub(crate) fn len(&self) -> usize {
|
||||||
self.first_indexes.len()
|
self.first_indexes.len()
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ use brk_error::Result;
|
|||||||
use brk_indexer::Indexer;
|
use brk_indexer::Indexer;
|
||||||
use brk_traversable::Traversable;
|
use brk_traversable::Traversable;
|
||||||
use brk_types::{
|
use brk_types::{
|
||||||
Day1, EmptyAddressData, EmptyAddressIndex, FundedAddressData, FundedAddressIndex, Height,
|
Cents, Day1, EmptyAddressData, EmptyAddressIndex, FundedAddressData, FundedAddressIndex,
|
||||||
SupplyState, TxIndex, Version,
|
Height, SupplyState, Timestamp, TxIndex, Version,
|
||||||
};
|
};
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info};
|
||||||
use vecdb::{
|
use vecdb::{
|
||||||
@@ -16,7 +16,10 @@ use vecdb::{
|
|||||||
use crate::{
|
use crate::{
|
||||||
ComputeIndexes, blocks,
|
ComputeIndexes, blocks,
|
||||||
distribution::{
|
distribution::{
|
||||||
compute::{StartMode, determine_start_mode, process_blocks, recover_state, reset_state},
|
compute::{
|
||||||
|
PriceRangeMax, StartMode, determine_start_mode, process_blocks, recover_state,
|
||||||
|
reset_state,
|
||||||
|
},
|
||||||
state::BlockState,
|
state::BlockState,
|
||||||
},
|
},
|
||||||
indexes, inputs, outputs, prices, transactions,
|
indexes, inputs, outputs, prices, transactions,
|
||||||
@@ -69,6 +72,16 @@ pub struct Vecs<M: StorageMode = Rw> {
|
|||||||
/// In-memory txindex→height reverse lookup. Kept across compute() calls.
|
/// In-memory txindex→height reverse lookup. Kept across compute() calls.
|
||||||
#[traversable(skip)]
|
#[traversable(skip)]
|
||||||
txindex_to_height: RangeMap<TxIndex, Height>,
|
txindex_to_height: RangeMap<TxIndex, Height>,
|
||||||
|
|
||||||
|
/// Cached height→price mapping. Incrementally extended, O(new_blocks) on resume.
|
||||||
|
#[traversable(skip)]
|
||||||
|
cached_prices: Vec<Cents>,
|
||||||
|
/// Cached height→timestamp mapping. Incrementally extended, O(new_blocks) on resume.
|
||||||
|
#[traversable(skip)]
|
||||||
|
cached_timestamps: Vec<Timestamp>,
|
||||||
|
/// Cached sparse table for O(1) range-max price queries. Incrementally extended.
|
||||||
|
#[traversable(skip)]
|
||||||
|
cached_price_range_max: PriceRangeMax,
|
||||||
}
|
}
|
||||||
|
|
||||||
const SAVED_STAMPED_CHANGES: u16 = 10;
|
const SAVED_STAMPED_CHANGES: u16 = 10;
|
||||||
@@ -159,6 +172,10 @@ impl Vecs {
|
|||||||
chain_state: Vec::new(),
|
chain_state: Vec::new(),
|
||||||
txindex_to_height: RangeMap::default(),
|
txindex_to_height: RangeMap::default(),
|
||||||
|
|
||||||
|
cached_prices: Vec::new(),
|
||||||
|
cached_timestamps: Vec::new(),
|
||||||
|
cached_price_range_max: PriceRangeMax::default(),
|
||||||
|
|
||||||
db,
|
db,
|
||||||
states_path,
|
states_path,
|
||||||
};
|
};
|
||||||
@@ -194,6 +211,32 @@ impl Vecs {
|
|||||||
starting_indexes: &mut ComputeIndexes,
|
starting_indexes: &mut ComputeIndexes,
|
||||||
exit: &Exit,
|
exit: &Exit,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let cache_target_len = prices
|
||||||
|
.price
|
||||||
|
.cents
|
||||||
|
.height
|
||||||
|
.len()
|
||||||
|
.min(blocks.time.timestamp_monotonic.len());
|
||||||
|
let cache_current_len = self.cached_prices.len();
|
||||||
|
if cache_target_len < cache_current_len {
|
||||||
|
self.cached_prices.truncate(cache_target_len);
|
||||||
|
self.cached_timestamps.truncate(cache_target_len);
|
||||||
|
self.cached_price_range_max.truncate(cache_target_len);
|
||||||
|
} else if cache_target_len > cache_current_len {
|
||||||
|
let new_prices = prices
|
||||||
|
.price
|
||||||
|
.cents
|
||||||
|
.height
|
||||||
|
.collect_range_at(cache_current_len, cache_target_len);
|
||||||
|
let new_timestamps = blocks
|
||||||
|
.time
|
||||||
|
.timestamp_monotonic
|
||||||
|
.collect_range_at(cache_current_len, cache_target_len);
|
||||||
|
self.cached_prices.extend(new_prices);
|
||||||
|
self.cached_timestamps.extend(new_timestamps);
|
||||||
|
}
|
||||||
|
self.cached_price_range_max.extend(&self.cached_prices);
|
||||||
|
|
||||||
// 1. Find minimum height we have data for across stateful vecs
|
// 1. Find minimum height we have data for across stateful vecs
|
||||||
let current_height = Height::from(self.supply_state.len());
|
let current_height = Height::from(self.supply_state.len());
|
||||||
let min_stateful = self.min_stateful_height_len();
|
let min_stateful = self.min_stateful_height_len();
|
||||||
@@ -268,15 +311,9 @@ impl Vecs {
|
|||||||
debug!("reusing in-memory chain_state ({} entries)", chain_state.len());
|
debug!("reusing in-memory chain_state ({} entries)", chain_state.len());
|
||||||
recovered_height
|
recovered_height
|
||||||
} else {
|
} else {
|
||||||
// Rollback or first run after restart: rebuild from supply_state
|
|
||||||
debug!("rebuilding chain_state from stored values");
|
debug!("rebuilding chain_state from stored values");
|
||||||
let height_to_timestamp = &blocks.time.timestamp_monotonic;
|
|
||||||
let height_to_price = &prices.price.cents.height;
|
|
||||||
|
|
||||||
let end = usize::from(recovered_height);
|
let end = usize::from(recovered_height);
|
||||||
let timestamp_data: Vec<_> = height_to_timestamp.collect_range_at(0, end);
|
|
||||||
let price_data: Vec<_> = height_to_price.collect_range_at(0, end);
|
|
||||||
|
|
||||||
debug!("building supply_state vec for {} heights", recovered_height);
|
debug!("building supply_state vec for {} heights", recovered_height);
|
||||||
let supply_state_data: Vec<_> = self.supply_state.collect_range_at(0, end);
|
let supply_state_data: Vec<_> = self.supply_state.collect_range_at(0, end);
|
||||||
chain_state = supply_state_data
|
chain_state = supply_state_data
|
||||||
@@ -284,8 +321,8 @@ impl Vecs {
|
|||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(h, supply)| BlockState {
|
.map(|(h, supply)| BlockState {
|
||||||
supply,
|
supply,
|
||||||
price: price_data[h],
|
price: self.cached_prices[h],
|
||||||
timestamp: timestamp_data[h],
|
timestamp: self.cached_timestamps[h],
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
debug!("chain_state rebuilt");
|
debug!("chain_state rebuilt");
|
||||||
@@ -329,6 +366,12 @@ impl Vecs {
|
|||||||
// 4. Process blocks
|
// 4. Process blocks
|
||||||
if starting_height <= last_height {
|
if starting_height <= last_height {
|
||||||
debug!("calling process_blocks");
|
debug!("calling process_blocks");
|
||||||
|
|
||||||
|
let cached_prices = std::mem::take(&mut self.cached_prices);
|
||||||
|
let cached_timestamps = std::mem::take(&mut self.cached_timestamps);
|
||||||
|
let cached_price_range_max =
|
||||||
|
std::mem::take(&mut self.cached_price_range_max);
|
||||||
|
|
||||||
process_blocks(
|
process_blocks(
|
||||||
self,
|
self,
|
||||||
indexer,
|
indexer,
|
||||||
@@ -337,13 +380,19 @@ impl Vecs {
|
|||||||
outputs,
|
outputs,
|
||||||
transactions,
|
transactions,
|
||||||
blocks,
|
blocks,
|
||||||
prices,
|
|
||||||
starting_height,
|
starting_height,
|
||||||
last_height,
|
last_height,
|
||||||
&mut chain_state,
|
&mut chain_state,
|
||||||
&mut txindex_to_height,
|
&mut txindex_to_height,
|
||||||
|
&cached_prices,
|
||||||
|
&cached_timestamps,
|
||||||
|
&cached_price_range_max,
|
||||||
exit,
|
exit,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
self.cached_prices = cached_prices;
|
||||||
|
self.cached_timestamps = cached_timestamps;
|
||||||
|
self.cached_price_range_max = cached_price_range_max;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put chain_state and txindex_to_height back
|
// Put chain_state and txindex_to_height back
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use vecdb::{AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, ReadableVec,
|
|||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
ComputeIndexes, blocks, indexes,
|
ComputeIndexes, blocks, indexes,
|
||||||
internal::{ComputedFromHeightStdDevExtended, Price},
|
internal::{ComputedFromHeightStdDevExtended, Price, TDigest},
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::super::ComputedFromHeight;
|
use super::super::ComputedFromHeight;
|
||||||
@@ -31,9 +31,12 @@ pub struct ComputedFromHeightRatioExtension<M: StorageMode = Rw> {
|
|||||||
pub ratio_4y_sd: ComputedFromHeightStdDevExtended<M>,
|
pub ratio_4y_sd: ComputedFromHeightStdDevExtended<M>,
|
||||||
pub ratio_2y_sd: ComputedFromHeightStdDevExtended<M>,
|
pub ratio_2y_sd: ComputedFromHeightStdDevExtended<M>,
|
||||||
pub ratio_1y_sd: ComputedFromHeightStdDevExtended<M>,
|
pub ratio_1y_sd: ComputedFromHeightStdDevExtended<M>,
|
||||||
|
|
||||||
|
#[traversable(skip)]
|
||||||
|
tdigest: TDigest,
|
||||||
}
|
}
|
||||||
|
|
||||||
const VERSION: Version = Version::new(3);
|
const VERSION: Version = Version::new(4);
|
||||||
|
|
||||||
impl ComputedFromHeightRatioExtension {
|
impl ComputedFromHeightRatioExtension {
|
||||||
pub(crate) fn forced_import(
|
pub(crate) fn forced_import(
|
||||||
@@ -92,6 +95,7 @@ impl ComputedFromHeightRatioExtension {
|
|||||||
ratio_pct5_price: import_price!("ratio_pct5"),
|
ratio_pct5_price: import_price!("ratio_pct5"),
|
||||||
ratio_pct2_price: import_price!("ratio_pct2"),
|
ratio_pct2_price: import_price!("ratio_pct2"),
|
||||||
ratio_pct1_price: import_price!("ratio_pct1"),
|
ratio_pct1_price: import_price!("ratio_pct1"),
|
||||||
|
tdigest: TDigest::default(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,8 +122,6 @@ impl ComputedFromHeightRatioExtension {
|
|||||||
exit,
|
exit,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
// Percentiles via order-statistic Fenwick tree with coordinate compression.
|
|
||||||
// O(n log n) total vs O(n²) for the naive sorted-insert approach.
|
|
||||||
let ratio_version = ratio_source.version();
|
let ratio_version = ratio_source.version();
|
||||||
self.mut_ratio_vecs()
|
self.mut_ratio_vecs()
|
||||||
.try_for_each(|v| -> Result<()> {
|
.try_for_each(|v| -> Result<()> {
|
||||||
@@ -138,53 +140,19 @@ impl ComputedFromHeightRatioExtension {
|
|||||||
let ratio_len = ratio_source.len();
|
let ratio_len = ratio_source.len();
|
||||||
|
|
||||||
if ratio_len > start {
|
if ratio_len > start {
|
||||||
let all_ratios = ratio_source.collect_range_at(0, ratio_len);
|
let tdigest_count = self.tdigest.count() as usize;
|
||||||
|
if tdigest_count != start {
|
||||||
// Coordinate compression: unique sorted values → integer ranks
|
self.tdigest.reset();
|
||||||
let coords = {
|
if start > 0 {
|
||||||
let mut c = all_ratios.clone();
|
let historical = ratio_source.collect_range_at(0, start);
|
||||||
c.sort_unstable();
|
for &v in &historical {
|
||||||
c.dedup();
|
self.tdigest.add(*v as f64);
|
||||||
c
|
|
||||||
};
|
|
||||||
let m = coords.len();
|
|
||||||
|
|
||||||
// Build Fenwick tree (BIT) from elements [0, start) in O(m)
|
|
||||||
let mut bit = vec![0u32; m + 1]; // 1-indexed
|
|
||||||
for &v in &all_ratios[..start] {
|
|
||||||
bit[coords.binary_search(&v).unwrap() + 1] += 1;
|
|
||||||
}
|
|
||||||
for i in 1..=m {
|
|
||||||
let j = i + (i & i.wrapping_neg());
|
|
||||||
if j <= m {
|
|
||||||
bit[j] += bit[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Highest power of 2 <= m (for binary-lifting kth query)
|
|
||||||
let log2 = {
|
|
||||||
let mut b = 1usize;
|
|
||||||
while b <= m {
|
|
||||||
b <<= 1;
|
|
||||||
}
|
|
||||||
b >> 1
|
|
||||||
};
|
|
||||||
|
|
||||||
// Find rank of k-th smallest element (k is 1-indexed) in O(log m)
|
|
||||||
let kth = |bit: &[u32], mut k: u32| -> usize {
|
|
||||||
let mut pos = 0;
|
|
||||||
let mut b = log2;
|
|
||||||
while b > 0 {
|
|
||||||
let next = pos + b;
|
|
||||||
if next <= m && bit[next] < k {
|
|
||||||
k -= bit[next];
|
|
||||||
pos = next;
|
|
||||||
}
|
}
|
||||||
b >>= 1;
|
|
||||||
}
|
}
|
||||||
pos
|
}
|
||||||
};
|
|
||||||
|
|
||||||
|
// Process new blocks [start, ratio_len)
|
||||||
|
let new_ratios = ratio_source.collect_range_at(start, ratio_len);
|
||||||
let mut pct_vecs: [&mut EagerVec<PcoVec<Height, StoredF32>>; 6] = [
|
let mut pct_vecs: [&mut EagerVec<PcoVec<Height, StoredF32>>; 6] = [
|
||||||
&mut self.ratio_pct1.height,
|
&mut self.ratio_pct1.height,
|
||||||
&mut self.ratio_pct2.height,
|
&mut self.ratio_pct2.height,
|
||||||
@@ -194,25 +162,14 @@ impl ComputedFromHeightRatioExtension {
|
|||||||
&mut self.ratio_pct99.height,
|
&mut self.ratio_pct99.height,
|
||||||
];
|
];
|
||||||
const PCTS: [f64; 6] = [0.01, 0.02, 0.05, 0.95, 0.98, 0.99];
|
const PCTS: [f64; 6] = [0.01, 0.02, 0.05, 0.95, 0.98, 0.99];
|
||||||
|
let mut out = [0.0f64; 6];
|
||||||
|
|
||||||
let mut count = start;
|
for (offset, &ratio) in new_ratios.iter().enumerate() {
|
||||||
for (offset, &ratio) in all_ratios[start..].iter().enumerate() {
|
self.tdigest.add(*ratio as f64);
|
||||||
count += 1;
|
self.tdigest.quantiles(&PCTS, &mut out);
|
||||||
|
|
||||||
// Insert into Fenwick tree: O(log m)
|
|
||||||
let mut i = coords.binary_search(&ratio).unwrap() + 1;
|
|
||||||
while i <= m {
|
|
||||||
bit[i] += 1;
|
|
||||||
i += i & i.wrapping_neg();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nearest-rank percentile: one kth query each
|
|
||||||
let idx = start + offset;
|
let idx = start + offset;
|
||||||
let cf = count as f64;
|
for (vec, &val) in pct_vecs.iter_mut().zip(out.iter()) {
|
||||||
for (vec, &pct) in pct_vecs.iter_mut().zip(PCTS.iter()) {
|
vec.truncate_push_at(idx, StoredF32::from(val as f32))?;
|
||||||
let k = (cf * pct).ceil().max(1.0) as u32;
|
|
||||||
let val = coords[kth(&bit, k)];
|
|
||||||
vec.truncate_push_at(idx, val)?;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ mod lazy_eager_indexes;
|
|||||||
mod lazy_value;
|
mod lazy_value;
|
||||||
mod rolling;
|
mod rolling;
|
||||||
pub(crate) mod sliding_window;
|
pub(crate) mod sliding_window;
|
||||||
|
mod tdigest;
|
||||||
mod traits;
|
mod traits;
|
||||||
mod transform;
|
mod transform;
|
||||||
mod tx_derived;
|
mod tx_derived;
|
||||||
@@ -28,6 +29,7 @@ pub(crate) use indexes::*;
|
|||||||
pub(crate) use lazy_eager_indexes::*;
|
pub(crate) use lazy_eager_indexes::*;
|
||||||
pub(crate) use lazy_value::*;
|
pub(crate) use lazy_value::*;
|
||||||
pub(crate) use rolling::*;
|
pub(crate) use rolling::*;
|
||||||
|
pub(crate) use tdigest::*;
|
||||||
pub(crate) use traits::*;
|
pub(crate) use traits::*;
|
||||||
pub use transform::*;
|
pub use transform::*;
|
||||||
pub(crate) use tx_derived::*;
|
pub(crate) use tx_derived::*;
|
||||||
|
|||||||
@@ -0,0 +1,263 @@
|
|||||||
|
/// Streaming t-digest for approximate quantile estimation.
|
||||||
|
///
|
||||||
|
/// Uses the merging algorithm with scale function k₂: `q * (1 - q)`.
|
||||||
|
/// Compression parameter δ controls accuracy vs memory (default 100 → ~200 centroids max).
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct TDigest {
|
||||||
|
centroids: Vec<Centroid>,
|
||||||
|
count: u64,
|
||||||
|
min: f64,
|
||||||
|
max: f64,
|
||||||
|
compression: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct Centroid {
|
||||||
|
mean: f64,
|
||||||
|
weight: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for TDigest {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new(100.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TDigest {
|
||||||
|
pub fn new(compression: f64) -> Self {
|
||||||
|
Self {
|
||||||
|
centroids: Vec::new(),
|
||||||
|
count: 0,
|
||||||
|
min: f64::INFINITY,
|
||||||
|
max: f64::NEG_INFINITY,
|
||||||
|
compression,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn count(&self) -> u64 {
|
||||||
|
self.count
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.centroids.clear();
|
||||||
|
self.count = 0;
|
||||||
|
self.min = f64::INFINITY;
|
||||||
|
self.max = f64::NEG_INFINITY;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add(&mut self, value: f64) {
|
||||||
|
if value.is_nan() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.count += 1;
|
||||||
|
if value < self.min {
|
||||||
|
self.min = value;
|
||||||
|
}
|
||||||
|
if value > self.max {
|
||||||
|
self.max = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.centroids.is_empty() {
|
||||||
|
self.centroids.push(Centroid {
|
||||||
|
mean: value,
|
||||||
|
weight: 1.0,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find nearest centroid by mean
|
||||||
|
let pos = self
|
||||||
|
.centroids
|
||||||
|
.binary_search_by(|c| c.mean.partial_cmp(&value).unwrap_or(std::cmp::Ordering::Equal))
|
||||||
|
.unwrap_or_else(|i| i.min(self.centroids.len() - 1));
|
||||||
|
|
||||||
|
// Check neighbors for the actual nearest
|
||||||
|
let nearest = if pos > 0
|
||||||
|
&& (value - self.centroids[pos - 1].mean).abs()
|
||||||
|
< (value - self.centroids[pos].mean).abs()
|
||||||
|
{
|
||||||
|
pos - 1
|
||||||
|
} else {
|
||||||
|
pos
|
||||||
|
};
|
||||||
|
|
||||||
|
// Compute quantile of nearest centroid
|
||||||
|
let cum_weight: f64 = self.centroids[..nearest]
|
||||||
|
.iter()
|
||||||
|
.map(|c| c.weight)
|
||||||
|
.sum::<f64>()
|
||||||
|
+ self.centroids[nearest].weight / 2.0;
|
||||||
|
let q = cum_weight / self.count as f64;
|
||||||
|
let limit = (4.0 * self.compression * q * (1.0 - q)).floor().max(1.0);
|
||||||
|
|
||||||
|
if self.centroids[nearest].weight + 1.0 <= limit {
|
||||||
|
// Merge into nearest centroid
|
||||||
|
let c = &mut self.centroids[nearest];
|
||||||
|
c.mean = (c.mean * c.weight + value) / (c.weight + 1.0);
|
||||||
|
c.weight += 1.0;
|
||||||
|
} else {
|
||||||
|
// Insert new centroid at sorted position
|
||||||
|
let insert_pos = self
|
||||||
|
.centroids
|
||||||
|
.binary_search_by(|c| {
|
||||||
|
c.mean
|
||||||
|
.partial_cmp(&value)
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|i| i);
|
||||||
|
self.centroids.insert(
|
||||||
|
insert_pos,
|
||||||
|
Centroid {
|
||||||
|
mean: value,
|
||||||
|
weight: 1.0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compress if too many centroids
|
||||||
|
let max_centroids = (2.0 * self.compression) as usize;
|
||||||
|
if self.centroids.len() > max_centroids {
|
||||||
|
self.compress();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compress(&mut self) {
|
||||||
|
if self.centroids.len() <= 1 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let total: f64 = self.centroids.iter().map(|c| c.weight).sum();
|
||||||
|
let mut merged: Vec<Centroid> = Vec::with_capacity(self.centroids.len());
|
||||||
|
let mut cum = 0.0;
|
||||||
|
|
||||||
|
for c in &self.centroids {
|
||||||
|
if let Some(last) = merged.last_mut() {
|
||||||
|
let q = (cum + last.weight / 2.0) / total;
|
||||||
|
let limit = (4.0 * self.compression * q * (1.0 - q)).floor().max(1.0);
|
||||||
|
if last.weight + c.weight <= limit {
|
||||||
|
let new_weight = last.weight + c.weight;
|
||||||
|
last.mean = (last.mean * last.weight + c.mean * c.weight) / new_weight;
|
||||||
|
last.weight = new_weight;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
cum += last.weight;
|
||||||
|
}
|
||||||
|
merged.push(*c);
|
||||||
|
}
|
||||||
|
self.centroids = merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn quantile(&self, q: f64) -> f64 {
|
||||||
|
if self.centroids.is_empty() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
if q <= 0.0 {
|
||||||
|
return self.min;
|
||||||
|
}
|
||||||
|
if q >= 1.0 {
|
||||||
|
return self.max;
|
||||||
|
}
|
||||||
|
if self.centroids.len() == 1 {
|
||||||
|
return self.centroids[0].mean;
|
||||||
|
}
|
||||||
|
|
||||||
|
let total: f64 = self.centroids.iter().map(|c| c.weight).sum();
|
||||||
|
let target = q * total;
|
||||||
|
let mut cum = 0.0;
|
||||||
|
|
||||||
|
for i in 0..self.centroids.len() {
|
||||||
|
let c = &self.centroids[i];
|
||||||
|
let mid = cum + c.weight / 2.0;
|
||||||
|
|
||||||
|
if target < mid {
|
||||||
|
// Interpolate between previous centroid (or min) and this one
|
||||||
|
if i == 0 {
|
||||||
|
// Between min and first centroid center
|
||||||
|
let first_mid = c.weight / 2.0;
|
||||||
|
if first_mid == 0.0 {
|
||||||
|
return self.min;
|
||||||
|
}
|
||||||
|
return self.min + (c.mean - self.min) * (target / first_mid);
|
||||||
|
}
|
||||||
|
let prev = &self.centroids[i - 1];
|
||||||
|
let prev_center = cum - prev.weight / 2.0;
|
||||||
|
let frac = if mid == prev_center {
|
||||||
|
0.5
|
||||||
|
} else {
|
||||||
|
(target - prev_center) / (mid - prev_center)
|
||||||
|
};
|
||||||
|
return prev.mean + (c.mean - prev.mean) * frac;
|
||||||
|
}
|
||||||
|
|
||||||
|
cum += c.weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Between last centroid center and max
|
||||||
|
let last = self.centroids.last().unwrap();
|
||||||
|
let last_mid = total - last.weight / 2.0;
|
||||||
|
let remaining = total - last_mid;
|
||||||
|
if remaining == 0.0 {
|
||||||
|
return self.max;
|
||||||
|
}
|
||||||
|
last.mean + (self.max - last.mean) * ((target - last_mid) / remaining)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Batch quantile query. `qs` must be sorted ascending.
|
||||||
|
pub fn quantiles(&self, qs: &[f64], out: &mut [f64]) {
|
||||||
|
for (i, &q) in qs.iter().enumerate() {
|
||||||
|
out[i] = self.quantile(q);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn basic_quantiles() {
|
||||||
|
let mut td = TDigest::default();
|
||||||
|
for i in 1..=1000 {
|
||||||
|
td.add(i as f64);
|
||||||
|
}
|
||||||
|
assert_eq!(td.count(), 1000);
|
||||||
|
|
||||||
|
let median = td.quantile(0.5);
|
||||||
|
assert!((median - 500.0).abs() < 10.0, "median was {median}");
|
||||||
|
|
||||||
|
let p99 = td.quantile(0.99);
|
||||||
|
assert!((p99 - 990.0).abs() < 15.0, "p99 was {p99}");
|
||||||
|
|
||||||
|
let p01 = td.quantile(0.01);
|
||||||
|
assert!((p01 - 10.0).abs() < 15.0, "p01 was {p01}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_digest() {
|
||||||
|
let td = TDigest::default();
|
||||||
|
assert_eq!(td.count(), 0);
|
||||||
|
assert_eq!(td.quantile(0.5), 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn single_value() {
|
||||||
|
let mut td = TDigest::default();
|
||||||
|
td.add(42.0);
|
||||||
|
assert_eq!(td.quantile(0.0), 42.0);
|
||||||
|
assert_eq!(td.quantile(0.5), 42.0);
|
||||||
|
assert_eq!(td.quantile(1.0), 42.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reset_works() {
|
||||||
|
let mut td = TDigest::default();
|
||||||
|
for i in 0..100 {
|
||||||
|
td.add(i as f64);
|
||||||
|
}
|
||||||
|
assert_eq!(td.count(), 100);
|
||||||
|
td.reset();
|
||||||
|
assert_eq!(td.count(), 0);
|
||||||
|
assert_eq!(td.quantile(0.5), 0.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
use brk_types::StoredF32;
|
||||||
|
use vecdb::UnaryTransform;
|
||||||
|
|
||||||
|
pub struct DaysToYears;
|
||||||
|
|
||||||
|
impl UnaryTransform<StoredF32, StoredF32> for DaysToYears {
|
||||||
|
#[inline(always)]
|
||||||
|
fn apply(v: StoredF32) -> StoredF32 {
|
||||||
|
StoredF32::from(*v / 365.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,7 +41,7 @@ mod sat_halve_to_bitcoin;
|
|||||||
mod sat_identity;
|
mod sat_identity;
|
||||||
mod sat_mask;
|
mod sat_mask;
|
||||||
mod sat_to_bitcoin;
|
mod sat_to_bitcoin;
|
||||||
mod u16_to_years;
|
mod days_to_years;
|
||||||
mod volatility_sqrt30;
|
mod volatility_sqrt30;
|
||||||
mod volatility_sqrt365;
|
mod volatility_sqrt365;
|
||||||
mod volatility_sqrt7;
|
mod volatility_sqrt7;
|
||||||
@@ -89,7 +89,7 @@ pub use sat_halve_to_bitcoin::*;
|
|||||||
pub use sat_identity::*;
|
pub use sat_identity::*;
|
||||||
pub use sat_mask::*;
|
pub use sat_mask::*;
|
||||||
pub use sat_to_bitcoin::*;
|
pub use sat_to_bitcoin::*;
|
||||||
pub use u16_to_years::*;
|
pub use days_to_years::*;
|
||||||
pub use volatility_sqrt7::*;
|
pub use volatility_sqrt7::*;
|
||||||
pub use volatility_sqrt30::*;
|
pub use volatility_sqrt30::*;
|
||||||
pub use volatility_sqrt365::*;
|
pub use volatility_sqrt365::*;
|
||||||
|
|||||||
@@ -1,12 +0,0 @@
|
|||||||
use brk_types::{StoredF32, StoredU16};
|
|
||||||
use vecdb::UnaryTransform;
|
|
||||||
|
|
||||||
/// StoredU16 / 365.0 -> StoredF32 (days to years conversion)
|
|
||||||
pub struct StoredU16ToYears;
|
|
||||||
|
|
||||||
impl UnaryTransform<StoredU16, StoredF32> for StoredU16ToYears {
|
|
||||||
#[inline(always)]
|
|
||||||
fn apply(v: StoredU16) -> StoredF32 {
|
|
||||||
StoredF32::from(*v as f64 / 365.0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_types::{Day1, StoredU16};
|
use brk_types::{StoredF32, Timestamp};
|
||||||
use vecdb::{Exit, ReadableVec, VecIndex};
|
use vecdb::{Exit, ReadableVec, VecIndex};
|
||||||
|
|
||||||
use super::Vecs;
|
use super::Vecs;
|
||||||
use crate::{ComputeIndexes, indexes, prices, traits::ComputeDrawdown};
|
use crate::{blocks, ComputeIndexes, prices, traits::ComputeDrawdown};
|
||||||
|
|
||||||
impl Vecs {
|
impl Vecs {
|
||||||
pub(crate) fn compute(
|
pub(crate) fn compute(
|
||||||
&mut self,
|
&mut self,
|
||||||
prices: &prices::Vecs,
|
prices: &prices::Vecs,
|
||||||
indexes: &indexes::Vecs,
|
blocks: &blocks::Vecs,
|
||||||
starting_indexes: &ComputeIndexes,
|
starting_indexes: &ComputeIndexes,
|
||||||
exit: &Exit,
|
exit: &Exit,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
@@ -19,28 +19,28 @@ impl Vecs {
|
|||||||
exit,
|
exit,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let mut ath_day: Option<Day1> = None;
|
let mut ath_ts: Option<Timestamp> = None;
|
||||||
self.days_since_price_ath.height.compute_transform3(
|
self.days_since_price_ath.height.compute_transform3(
|
||||||
starting_indexes.height,
|
starting_indexes.height,
|
||||||
&self.price_ath.cents.height,
|
&self.price_ath.cents.height,
|
||||||
&prices.price.cents.height,
|
&prices.price.cents.height,
|
||||||
&indexes.height.day1,
|
&blocks.time.timestamp_monotonic,
|
||||||
|(i, ath, price, day, slf)| {
|
|(i, ath, price, ts, slf)| {
|
||||||
if ath_day.is_none() {
|
if ath_ts.is_none() {
|
||||||
let idx = i.to_usize();
|
let idx = i.to_usize();
|
||||||
ath_day = Some(if idx > 0 {
|
ath_ts = Some(if idx > 0 {
|
||||||
let prev_days_since = slf.collect_one_at(idx - 1).unwrap();
|
let prev_days: StoredF32 = slf.collect_one_at(idx - 1).unwrap();
|
||||||
Day1::from(day.to_usize().saturating_sub(usize::from(prev_days_since)))
|
Timestamp::from((*ts as f64 - *prev_days as f64 * 86400.0) as u32)
|
||||||
} else {
|
} else {
|
||||||
day
|
ts
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if price == ath {
|
if price == ath {
|
||||||
ath_day = Some(day);
|
ath_ts = Some(ts);
|
||||||
(i, StoredU16::default())
|
(i, StoredF32::default())
|
||||||
} else {
|
} else {
|
||||||
let days_since = (day.to_usize() - ath_day.unwrap().to_usize()) as u16;
|
let days = ts.difference_in_days_between_float(ath_ts.unwrap());
|
||||||
(i, StoredU16::from(days_since))
|
(i, StoredF32::from(days as f32))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
exit,
|
exit,
|
||||||
@@ -56,7 +56,7 @@ impl Vecs {
|
|||||||
prev.replace(if i > 0 {
|
prev.replace(if i > 0 {
|
||||||
slf.collect_one_at(i - 1).unwrap()
|
slf.collect_one_at(i - 1).unwrap()
|
||||||
} else {
|
} else {
|
||||||
StoredU16::ZERO
|
StoredF32::default()
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let max = prev.unwrap().max(days);
|
let max = prev.unwrap().max(days);
|
||||||
|
|||||||
@@ -5,45 +5,42 @@ use vecdb::Database;
|
|||||||
use super::Vecs;
|
use super::Vecs;
|
||||||
use crate::{
|
use crate::{
|
||||||
indexes,
|
indexes,
|
||||||
internal::{
|
internal::{ComputedFromHeight, DaysToYears, LazyHeightDerived, Price},
|
||||||
ComputedFromHeight, LazyHeightDerived,
|
|
||||||
Price, StoredU16ToYears,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const VERSION: Version = Version::ONE;
|
||||||
|
|
||||||
impl Vecs {
|
impl Vecs {
|
||||||
pub(crate) fn forced_import(
|
pub(crate) fn forced_import(
|
||||||
db: &Database,
|
db: &Database,
|
||||||
version: Version,
|
version: Version,
|
||||||
indexes: &indexes::Vecs,
|
indexes: &indexes::Vecs,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let price_ath = Price::forced_import(db, "price_ath", version, indexes)?;
|
let v = version + VERSION;
|
||||||
|
|
||||||
let max_days_between_price_aths = ComputedFromHeight::forced_import(
|
let price_ath = Price::forced_import(db, "price_ath", v, indexes)?;
|
||||||
db,
|
|
||||||
"max_days_between_price_aths",
|
let max_days_between_price_aths =
|
||||||
version,
|
ComputedFromHeight::forced_import(db, "max_days_between_price_aths", v, indexes)?;
|
||||||
indexes,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
let max_years_between_price_aths =
|
let max_years_between_price_aths =
|
||||||
LazyHeightDerived::from_computed::<StoredU16ToYears>(
|
LazyHeightDerived::from_computed::<DaysToYears>(
|
||||||
"max_years_between_price_aths",
|
"max_years_between_price_aths",
|
||||||
version,
|
v,
|
||||||
&max_days_between_price_aths,
|
&max_days_between_price_aths,
|
||||||
);
|
);
|
||||||
|
|
||||||
let days_since_price_ath =
|
let days_since_price_ath =
|
||||||
ComputedFromHeight::forced_import(db, "days_since_price_ath", version, indexes)?;
|
ComputedFromHeight::forced_import(db, "days_since_price_ath", v, indexes)?;
|
||||||
|
|
||||||
let years_since_price_ath = LazyHeightDerived::from_computed::<StoredU16ToYears>(
|
let years_since_price_ath = LazyHeightDerived::from_computed::<DaysToYears>(
|
||||||
"years_since_price_ath",
|
"years_since_price_ath",
|
||||||
version,
|
v,
|
||||||
&days_since_price_ath,
|
&days_since_price_ath,
|
||||||
);
|
);
|
||||||
|
|
||||||
let price_drawdown =
|
let price_drawdown =
|
||||||
ComputedFromHeight::forced_import(db, "price_drawdown", version, indexes)?;
|
ComputedFromHeight::forced_import(db, "price_drawdown", v, indexes)?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
price_ath,
|
price_ath,
|
||||||
|
|||||||
@@ -1,16 +1,15 @@
|
|||||||
use brk_traversable::Traversable;
|
use brk_traversable::Traversable;
|
||||||
use brk_types::{Cents, StoredF32, StoredU16};
|
use brk_types::{Cents, StoredF32};
|
||||||
use vecdb::{Rw, StorageMode};
|
use vecdb::{Rw, StorageMode};
|
||||||
|
|
||||||
use crate::internal::{ComputedFromHeight, LazyHeightDerived, Price};
|
use crate::internal::{ComputedFromHeight, LazyHeightDerived, Price};
|
||||||
|
|
||||||
/// All-time high related metrics
|
|
||||||
#[derive(Traversable)]
|
#[derive(Traversable)]
|
||||||
pub struct Vecs<M: StorageMode = Rw> {
|
pub struct Vecs<M: StorageMode = Rw> {
|
||||||
pub price_ath: Price<ComputedFromHeight<Cents, M>>,
|
pub price_ath: Price<ComputedFromHeight<Cents, M>>,
|
||||||
pub price_drawdown: ComputedFromHeight<StoredF32, M>,
|
pub price_drawdown: ComputedFromHeight<StoredF32, M>,
|
||||||
pub days_since_price_ath: ComputedFromHeight<StoredU16, M>,
|
pub days_since_price_ath: ComputedFromHeight<StoredF32, M>,
|
||||||
pub years_since_price_ath: LazyHeightDerived<StoredF32, StoredU16>,
|
pub years_since_price_ath: LazyHeightDerived<StoredF32, StoredF32>,
|
||||||
pub max_days_between_price_aths: ComputedFromHeight<StoredU16, M>,
|
pub max_days_between_price_aths: ComputedFromHeight<StoredF32, M>,
|
||||||
pub max_years_between_price_aths: LazyHeightDerived<StoredF32, StoredU16>,
|
pub max_years_between_price_aths: LazyHeightDerived<StoredF32, StoredF32>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,8 +18,7 @@ impl Vecs {
|
|||||||
starting_indexes: &ComputeIndexes,
|
starting_indexes: &ComputeIndexes,
|
||||||
exit: &Exit,
|
exit: &Exit,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// ATH metrics (independent)
|
self.ath.compute(prices, blocks, starting_indexes, exit)?;
|
||||||
self.ath.compute(prices, indexes, starting_indexes, exit)?;
|
|
||||||
|
|
||||||
// Lookback metrics (independent)
|
// Lookback metrics (independent)
|
||||||
self.lookback
|
self.lookback
|
||||||
@@ -46,7 +45,6 @@ impl Vecs {
|
|||||||
.compute(indexes, prices, blocks, &self.lookback, starting_indexes, exit)?;
|
.compute(indexes, prices, blocks, &self.lookback, starting_indexes, exit)?;
|
||||||
|
|
||||||
self.indicators.compute(
|
self.indicators.compute(
|
||||||
indexes,
|
|
||||||
&mining.rewards,
|
&mining.rewards,
|
||||||
&self.returns,
|
&self.returns,
|
||||||
&self.range,
|
&self.range,
|
||||||
|
|||||||
@@ -4,10 +4,7 @@ use vecdb::{AnyVec, Exit, ReadableOptionVec, ReadableVec, VecIndex};
|
|||||||
|
|
||||||
use super::Vecs;
|
use super::Vecs;
|
||||||
use crate::{
|
use crate::{
|
||||||
ComputeIndexes, blocks, indexes,
|
ComputeIndexes, blocks, indexes, internal::PercentageDiffCents, market::lookback, prices,
|
||||||
internal::{ComputedFromHeight, PercentageDiffCents},
|
|
||||||
market::lookback,
|
|
||||||
prices,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
|
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
|
||||||
@@ -25,9 +22,7 @@ impl Vecs {
|
|||||||
let h2d = &indexes.height.day1;
|
let h2d = &indexes.height.day1;
|
||||||
let close = &prices.split.close.usd.day1;
|
let close = &prices.split.close.usd.day1;
|
||||||
|
|
||||||
let first_price_di = Day1::try_from(Date::new(2010, 7, 12))
|
let first_price_di = Day1::try_from(Date::new(2010, 7, 12)).unwrap().to_usize();
|
||||||
.unwrap()
|
|
||||||
.to_usize();
|
|
||||||
|
|
||||||
// Compute per-height DCA sats contribution once (reused by all periods).
|
// Compute per-height DCA sats contribution once (reused by all periods).
|
||||||
// Value = sats_from_dca(close_price) on day-boundary blocks, Sats::ZERO otherwise.
|
// Value = sats_from_dca(close_price) on day-boundary blocks, Sats::ZERO otherwise.
|
||||||
@@ -42,7 +37,10 @@ impl Vecs {
|
|||||||
if same_day {
|
if same_day {
|
||||||
(h, Sats::ZERO)
|
(h, Sats::ZERO)
|
||||||
} else {
|
} else {
|
||||||
let s = close.collect_one_flat(di).map(sats_from_dca).unwrap_or(Sats::ZERO);
|
let s = close
|
||||||
|
.collect_one_flat(di)
|
||||||
|
.map(sats_from_dca)
|
||||||
|
.unwrap_or(Sats::ZERO);
|
||||||
(h, s)
|
(h, s)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -68,7 +66,10 @@ impl Vecs {
|
|||||||
.zip_mut_with_days(&self.period_stack)
|
.zip_mut_with_days(&self.period_stack)
|
||||||
{
|
{
|
||||||
let days = days as usize;
|
let days = days as usize;
|
||||||
let stack_data = stack.sats.height.collect_range_at(sh, stack.sats.height.len());
|
let stack_data = stack
|
||||||
|
.sats
|
||||||
|
.height
|
||||||
|
.collect_range_at(sh, stack.sats.height.len());
|
||||||
average_price.cents.height.compute_transform(
|
average_price.cents.height.compute_transform(
|
||||||
starting_indexes.height,
|
starting_indexes.height,
|
||||||
h2d,
|
h2d,
|
||||||
@@ -76,9 +77,7 @@ impl Vecs {
|
|||||||
let di_usize = di.to_usize();
|
let di_usize = di.to_usize();
|
||||||
let stack_sats = stack_data[h.to_usize() - sh];
|
let stack_sats = stack_data[h.to_usize() - sh];
|
||||||
let avg = if di_usize > first_price_di {
|
let avg = if di_usize > first_price_di {
|
||||||
let num_days = days
|
let num_days = days.min(di_usize + 1).min(di_usize + 1 - first_price_di);
|
||||||
.min(di_usize + 1)
|
|
||||||
.min(di_usize + 1 - first_price_di);
|
|
||||||
Cents::from(DCA_AMOUNT * num_days / Bitcoin::from(stack_sats))
|
Cents::from(DCA_AMOUNT * num_days / Bitcoin::from(stack_sats))
|
||||||
} else {
|
} else {
|
||||||
Cents::ZERO
|
Cents::ZERO
|
||||||
@@ -123,7 +122,10 @@ impl Vecs {
|
|||||||
self.period_lump_sum_stack.zip_mut_with_days(&lookback_dca)
|
self.period_lump_sum_stack.zip_mut_with_days(&lookback_dca)
|
||||||
{
|
{
|
||||||
let total_invested = DCA_AMOUNT * days as usize;
|
let total_invested = DCA_AMOUNT * days as usize;
|
||||||
let lookback_data = lookback_price.cents.height.collect_range_at(sh, lookback_price.cents.height.len());
|
let lookback_data = lookback_price
|
||||||
|
.cents
|
||||||
|
.height
|
||||||
|
.collect_range_at(sh, lookback_price.cents.height.len());
|
||||||
stack.sats.height.compute_transform(
|
stack.sats.height.compute_transform(
|
||||||
starting_indexes.height,
|
starting_indexes.height,
|
||||||
h2d,
|
h2d,
|
||||||
@@ -193,7 +195,10 @@ impl Vecs {
|
|||||||
} else {
|
} else {
|
||||||
Sats::ZERO
|
Sats::ZERO
|
||||||
};
|
};
|
||||||
let s = close.collect_one_flat(di).map(sats_from_dca).unwrap_or(Sats::ZERO);
|
let s = close
|
||||||
|
.collect_one_flat(di)
|
||||||
|
.map(sats_from_dca)
|
||||||
|
.unwrap_or(Sats::ZERO);
|
||||||
prev + s
|
prev + s
|
||||||
};
|
};
|
||||||
prev_value = result;
|
prev_value = result;
|
||||||
@@ -212,7 +217,10 @@ impl Vecs {
|
|||||||
.zip(start_days)
|
.zip(start_days)
|
||||||
{
|
{
|
||||||
let from_usize = from.to_usize();
|
let from_usize = from.to_usize();
|
||||||
let stack_data = stack.sats.height.collect_range_at(sh, stack.sats.height.len());
|
let stack_data = stack
|
||||||
|
.sats
|
||||||
|
.height
|
||||||
|
.collect_range_at(sh, stack.sats.height.len());
|
||||||
average_price.cents.height.compute_transform(
|
average_price.cents.height.compute_transform(
|
||||||
starting_indexes.height,
|
starting_indexes.height,
|
||||||
h2d,
|
h2d,
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_types::{Day1, Dollars, StoredF32};
|
use brk_types::{Dollars, StoredF32};
|
||||||
use vecdb::{Exit, ReadableVec};
|
use vecdb::Exit;
|
||||||
|
|
||||||
use super::{super::range, Vecs};
|
use super::{super::range, Vecs};
|
||||||
use crate::{
|
use crate::{
|
||||||
ComputeIndexes, blocks, distribution, indexes,
|
ComputeIndexes, blocks, distribution,
|
||||||
internal::Ratio32,
|
internal::Ratio32,
|
||||||
mining, prices, transactions,
|
mining, prices, transactions,
|
||||||
};
|
};
|
||||||
@@ -23,7 +23,6 @@ impl Vecs {
|
|||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) fn compute(
|
pub(crate) fn compute(
|
||||||
&mut self,
|
&mut self,
|
||||||
indexes: &indexes::Vecs,
|
|
||||||
rewards: &mining::RewardsVecs,
|
rewards: &mining::RewardsVecs,
|
||||||
returns: &super::super::returns::Vecs,
|
returns: &super::super::returns::Vecs,
|
||||||
range: &range::Vecs,
|
range: &range::Vecs,
|
||||||
@@ -106,14 +105,10 @@ impl Vecs {
|
|||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gini (daily, expanded to Height)
|
// Gini (per height)
|
||||||
let h2d: Vec<Day1> = indexes.height.day1.collect();
|
|
||||||
let total_heights = h2d.len();
|
|
||||||
super::gini::compute(
|
super::gini::compute(
|
||||||
&mut self.gini,
|
&mut self.gini,
|
||||||
distribution,
|
distribution,
|
||||||
&h2d,
|
|
||||||
total_heights,
|
|
||||||
starting_indexes,
|
starting_indexes,
|
||||||
exit,
|
exit,
|
||||||
)?;
|
)?;
|
||||||
|
|||||||
@@ -1,14 +1,12 @@
|
|||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_types::{Day1, StoredF32, Version};
|
use brk_types::{Sats, StoredF32, StoredU64, Version};
|
||||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableOptionVec, VecIndex, WritableVec};
|
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||||
|
|
||||||
use crate::{ComputeIndexes, distribution, internal::ComputedFromHeight};
|
use crate::{ComputeIndexes, distribution, internal::ComputedFromHeight};
|
||||||
|
|
||||||
pub(super) fn compute(
|
pub(super) fn compute(
|
||||||
gini: &mut ComputedFromHeight<StoredF32>,
|
gini: &mut ComputedFromHeight<StoredF32>,
|
||||||
distribution: &distribution::Vecs,
|
distribution: &distribution::Vecs,
|
||||||
h2d: &[Day1],
|
|
||||||
total_heights: usize,
|
|
||||||
starting_indexes: &ComputeIndexes,
|
starting_indexes: &ComputeIndexes,
|
||||||
exit: &Exit,
|
exit: &Exit,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
@@ -16,11 +14,11 @@ pub(super) fn compute(
|
|||||||
|
|
||||||
let supply_vecs: Vec<&_> = amount_range
|
let supply_vecs: Vec<&_> = amount_range
|
||||||
.iter()
|
.iter()
|
||||||
.map(|c| &c.metrics.supply.total.sats.day1)
|
.map(|c| &c.metrics.supply.total.sats.height)
|
||||||
.collect();
|
.collect();
|
||||||
let count_vecs: Vec<&_> = amount_range
|
let count_vecs: Vec<&_> = amount_range
|
||||||
.iter()
|
.iter()
|
||||||
.map(|c| &c.metrics.outputs.utxo_count.day1)
|
.map(|c| &c.metrics.outputs.utxo_count.height)
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if supply_vecs.is_empty() || supply_vecs.len() != count_vecs.len() {
|
if supply_vecs.is_empty() || supply_vecs.len() != count_vecs.len() {
|
||||||
@@ -39,49 +37,40 @@ pub(super) fn compute(
|
|||||||
gini.height
|
gini.height
|
||||||
.truncate_if_needed_at(gini.height.len().min(starting_indexes.height.to_usize()))?;
|
.truncate_if_needed_at(gini.height.len().min(starting_indexes.height.to_usize()))?;
|
||||||
|
|
||||||
let start_height = gini.height.len();
|
let total_heights = supply_vecs
|
||||||
if start_height >= total_heights {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
let num_days = supply_vecs
|
|
||||||
.iter()
|
.iter()
|
||||||
.map(|v| v.len())
|
.map(|v| v.len())
|
||||||
.min()
|
.min()
|
||||||
.unwrap_or(0)
|
.unwrap_or(0)
|
||||||
.min(count_vecs.iter().map(|v| v.len()).min().unwrap_or(0));
|
.min(count_vecs.iter().map(|v| v.len()).min().unwrap_or(0));
|
||||||
|
|
||||||
// Only compute gini for new days (each day is independent)
|
let start_height = gini.height.len();
|
||||||
let start_day = if start_height > 0 {
|
if start_height >= total_heights {
|
||||||
h2d[start_height].to_usize()
|
return Ok(());
|
||||||
} else {
|
|
||||||
0
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut gini_new: Vec<f32> = Vec::with_capacity(num_days.saturating_sub(start_day));
|
|
||||||
let mut buckets: Vec<(u64, u64)> = Vec::with_capacity(supply_vecs.len());
|
|
||||||
for di in start_day..num_days {
|
|
||||||
buckets.clear();
|
|
||||||
let day = Day1::from(di);
|
|
||||||
for (sv, cv) in supply_vecs.iter().zip(count_vecs.iter()) {
|
|
||||||
let supply: u64 = sv.collect_one_flat(day).unwrap_or_default().into();
|
|
||||||
let count: u64 = cv.collect_one_flat(day).unwrap_or_default().into();
|
|
||||||
buckets.push((count, supply));
|
|
||||||
}
|
|
||||||
gini_new.push(gini_from_lorenz(&buckets));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Expand to Height
|
// Batch-collect all cohort data for the range [start_height, total_heights)
|
||||||
(start_height..total_heights).for_each(|h| {
|
let n_cohorts = supply_vecs.len();
|
||||||
let di = h2d[h].to_usize();
|
let supply_data: Vec<Vec<Sats>> = supply_vecs
|
||||||
let offset = di.saturating_sub(start_day);
|
.iter()
|
||||||
let val = if offset < gini_new.len() {
|
.map(|v| v.collect_range_at(start_height, total_heights))
|
||||||
StoredF32::from(gini_new[offset])
|
.collect();
|
||||||
} else {
|
let count_data: Vec<Vec<StoredU64>> = count_vecs
|
||||||
StoredF32::NAN
|
.iter()
|
||||||
};
|
.map(|v| v.collect_range_at(start_height, total_heights))
|
||||||
gini.height.push(val);
|
.collect();
|
||||||
});
|
|
||||||
|
let mut buckets: Vec<(u64, u64)> = Vec::with_capacity(n_cohorts);
|
||||||
|
for offset in 0..total_heights - start_height {
|
||||||
|
buckets.clear();
|
||||||
|
for c in 0..n_cohorts {
|
||||||
|
let supply: u64 = supply_data[c][offset].into();
|
||||||
|
let count: u64 = count_data[c][offset].into();
|
||||||
|
buckets.push((count, supply));
|
||||||
|
}
|
||||||
|
gini.height
|
||||||
|
.push(StoredF32::from(gini_from_lorenz(&buckets)));
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
let _lock = exit.lock();
|
let _lock = exit.lock();
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use vecdb::Exit;
|
|||||||
use super::MacdChain;
|
use super::MacdChain;
|
||||||
use crate::{ComputeIndexes, blocks, prices};
|
use crate::{ComputeIndexes, blocks, prices};
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(super) fn compute(
|
pub(super) fn compute(
|
||||||
chain: &mut MacdChain,
|
chain: &mut MacdChain,
|
||||||
blocks: &blocks::Vecs,
|
blocks: &blocks::Vecs,
|
||||||
@@ -19,19 +20,15 @@ pub(super) fn compute(
|
|||||||
let ws_slow = blocks.count.start_vec(slow_days);
|
let ws_slow = blocks.count.start_vec(slow_days);
|
||||||
let ws_signal = blocks.count.start_vec(signal_days);
|
let ws_signal = blocks.count.start_vec(signal_days);
|
||||||
|
|
||||||
chain.ema_fast.height.compute_rolling_ema(
|
chain
|
||||||
starting_indexes.height,
|
.ema_fast
|
||||||
ws_fast,
|
.height
|
||||||
close,
|
.compute_rolling_ema(starting_indexes.height, ws_fast, close, exit)?;
|
||||||
exit,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
chain.ema_slow.height.compute_rolling_ema(
|
chain
|
||||||
starting_indexes.height,
|
.ema_slow
|
||||||
ws_slow,
|
.height
|
||||||
close,
|
.compute_rolling_ema(starting_indexes.height, ws_slow, close, exit)?;
|
||||||
exit,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
// MACD line = ema_fast - ema_slow
|
// MACD line = ema_fast - ema_slow
|
||||||
chain.line.height.compute_subtract(
|
chain.line.height.compute_subtract(
|
||||||
|
|||||||
+15
-15
@@ -239,9 +239,9 @@
|
|||||||
* Data range with output format for API query parameters
|
* Data range with output format for API query parameters
|
||||||
*
|
*
|
||||||
* @typedef {Object} DataRangeFormat
|
* @typedef {Object} DataRangeFormat
|
||||||
* @property {?number=} start - Inclusive starting index, if negative counts from end
|
* @property {?number=} start - Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
* @property {?number=} end - Exclusive ending index, if negative counts from end
|
* @property {?number=} end - Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
* @property {(Limit|null)=} limit - Maximum number of values to return (ignored if `end` is set)
|
* @property {(Limit|null)=} limit - Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
* @property {Format=} format - Format of the output
|
* @property {Format=} format - Format of the output
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
@@ -461,9 +461,9 @@
|
|||||||
* @typedef {Object} MetricSelection
|
* @typedef {Object} MetricSelection
|
||||||
* @property {Metrics} metrics - Requested metrics
|
* @property {Metrics} metrics - Requested metrics
|
||||||
* @property {Index} index - Index to query
|
* @property {Index} index - Index to query
|
||||||
* @property {?number=} start - Inclusive starting index, if negative counts from end
|
* @property {?number=} start - Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
* @property {?number=} end - Exclusive ending index, if negative counts from end
|
* @property {?number=} end - Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
* @property {(Limit|null)=} limit - Maximum number of values to return (ignored if `end` is set)
|
* @property {(Limit|null)=} limit - Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
* @property {Format=} format - Format of the output
|
* @property {Format=} format - Format of the output
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
@@ -472,9 +472,9 @@
|
|||||||
* @typedef {Object} MetricSelectionLegacy
|
* @typedef {Object} MetricSelectionLegacy
|
||||||
* @property {Index} index
|
* @property {Index} index
|
||||||
* @property {Metrics} ids
|
* @property {Metrics} ids
|
||||||
* @property {?number=} start - Inclusive starting index, if negative counts from end
|
* @property {?number=} start - Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
* @property {?number=} end - Exclusive ending index, if negative counts from end
|
* @property {?number=} end - Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
* @property {(Limit|null)=} limit - Maximum number of values to return (ignored if `end` is set)
|
* @property {(Limit|null)=} limit - Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
* @property {Format=} format - Format of the output
|
* @property {Format=} format - Format of the output
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
@@ -7994,9 +7994,9 @@ class BrkClient extends BrkClientBase {
|
|||||||
*
|
*
|
||||||
* @param {Metric} metric - Metric name
|
* @param {Metric} metric - Metric name
|
||||||
* @param {Index} index - Aggregation index
|
* @param {Index} index - Aggregation index
|
||||||
* @param {number=} [start] - Inclusive starting index, if negative counts from end
|
* @param {number=} [start] - Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
* @param {number=} [end] - Exclusive ending index, if negative counts from end
|
* @param {number=} [end] - Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
* @param {string=} [limit] - Maximum number of values to return (ignored if `end` is set)
|
* @param {string=} [limit] - Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
* @param {Format=} [format] - Format of the output
|
* @param {Format=} [format] - Format of the output
|
||||||
* @returns {Promise<AnyMetricData | string>}
|
* @returns {Promise<AnyMetricData | string>}
|
||||||
*/
|
*/
|
||||||
@@ -8035,9 +8035,9 @@ class BrkClient extends BrkClientBase {
|
|||||||
*
|
*
|
||||||
* @param {Metrics} [metrics] - Requested metrics
|
* @param {Metrics} [metrics] - Requested metrics
|
||||||
* @param {Index} [index] - Index to query
|
* @param {Index} [index] - Index to query
|
||||||
* @param {number=} [start] - Inclusive starting index, if negative counts from end
|
* @param {number=} [start] - Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
* @param {number=} [end] - Exclusive ending index, if negative counts from end
|
* @param {number=} [end] - Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
* @param {string=} [limit] - Maximum number of values to return (ignored if `end` is set)
|
* @param {string=} [limit] - Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
* @param {Format=} [format] - Format of the output
|
* @param {Format=} [format] - Format of the output
|
||||||
* @returns {Promise<AnyMetricData[] | string>}
|
* @returns {Promise<AnyMetricData[] | string>}
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -413,9 +413,9 @@ class DataRangeFormat(TypedDict):
|
|||||||
Data range with output format for API query parameters
|
Data range with output format for API query parameters
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
start: Inclusive starting index, if negative counts from end
|
start: Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
end: Exclusive ending index, if negative counts from end
|
end: Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
limit: Maximum number of values to return (ignored if `end` is set)
|
limit: Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
format: Format of the output
|
format: Format of the output
|
||||||
"""
|
"""
|
||||||
start: Optional[int]
|
start: Optional[int]
|
||||||
@@ -638,9 +638,9 @@ class MetricSelection(TypedDict):
|
|||||||
Attributes:
|
Attributes:
|
||||||
metrics: Requested metrics
|
metrics: Requested metrics
|
||||||
index: Index to query
|
index: Index to query
|
||||||
start: Inclusive starting index, if negative counts from end
|
start: Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
end: Exclusive ending index, if negative counts from end
|
end: Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
limit: Maximum number of values to return (ignored if `end` is set)
|
limit: Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
format: Format of the output
|
format: Format of the output
|
||||||
"""
|
"""
|
||||||
metrics: Metrics
|
metrics: Metrics
|
||||||
@@ -655,9 +655,9 @@ class MetricSelectionLegacy(TypedDict):
|
|||||||
Legacy metric selection parameters (deprecated)
|
Legacy metric selection parameters (deprecated)
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
start: Inclusive starting index, if negative counts from end
|
start: Inclusive starting index, if negative counts from end. Aliases: `from`, `f`, `s`
|
||||||
end: Exclusive ending index, if negative counts from end
|
end: Exclusive ending index, if negative counts from end. Aliases: `to`, `t`, `e`
|
||||||
limit: Maximum number of values to return (ignored if `end` is set)
|
limit: Maximum number of values to return (ignored if `end` is set). Aliases: `count`, `c`, `l`
|
||||||
format: Format of the output
|
format: Format of the output
|
||||||
"""
|
"""
|
||||||
index: Index
|
index: Index
|
||||||
|
|||||||
@@ -37,6 +37,10 @@ Get a metric by name and index:
|
|||||||
GET /api/metric/{metric}/{index}
|
GET /api/metric/{metric}/{index}
|
||||||
GET /api/metric/{metric}/{index}?start=-30
|
GET /api/metric/{metric}/{index}?start=-30
|
||||||
|
|
||||||
|
Example — last 30 days of Bitcoin closing price:
|
||||||
|
|
||||||
|
GET /api/metric/close/1d?start=-30
|
||||||
|
|
||||||
Fetch multiple metrics at once:
|
Fetch multiple metrics at once:
|
||||||
|
|
||||||
GET /api/metrics/bulk?index={index}&metrics={metric1},{metric2}
|
GET /api/metrics/bulk?index={index}&metrics={metric1},{metric2}
|
||||||
|
|||||||
Reference in New Issue
Block a user