global: snapshot

This commit is contained in:
nym21
2026-01-16 23:49:49 +01:00
parent 3b00a92fa4
commit 6bb1a2a311
34 changed files with 2600 additions and 5071 deletions

View File

@@ -26,7 +26,8 @@ impl Vecs {
info!("Computing oracle prices...");
let i = Instant::now();
self.oracle.compute(indexer, indexes, starting_indexes, exit)?;
self.oracle
.compute(indexer, indexes, &self.cents, starting_indexes, exit)?;
info!("Computed oracle prices in {:?}", i.elapsed());
}

View File

@@ -53,8 +53,8 @@ use std::collections::VecDeque;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{
Cents, Close, Date, DateIndex, Height, High, Low, OHLCCents, Open, OracleBins, OutputType,
PHASE_BINS, PairOutputIndex, Sats, StoredU32, StoredU64, TxIndex,
Cents, Close, Date, DateIndex, Height, High, Low, OHLCCents, Open, OracleBins, OracleBinsV2,
OutputType, PHASE_BINS, PairOutputIndex, Sats, StoredU32, StoredU64, TxIndex,
};
use tracing::info;
use vecdb::{
@@ -66,9 +66,10 @@ use super::{
Vecs,
config::OracleConfig,
histogram::{Histogram, TOTAL_BINS},
phase_v2::{PhaseHistogramV2, find_best_phase, phase_range_from_anchor, phase_to_price},
stencil::{find_best_price, is_round_sats, refine_price},
};
use crate::{ComputeIndexes, indexes};
use crate::{ComputeIndexes, indexes, price::cents};
/// Flush interval for periodic writes during oracle computation.
const FLUSH_INTERVAL: usize = 10_000;
@@ -79,6 +80,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
price_cents: &cents::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
@@ -100,6 +102,32 @@ impl Vecs {
// Step 7: Aggregate to daily OHLC
self.compute_daily_ohlc(indexes, starting_indexes, exit)?;
// Step 8: Compute Phase Oracle V2 (round USD template matching)
// 8a: Per-block 200-bin histograms (uses ALL outputs, not pair-filtered)
self.compute_phase_v2_histograms(indexer, indexes, starting_indexes, exit)?;
// 8b: Per-block prices using cross-correlation with weekly anchors
self.compute_phase_v2_prices(indexes, price_cents, starting_indexes, exit)?;
// 8c: Per-block prices using direct peak finding (like V1)
self.compute_phase_v2_peak_prices(indexes, price_cents, starting_indexes, exit)?;
// 8d: Daily distributions from per-block prices
self.compute_phase_v2_daily(indexes, starting_indexes, exit)?;
// Step 9: Compute Phase Oracle V3 (BASE + uniqueVal filter)
// 9a: Per-block histograms with uniqueVal filtering (only outputs with unique values in tx)
self.compute_phase_v3_histograms(indexer, indexes, starting_indexes, exit)?;
// 9b: Per-block prices using cross-correlation
self.compute_phase_v3_prices(indexes, price_cents, starting_indexes, exit)?;
// 9c: Per-block prices using direct peak finding (like V1)
self.compute_phase_v3_peak_prices(indexes, price_cents, starting_indexes, exit)?;
// 9d: Daily distributions from per-block prices
self.compute_phase_v3_daily(indexes, starting_indexes, exit)?;
Ok(())
}
@@ -1091,4 +1119,898 @@ impl Vecs {
Ok(())
}
/// Compute Phase Oracle V2 - Step 1: Per-block 200-bin phase histograms
///
/// Uses ALL outputs (like Python test), filtered only by sats range (1k-100k BTC).
/// This is different from the pair-filtered approach used by UTXOracle.
fn compute_phase_v2_histograms(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version = indexer.vecs.outputs.value.version();
self.phase_v2_histogram
.validate_computed_version_or_reset(source_version)?;
let total_heights = indexer.vecs.blocks.timestamp.len();
let start_height = self
.phase_v2_histogram
.len()
.min(starting_indexes.height.to_usize());
self.phase_v2_histogram
.truncate_if_needed_at(start_height)?;
if start_height >= total_heights {
return Ok(());
}
info!(
"Computing phase V2 histograms from height {} to {}",
start_height, total_heights
);
let mut height_to_first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter();
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.transactions.first_txoutindex.into_iter();
let mut txindex_to_output_count_iter = indexes.txindex.output_count.iter();
let mut txoutindex_to_value_iter = indexer.vecs.outputs.value.into_iter();
let total_txs = indexer.vecs.transactions.height.len();
let mut last_progress = (start_height * 100 / total_heights.max(1)) as u8;
for height in start_height..total_heights {
// Get transaction range for this block
let first_txindex = height_to_first_txindex_iter.get_at_unwrap(height);
let next_first_txindex = height_to_first_txindex_iter
.get_at(height + 1)
.unwrap_or(TxIndex::from(total_txs));
// Build phase histogram from ALL outputs in this block
let mut histogram = OracleBinsV2::ZERO;
for txindex in first_txindex.to_usize()..next_first_txindex.to_usize() {
// Get output count and first output for this transaction
let first_txoutindex = txindex_to_first_txoutindex_iter.get_at_unwrap(txindex);
let output_count: StoredU64 =
txindex_to_output_count_iter.get_unwrap(TxIndex::from(txindex));
for i in 0..*output_count as usize {
let txoutindex = first_txoutindex.to_usize() + i;
let sats: Sats = txoutindex_to_value_iter.get_at_unwrap(txoutindex);
// OracleBinsV2::add already filters by sats range (1k to 100k BTC)
histogram.add(sats);
}
}
self.phase_v2_histogram.push(histogram);
// Progress logging
let progress = (height * 100 / total_heights.max(1)) as u8;
if progress > last_progress {
last_progress = progress;
info!("Phase V2 histogram computation: {}%", progress);
let _lock = exit.lock();
self.phase_v2_histogram.write()?;
}
}
// Final write
{
let _lock = exit.lock();
self.phase_v2_histogram.write()?;
}
info!(
"Phase V2 histograms complete: {} blocks",
self.phase_v2_histogram.len()
);
Ok(())
}
/// Compute Phase Oracle V2 - Step 2: Per-block prices using cross-correlation
fn compute_phase_v2_prices(
&mut self,
indexes: &indexes::Vecs,
price_cents: &cents::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version = self.phase_v2_histogram.version();
self.phase_v2_price_cents
.validate_computed_version_or_reset(source_version)?;
let total_heights = self.phase_v2_histogram.len();
let start_height = self
.phase_v2_price_cents
.len()
.min(starting_indexes.height.to_usize());
self.phase_v2_price_cents
.truncate_if_needed_at(start_height)?;
if start_height >= total_heights {
return Ok(());
}
info!(
"Computing phase V2 prices from height {} to {}",
start_height, total_heights
);
let mut histogram_iter = self.phase_v2_histogram.iter()?;
let mut height_to_dateindex_iter = indexes.height.dateindex.iter();
// For weekly OHLC anchors
let mut price_ohlc_iter = price_cents.ohlc.dateindex.iter()?;
let mut dateindex_to_weekindex_iter = indexes.dateindex.weekindex.iter();
let mut weekindex_to_first_dateindex_iter = indexes.weekindex.first_dateindex.iter();
let mut weekindex_dateindex_count_iter = indexes.weekindex.dateindex_count.iter();
let mut last_progress = (start_height * 100 / total_heights.max(1)) as u8;
// Track previous price for fallback
let mut prev_price_cents = if start_height > 0 {
self.phase_v2_price_cents
.iter()?
.get(Height::from(start_height - 1))
.unwrap_or(Cents::from(10_000_000i64))
} else {
Cents::from(10_000_000i64) // Default ~$100k
};
for height in start_height..total_heights {
let height_idx = Height::from(height);
let histogram: OracleBinsV2 = histogram_iter.get_unwrap(height_idx);
// Get weekly anchor for this block's date
let dateindex = height_to_dateindex_iter.get(height_idx);
let weekly_bounds: Option<(f64, f64)> = dateindex.and_then(|di| {
let wi = dateindex_to_weekindex_iter.get(di)?;
let first_di = weekindex_to_first_dateindex_iter.get(wi)?;
let count = weekindex_dateindex_count_iter
.get(wi)
.map(|c| *c as usize)?;
let mut low = Cents::from(i64::MAX);
let mut high = Cents::from(0i64);
for i in 0..count {
let di = DateIndex::from(first_di.to_usize() + i);
if let Some(ohlc) = price_ohlc_iter.get(di) {
if *ohlc.low < low {
low = *ohlc.low;
}
if *ohlc.high > high {
high = *ohlc.high;
}
}
}
if i64::from(low) > 0 && i64::from(high) > 0 {
Some((
i64::from(low) as f64 / 100.0,
i64::from(high) as f64 / 100.0,
))
} else {
None
}
});
// Compute price using cross-correlation
let price_cents = if histogram.total_count() >= 10 {
// Convert OracleBinsV2 to PhaseHistogramV2
let mut phase_hist = PhaseHistogramV2::new();
for (i, &count) in histogram.bins.iter().enumerate() {
if count > 0 {
let phase = (i as f64 + 0.5) / 200.0;
let log_sats = 6.0 + phase;
let sats = 10.0_f64.powf(log_sats);
for _ in 0..count {
phase_hist.add(Sats::from(sats as u64));
}
}
}
if let Some((low, high)) = weekly_bounds {
// Have weekly anchor - constrained search
let (phase_min, phase_max) = phase_range_from_anchor(low, high, 0.05);
let (best_phase, _corr) =
find_best_phase(&phase_hist, 2, Some(phase_min), Some(phase_max));
let price = phase_to_price(best_phase, low, high);
Cents::from((price * 100.0) as i64)
} else {
// No anchor - use previous price as reference
let anchor_low = (i64::from(prev_price_cents) as f64 / 100.0) * 0.5;
let anchor_high = (i64::from(prev_price_cents) as f64 / 100.0) * 2.0;
let (best_phase, _corr) = find_best_phase(&phase_hist, 2, None, None);
let price = phase_to_price(best_phase, anchor_low, anchor_high);
Cents::from((price * 100.0) as i64)
}
} else {
// Too few outputs - use previous price
prev_price_cents
};
prev_price_cents = price_cents;
self.phase_v2_price_cents.push(price_cents);
// Progress logging
let progress = (height * 100 / total_heights.max(1)) as u8;
if progress > last_progress {
last_progress = progress;
info!("Phase V2 price computation: {}%", progress);
let _lock = exit.lock();
self.phase_v2_price_cents.write()?;
}
}
// Final write
{
let _lock = exit.lock();
self.phase_v2_price_cents.write()?;
}
info!(
"Phase V2 prices complete: {} blocks",
self.phase_v2_price_cents.len()
);
Ok(())
}
/// Compute Phase Oracle V2 - Peak prices using direct peak finding (like V1)
fn compute_phase_v2_peak_prices(
&mut self,
indexes: &indexes::Vecs,
price_cents: &cents::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version = self.phase_v2_histogram.version();
self.phase_v2_peak_price_cents
.validate_computed_version_or_reset(source_version)?;
let total_heights = self.phase_v2_histogram.len();
let start_height = self
.phase_v2_peak_price_cents
.len()
.min(starting_indexes.height.to_usize());
self.phase_v2_peak_price_cents
.truncate_if_needed_at(start_height)?;
if start_height >= total_heights {
return Ok(());
}
info!(
"Computing phase V2 peak prices from height {} to {}",
start_height, total_heights
);
let mut histogram_iter = self.phase_v2_histogram.iter()?;
let mut height_to_dateindex_iter = indexes.height.dateindex.iter();
// For weekly OHLC anchors
let mut price_ohlc_iter = price_cents.ohlc.dateindex.iter()?;
let mut dateindex_to_weekindex_iter = indexes.dateindex.weekindex.iter();
let mut weekindex_to_first_dateindex_iter = indexes.weekindex.first_dateindex.iter();
let mut weekindex_dateindex_count_iter = indexes.weekindex.dateindex_count.iter();
let mut last_progress = (start_height * 100 / total_heights.max(1)) as u8;
// Track previous price for fallback
let mut prev_price_cents = if start_height > 0 {
self.phase_v2_peak_price_cents
.iter()?
.get(Height::from(start_height - 1))
.unwrap_or(Cents::from(10_000_000i64))
} else {
Cents::from(10_000_000i64)
};
for height in start_height..total_heights {
let height_idx = Height::from(height);
let histogram: OracleBinsV2 = histogram_iter.get_unwrap(height_idx);
// Get weekly anchor for decade selection
let dateindex = height_to_dateindex_iter.get(height_idx);
let anchor_price: Option<f64> = dateindex.and_then(|di| {
let wi = dateindex_to_weekindex_iter.get(di)?;
let first_di = weekindex_to_first_dateindex_iter.get(wi)?;
let count = weekindex_dateindex_count_iter
.get(wi)
.map(|c| *c as usize)?;
let mut sum = 0i64;
let mut cnt = 0;
for i in 0..count {
let di = DateIndex::from(first_di.to_usize() + i);
if let Some(ohlc) = price_ohlc_iter.get(di) {
sum += i64::from(*ohlc.close);
cnt += 1;
}
}
if cnt > 0 {
Some(sum as f64 / cnt as f64 / 100.0)
} else {
None
}
});
// Use anchor or previous price for decade selection
let anchor = anchor_price.unwrap_or(i64::from(prev_price_cents) as f64 / 100.0);
// Find peak bin directly (like V1) using 100 bins (downsample from 200)
let price_cents = if histogram.total_count() >= 10 {
// Downsample 200 bins to 100 bins
let mut bins100 = [0u32; 100];
for i in 0..100 {
bins100[i] = histogram.bins[i * 2] as u32 + histogram.bins[i * 2 + 1] as u32;
}
// Find peak bin, skipping bin 0 (round BTC amounts cluster there)
let peak_bin = bins100
.iter()
.enumerate()
.filter(|(bin, _)| *bin != 0)
.max_by_key(|(_, count)| *count)
.map(|(bin, _)| bin)
.unwrap_or(0);
// Convert bin to price using anchor for decade (100 bins)
let phase = (peak_bin as f64 + 0.5) / 100.0;
let base_price = 10.0_f64.powf(phase);
// Find best decade
let mut best_price = base_price;
let mut best_dist = f64::MAX;
for decade in -2..=6 {
let candidate = base_price * 10.0_f64.powi(decade);
let dist = (candidate - anchor).abs();
if dist < best_dist {
best_dist = dist;
best_price = candidate;
}
}
Cents::from((best_price.clamp(0.01, 10_000_000.0) * 100.0) as i64)
} else {
prev_price_cents
};
prev_price_cents = price_cents;
self.phase_v2_peak_price_cents.push(price_cents);
// Progress logging
let progress = (height * 100 / total_heights.max(1)) as u8;
if progress > last_progress {
last_progress = progress;
info!("Phase V2 peak price computation: {}%", progress);
let _lock = exit.lock();
self.phase_v2_peak_price_cents.write()?;
}
}
// Final write
{
let _lock = exit.lock();
self.phase_v2_peak_price_cents.write()?;
}
info!(
"Phase V2 peak prices complete: {} blocks",
self.phase_v2_peak_price_cents.len()
);
Ok(())
}
/// Compute Phase Oracle V2 - Daily distributions from per-block prices
fn compute_phase_v2_daily(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
info!("Computing phase V2 daily distributions");
// Cross-correlation based
self.phase_v2_daily_cents.compute(
starting_indexes.dateindex,
&self.phase_v2_price_cents,
&indexes.dateindex.first_height,
&indexes.dateindex.height_count,
exit,
)?;
// Peak-based
self.phase_v2_peak_daily_cents.compute(
starting_indexes.dateindex,
&self.phase_v2_peak_price_cents,
&indexes.dateindex.first_height,
&indexes.dateindex.height_count,
exit,
)?;
info!(
"Phase V2 daily distributions complete: {} days",
self.phase_v2_daily_cents.len()
);
Ok(())
}
/// Compute Phase Oracle V3 - Step 1: Per-block histograms with uniqueVal filtering
///
/// Filters: >= 1000 sats, only outputs with unique values within their transaction.
/// This reduces spurious peaks from exchange batched payouts and inscription spam.
fn compute_phase_v3_histograms(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version = indexer.vecs.outputs.value.version();
self.phase_v3_histogram
.validate_computed_version_or_reset(source_version)?;
let total_heights = indexer.vecs.blocks.timestamp.len();
let start_height = self
.phase_v3_histogram
.len()
.min(starting_indexes.height.to_usize());
self.phase_v3_histogram
.truncate_if_needed_at(start_height)?;
if start_height >= total_heights {
return Ok(());
}
info!(
"Computing phase V3 histograms from height {} to {}",
start_height, total_heights
);
let mut height_to_first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter();
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.transactions.first_txoutindex.into_iter();
let mut txindex_to_output_count_iter = indexes.txindex.output_count.iter();
let mut txoutindex_to_value_iter = indexer.vecs.outputs.value.into_iter();
let total_txs = indexer.vecs.transactions.height.len();
let mut last_progress = (start_height * 100 / total_heights.max(1)) as u8;
// Reusable buffer for collecting output values per transaction
let mut tx_values: Vec<Sats> = Vec::with_capacity(16);
for height in start_height..total_heights {
// Get transaction range for this block
let first_txindex = height_to_first_txindex_iter.get_at_unwrap(height);
let next_first_txindex = height_to_first_txindex_iter
.get_at(height + 1)
.unwrap_or(TxIndex::from(total_txs));
// Build phase histogram with uniqueVal filtering
let mut histogram = OracleBinsV2::ZERO;
// Skip coinbase (first tx in block)
for txindex in (first_txindex.to_usize() + 1)..next_first_txindex.to_usize() {
// Get output count and first output for this transaction
let first_txoutindex = txindex_to_first_txoutindex_iter.get_at_unwrap(txindex);
let output_count: StoredU64 =
txindex_to_output_count_iter.get_unwrap(TxIndex::from(txindex));
// Collect all output values for this transaction
tx_values.clear();
for i in 0..*output_count as usize {
let txoutindex = first_txoutindex.to_usize() + i;
let sats: Sats = txoutindex_to_value_iter.get_at_unwrap(txoutindex);
tx_values.push(sats);
}
// Count occurrences of each value to determine uniqueness
// For small output counts, simple nested loop is faster than HashMap
for (i, &sats) in tx_values.iter().enumerate() {
// Skip if below minimum (BASE filter: >= 1000 sats)
if sats < Sats::_1K {
continue;
}
// Check if this value is unique within the transaction
let mut is_unique = true;
for (j, &other_sats) in tx_values.iter().enumerate() {
if i != j && sats == other_sats {
is_unique = false;
break;
}
}
// Only add unique values to histogram
if is_unique {
histogram.add(sats);
}
}
}
self.phase_v3_histogram.push(histogram);
// Progress logging
let progress = (height * 100 / total_heights.max(1)) as u8;
if progress > last_progress {
last_progress = progress;
info!("Phase V3 histogram computation: {}%", progress);
let _lock = exit.lock();
self.phase_v3_histogram.write()?;
}
}
// Final write
{
let _lock = exit.lock();
self.phase_v3_histogram.write()?;
}
info!(
"Phase V3 histograms complete: {} blocks",
self.phase_v3_histogram.len()
);
Ok(())
}
/// Compute Phase Oracle V3 - Step 2: Per-block prices using cross-correlation
fn compute_phase_v3_prices(
&mut self,
indexes: &indexes::Vecs,
price_cents: &cents::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version = self.phase_v3_histogram.version();
self.phase_v3_price_cents
.validate_computed_version_or_reset(source_version)?;
let total_heights = self.phase_v3_histogram.len();
let start_height = self
.phase_v3_price_cents
.len()
.min(starting_indexes.height.to_usize());
self.phase_v3_price_cents
.truncate_if_needed_at(start_height)?;
if start_height >= total_heights {
return Ok(());
}
info!(
"Computing phase V3 prices from height {} to {}",
start_height, total_heights
);
let mut histogram_iter = self.phase_v3_histogram.iter()?;
let mut height_to_dateindex_iter = indexes.height.dateindex.iter();
// For weekly OHLC anchors
let mut price_ohlc_iter = price_cents.ohlc.dateindex.iter()?;
let mut dateindex_to_weekindex_iter = indexes.dateindex.weekindex.iter();
let mut weekindex_to_first_dateindex_iter = indexes.weekindex.first_dateindex.iter();
let mut weekindex_dateindex_count_iter = indexes.weekindex.dateindex_count.iter();
let mut last_progress = (start_height * 100 / total_heights.max(1)) as u8;
// Track previous price for fallback
let mut prev_price_cents = if start_height > 0 {
self.phase_v3_price_cents
.iter()?
.get(Height::from(start_height - 1))
.unwrap_or(Cents::from(10_000_000i64))
} else {
Cents::from(10_000_000i64) // Default ~$100k
};
for height in start_height..total_heights {
let height_idx = Height::from(height);
let histogram: OracleBinsV2 = histogram_iter.get_unwrap(height_idx);
// Get weekly anchor for this block's date
let dateindex = height_to_dateindex_iter.get(height_idx);
let weekly_bounds: Option<(f64, f64)> = dateindex.and_then(|di| {
let wi = dateindex_to_weekindex_iter.get(di)?;
let first_di = weekindex_to_first_dateindex_iter.get(wi)?;
let count = weekindex_dateindex_count_iter
.get(wi)
.map(|c| *c as usize)?;
let mut low = Cents::from(i64::MAX);
let mut high = Cents::from(0i64);
for i in 0..count {
let di = DateIndex::from(first_di.to_usize() + i);
if let Some(ohlc) = price_ohlc_iter.get(di) {
if *ohlc.low < low {
low = *ohlc.low;
}
if *ohlc.high > high {
high = *ohlc.high;
}
}
}
if i64::from(low) > 0 && i64::from(high) > 0 {
Some((
i64::from(low) as f64 / 100.0,
i64::from(high) as f64 / 100.0,
))
} else {
None
}
});
// Compute price using cross-correlation
let price_cents = if histogram.total_count() >= 10 {
// Convert OracleBinsV2 to PhaseHistogramV2
let mut phase_hist = PhaseHistogramV2::new();
for (i, &count) in histogram.bins.iter().enumerate() {
if count > 0 {
let phase = (i as f64 + 0.5) / 200.0;
let log_sats = 6.0 + phase;
let sats = 10.0_f64.powf(log_sats);
for _ in 0..count {
phase_hist.add(Sats::from(sats as u64));
}
}
}
if let Some((low, high)) = weekly_bounds {
// Have weekly anchor - constrained search
let (phase_min, phase_max) = phase_range_from_anchor(low, high, 0.05);
let (best_phase, _corr) =
find_best_phase(&phase_hist, 2, Some(phase_min), Some(phase_max));
let price = phase_to_price(best_phase, low, high);
Cents::from((price * 100.0) as i64)
} else {
// No anchor - use previous price as reference
let anchor_low = (i64::from(prev_price_cents) as f64 / 100.0) * 0.5;
let anchor_high = (i64::from(prev_price_cents) as f64 / 100.0) * 2.0;
let (best_phase, _corr) = find_best_phase(&phase_hist, 2, None, None);
let price = phase_to_price(best_phase, anchor_low, anchor_high);
Cents::from((price * 100.0) as i64)
}
} else {
// Too few outputs - use previous price
prev_price_cents
};
prev_price_cents = price_cents;
self.phase_v3_price_cents.push(price_cents);
// Progress logging
let progress = (height * 100 / total_heights.max(1)) as u8;
if progress > last_progress {
last_progress = progress;
info!("Phase V3 price computation: {}%", progress);
let _lock = exit.lock();
self.phase_v3_price_cents.write()?;
}
}
// Final write
{
let _lock = exit.lock();
self.phase_v3_price_cents.write()?;
}
info!(
"Phase V3 prices complete: {} blocks",
self.phase_v3_price_cents.len()
);
Ok(())
}
/// Compute Phase Oracle V3 - Peak prices using direct peak finding (like V1)
fn compute_phase_v3_peak_prices(
&mut self,
indexes: &indexes::Vecs,
price_cents: &cents::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let source_version = self.phase_v3_histogram.version();
self.phase_v3_peak_price_cents
.validate_computed_version_or_reset(source_version)?;
let total_heights = self.phase_v3_histogram.len();
let start_height = self
.phase_v3_peak_price_cents
.len()
.min(starting_indexes.height.to_usize());
self.phase_v3_peak_price_cents
.truncate_if_needed_at(start_height)?;
if start_height >= total_heights {
return Ok(());
}
info!(
"Computing phase V3 peak prices from height {} to {}",
start_height, total_heights
);
let mut histogram_iter = self.phase_v3_histogram.iter()?;
let mut height_to_dateindex_iter = indexes.height.dateindex.iter();
// For weekly OHLC anchors
let mut price_ohlc_iter = price_cents.ohlc.dateindex.iter()?;
let mut dateindex_to_weekindex_iter = indexes.dateindex.weekindex.iter();
let mut weekindex_to_first_dateindex_iter = indexes.weekindex.first_dateindex.iter();
let mut weekindex_dateindex_count_iter = indexes.weekindex.dateindex_count.iter();
let mut last_progress = (start_height * 100 / total_heights.max(1)) as u8;
// Track previous price for fallback
let mut prev_price_cents = if start_height > 0 {
self.phase_v3_peak_price_cents
.iter()?
.get(Height::from(start_height - 1))
.unwrap_or(Cents::from(10_000_000i64))
} else {
Cents::from(10_000_000i64)
};
for height in start_height..total_heights {
let height_idx = Height::from(height);
let histogram: OracleBinsV2 = histogram_iter.get_unwrap(height_idx);
// Get weekly anchor for decade selection
let dateindex = height_to_dateindex_iter.get(height_idx);
let anchor_price: Option<f64> = dateindex.and_then(|di| {
let wi = dateindex_to_weekindex_iter.get(di)?;
let first_di = weekindex_to_first_dateindex_iter.get(wi)?;
let count = weekindex_dateindex_count_iter
.get(wi)
.map(|c| *c as usize)?;
let mut sum = 0i64;
let mut cnt = 0;
for i in 0..count {
let di = DateIndex::from(first_di.to_usize() + i);
if let Some(ohlc) = price_ohlc_iter.get(di) {
sum += i64::from(*ohlc.close);
cnt += 1;
}
}
if cnt > 0 {
Some(sum as f64 / cnt as f64 / 100.0)
} else {
None
}
});
// Use anchor or previous price for decade selection
let anchor = anchor_price.unwrap_or(i64::from(prev_price_cents) as f64 / 100.0);
// Find peak bin directly (like V1) using 100 bins (downsample from 200)
let price_cents = if histogram.total_count() >= 10 {
// Downsample 200 bins to 100 bins
let mut bins100 = [0u32; 100];
(0..100).for_each(|i| {
bins100[i] = histogram.bins[i * 2] as u32 + histogram.bins[i * 2 + 1] as u32;
});
// Find peak bin, skipping bin 0 (round BTC amounts cluster there)
let peak_bin = bins100
.iter()
.enumerate()
.filter(|(bin, _)| *bin != 0)
.max_by_key(|(_, count)| *count)
.map(|(bin, _)| bin)
.unwrap_or(0);
// Convert bin to price using anchor for decade (100 bins)
let phase = (peak_bin as f64 + 0.5) / 100.0;
let base_price = 10.0_f64.powf(phase);
// Find best decade
let mut best_price = base_price;
let mut best_dist = f64::MAX;
for decade in -2..=6 {
let candidate = base_price * 10.0_f64.powi(decade);
let dist = (candidate - anchor).abs();
if dist < best_dist {
best_dist = dist;
best_price = candidate;
}
}
Cents::from((best_price.clamp(0.01, 10_000_000.0) * 100.0) as i64)
} else {
prev_price_cents
};
prev_price_cents = price_cents;
self.phase_v3_peak_price_cents.push(price_cents);
// Progress logging
let progress = (height * 100 / total_heights.max(1)) as u8;
if progress > last_progress {
last_progress = progress;
info!("Phase V3 peak price computation: {}%", progress);
let _lock = exit.lock();
self.phase_v3_peak_price_cents.write()?;
}
}
// Final write
{
let _lock = exit.lock();
self.phase_v3_peak_price_cents.write()?;
}
info!(
"Phase V3 peak prices complete: {} blocks",
self.phase_v3_peak_price_cents.len()
);
Ok(())
}
/// Compute Phase Oracle V3 - Daily distributions from per-block prices
fn compute_phase_v3_daily(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
info!("Computing phase V3 daily distributions");
// Cross-correlation based
self.phase_v3_daily_cents.compute(
starting_indexes.dateindex,
&self.phase_v3_price_cents,
&indexes.dateindex.first_height,
&indexes.dateindex.height_count,
exit,
)?;
// Peak-based
self.phase_v3_peak_daily_cents.compute(
starting_indexes.dateindex,
&self.phase_v3_peak_price_cents,
&indexes.dateindex.first_height,
&indexes.dateindex.height_count,
exit,
)?;
info!(
"Phase V3 daily distributions complete: {} days",
self.phase_v3_daily_cents.len()
);
Ok(())
}
}

View File

@@ -46,6 +46,58 @@ impl Vecs {
|di: DateIndex, iter| iter.get(di).map(|o: OHLCCents| OHLCDollars::from(o)),
);
// Phase Oracle V2 (round USD template matching)
// v3: Peak prices use 100 bins (downsampled from 200)
let phase_v2_version = version + Version::new(3);
let phase_v2_histogram =
BytesVec::forced_import(db, "phase_v2_histogram", phase_v2_version)?;
let phase_v2_price_cents =
PcoVec::forced_import(db, "phase_v2_price_cents", phase_v2_version)?;
let phase_v2_peak_price_cents =
PcoVec::forced_import(db, "phase_v2_peak_price_cents", phase_v2_version)?;
let phase_v2_daily_cents =
Distribution::forced_import(db, "phase_v2_daily", phase_v2_version)?;
let phase_v2_daily_dollars =
LazyTransformDistribution::from_distribution::<CentsToDollars>(
"phase_v2_daily_dollars",
phase_v2_version,
&phase_v2_daily_cents,
);
let phase_v2_peak_daily_cents =
Distribution::forced_import(db, "phase_v2_peak_daily", phase_v2_version)?;
let phase_v2_peak_daily_dollars =
LazyTransformDistribution::from_distribution::<CentsToDollars>(
"phase_v2_peak_daily_dollars",
phase_v2_version,
&phase_v2_peak_daily_cents,
);
// Phase Oracle V3 (BASE + uniqueVal filter)
// v4: Peak prices use 100 bins (downsampled from 200)
let phase_v3_version = version + Version::new(4);
let phase_v3_histogram =
BytesVec::forced_import(db, "phase_v3_histogram", phase_v3_version)?;
let phase_v3_price_cents =
PcoVec::forced_import(db, "phase_v3_price_cents", phase_v3_version)?;
let phase_v3_peak_price_cents =
PcoVec::forced_import(db, "phase_v3_peak_price_cents", phase_v3_version)?;
let phase_v3_daily_cents =
Distribution::forced_import(db, "phase_v3_daily", phase_v3_version)?;
let phase_v3_daily_dollars =
LazyTransformDistribution::from_distribution::<CentsToDollars>(
"phase_v3_daily_dollars",
phase_v3_version,
&phase_v3_daily_cents,
);
let phase_v3_peak_daily_cents =
Distribution::forced_import(db, "phase_v3_peak_daily", phase_v3_version)?;
let phase_v3_peak_daily_dollars =
LazyTransformDistribution::from_distribution::<CentsToDollars>(
"phase_v3_peak_daily_dollars",
phase_v3_version,
&phase_v3_peak_daily_cents,
);
Ok(Self {
pairoutputindex_to_txindex,
height_to_first_pairoutputindex,
@@ -59,6 +111,20 @@ impl Vecs {
ohlc_cents,
ohlc_dollars,
tx_count,
phase_v2_histogram,
phase_v2_price_cents,
phase_v2_peak_price_cents,
phase_v2_daily_cents,
phase_v2_daily_dollars,
phase_v2_peak_daily_cents,
phase_v2_peak_daily_dollars,
phase_v3_histogram,
phase_v3_price_cents,
phase_v3_peak_price_cents,
phase_v3_daily_cents,
phase_v3_daily_dollars,
phase_v3_peak_daily_cents,
phase_v3_peak_daily_dollars,
})
}
}

View File

@@ -158,6 +158,7 @@ mod compute;
mod config;
mod histogram;
mod import;
mod phase_v2;
mod stencil;
mod vecs;

View File

@@ -0,0 +1,296 @@
//! Phase Oracle V2 - Round USD Template Cross-Correlation
//!
//! Detects Bitcoin prices by finding where round USD amounts ($1, $5, $10, etc.)
//! cluster in the phase histogram. Uses weekly OHLC anchors to constrain search.
//!
//! ## Algorithm
//!
//! 1. Build 200-bin phase histogram: bin = frac(log10(sats)) * 200
//! 2. Cross-correlate with weighted round USD template
//! 3. Use weekly OHLC anchor to constrain phase search range
//! 4. Return best-matching phase, convert to price
//!
//! ## Key Insight
//!
//! Round USD amounts create a fixed "fingerprint" pattern in phase space:
//! - $1, $10, $100, $1000 → phase 0.00 (weight 10)
//! - $5, $50, $500 → phase 0.70 (weight 9)
//! - $2, $20, $200 → phase 0.30 (weight 7)
//! - etc.
//!
//! The pattern shifts based on price: sats_phase = usd_phase - price_phase (mod 1)
//! Finding the shift that best matches the template reveals the price phase.
use brk_types::Sats;
/// Number of phase bins (0.5% resolution)
pub const PHASE_BINS_V2: usize = 200;
/// Round USD template: (phase, weight) pairs
/// Phase = frac(log10(usd_cents)) for round USD values
/// Weight reflects expected popularity (higher = more common)
pub const ROUND_USD_TEMPLATE: [(f64, u32); 11] = [
(0.00, 10), // $1, $10, $100, $1000 - VERY common
(0.18, 3), // $1.50, $15, $150 - uncommon
(0.30, 7), // $2, $20, $200 - common
(0.40, 4), // $2.50, $25, $250 - moderate
(0.48, 5), // $3, $30, $300 - moderate
(0.60, 4), // $4, $40, $400 - moderate
(0.70, 9), // $5, $50, $500 - VERY common
(0.78, 2), // $6, $60, $600 - rare
(0.85, 2), // $7, $70, $700 - rare
(0.90, 2), // $8, $80, $800 - rare
(0.95, 2), // $9, $90, $900 - rare
];
/// Pre-computed template bins: (bin_index, weight)
pub fn template_bins() -> Vec<(usize, u32)> {
ROUND_USD_TEMPLATE
.iter()
.map(|&(phase, weight)| {
let bin = ((phase * PHASE_BINS_V2 as f64) as usize) % PHASE_BINS_V2;
(bin, weight)
})
.collect()
}
/// Phase histogram for V2 oracle (200 bins)
#[derive(Clone)]
pub struct PhaseHistogramV2 {
bins: [u32; PHASE_BINS_V2],
total: u32,
}
impl Default for PhaseHistogramV2 {
fn default() -> Self {
Self::new()
}
}
impl PhaseHistogramV2 {
pub fn new() -> Self {
Self {
bins: [0; PHASE_BINS_V2],
total: 0,
}
}
/// Convert sats value to phase bin index
/// Filters: min 1k sats, max 100k BTC
#[inline]
pub fn sats_to_bin(sats: Sats) -> Option<usize> {
if sats < Sats::_1K || sats > Sats::_100K_BTC {
return None;
}
let log_sats = f64::from(sats).log10();
let phase = log_sats.fract();
let phase = if phase < 0.0 { phase + 1.0 } else { phase };
Some(((phase * PHASE_BINS_V2 as f64) as usize).min(PHASE_BINS_V2 - 1))
}
/// Add a sats value to the histogram
#[inline]
pub fn add(&mut self, sats: Sats) {
if let Some(bin) = Self::sats_to_bin(sats) {
self.bins[bin] = self.bins[bin].saturating_add(1);
self.total += 1;
}
}
/// Add another histogram to this one
pub fn add_histogram(&mut self, other: &PhaseHistogramV2) {
for (i, &count) in other.bins.iter().enumerate() {
self.bins[i] = self.bins[i].saturating_add(count);
}
self.total = self.total.saturating_add(other.total);
}
/// Get total count
pub fn total(&self) -> u32 {
self.total
}
/// Get bins array
pub fn bins(&self) -> &[u32; PHASE_BINS_V2] {
&self.bins
}
/// Clear the histogram
pub fn clear(&mut self) {
self.bins.fill(0);
self.total = 0;
}
}
/// Find the best price phase using cross-correlation with weighted template
///
/// # Arguments
/// * `histogram` - Phase histogram to analyze
/// * `tolerance_bins` - Number of bins tolerance for template matching (e.g., 4 = ±2%)
/// * `phase_min` - Optional minimum phase from anchor (0.0-1.0)
/// * `phase_max` - Optional maximum phase from anchor (0.0-1.0)
///
/// # Returns
/// * `(best_phase, best_correlation)` - Best matching phase (0.0-1.0) and correlation score
pub fn find_best_phase(
histogram: &PhaseHistogramV2,
tolerance_bins: usize,
phase_min: Option<f64>,
phase_max: Option<f64>,
) -> (f64, u64) {
let template = template_bins();
let bins = histogram.bins();
let mut best_phase = 0.0;
let mut best_corr: u64 = 0;
// Determine valid shifts based on anchor constraints
let valid_shifts: Vec<usize> = if let (Some(p_min), Some(p_max)) = (phase_min, phase_max) {
let min_bin = ((p_min * PHASE_BINS_V2 as f64) as usize) % PHASE_BINS_V2;
let max_bin = ((p_max * PHASE_BINS_V2 as f64) as usize) % PHASE_BINS_V2;
if min_bin <= max_bin {
(min_bin..=max_bin).collect()
} else {
// Wraps around
(min_bin..PHASE_BINS_V2)
.chain(0..=max_bin)
.collect()
}
} else {
(0..PHASE_BINS_V2).collect()
};
// Cross-correlation: slide template across histogram
for shift in valid_shifts {
let mut corr: u64 = 0;
for &(template_bin, weight) in &template {
// Where would this template bin appear at this price phase shift?
let expected_bin = (template_bin + PHASE_BINS_V2 - shift) % PHASE_BINS_V2;
// Sum bins within tolerance, weighted
for t in 0..=(2 * tolerance_bins) {
let check_bin = (expected_bin + PHASE_BINS_V2 - tolerance_bins + t) % PHASE_BINS_V2;
corr += bins[check_bin] as u64 * weight as u64;
}
}
if corr > best_corr {
best_corr = corr;
best_phase = shift as f64 / PHASE_BINS_V2 as f64;
}
}
(best_phase, best_corr)
}
/// Get phase range from price anchor (low, high)
///
/// Returns (phase_min, phase_max) with tolerance added
pub fn phase_range_from_anchor(price_low: f64, price_high: f64, tolerance_pct: f64) -> (f64, f64) {
let low_adj = price_low * (1.0 - tolerance_pct);
let high_adj = price_high * (1.0 + tolerance_pct);
let phase_low = low_adj.log10().fract();
let phase_high = high_adj.log10().fract();
let phase_low = if phase_low < 0.0 {
phase_low + 1.0
} else {
phase_low
};
let phase_high = if phase_high < 0.0 {
phase_high + 1.0
} else {
phase_high
};
(phase_low, phase_high)
}
/// Convert detected phase to price using anchor for decade selection
///
/// The phase alone is ambiguous ($6.3, $63, $630, $6300 all have same phase).
/// Use the anchor price range to select the correct decade.
pub fn phase_to_price(phase: f64, anchor_low: f64, anchor_high: f64) -> f64 {
// Base price from phase (arbitrary decade, we'll adjust)
// phase = frac(log10(price)), so price = 10^(decade + phase)
// Start with decade 0 (prices 1-10)
let base_price = 10.0_f64.powf(phase);
// Find which decade puts us in the anchor range
let anchor_mid = (anchor_low + anchor_high) / 2.0;
// Try decades -2 to 6 ($0.01 to $1,000,000)
let mut best_price = base_price;
let mut best_dist = f64::MAX;
for decade in -2..=6 {
let candidate = base_price * 10.0_f64.powi(decade);
let dist = (candidate - anchor_mid).abs();
if dist < best_dist {
best_dist = dist;
best_price = candidate;
}
}
// Clamp to reasonable range
best_price.clamp(0.01, 10_000_000.0)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_template_bins() {
let template = template_bins();
assert_eq!(template.len(), 11);
// Check $1/$10/$100 maps to bin 0
assert_eq!(template[0].0, 0);
assert_eq!(template[0].1, 10);
// Check $5/$50 maps to bin 140 (0.70 * 200)
assert_eq!(template[6].0, 140);
assert_eq!(template[6].1, 9);
}
#[test]
fn test_sats_to_bin() {
// 1 BTC = 100M sats, log10(100M) = 8.0, frac = 0.0 → bin 0
let bin = PhaseHistogramV2::sats_to_bin(Sats::_1BTC).unwrap();
assert_eq!(bin, 0);
// 10M sats, log10(10M) = 7.0, frac = 0.0 → bin 0
let bin = PhaseHistogramV2::sats_to_bin(Sats::_10M).unwrap();
assert_eq!(bin, 0);
// 5M sats, log10(5M) ≈ 6.699, frac ≈ 0.699 → bin ~140
let bin = PhaseHistogramV2::sats_to_bin(Sats::from(5_000_000u64)).unwrap();
assert!((138..=142).contains(&bin), "5M sats bin = {}", bin);
}
#[test]
fn test_phase_range_from_anchor() {
// $6000-$8000 range
let (p_min, p_max) = phase_range_from_anchor(6000.0, 8000.0, 0.05);
// $6000 → log10 = 3.778, phase = 0.778
// $8000 → log10 = 3.903, phase = 0.903
assert!(p_min > 0.7 && p_min < 0.8, "p_min = {}", p_min);
assert!(p_max > 0.85 && p_max < 0.95, "p_max = {}", p_max);
}
#[test]
fn test_phase_to_price() {
// Phase 0.0 with anchor $50-150 should give ~$100
let price = phase_to_price(0.0, 50.0, 150.0);
assert!(price > 80.0 && price < 120.0, "price = {}", price);
// Phase 0.70 with anchor $4000-6000 should give ~$5000
let price = phase_to_price(0.70, 4000.0, 6000.0);
assert!(price > 4000.0 && price < 6000.0, "price = {}", price);
}
}

View File

@@ -1,7 +1,7 @@
use brk_traversable::Traversable;
use brk_types::{
Cents, DateIndex, Dollars, Height, OHLCCents, OHLCDollars, OracleBins, PairOutputIndex, Sats,
StoredU32, TxIndex,
Cents, DateIndex, Dollars, Height, OHLCCents, OHLCDollars, OracleBins, OracleBinsV2,
PairOutputIndex, Sats, StoredU32, TxIndex,
};
use vecdb::{BytesVec, LazyVecFrom1, PcoVec};
@@ -55,4 +55,49 @@ pub struct Vecs {
/// Number of qualifying transactions per day (for confidence)
pub tx_count: PcoVec<DateIndex, StoredU32>,
// ========== Phase Oracle V2 (round USD template matching) ==========
/// Per-block 200-bin phase histogram
pub phase_v2_histogram: BytesVec<Height, OracleBinsV2>,
/// Per-block price in cents from phase oracle V2 (cross-correlation with round USD template)
pub phase_v2_price_cents: PcoVec<Height, Cents>,
/// Per-block price in cents using direct peak finding (like V1)
pub phase_v2_peak_price_cents: PcoVec<Height, Cents>,
/// Daily distribution (min, max, average, percentiles) from phase oracle V2
pub phase_v2_daily_cents: Distribution<DateIndex, Cents>,
/// Daily distribution in dollars (lazy conversion from cents)
pub phase_v2_daily_dollars: LazyTransformDistribution<DateIndex, Dollars, Cents>,
/// Daily distribution from peak-based prices
pub phase_v2_peak_daily_cents: Distribution<DateIndex, Cents>,
/// Daily distribution in dollars (lazy conversion from cents)
pub phase_v2_peak_daily_dollars: LazyTransformDistribution<DateIndex, Dollars, Cents>,
// ========== Phase Oracle V3 (BASE + uniqueVal filter) ==========
/// Per-block 200-bin phase histogram with uniqueVal filtering
/// Only includes outputs with unique values within their transaction
pub phase_v3_histogram: BytesVec<Height, OracleBinsV2>,
/// Per-block price in cents from phase oracle V3 (cross-correlation)
pub phase_v3_price_cents: PcoVec<Height, Cents>,
/// Per-block price in cents using direct peak finding (like V1)
pub phase_v3_peak_price_cents: PcoVec<Height, Cents>,
/// Daily distribution from phase oracle V3
pub phase_v3_daily_cents: Distribution<DateIndex, Cents>,
/// Daily distribution in dollars (lazy conversion from cents)
pub phase_v3_daily_dollars: LazyTransformDistribution<DateIndex, Dollars, Cents>,
/// Daily distribution from peak-based prices
pub phase_v3_peak_daily_cents: Distribution<DateIndex, Cents>,
/// Daily distribution in dollars (lazy conversion from cents)
pub phase_v3_peak_daily_dollars: LazyTransformDistribution<DateIndex, Dollars, Cents>,
}