mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-11 07:23:32 -07:00
oracle: snap pre 340k patch
This commit is contained in:
nym21
@@ -9858,7 +9858,7 @@ impl BrkClient {
|
||||
|
||||
/// Live BTC/USD price
|
||||
///
|
||||
/// Current BTC/USD price in dollars, derived purely from on-chain round-dollar output patterns over the last 12 blocks plus the forming mempool block. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
/// Current BTC/USD price in dollars. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/price`
|
||||
pub fn get_oracle_price(&self) -> Result<Dollars> {
|
||||
@@ -9867,7 +9867,7 @@ impl BrkClient {
|
||||
|
||||
/// Live EMA histogram
|
||||
///
|
||||
/// Smoothed round-dollar payment histogram at the live tip: the committed 12-block EMA with the forming mempool block blended in as a final slot. A flat array of 2400 log-scale bins, quantized to `u16` for the wire. This is the heatmap column you render.
|
||||
/// Smoothed round-dollar payment histogram at the live tip: the committed EMA with the forming mempool block blended in. A flat array of log-scale bins.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/ema/live`
|
||||
pub fn get_oracle_histogram_ema_live(&self) -> Result<Histogram_uint16> {
|
||||
@@ -9876,7 +9876,7 @@ impl BrkClient {
|
||||
|
||||
/// EMA histogram at height
|
||||
///
|
||||
/// Smoothed round-dollar payment histogram for a confirmed height, deterministically reconstructed by replaying the 12-block window ending at that height. Immutable once buried, so repeated requests return byte-identical results. A flat array of 2400 log-scale bins, quantized to `u16`.
|
||||
/// Smoothed round-dollar payment histogram for a confirmed height. A flat array of log-scale bins.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/ema/{height}`
|
||||
pub fn get_oracle_histogram_ema(&self, height: Height) -> Result<Histogram_uint16> {
|
||||
@@ -9885,7 +9885,7 @@ impl BrkClient {
|
||||
|
||||
/// Live raw histogram
|
||||
///
|
||||
/// Un-smoothed per-block round-dollar counts for the forming mempool block: the spiky primitive the EMA smooths over. A flat array of 2400 log-scale bins (`u32` counts), all zero when no mempool is configured.
|
||||
/// Un-smoothed per-block round-dollar counts for the forming mempool block. A flat array of log-scale bins, all zero when no mempool is configured.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/raw/live`
|
||||
pub fn get_oracle_histogram_raw_live(&self) -> Result<Histogram_uint32> {
|
||||
@@ -9894,7 +9894,7 @@ impl BrkClient {
|
||||
|
||||
/// Raw histogram at height
|
||||
///
|
||||
/// Un-smoothed round-dollar counts for a single confirmed block. A flat array of 2400 log-scale bins (`u32` counts).
|
||||
/// Un-smoothed round-dollar counts for a single confirmed block. A flat array of log-scale bins.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/raw/{height}`
|
||||
pub fn get_oracle_histogram_raw(&self, height: Height) -> Result<Histogram_uint32> {
|
||||
|
||||
@@ -3,7 +3,8 @@ use std::ops::Range;
|
||||
use brk_error::Result;
|
||||
use brk_indexer::{Indexer, Lengths};
|
||||
use brk_oracle::{
|
||||
Config, HistogramRaw, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin, for_each_round_dollar_bin,
|
||||
Config, HistogramRaw, Oracle, START_HEIGHT, START_HEIGHT_SLOW, bin_to_cents, cents_to_bin,
|
||||
for_each_round_dollar_bin,
|
||||
};
|
||||
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use tracing::info;
|
||||
@@ -73,7 +74,7 @@ impl Vecs {
|
||||
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
|
||||
if total_heights <= START_HEIGHT {
|
||||
if total_heights <= START_HEIGHT_SLOW {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -85,12 +86,12 @@ impl Vecs {
|
||||
.inner
|
||||
.truncate_if_needed_at(truncate_to)?;
|
||||
|
||||
if self.spot.cents.height.len() < START_HEIGHT {
|
||||
if self.spot.cents.height.len() < START_HEIGHT_SLOW {
|
||||
for line in brk_oracle::PRICES
|
||||
.lines()
|
||||
.skip(self.spot.cents.height.len())
|
||||
{
|
||||
if self.spot.cents.height.len() >= START_HEIGHT {
|
||||
if self.spot.cents.height.len() >= START_HEIGHT_SLOW {
|
||||
break;
|
||||
}
|
||||
let dollars: f64 = line.parse().unwrap_or(0.0);
|
||||
@@ -103,8 +104,8 @@ impl Vecs {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let config = Config::default();
|
||||
let committed = self.spot.cents.height.len();
|
||||
let config = Config::for_height(committed);
|
||||
let prev_cents = self
|
||||
.spot
|
||||
.cents
|
||||
@@ -112,7 +113,7 @@ impl Vecs {
|
||||
.collect_one_at(committed - 1)
|
||||
.unwrap();
|
||||
let seed_bin = cents_to_bin(prev_cents.inner() as f64);
|
||||
let warmup = config.window_size.min(committed - START_HEIGHT);
|
||||
let warmup = config.window_size.min(committed - START_HEIGHT_SLOW);
|
||||
let mut oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
|
||||
Self::feed_blocks(o, indexer, (committed - warmup)..committed, None);
|
||||
});
|
||||
@@ -123,7 +124,26 @@ impl Vecs {
|
||||
committed, total_heights
|
||||
);
|
||||
|
||||
let ref_bins = Self::feed_blocks(&mut oracle, indexer, committed..total_heights, None);
|
||||
// Slow cold-start EMA up to START_HEIGHT, then switch to the fast
|
||||
// mature-market EMA. Steady-state runs start past START_HEIGHT and skip
|
||||
// the slow segment entirely.
|
||||
let mut ref_bins = Vec::with_capacity(num_new);
|
||||
if committed < START_HEIGHT {
|
||||
let slow_end = START_HEIGHT.min(total_heights);
|
||||
ref_bins.extend(Self::feed_blocks(&mut oracle, indexer, committed..slow_end, None));
|
||||
if slow_end == START_HEIGHT {
|
||||
oracle.reconfigure(Config::default());
|
||||
}
|
||||
}
|
||||
let fast_start = committed.max(START_HEIGHT);
|
||||
if fast_start < total_heights {
|
||||
ref_bins.extend(Self::feed_blocks(
|
||||
&mut oracle,
|
||||
indexer,
|
||||
fast_start..total_heights,
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
for (i, ref_bin) in ref_bins.into_iter().enumerate() {
|
||||
self.spot
|
||||
|
||||
+30
-27
@@ -1,8 +1,8 @@
|
||||
# brk_oracle
|
||||
|
||||
**Version 2**
|
||||
**Version 3**
|
||||
|
||||
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 508,000 (February 2018) onward.
|
||||
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 470,000 (June 2017) onward.
|
||||
|
||||
Inspired by [UTXOracle](https://utxo.live/oracle/) by [@SteveSimple](https://x.com/SteveSimple), which proved the concept. brk_oracle takes the same core insight and redesigns the algorithm for per-block resolution and rolling operation. See [comparison](#comparison-with-utxoracle) below.
|
||||
|
||||
@@ -87,7 +87,7 @@ The fixed ratios between round-dollar amounts ($1, $2, $3, $5, ... $10,000) crea
|
||||
The oracle slides this stencil across the EMA histogram within the search window. At each candidate position:
|
||||
|
||||
1. **Read** the EMA value at all 19 expected spike locations
|
||||
2. **Normalize** each value by dividing by that offset's peak within the search window — this gives rare amounts like $3 equal voting weight to common amounts like $100
|
||||
2. **Normalize** each value by dividing by that offset's peak within the search window: this gives rare amounts like $3 equal voting weight to common amounts like $100
|
||||
3. **Sum** the 19 normalized values into a single score
|
||||
|
||||
The position with the highest score is where the fingerprint best matches the histogram.
|
||||
@@ -102,7 +102,7 @@ A $100 purchase at price P produces `$100 / P × 10⁸` sats, which lands in bin
|
||||
= (10 − log₁₀(P)) × 200
|
||||
```
|
||||
|
||||
So the stencil's winning position — the bin where $100 purchases land — directly encodes the price:
|
||||
So the stencil's winning position, the bin where $100 purchases land, directly encodes the price:
|
||||
|
||||
```
|
||||
price = 10^(10 − bin / 200) dollars
|
||||
@@ -124,7 +124,7 @@ The oracle consumes one pre-built histogram per block via `process_histogram(&hi
|
||||
|
||||
The caller does the filtering when it builds the histogram. For each block it skips the coinbase, drops every output of a transaction carrying an `OP_RETURN` (and, below height 630,000, every output of a transaction with more than 100 outputs), then bins the rest. `default_eligible_bin(sats, output_type)` (or `Oracle::output_to_bin` for a non-default `Config`) applies the per-output rules: excluded script types, dust, and round-BTC values. It returns the bin index, or `None` for a filtered output.
|
||||
|
||||
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for heights 0..508,000. Its last entry, height 507,999 (one below `START_HEIGHT`), seeds the oracle's first on-chain computation at height 508,000.
|
||||
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for heights 0..470,000. Its last entry, height 469,999 (one below `START_HEIGHT_SLOW`), seeds the oracle's first on-chain computation at height 470,000.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -139,6 +139,8 @@ All parameters via `Config` with sensible defaults:
|
||||
| `exclude_common_round_values` | true | Filter d × 10ⁿ (d ∈ {1,2,3,5,6}) to prevent false stencil matches |
|
||||
| `excluded_output_types` | P2TR | Script types dominated by protocol activity |
|
||||
|
||||
Between height 470,000 and 508,000 the oracle runs a slower cold-start configuration (`Config::slow()`: `alpha` = 0.10, ~19-block span, `window_size` = 40). The thinner pre-2018 output mix lets the fast default octave-lock onto the round-dollar half-price pattern, and the slow EMA resists that drift. At height 508,000 `Oracle::reconfigure` switches to the defaults above. `Config::for_height` returns the right one for any height.
|
||||
|
||||
## Comparison with UTXOracle
|
||||
|
||||
[UTXOracle](https://utxo.live/oracle/) by [@SteveSimple](https://x.com/SteveSimple) proved that BTC/USD can be derived purely from on-chain data. Both projects share the same core insight (round-dollar detection via log-scale histogram) but make different engineering choices:
|
||||
@@ -152,29 +154,29 @@ All parameters via `Config` with sensible defaults:
|
||||
| Stencil | 19 round-USD offsets ($1 to $10k), each normalized to its own peak | 803-point Gaussian + weighted spike template targeting 17 round-USD amounts |
|
||||
| Round BTC handling | Excluded from histogram entirely | Histogram bins smoothed by averaging neighbors |
|
||||
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: not coinbase, no OP_RETURN, exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
|
||||
| Validated from | Height 508,000 (February 2018) | Dec 15, 2023 |
|
||||
| Validated from | Height 470,000 (June 2017) | Dec 15, 2023 |
|
||||
| Language | Rust | Python |
|
||||
| Dependencies | None (pure computation, caller provides block data) | bitcoin-cli + direct blk file reads |
|
||||
| Bins per decade | 200 | 200 |
|
||||
|
||||
## Accuracy
|
||||
|
||||
Tested over 428,251 blocks (heights 508,000 to 950,490, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
|
||||
Tested over 466,251 blocks (heights 470,000 to 950,694, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
|
||||
|
||||
### Per-block
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Median error | 0.11% |
|
||||
| 95th percentile | 0.68% |
|
||||
| 99th percentile | 1.7% |
|
||||
| 99.9th percentile | 4.7% |
|
||||
| RMSE | 0.46% |
|
||||
| Max error | 28.8% |
|
||||
| Bias | +0.00 bins (essentially zero) |
|
||||
| Blocks > 5% error | 367 (0.086%) |
|
||||
| Blocks > 10% error | 116 |
|
||||
| Blocks > 20% error | 1 |
|
||||
| Median error | 0.12% |
|
||||
| 95th percentile | 0.91% |
|
||||
| 99th percentile | 2.6% |
|
||||
| 99.9th percentile | 12.0% |
|
||||
| RMSE | 0.85% |
|
||||
| Max error | 47.7% |
|
||||
| Bias | +0.03 bins (essentially zero) |
|
||||
| Blocks > 5% error | 1,605 (0.344%) |
|
||||
| Blocks > 10% error | 609 |
|
||||
| Blocks > 20% error | 189 |
|
||||
|
||||
### Daily candles
|
||||
|
||||
@@ -182,30 +184,31 @@ Oracle daily OHLC built from per-block prices vs exchange daily OHLC:
|
||||
|
||||
| | Median | RMSE | Max |
|
||||
|-------|--------|------|-----|
|
||||
| Open | 0.21% | 0.59% | 15.3% |
|
||||
| High | 0.53% | 0.99% | 15.4% |
|
||||
| Low | 0.52% | 1.39% | 21.5% |
|
||||
| Close | 0.24% | 0.73% | 15.4% |
|
||||
| Open | 0.22% | 0.90% | 21.1% |
|
||||
| High | 0.55% | 1.08% | 15.4% |
|
||||
| Low | 0.54% | 1.65% | 20.5% |
|
||||
| Close | 0.26% | 0.98% | 21.1% |
|
||||
|
||||
### By year
|
||||
|
||||
| Year | Blocks | Median | RMSE | Max | >5% | >10% | >20% | Price range |
|
||||
|------|--------|--------|------|-----|-----|------|------|-------------|
|
||||
| 2018 | 48,492 | 0.17% | 0.84% | 28.8% | 136 | 87 | 1 | $3,129–$11,775 |
|
||||
| 2017 | 31,961 | 0.39% | 2.37% | 47.7% | 980 | 373 | 116 | $1,758–$19,892 |
|
||||
| 2018 | 54,531 | 0.18% | 1.35% | 32.2% | 394 | 207 | 73 | $3,129–$17,178 |
|
||||
| 2019 | 54,272 | 0.16% | 0.59% | 17.4% | 100 | 16 | 0 | $3,338–$13,868 |
|
||||
| 2020 | 53,102 | 0.10% | 0.42% | 11.6% | 61 | 3 | 0 | $3,858–$29,322 |
|
||||
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 43 | 10 | 0 | $27,678–$69,000 |
|
||||
| 2022 | 53,230 | 0.07% | 0.32% | 6.8% | 10 | 0 | 0 | $15,460–$48,240 |
|
||||
| 2023 | 54,032 | 0.10% | 0.25% | 6.6% | 5 | 0 | 0 | $16,490–$44,700 |
|
||||
| 2024 | 53,367 | 0.10% | 0.29% | 7.1% | 8 | 0 | 0 | $38,555–$108,298 |
|
||||
| 2024 | 53,367 | 0.10% | 0.28% | 7.1% | 8 | 0 | 0 | $38,555–$108,298 |
|
||||
| 2025 | 53,113 | 0.11% | 0.25% | 5.8% | 4 | 0 | 0 | $74,409–$126,198 |
|
||||
| 2026 | 5,910 | 0.11% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000–$97,900 |
|
||||
| 2026 | 5,910 | 0.10% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000–$97,900 |
|
||||
|
||||
The oracle is only as good as the signal it reads. The largest errors cluster in late 2018: the November price crash fell faster than the narrow search window could follow (28.8% max error, at height 550,890), and on-chain volume was lower then, so the round-dollar pattern was weaker (0.84% RMSE for the year). By 2020 the signal is strong enough for 0.1% median accuracy, and since 2022 no block exceeds 10% error.
|
||||
The oracle is only as good as the signal it reads. The largest errors cluster in late 2017: the parabolic December run-up toward $20,000 rose faster than the slow cold-start EMA could follow, so the oracle lagged low (47.7% max error, at height 498,246, oracle ~$11,100 vs exchange ~$16,400). The thinner pre-2018 on-chain volume also weakens the round-dollar pattern, so 2017 and 2018 carry the bulk of the error (2.37% and 1.35% RMSE). From 2019 the signal strengthens: by 2020 the oracle reaches 0.1% median accuracy, and since 2022 no block exceeds 10% error.
|
||||
|
||||
### Why no outlier smoothing?
|
||||
|
||||
Post-hoc smoothing — for example, correcting any block whose price deviates more than 5% from both its neighbors — would improve the aggregate numbers. This is deliberately not done, for two reasons:
|
||||
Post-hoc smoothing, for example correcting any block whose price deviates more than 5% from both its neighbors, would improve the aggregate numbers. This is deliberately not done, for two reasons:
|
||||
|
||||
1. **Simplicity**: The oracle is a single forward pass with no lookback corrections. Adding smoothing means defining thresholds, neighbor windows, and replacement strategies, all of which add complexity for marginal gain.
|
||||
2. **Finality**: Each block's price is produced once and never revised (unless the block itself is reorged). Downstream consumers can treat the oracle output as append-only. Smoothing would require retroactively changing already-published prices, breaking that property.
|
||||
@@ -216,7 +219,7 @@ Post-hoc smoothing — for example, correcting any block whose price deviates mo
|
||||
|
||||
Changes from v2:
|
||||
|
||||
- **Earlier start**: on-chain tracking begins at height 508,000 (February 2018) instead of 525,000, adding about 17,000 blocks of history.
|
||||
- **Earlier start with a cold-start regime**: on-chain tracking begins at height 470,000 (June 2017) instead of 525,000, adding about 55,000 blocks of history. Below height 508,000 the oracle runs a slower EMA (`Config::slow()`, ~19-block span, window 40) that resists the round-dollar half-price drift the fast default octave-locks onto in the thinner pre-2018 output mix, then switches to the fast default at 508,000 via `Oracle::reconfigure`.
|
||||
- **Max-outputs filter**: a transaction with more than 100 outputs is dropped from the histogram below height 630,000. Large fan-outs (exchange sweeps, mixer payouts) are batch machinery, not round-dollar payments, and the thin 2018-2020 signal needs them removed to stay locked onto the pattern. Above 630,000 on-chain volume is dense enough that the cap removes more genuine signal than noise, so it is lifted.
|
||||
- **Wider up-reach**: `search_below` raised from 9 to 12 bins. The sharp 2018 reversal candles need extra room to follow a fast move upward in price.
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Dump the RAW per-output data over a height range for fully offline analysis.
|
||||
//! Nothing is filtered or binned, so any downstream filter (round-BTC tolerance,
|
||||
//! dust floor, type exclusion, OP_RETURN / batch-payout tx drops, log-bin
|
||||
//! resolution) can be reconstructed in analysis WITHOUT re-dumping.
|
||||
//!
|
||||
//! For every non-coinbase output in [ORACLE_START, ORACLE_END) (default
|
||||
//! 500000..510000) one row is written:
|
||||
//! oracle_outputs_{start}_{end}.csv height,tx,sats,otype
|
||||
//! where `tx` is the 0-based index of the (non-coinbase) transaction within the
|
||||
//! block (so OP_RETURN-tx and >N-output-tx drops can be reapplied by grouping),
|
||||
//! `sats` is the exact output value, and `otype` is `OutputType as u8`.
|
||||
//!
|
||||
//! Plus per-block metadata:
|
||||
//! oracle_meta_{start}_{end}.csv height,timestamp,ex_low,ex_high,ex_close
|
||||
//!
|
||||
//! Run: cargo run -p brk_oracle --example dump_hist --release
|
||||
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{BufWriter, Write},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
fn main() {
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from(std::env::var("HOME").unwrap()).join(".brk"));
|
||||
let out_dir = std::env::var("DUMP_DIR").unwrap_or_else(|_| "/tmp".to_string());
|
||||
let start: usize = std::env::var("ORACLE_START")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(500_000);
|
||||
let end: usize = std::env::var("ORACLE_END")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(510_000);
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
let end = end.min(total_heights);
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("parse height OHLC");
|
||||
|
||||
let timestamps: Vec<brk_types::Timestamp> = indexer.vecs.blocks.timestamp.collect();
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
|
||||
let mut tx_starts: Vec<usize> = Vec::new();
|
||||
|
||||
let out_path = format!("{out_dir}/oracle_outputs_{start}_{end}.csv");
|
||||
let meta_path = format!("{out_dir}/oracle_meta_{start}_{end}.csv");
|
||||
let mut out_w = BufWriter::new(File::create(&out_path).expect("create outputs csv"));
|
||||
let mut meta_w = BufWriter::new(File::create(&meta_path).expect("create meta csv"));
|
||||
writeln!(out_w, "height,tx,sats,otype").unwrap();
|
||||
writeln!(meta_w, "height,timestamp,ex_low,ex_high,ex_close").unwrap();
|
||||
|
||||
eprintln!(
|
||||
"otype legend: OpReturn={} P2TR={}",
|
||||
OutputType::OpReturn as u8,
|
||||
OutputType::P2TR as u8
|
||||
);
|
||||
|
||||
let mut rows: u64 = 0;
|
||||
for h in start..end {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
let block_first_tx = ft.to_usize() + 1; // skip coinbase
|
||||
let tx_count = next_ft.to_usize() - block_first_tx;
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
txout_cursor.advance(block_first_tx - txout_cursor.position());
|
||||
tx_starts.clear();
|
||||
for _ in 0..tx_count {
|
||||
tx_starts.push(txout_cursor.next().unwrap().to_usize());
|
||||
}
|
||||
let out_start = tx_starts.first().copied().unwrap_or(out_end);
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
for i in lo..hi {
|
||||
writeln!(out_w, "{h},{tx},{},{}", *values[i], output_types[i] as u8).unwrap();
|
||||
rows += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let o = height_ohlc.get(h).copied().unwrap_or([0.0; 4]);
|
||||
writeln!(
|
||||
meta_w,
|
||||
"{h},{},{:.2},{:.2},{:.2}",
|
||||
*timestamps[h], o[2], o[1], o[3]
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
out_w.flush().unwrap();
|
||||
meta_w.flush().unwrap();
|
||||
eprintln!("wrote {out_path} ({rows} output rows)");
|
||||
eprintln!("wrote {meta_path}");
|
||||
}
|
||||
@@ -26,6 +26,80 @@ const STENCIL_OFFSETS: [i32; 19] = [
|
||||
-400, -340, -305, -260, -200, -165, -140, -120, -105, -60, 0, 35, 60, 95, 140, 200, 260, 340,
|
||||
400,
|
||||
];
|
||||
const N_ARMS: usize = STENCIL_OFFSETS.len();
|
||||
|
||||
/// Canonical L1-normalized payment shape across the 19 stencil arms, estimated
|
||||
/// from true-center arm vectors over a validated block range (~$1.8k era).
|
||||
/// The real price center reproduces this profile; a ½×/2× alias distorts it
|
||||
/// (dark holes at no-ladder-partner arms, spurious mass from between-rung
|
||||
/// payments), so correlation against it discriminates octaves the raw stencil
|
||||
/// sum cannot. Order matches STENCIL_OFFSETS / the $1..$10k ladder.
|
||||
const ARM_PROFILE: [f64; N_ARMS] = [
|
||||
0.022, 0.029, 0.021, 0.045, 0.060, 0.053, 0.092, 0.066, 0.077, 0.075, 0.105, 0.052, 0.075,
|
||||
0.049, 0.059, 0.043, 0.044, 0.021, 0.014,
|
||||
];
|
||||
|
||||
/// Raw EMA arm vector at `center` (mass on each of the 19 stencil offsets).
|
||||
fn arms_at(ema: &HistogramEma, center: i64) -> [f64; N_ARMS] {
|
||||
let mut arms = [0.0f64; N_ARMS];
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = center + off as i64;
|
||||
if idx >= 0 && (idx as usize) < NUM_BINS {
|
||||
arms[i] = ema[idx as usize];
|
||||
}
|
||||
}
|
||||
arms
|
||||
}
|
||||
|
||||
/// Pearson correlation between the raw EMA arm vector at `center` and a payment
|
||||
/// shape `profile`. High when the local shape matches real payments, low at a
|
||||
/// ½×/2× alias whose holes and spurious arms distort the shape.
|
||||
fn arm_profile_corr(ema: &HistogramEma, center: i64, profile: &[f64; N_ARMS]) -> f64 {
|
||||
let arms = arms_at(ema, center);
|
||||
let n = N_ARMS as f64;
|
||||
let ma = arms.iter().sum::<f64>() / n;
|
||||
let mb = profile.iter().sum::<f64>() / n;
|
||||
let (mut num, mut da, mut db) = (0.0, 0.0, 0.0);
|
||||
for i in 0..N_ARMS {
|
||||
let (xa, xb) = (arms[i] - ma, profile[i] - mb);
|
||||
num += xa * xb;
|
||||
da += xa * xa;
|
||||
db += xb * xb;
|
||||
}
|
||||
if da > 0.0 && db > 0.0 {
|
||||
num / (da * db).sqrt()
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Stencil-arm indices whose value v has 2v NOT on the round-USD ladder
|
||||
/// ($2 $3 $20 $30 $200 $300 $2000 $10000). A half-price hypothesis shifts the
|
||||
/// center +60 bins; an arm is lit there only if 2v is itself a round-USD amount
|
||||
/// people pay, so these eight are the only arms that fall dark at the ½x alias.
|
||||
/// They carry the entire octave discrimination; the other eleven alias cleanly.
|
||||
const DISC_ARMS: [usize; 8] = [1, 2, 6, 8, 12, 13, 16, 18];
|
||||
|
||||
/// The four "decade-anchor" arms ($10 $50 $100 $1000) whose value has BOTH 2v
|
||||
/// and v/2 on the round-USD ladder, so they alias across the octave in either
|
||||
/// direction and carry zero up/down information. Down-weighting them is the
|
||||
/// symmetric counterpart to up-weighting the half-only DISC_ARMS, meant to
|
||||
/// resist the 2x climb as well as the 1/2x slide.
|
||||
const ALIAS_ARMS: [usize; 4] = [4, 9, 10, 15];
|
||||
|
||||
/// Sum of EMA mass on a chosen subset of stencil arms at `center`.
|
||||
fn arm_subset_sum(ema: &HistogramEma, center: i64, arms: &[usize]) -> f64 {
|
||||
arms.iter()
|
||||
.map(|&i| {
|
||||
let idx = center + STENCIL_OFFSETS[i] as i64;
|
||||
if idx >= 0 && (idx as usize) < NUM_BINS {
|
||||
ema[idx as usize]
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Raw sum of EMA mass landing on the 19 stencil arms when centered at `center`.
|
||||
fn ema_stencil_sum(ema: &HistogramEma, center: i64) -> f64 {
|
||||
@@ -73,7 +147,7 @@ impl GuardCfg {
|
||||
enabled: std::env::var("OCTAVE_GUARD")
|
||||
.ok()
|
||||
.map(|v| v != "0")
|
||||
.unwrap_or(true),
|
||||
.unwrap_or(false),
|
||||
tau: g("GUARD_TAU", 0.15),
|
||||
raw_margin: g("GUARD_RAW", 1.0),
|
||||
q_margin: g("GUARD_QMARGIN", 4.0) as usize,
|
||||
@@ -92,7 +166,7 @@ impl GuardCfg {
|
||||
/// partner one octave away), so this count separates truth from alias even when
|
||||
/// the normalized score-sum cannot.
|
||||
fn arm_count(ema: &HistogramEma, center: i64, tau: f64) -> usize {
|
||||
let mut arms = [0.0f64; 19];
|
||||
let mut arms = [0.0f64; N_ARMS];
|
||||
let mut peak = 0.0f64;
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = center + off as i64;
|
||||
@@ -116,7 +190,7 @@ fn arm_count(ema: &HistogramEma, center: i64, tau: f64) -> usize {
|
||||
/// EMA mass >= tau * peak arm). Order: $1 $2 $3 $5 $10 $15 $20 $25 $30 $50 $100
|
||||
/// $150 $200 $300 $500 $1k $2k $5k $10k. Reveals WHICH amounts are present.
|
||||
fn arm_pattern(ema: &HistogramEma, center: i64, tau: f64) -> String {
|
||||
let mut arms = [0.0f64; 19];
|
||||
let mut arms = [0.0f64; N_ARMS];
|
||||
let mut peak = 0.0f64;
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = center + off as i64;
|
||||
@@ -151,6 +225,9 @@ fn guarded_best_bin(
|
||||
search_below: usize,
|
||||
search_above: usize,
|
||||
guard: &GuardCfg,
|
||||
arm_weights: &[f64; N_ARMS],
|
||||
corr_weight: f64,
|
||||
profile: &[f64; N_ARMS],
|
||||
) -> f64 {
|
||||
let center = prev_bin.round() as usize;
|
||||
let search_start = center.saturating_sub(search_below);
|
||||
@@ -159,7 +236,7 @@ fn guarded_best_bin(
|
||||
return prev_bin;
|
||||
}
|
||||
|
||||
let mut track_norm = [0.0f64; 19];
|
||||
let mut track_norm = [0.0f64; N_ARMS];
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
for bin in search_start..search_end {
|
||||
let idx = bin as i32 + off;
|
||||
@@ -173,9 +250,12 @@ fn guarded_best_bin(
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = bin as i32 + off;
|
||||
if idx >= 0 && (idx as usize) < brk_oracle::NUM_BINS && track_norm[i] > 0.0 {
|
||||
total += ema[idx as usize] / track_norm[i];
|
||||
total += arm_weights[i] * ema[idx as usize] / track_norm[i];
|
||||
}
|
||||
}
|
||||
if corr_weight != 0.0 {
|
||||
total += corr_weight * arm_profile_corr(ema, bin as i64, profile);
|
||||
}
|
||||
total
|
||||
};
|
||||
|
||||
@@ -454,12 +534,22 @@ fn main() {
|
||||
.map(|ts| (**ts / 86400).saturating_sub(GENESIS_DAY) as usize)
|
||||
.collect();
|
||||
|
||||
// Seed price at height `start - 1`. The baked prices.txt only covers up to
|
||||
// 508k (the cold-start seed); past it we warm-start from the exchange close
|
||||
// so any later start height gets a primed ref_bin without the cold-start
|
||||
// alias zone. start <= 508k stays bit-identical to the old baseline.
|
||||
let start_price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
.and_then(|l| l.parse().ok())
|
||||
.unwrap_or_else(|| {
|
||||
let o = height_ohlc.get(start - 1).copied().unwrap_or([0.0; 4]);
|
||||
if o[3] > 0.0 { o[3] } else { (o[1] + o[2]) / 2.0 }
|
||||
});
|
||||
// Exact seed override (reproduce the committed prices.txt seed at a start the
|
||||
// truncated working-tree prices.txt no longer covers).
|
||||
let start_price =
|
||||
std::env::var("SEED").ok().and_then(|s| s.parse().ok()).unwrap_or(start_price);
|
||||
|
||||
let mut config = Config::default();
|
||||
if let Some(w) = std::env::var("EMA_WINDOW")
|
||||
@@ -484,6 +574,57 @@ fn main() {
|
||||
config.search_above = sa;
|
||||
}
|
||||
let guard = GuardCfg::from_env();
|
||||
// Lever 3: up-weight the 8 octave-discriminating arms (2v not on the ladder)
|
||||
// in the stencil score. They alone separate a center from its half-price
|
||||
// alias; the other 11 alias cleanly and only dilute the up/down decision.
|
||||
let disc_weight: f64 = std::env::var("DISC_WEIGHT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1.0);
|
||||
let alias_weight: f64 = std::env::var("ALIAS_WEIGHT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1.0);
|
||||
// Shape-correlation restoring force: add corr_weight * Pearson(arms, profile)
|
||||
// to each candidate bin's stencil score. Pulls the ±window pick toward the
|
||||
// octave whose arm-shape matches real payments, resisting the ½×/2× slide
|
||||
// without a hard continuity clamp. 0 = off (bit-identical to baseline).
|
||||
let corr_weight: f64 = std::env::var("CORR_WEIGHT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0.0);
|
||||
// EMA rate for the adaptive shape template. The profile tracks the current
|
||||
// price regime (which arms are tall) so correlation stays meaningful as the
|
||||
// price moves an octave over months, while remaining slow enough to ride
|
||||
// through a transient ½×/2× slide (tens of blocks) without adapting to it.
|
||||
let corr_beta: f64 = std::env::var("CORR_BETA")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0.002);
|
||||
// Mid-run regime switch, mirrors production Oracle::reconfigure at START_HEIGHT:
|
||||
// at SWITCH_AT rebuild the EMA to SWITCH_WINDOW/SWITCH_ALPHA and warm-start fresh
|
||||
// (ring reset, ref_bin kept) - the same state as a fresh warm-up. Search window
|
||||
// is unchanged (both regimes share it). 0 = no switch (single-config baseline).
|
||||
let switch_at: usize = std::env::var("SWITCH_AT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0);
|
||||
let switch_window: usize = std::env::var("SWITCH_WINDOW")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(12);
|
||||
let switch_alpha: f64 = std::env::var("SWITCH_ALPHA")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(2.0 / 7.0);
|
||||
let mut arm_weights = [1.0f64; N_ARMS];
|
||||
for &i in &DISC_ARMS {
|
||||
arm_weights[i] = disc_weight;
|
||||
}
|
||||
for &i in &ALIAS_ARMS {
|
||||
arm_weights[i] = alias_weight;
|
||||
}
|
||||
eprintln!(" disc_weight={disc_weight} on {DISC_ARMS:?}; alias_weight={alias_weight} on {ALIAS_ARMS:?}; corr_weight={corr_weight}");
|
||||
let anom_thresh: f64 = std::env::var("ANOM_THRESH")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
@@ -539,10 +680,13 @@ fn main() {
|
||||
guard.global,
|
||||
guard.global_radius,
|
||||
);
|
||||
if switch_at != 0 {
|
||||
eprintln!(" switch: at height {switch_at} -> window={switch_window} alpha={switch_alpha:.5}");
|
||||
}
|
||||
let (sb, sa) = (config.search_below, config.search_above);
|
||||
let window_size = config.window_size;
|
||||
let mut window_size = config.window_size;
|
||||
let alpha = config.alpha;
|
||||
let weights: Vec<f64> = (0..window_size)
|
||||
let mut weights: Vec<f64> = (0..window_size)
|
||||
.map(|i| alpha * (1.0 - alpha).powi(i as i32))
|
||||
.collect();
|
||||
let mut ring: Vec<Vec<f64>> = vec![vec![0.0; NUM_BINS]; window_size];
|
||||
@@ -550,6 +694,9 @@ fn main() {
|
||||
let mut filled = 0usize;
|
||||
let mut ema = HistogramEma::zeros();
|
||||
let mut ref_bin = cents_to_bin(start_price * 100.0);
|
||||
// Adaptive shape template, seeded with the static profile and re-estimated
|
||||
// each block from the L1-normalized arm vector at the pick.
|
||||
let mut profile = ARM_PROFILE;
|
||||
|
||||
// Lever 4: a parallel "sharp" detection EMA (fast span, short window) folded
|
||||
// from the same per-block hists. The slow EMA above still sets the price; this
|
||||
@@ -597,6 +744,15 @@ fn main() {
|
||||
|
||||
let loop_end = end_override.unwrap_or(total_heights).min(total_heights);
|
||||
for h in start..loop_end {
|
||||
if switch_at != 0 && h == switch_at {
|
||||
window_size = switch_window;
|
||||
weights = (0..window_size)
|
||||
.map(|i| switch_alpha * (1.0 - switch_alpha).powi(i as i32))
|
||||
.collect();
|
||||
ring = vec![vec![0.0; NUM_BINS]; window_size];
|
||||
ring_cursor = 0;
|
||||
filled = 0;
|
||||
}
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
@@ -692,9 +848,21 @@ fn main() {
|
||||
sharp_ema[b] += w * block[b];
|
||||
}
|
||||
}
|
||||
ref_bin = guarded_best_bin(&ema, ref_bin, sb, sa, &guard);
|
||||
ref_bin = guarded_best_bin(&ema, ref_bin, sb, sa, &guard, &arm_weights, corr_weight, &profile);
|
||||
let oracle_price = bin_to_cents(ref_bin) as f64 / 100.0;
|
||||
|
||||
// Re-estimate the shape template from the L1-normalized arm vector at the
|
||||
// new pick, blended in slowly so a transient octave slide cannot corrupt it.
|
||||
if corr_weight != 0.0 {
|
||||
let arms = arms_at(&ema, ref_bin.round() as i64);
|
||||
let s: f64 = arms.iter().sum();
|
||||
if s > 0.0 {
|
||||
for i in 0..N_ARMS {
|
||||
profile[i] = (1.0 - corr_beta) * profile[i] + corr_beta * (arms[i] / s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let o = height_ohlc.get(h).copied().unwrap_or([0.0; 4]);
|
||||
let (ex_high, ex_low, ex_close) = (o[1], o[2], o[3]);
|
||||
let band_err = if ex_high > 0.0 && ex_low > 0.0 {
|
||||
@@ -724,6 +892,9 @@ fn main() {
|
||||
let qh = arm_count(&ema, true_bin + 60, guard.tau);
|
||||
let qd = arm_count(&ema, true_bin - 60, guard.tau);
|
||||
let pat = arm_pattern(&ema, true_bin, guard.tau);
|
||||
// Octave-discriminating subset only: mass at true vs half center.
|
||||
let dt = arm_subset_sum(&ema, true_bin, &DISC_ARMS);
|
||||
let dh = arm_subset_sum(&ema, true_bin + 60, &DISC_ARMS);
|
||||
// Same arm-count contrast measured on the sharp detection EMA.
|
||||
let qst = arm_count(&sharp_ema, true_bin, guard.tau);
|
||||
let qsh = arm_count(&sharp_ema, true_bin + 60, guard.tau);
|
||||
@@ -731,7 +902,7 @@ fn main() {
|
||||
let spat = arm_pattern(&sharp_ema, true_bin, guard.tau);
|
||||
let ts_secs: u32 = *timestamps[h];
|
||||
eprintln!(
|
||||
"{h}\t{ts_secs}\t{oracle_price:.0}\t{ex_close:.0}\t{band_err:+.2}\t{eligible}\tT={s_true:.1}\tH={s_half:.1}\tD={s_dbl:.1}\tQt={qt}\tQh={qh}\tQd={qd}\t{pat}\t|sharp Qt={qst} Qh={qsh} Qd={qsd}\t{spat}"
|
||||
"{h}\t{ts_secs}\t{oracle_price:.0}\t{ex_close:.0}\t{band_err:+.2}\t{eligible}\tT={s_true:.1}\tH={s_half:.1}\tD={s_dbl:.1}\tQt={qt}\tQh={qh}\tQd={qd}\tDt={dt:.1}\tDh={dh:.1}\t{pat}\t|sharp Qt={qst} Qh={qsh} Qd={qsd}\t{spat}"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -37,3 +37,27 @@ impl Default for Config {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Cold-start config below [`START_HEIGHT`](crate::START_HEIGHT): a slow EMA
|
||||
/// (span ~19) that resists the round-USD half-price drift the fast default
|
||||
/// octave-locks onto in the thin pre-2018 output mix. Window grows to 40 to
|
||||
/// hold the decay.
|
||||
pub fn slow() -> Self {
|
||||
Self {
|
||||
alpha: 0.10,
|
||||
window_size: 40,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Config for `height`: [`slow`](Self::slow) below
|
||||
/// [`START_HEIGHT`](crate::START_HEIGHT), else [`default`](Self::default).
|
||||
pub fn for_height(height: usize) -> Self {
|
||||
if height < crate::START_HEIGHT {
|
||||
Self::slow()
|
||||
} else {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,12 +14,17 @@ use config::{DEFAULT_EXCLUDED_OUTPUT_TYPES, DEFAULT_MIN_SATS};
|
||||
/// so downstream consumers can invalidate cached results.
|
||||
pub const VERSION: u32 = 3;
|
||||
|
||||
/// Pre-oracle dollar prices, one per line, heights 0..508_000. The last entry
|
||||
/// (height `START_HEIGHT - 1`) seeds the oracle's first on-chain computation at
|
||||
/// `START_HEIGHT`.
|
||||
/// Pre-oracle dollar prices, one per line, heights 0..470_000. The last entry
|
||||
/// seeds the oracle's first on-chain computation at `START_HEIGHT_SLOW`.
|
||||
pub const PRICES: &str = include_str!("prices.txt");
|
||||
|
||||
/// First height where the oracle computes from on-chain data.
|
||||
/// First height the oracle computes on-chain, with the slow cold-start EMA
|
||||
/// ([`Config::slow`]). Below it, prices come from [`PRICES`].
|
||||
pub const START_HEIGHT_SLOW: usize = 470_000;
|
||||
|
||||
/// Height where the oracle switches slow -> fast EMA ([`Config::default`]).
|
||||
/// The regimes are complementary: slow resists the round-USD half-price drift
|
||||
/// that locks fast below here; fast tracks the 2018-2019 crashes that lock slow.
|
||||
pub const START_HEIGHT: usize = 508_000;
|
||||
|
||||
/// A transaction with more than this many outputs is a batch payout (exchange
|
||||
@@ -300,6 +305,23 @@ impl Oracle {
|
||||
self.ref_bin
|
||||
}
|
||||
|
||||
/// Switch EMA regime mid-stream (slow -> fast at [`START_HEIGHT`]) by
|
||||
/// re-warming under `config` over the most recent `config.window_size` raw
|
||||
/// histograms, so a continuous build and an incremental warm-up reach the
|
||||
/// same state; `ref_bin` carries over.
|
||||
pub fn reconfigure(&mut self, config: Config) {
|
||||
let window = self.config.window_size;
|
||||
let kept: Vec<HistogramRaw> = (0..self.filled.min(config.window_size))
|
||||
.rev()
|
||||
.map(|age| self.histograms[(self.cursor + window - 1 - age) % window].clone())
|
||||
.collect();
|
||||
*self = Self::from_checkpoint(self.ref_bin, config, |o| {
|
||||
kept.iter().for_each(|h| {
|
||||
o.process_histogram(h);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
pub fn ref_bin(&self) -> f64 {
|
||||
self.ref_bin
|
||||
}
|
||||
@@ -380,4 +402,35 @@ mod tests {
|
||||
assert_eq!(oracle.ref_bin(), 1600.0);
|
||||
assert_eq!(oracle.price_cents(), bin_to_cents(1600.0).into());
|
||||
}
|
||||
|
||||
// reconfigure must leave the oracle in the same state as a fresh warm-up
|
||||
// over the most recent window of raw histograms; the continuous build and
|
||||
// the incremental resume rely on this agreeing at the slow -> fast seam.
|
||||
#[test]
|
||||
fn reconfigure_matches_fresh_warmup() {
|
||||
let hists: Vec<HistogramRaw> = (0..60)
|
||||
.map(|i| {
|
||||
let mut h = HistogramRaw::zeros();
|
||||
h.increment(1200 + i % 7);
|
||||
h.increment(1600 + i % 5);
|
||||
h
|
||||
})
|
||||
.collect();
|
||||
|
||||
let fast = Config::default();
|
||||
let mut switched = Oracle::new(1600.0, Config::slow());
|
||||
hists.iter().for_each(|h| {
|
||||
switched.process_histogram(h);
|
||||
});
|
||||
switched.reconfigure(fast.clone());
|
||||
|
||||
let keep = fast.window_size;
|
||||
let fresh = Oracle::from_checkpoint(switched.ref_bin(), fast, |o| {
|
||||
hists[hists.len() - keep..].iter().for_each(|h| {
|
||||
o.process_histogram(h);
|
||||
});
|
||||
});
|
||||
|
||||
assert!(switched.ema().iter().eq(fresh.ema().iter()));
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ use brk_computer::prices::Vecs as PricesVecs;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_indexer::Lengths;
|
||||
use brk_oracle::{
|
||||
Config, HistogramEmaCompact, HistogramRaw, Oracle, START_HEIGHT, cents_to_bin,
|
||||
Config, HistogramEmaCompact, HistogramRaw, Oracle, START_HEIGHT_SLOW, cents_to_bin,
|
||||
for_each_round_dollar_bin,
|
||||
};
|
||||
use brk_types::{Dollars, OutputType, Sats, TxIndex, TxOutIndex};
|
||||
@@ -90,7 +90,7 @@ impl Query {
|
||||
/// committed blocks ending just before `end`. Reads are capped at `safe` so
|
||||
/// concurrent indexer writes past the cap stay invisible.
|
||||
fn warm_oracle(&self, seed_bin: f64, end: usize, safe: &Lengths) -> Oracle {
|
||||
let config = Config::default();
|
||||
let config = Config::for_height(end.saturating_sub(1));
|
||||
let start = end.saturating_sub(config.window_size);
|
||||
Oracle::from_checkpoint(seed_bin, config, |o| {
|
||||
PricesVecs::feed_blocks(o, self.indexer(), start..end, Some(safe));
|
||||
@@ -113,7 +113,7 @@ impl Query {
|
||||
Ok(cents_to_bin(cents.inner() as f64))
|
||||
}
|
||||
|
||||
/// `START_HEIGHT <= height < min(spot price len, safe height)` or 404.
|
||||
/// `START_HEIGHT_SLOW <= height < min(spot price len, safe height)` or 404.
|
||||
/// Returns the safe lengths so callers cap reads at the same bound.
|
||||
fn check_histogram_height(&self, height: usize) -> Result<Lengths> {
|
||||
let safe = self.safe_lengths();
|
||||
@@ -125,7 +125,7 @@ impl Query {
|
||||
.height
|
||||
.len()
|
||||
.min(safe.height.to_usize());
|
||||
if height < START_HEIGHT || height >= bound {
|
||||
if height < START_HEIGHT_SLOW || height >= bound {
|
||||
return Err(Error::NotFound(format!(
|
||||
"oracle histogram unavailable for height {height}"
|
||||
)));
|
||||
|
||||
@@ -11870,7 +11870,7 @@ class BrkClient extends BrkClientBase {
|
||||
/**
|
||||
* Live BTC/USD price
|
||||
*
|
||||
* Current BTC/USD price in dollars, derived purely from on-chain round-dollar output patterns over the last 12 blocks plus the forming mempool block. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
* Current BTC/USD price in dollars. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/price`
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Dollars) => void }} [options]
|
||||
@@ -11884,7 +11884,7 @@ class BrkClient extends BrkClientBase {
|
||||
/**
|
||||
* Live EMA histogram
|
||||
*
|
||||
* Smoothed round-dollar payment histogram at the live tip: the committed 12-block EMA with the forming mempool block blended in as a final slot. A flat array of 2400 log-scale bins, quantized to `u16` for the wire. This is the heatmap column you render.
|
||||
* Smoothed round-dollar payment histogram at the live tip: the committed EMA with the forming mempool block blended in. A flat array of log-scale bins.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/ema/live`
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Histogram_uint16) => void }} [options]
|
||||
@@ -11898,7 +11898,7 @@ class BrkClient extends BrkClientBase {
|
||||
/**
|
||||
* EMA histogram at height
|
||||
*
|
||||
* Smoothed round-dollar payment histogram for a confirmed height, deterministically reconstructed by replaying the 12-block window ending at that height. Immutable once buried, so repeated requests return byte-identical results. A flat array of 2400 log-scale bins, quantized to `u16`.
|
||||
* Smoothed round-dollar payment histogram for a confirmed height. A flat array of log-scale bins.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/ema/{height}`
|
||||
*
|
||||
@@ -11914,7 +11914,7 @@ class BrkClient extends BrkClientBase {
|
||||
/**
|
||||
* Live raw histogram
|
||||
*
|
||||
* Un-smoothed per-block round-dollar counts for the forming mempool block: the spiky primitive the EMA smooths over. A flat array of 2400 log-scale bins (`u32` counts), all zero when no mempool is configured.
|
||||
* Un-smoothed per-block round-dollar counts for the forming mempool block. A flat array of log-scale bins, all zero when no mempool is configured.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/raw/live`
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Histogram_uint32) => void }} [options]
|
||||
@@ -11928,7 +11928,7 @@ class BrkClient extends BrkClientBase {
|
||||
/**
|
||||
* Raw histogram at height
|
||||
*
|
||||
* Un-smoothed round-dollar counts for a single confirmed block. A flat array of 2400 log-scale bins (`u32` counts).
|
||||
* Un-smoothed round-dollar counts for a single confirmed block. A flat array of log-scale bins.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/raw/{height}`
|
||||
*
|
||||
|
||||
@@ -8663,7 +8663,7 @@ class BrkClient(BrkClientBase):
|
||||
def get_oracle_price(self) -> Dollars:
|
||||
"""Live BTC/USD price.
|
||||
|
||||
Current BTC/USD price in dollars, derived purely from on-chain round-dollar output patterns over the last 12 blocks plus the forming mempool block. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
Current BTC/USD price in dollars. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
|
||||
Endpoint: `GET /api/oracle/price`"""
|
||||
return self.get_json('/api/oracle/price')
|
||||
@@ -8671,7 +8671,7 @@ class BrkClient(BrkClientBase):
|
||||
def get_oracle_histogram_ema_live(self) -> Histogram_uint16:
|
||||
"""Live EMA histogram.
|
||||
|
||||
Smoothed round-dollar payment histogram at the live tip: the committed 12-block EMA with the forming mempool block blended in as a final slot. A flat array of 2400 log-scale bins, quantized to `u16` for the wire. This is the heatmap column you render.
|
||||
Smoothed round-dollar payment histogram at the live tip: the committed EMA with the forming mempool block blended in. A flat array of log-scale bins.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/ema/live`"""
|
||||
return self.get_json('/api/oracle/histogram/ema/live')
|
||||
@@ -8679,7 +8679,7 @@ class BrkClient(BrkClientBase):
|
||||
def get_oracle_histogram_ema(self, height: Height) -> Histogram_uint16:
|
||||
"""EMA histogram at height.
|
||||
|
||||
Smoothed round-dollar payment histogram for a confirmed height, deterministically reconstructed by replaying the 12-block window ending at that height. Immutable once buried, so repeated requests return byte-identical results. A flat array of 2400 log-scale bins, quantized to `u16`.
|
||||
Smoothed round-dollar payment histogram for a confirmed height. A flat array of log-scale bins.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/ema/{height}`"""
|
||||
return self.get_json(f'/api/oracle/histogram/ema/{height}')
|
||||
@@ -8687,7 +8687,7 @@ class BrkClient(BrkClientBase):
|
||||
def get_oracle_histogram_raw_live(self) -> Histogram_uint32:
|
||||
"""Live raw histogram.
|
||||
|
||||
Un-smoothed per-block round-dollar counts for the forming mempool block: the spiky primitive the EMA smooths over. A flat array of 2400 log-scale bins (`u32` counts), all zero when no mempool is configured.
|
||||
Un-smoothed per-block round-dollar counts for the forming mempool block. A flat array of log-scale bins, all zero when no mempool is configured.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/raw/live`"""
|
||||
return self.get_json('/api/oracle/histogram/raw/live')
|
||||
@@ -8695,7 +8695,7 @@ class BrkClient(BrkClientBase):
|
||||
def get_oracle_histogram_raw(self, height: Height) -> Histogram_uint32:
|
||||
"""Raw histogram at height.
|
||||
|
||||
Un-smoothed round-dollar counts for a single confirmed block. A flat array of 2400 log-scale bins (`u32` counts).
|
||||
Un-smoothed round-dollar counts for a single confirmed block. A flat array of log-scale bins.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/raw/{height}`"""
|
||||
return self.get_json(f'/api/oracle/histogram/raw/{height}')
|
||||
|
||||
Reference in New Issue
Block a user