mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 06:01:57 -07:00
oracle: snapshot + start at 508k
This commit is contained in:
nym21
Generated
+27
-26
@@ -103,9 +103,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
@@ -618,6 +618,7 @@ dependencies = [
|
||||
"brk_indexer",
|
||||
"brk_logger",
|
||||
"brk_mempool",
|
||||
"brk_oracle",
|
||||
"brk_query",
|
||||
"brk_reader",
|
||||
"brk_rpc",
|
||||
@@ -732,9 +733,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.2"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
@@ -1186,9 +1187,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
||||
|
||||
[[package]]
|
||||
name = "enum_dispatch"
|
||||
@@ -1890,9 +1891,9 @@ checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.98"
|
||||
version = "0.3.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
|
||||
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -1947,9 +1948,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.47"
|
||||
version = "0.1.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
|
||||
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
@@ -2062,9 +2063,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.50"
|
||||
version = "0.1.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
|
||||
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
@@ -2127,9 +2128,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
||||
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
@@ -2767,9 +2768,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
@@ -3420,9 +3421,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
|
||||
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
@@ -3433,9 +3434,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
|
||||
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -3443,9 +3444,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
|
||||
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
@@ -3456,9 +3457,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.121"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
|
||||
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -3499,9 +3500,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.98"
|
||||
version = "0.3.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
|
||||
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
||||
+1
-1
@@ -76,7 +76,7 @@ schemars = { version = "1.2.1", features = ["indexmap2"] }
|
||||
serde = "1.0.228"
|
||||
serde_bytes = "0.11.19"
|
||||
serde_derive = "1.0.228"
|
||||
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["float_roundtrip", "preserve_order"] }
|
||||
smallvec = "1.15.1"
|
||||
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
|
||||
tower-http = { version = "0.6.11", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
|
||||
|
||||
@@ -8,5 +8,5 @@ homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
|
||||
mimalloc = { version = "0.1.50" }
|
||||
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
|
||||
mimalloc = { version = "0.1.52" }
|
||||
|
||||
@@ -9856,6 +9856,51 @@ impl BrkClient {
|
||||
self.base.get_json(&format!("/api/mempool/price"))
|
||||
}
|
||||
|
||||
/// Live BTC/USD price
|
||||
///
|
||||
/// Current BTC/USD price in dollars, derived purely from on-chain round-dollar output patterns over the last 12 blocks plus the forming mempool block. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/price`
|
||||
pub fn get_oracle_price(&self) -> Result<Dollars> {
|
||||
self.base.get_json(&format!("/api/oracle/price"))
|
||||
}
|
||||
|
||||
/// Live EMA histogram
|
||||
///
|
||||
/// Smoothed round-dollar payment histogram at the live tip: the committed 12-block EMA with the forming mempool block blended in as a final slot. A flat array of 2400 log-scale bins, quantized to `u16` for the wire. This is the heatmap column you render.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/ema/live`
|
||||
pub fn get_oracle_histogram_ema_live(&self) -> Result<Histogram_uint16> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/ema/live"))
|
||||
}
|
||||
|
||||
/// EMA histogram at height
|
||||
///
|
||||
/// Smoothed round-dollar payment histogram for a confirmed height, deterministically reconstructed by replaying the 12-block window ending at that height. Immutable once buried, so repeated requests return byte-identical results. A flat array of 2400 log-scale bins, quantized to `u16`.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/ema/{height}`
|
||||
pub fn get_oracle_histogram_ema(&self, height: Height) -> Result<Histogram_uint16> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/ema/{height}"))
|
||||
}
|
||||
|
||||
/// Live raw histogram
|
||||
///
|
||||
/// Un-smoothed per-block round-dollar counts for the forming mempool block: the spiky primitive the EMA smooths over. A flat array of 2400 log-scale bins (`u32` counts), all zero when no mempool is configured.
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/raw/live`
|
||||
pub fn get_oracle_histogram_raw_live(&self) -> Result<Histogram_uint32> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/raw/live"))
|
||||
}
|
||||
|
||||
/// Raw histogram at height
|
||||
///
|
||||
/// Un-smoothed round-dollar counts for a single confirmed block. A flat array of 2400 log-scale bins (`u32` counts).
|
||||
///
|
||||
/// Endpoint: `GET /api/oracle/histogram/raw/{height}`
|
||||
pub fn get_oracle_histogram_raw(&self, height: Height) -> Result<Histogram_uint32> {
|
||||
self.base.get_json(&format!("/api/oracle/histogram/raw/{height}"))
|
||||
}
|
||||
|
||||
/// Txid by index
|
||||
///
|
||||
/// Retrieve the transaction ID (txid) at a given global transaction index. Returns the txid as plain text.
|
||||
|
||||
@@ -2,7 +2,9 @@ use std::ops::Range;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_indexer::{Indexer, Lengths};
|
||||
use brk_oracle::{Config, Histogram, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
|
||||
use brk_oracle::{
|
||||
Config, HistogramRaw, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin, for_each_round_dollar_bin,
|
||||
};
|
||||
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use tracing::info;
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, StorageMode, VecIndex, WritableVec};
|
||||
@@ -61,8 +63,8 @@ impl Vecs {
|
||||
fn compute_prices(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
|
||||
let source_version = indexer.vecs.outputs.value.version()
|
||||
+ indexer.vecs.outputs.output_type.version();
|
||||
let source_version =
|
||||
indexer.vecs.outputs.value.version() + indexer.vecs.outputs.output_type.version();
|
||||
self.spot
|
||||
.cents
|
||||
.height
|
||||
@@ -121,8 +123,7 @@ impl Vecs {
|
||||
committed, total_heights
|
||||
);
|
||||
|
||||
let ref_bins =
|
||||
Self::feed_blocks(&mut oracle, indexer, committed..total_heights, None);
|
||||
let ref_bins = Self::feed_blocks(&mut oracle, indexer, committed..total_heights, None);
|
||||
|
||||
for (i, ref_bin) in ref_bins.into_iter().enumerate() {
|
||||
self.spot
|
||||
@@ -151,11 +152,8 @@ impl Vecs {
|
||||
}
|
||||
|
||||
/// Feed a range of blocks from the indexer into an Oracle (skipping coinbase),
|
||||
/// returning per-block ref_bin values.
|
||||
///
|
||||
/// A transaction carrying an `OP_RETURN` output is protocol machinery, not a
|
||||
/// dollar-denominated payment, so all of its outputs are dropped from the
|
||||
/// histogram. This needs per-transaction grouping of a block's outputs.
|
||||
/// returning per-block ref_bin values. Outputs are grouped per transaction
|
||||
/// because `for_each_round_dollar_bin` drops a whole tx on any OP_RETURN.
|
||||
///
|
||||
/// Pass `cap = None` from compute paths, when the indexer is quiescent and
|
||||
/// raw vec lengths are authoritative. Pass `cap = Some(&safe_lengths)` from
|
||||
@@ -239,21 +237,20 @@ impl Vecs {
|
||||
&mut output_types,
|
||||
);
|
||||
|
||||
let mut hist = Histogram::zeros();
|
||||
let mut hist = HistogramRaw::zeros();
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
if output_types[lo..hi].contains(&OutputType::OpReturn) {
|
||||
continue;
|
||||
}
|
||||
for i in lo..hi {
|
||||
if let Some(bin) = oracle.output_to_bin(values[i], output_types[i]) {
|
||||
hist.increment(bin);
|
||||
}
|
||||
}
|
||||
let outputs = values[lo..hi]
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(output_types[lo..hi].iter().copied());
|
||||
for_each_round_dollar_bin(range.start + idx, outputs, |bin| {
|
||||
hist.increment(bin as usize)
|
||||
});
|
||||
}
|
||||
|
||||
ref_bins.push(oracle.process_histogram(&hist));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Mempool info + price-blending output histogram.
|
||||
|
||||
use brk_oracle::Histogram;
|
||||
use brk_oracle::HistogramRaw;
|
||||
use brk_types::MempoolInfo;
|
||||
|
||||
use crate::Mempool;
|
||||
@@ -17,7 +17,7 @@ impl Mempool {
|
||||
/// of pool size. Used by `live_price` to blend the mempool into the
|
||||
/// committed oracle without re-parsing scripts per request.
|
||||
#[must_use]
|
||||
pub fn live_histogram(&self) -> Histogram {
|
||||
pub fn live_histogram(&self) -> HistogramRaw {
|
||||
self.read().txs.live_histogram()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use brk_oracle::default_eligible_bin;
|
||||
use brk_oracle::for_each_round_dollar_bin;
|
||||
use brk_types::Transaction;
|
||||
use smallvec::SmallVec;
|
||||
|
||||
@@ -9,12 +9,15 @@ pub struct OutputBins(SmallVec<[u16; 4]>);
|
||||
|
||||
impl OutputBins {
|
||||
pub fn from_tx(tx: &Transaction) -> Self {
|
||||
Self(
|
||||
tx.output
|
||||
.iter()
|
||||
.filter_map(|o| default_eligible_bin(o.value, o.type_()))
|
||||
.collect(),
|
||||
)
|
||||
let mut bins = SmallVec::new();
|
||||
// Live mempool txs are post-tip, always above the historical max-outputs
|
||||
// cap window, so the cap never applies here.
|
||||
for_each_round_dollar_bin(
|
||||
usize::MAX,
|
||||
tx.output.iter().map(|o| (o.value, o.type_())),
|
||||
|bin| bins.push(bin),
|
||||
);
|
||||
Self(bins)
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = u16> + '_ {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use brk_oracle::Histogram;
|
||||
use brk_oracle::HistogramRaw;
|
||||
use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, TxidPrefix, Vin};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
|
||||
@@ -40,7 +40,7 @@ pub struct TxStore {
|
||||
records: FxHashMap<TxidPrefix, TxRecord>,
|
||||
recent: Vec<MempoolRecentTx>,
|
||||
unresolved: FxHashSet<TxidPrefix>,
|
||||
live_histogram: Histogram,
|
||||
live_histogram: HistogramRaw,
|
||||
}
|
||||
|
||||
impl TxStore {
|
||||
@@ -120,7 +120,7 @@ impl TxStore {
|
||||
|
||||
/// Snapshot the live oracle-bin histogram. Maintained incrementally
|
||||
/// on insert/remove, so this is `O(NUM_BINS)`, not `O(live_outputs)`.
|
||||
pub fn live_histogram(&self) -> Histogram {
|
||||
pub fn live_histogram(&self) -> HistogramRaw {
|
||||
self.live_histogram.clone()
|
||||
}
|
||||
|
||||
@@ -263,7 +263,10 @@ mod tests {
|
||||
assert_eq!(applied[0].value, new_prevout.value);
|
||||
|
||||
let record = store.record_by_prefix(&prefix).expect("record present");
|
||||
assert_eq!(record.tx.input[0].prevout.as_ref().unwrap().value, new_prevout.value);
|
||||
assert_eq!(
|
||||
record.tx.input[0].prevout.as_ref().unwrap().value,
|
||||
new_prevout.value
|
||||
);
|
||||
assert_eq!(
|
||||
record.tx.input[1].prevout.as_ref().unwrap().value,
|
||||
prev_present.value
|
||||
@@ -277,7 +280,10 @@ mod tests {
|
||||
let stray_prefix = TxidPrefix::from(&fake_txid(0xFF));
|
||||
let applied = store.apply_fills(
|
||||
&stray_prefix,
|
||||
vec![(Vin::from(0u32), TxOut::from((ScriptBuf::new(), Sats::from(1u64))))],
|
||||
vec![(
|
||||
Vin::from(0u32),
|
||||
TxOut::from((ScriptBuf::new(), Sats::from(1u64))),
|
||||
)],
|
||||
);
|
||||
assert!(applied.is_empty());
|
||||
}
|
||||
@@ -319,10 +325,7 @@ mod tests {
|
||||
let tx_a = fake_tx(
|
||||
20,
|
||||
&[Some(TxOut::from((p2wpkh_script(8), Sats::from(1_234u64))))],
|
||||
&[
|
||||
(p2wpkh_script(9), 2_345),
|
||||
(p2wpkh_script(10), 3_456),
|
||||
],
|
||||
&[(p2wpkh_script(9), 2_345), (p2wpkh_script(10), 3_456)],
|
||||
);
|
||||
let tx_b = fake_tx(
|
||||
21,
|
||||
|
||||
+34
-24
@@ -2,7 +2,7 @@
|
||||
|
||||
**Version 2**
|
||||
|
||||
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 525,000 (May 2018) onward.
|
||||
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 508,000 (February 2018) onward.
|
||||
|
||||
Inspired by [UTXOracle](https://utxo.live/oracle/) by [@SteveSimple](https://x.com/SteveSimple), which proved the concept. brk_oracle takes the same core insight and redesigns the algorithm for per-block resolution and rolling operation. See [comparison](#comparison-with-utxoracle) below.
|
||||
|
||||
@@ -42,13 +42,13 @@ The spacing between spikes is constant (set by the ratios between dollar amounts
|
||||
|
||||
## How it works
|
||||
|
||||
The oracle tracks the price incrementally, block by block, starting from a known seed price. Each new block nudges the estimate. The search window is narrow (about ±10 bins, or ±12%), so the oracle can only follow gradual movement — it cannot jump to an arbitrary price from scratch. This is by design: it makes the algorithm resistant to noise.
|
||||
The oracle tracks the price incrementally, block by block, starting from a known seed price. Each new block nudges the estimate. The search window is narrow (about 12 bins, or +15% / -12% in price), so the oracle can only follow gradual movement, not jump to an arbitrary price from scratch. This is by design: it makes the algorithm resistant to noise.
|
||||
|
||||
For each new block:
|
||||
|
||||
### 1. Filter outputs
|
||||
|
||||
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
|
||||
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Below height 630,000, also skip every output of a transaction with more than 100 outputs: a large fan-out is a batch payout (exchange sweep, mixer), not a round-dollar payment, and the thin early signal needs it removed. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
|
||||
|
||||
### 2. Build a log-scale histogram
|
||||
|
||||
@@ -122,9 +122,9 @@ Parabolic interpolation between the best bin and its two neighbors refines the e
|
||||
|
||||
The oracle consumes one pre-built histogram per block via `process_histogram(&hist)`, a `[u32; 2400]` bin-count array, and returns the updated reference bin.
|
||||
|
||||
The caller does the filtering when it builds the histogram. For each block it skips the coinbase, drops every output of a transaction carrying an `OP_RETURN`, then bins the rest. `default_eligible_bin(sats, output_type)` (or `Oracle::output_to_bin` for a non-default `Config`) applies the per-output rules: excluded script types, dust, and round-BTC values. It returns the bin index, or `None` for a filtered output.
|
||||
The caller does the filtering when it builds the histogram. For each block it skips the coinbase, drops every output of a transaction carrying an `OP_RETURN` (and, below height 630,000, every output of a transaction with more than 100 outputs), then bins the rest. `default_eligible_bin(sats, output_type)` (or `Oracle::output_to_bin` for a non-default `Config`) applies the per-output rules: excluded script types, dust, and round-BTC values. It returns the bin index, or `None` for a filtered output.
|
||||
|
||||
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for heights 0..525,000; the last entry (height 524,999) seeds the oracle's first on-chain computation at `START_HEIGHT`.
|
||||
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for heights 0..508,000. Its last entry, height 507,999 (one below `START_HEIGHT`), seeds the oracle's first on-chain computation at height 508,000.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -134,7 +134,7 @@ All parameters via `Config` with sensible defaults:
|
||||
|-----------|---------|---------|
|
||||
| `alpha` | 2/7 | EMA decay rate (~6-block span) |
|
||||
| `window_size` | 12 | Ring buffer depth in blocks |
|
||||
| `search_below` / `search_above` | 9 / 11 | Search window around previous estimate (bins) |
|
||||
| `search_below` / `search_above` | 12 / 11 | Search window around previous estimate (bins) |
|
||||
| `min_sats` | 1,000 | Dust threshold |
|
||||
| `exclude_common_round_values` | true | Filter d × 10ⁿ (d ∈ {1,2,3,5,6}) to prevent false stencil matches |
|
||||
| `excluded_output_types` | P2TR | Script types dominated by protocol activity |
|
||||
@@ -152,29 +152,29 @@ All parameters via `Config` with sensible defaults:
|
||||
| Stencil | 19 round-USD offsets ($1 to $10k), each normalized to its own peak | 803-point Gaussian + weighted spike template targeting 17 round-USD amounts |
|
||||
| Round BTC handling | Excluded from histogram entirely | Histogram bins smoothed by averaging neighbors |
|
||||
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: not coinbase, no OP_RETURN, exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
|
||||
| Validated from | Height 525,000 (May 2018) | Dec 15, 2023 |
|
||||
| Validated from | Height 508,000 (February 2018) | Dec 15, 2023 |
|
||||
| Language | Rust | Python |
|
||||
| Dependencies | None (pure computation, caller provides block data) | bitcoin-cli + direct blk file reads |
|
||||
| Bins per decade | 200 | 200 |
|
||||
|
||||
## Accuracy
|
||||
|
||||
Tested over 411,251 blocks (heights 525,000 to 949,800, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
|
||||
Tested over 428,251 blocks (heights 508,000 to 950,490, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
|
||||
|
||||
### Per-block
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Median error | 0.11% |
|
||||
| 95th percentile | 0.67% |
|
||||
| 95th percentile | 0.68% |
|
||||
| 99th percentile | 1.7% |
|
||||
| 99.9th percentile | 5.4% |
|
||||
| RMSE | 0.50% |
|
||||
| Max error | 33.4% |
|
||||
| 99.9th percentile | 4.7% |
|
||||
| RMSE | 0.46% |
|
||||
| Max error | 28.8% |
|
||||
| Bias | +0.00 bins (essentially zero) |
|
||||
| Blocks > 5% error | 472 (0.11%) |
|
||||
| Blocks > 10% error | 177 |
|
||||
| Blocks > 20% error | 3 |
|
||||
| Blocks > 5% error | 367 (0.086%) |
|
||||
| Blocks > 10% error | 116 |
|
||||
| Blocks > 20% error | 1 |
|
||||
|
||||
### Daily candles
|
||||
|
||||
@@ -182,26 +182,26 @@ Oracle daily OHLC built from per-block prices vs exchange daily OHLC:
|
||||
|
||||
| | Median | RMSE | Max |
|
||||
|-------|--------|------|-----|
|
||||
| Open | 0.21% | 0.65% | 15.3% |
|
||||
| High | 0.53% | 1.12% | 28.0% |
|
||||
| Low | 0.51% | 1.38% | 19.7% |
|
||||
| Open | 0.21% | 0.59% | 15.3% |
|
||||
| High | 0.53% | 0.99% | 15.4% |
|
||||
| Low | 0.52% | 1.39% | 21.5% |
|
||||
| Close | 0.24% | 0.73% | 15.4% |
|
||||
|
||||
### By year
|
||||
|
||||
| Year | Blocks | Median | RMSE | Max | >5% | >10% | >20% | Price range |
|
||||
|------|--------|--------|------|-----|-----|------|------|-------------|
|
||||
| 2018 | 31,492 | 0.21% | 1.11% | 33.4% | 169 | 109 | 3 | $3,129–$8,488 |
|
||||
| 2019 | 54,272 | 0.16% | 0.69% | 17.4% | 165 | 53 | 0 | $3,338–$13,868 |
|
||||
| 2020 | 53,102 | 0.10% | 0.44% | 12.6% | 70 | 6 | 0 | $3,858–$29,322 |
|
||||
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 42 | 9 | 0 | $27,678–$69,000 |
|
||||
| 2018 | 48,492 | 0.17% | 0.84% | 28.8% | 136 | 87 | 1 | $3,129–$11,775 |
|
||||
| 2019 | 54,272 | 0.16% | 0.59% | 17.4% | 100 | 16 | 0 | $3,338–$13,868 |
|
||||
| 2020 | 53,102 | 0.10% | 0.42% | 11.6% | 61 | 3 | 0 | $3,858–$29,322 |
|
||||
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 43 | 10 | 0 | $27,678–$69,000 |
|
||||
| 2022 | 53,230 | 0.07% | 0.32% | 6.8% | 10 | 0 | 0 | $15,460–$48,240 |
|
||||
| 2023 | 54,032 | 0.10% | 0.25% | 6.6% | 5 | 0 | 0 | $16,490–$44,700 |
|
||||
| 2024 | 53,367 | 0.10% | 0.28% | 6.7% | 7 | 0 | 0 | $38,555–$108,298 |
|
||||
| 2024 | 53,367 | 0.10% | 0.29% | 7.1% | 8 | 0 | 0 | $38,555–$108,298 |
|
||||
| 2025 | 53,113 | 0.11% | 0.25% | 5.8% | 4 | 0 | 0 | $74,409–$126,198 |
|
||||
| 2026 | 5,910 | 0.11% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000–$97,900 |
|
||||
|
||||
The oracle is only as good as the signal it reads. The largest errors cluster in late 2018: the November price crash fell faster than the narrow search window could follow (33% max error), and on-chain volume was lower then, so the round-dollar pattern was weaker (1.1% RMSE for the year). By 2020 the signal is strong enough for 0.1% median accuracy, and since 2022 no block exceeds 10% error.
|
||||
The oracle is only as good as the signal it reads. The largest errors cluster in late 2018: the November price crash fell faster than the narrow search window could follow (28.8% max error, at height 550,890), and on-chain volume was lower then, so the round-dollar pattern was weaker (0.84% RMSE for the year). By 2020 the signal is strong enough for 0.1% median accuracy, and since 2022 no block exceeds 10% error.
|
||||
|
||||
### Why no outlier smoothing?
|
||||
|
||||
@@ -212,6 +212,16 @@ Post-hoc smoothing — for example, correcting any block whose price deviates mo
|
||||
|
||||
## Changelog
|
||||
|
||||
### v3
|
||||
|
||||
Changes from v2:
|
||||
|
||||
- **Earlier start**: on-chain tracking begins at height 508,000 (February 2018) instead of 525,000, adding about 17,000 blocks of history.
|
||||
- **Max-outputs filter**: a transaction with more than 100 outputs is dropped from the histogram below height 630,000. Large fan-outs (exchange sweeps, mixer payouts) are batch machinery, not round-dollar payments, and the thin 2018-2020 signal needs them removed to stay locked onto the pattern. Above 630,000 on-chain volume is dense enough that the cap removes more genuine signal than noise, so it is lifted.
|
||||
- **Wider up-reach**: `search_below` raised from 9 to 12 bins. The sharp 2018 reversal candles need extra room to follow a fast move upward in price.
|
||||
|
||||
`VERSION` is bumped to 3 so downstream consumers invalidate prices computed by an earlier algorithm.
|
||||
|
||||
### v2
|
||||
|
||||
Changes from v1:
|
||||
|
||||
@@ -14,8 +14,8 @@ use std::path::PathBuf;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
Config, Histogram, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
|
||||
default_eligible_bin,
|
||||
Config, HistogramRaw, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
|
||||
for_each_round_dollar_bin,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
@@ -31,6 +31,7 @@ fn seed_bin_for_start_height() -> f64 {
|
||||
}
|
||||
|
||||
struct Block {
|
||||
height: usize,
|
||||
values: Vec<Sats>,
|
||||
output_types: Vec<OutputType>,
|
||||
tx_starts: Vec<usize>,
|
||||
@@ -38,8 +39,8 @@ struct Block {
|
||||
out_end: usize,
|
||||
}
|
||||
|
||||
fn build_histogram(block: &Block) -> Histogram {
|
||||
let mut hist = Histogram::zeros();
|
||||
fn build_histogram(block: &Block) -> HistogramRaw {
|
||||
let mut hist = HistogramRaw::zeros();
|
||||
for tx in 0..block.tx_starts.len() {
|
||||
let lo = block.tx_starts[tx] - block.out_start;
|
||||
let hi = block
|
||||
@@ -47,14 +48,11 @@ fn build_histogram(block: &Block) -> Histogram {
|
||||
.get(tx + 1)
|
||||
.map(|s| s - block.out_start)
|
||||
.unwrap_or(block.out_end - block.out_start);
|
||||
if block.output_types[lo..hi].contains(&OutputType::OpReturn) {
|
||||
continue;
|
||||
}
|
||||
for i in lo..hi {
|
||||
if let Some(bin) = default_eligible_bin(block.values[i], block.output_types[i]) {
|
||||
hist.increment(bin as usize);
|
||||
}
|
||||
}
|
||||
let outputs = block.values[lo..hi]
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(block.output_types[lo..hi].iter().copied());
|
||||
for_each_round_dollar_bin(block.height, outputs, |bin| hist.increment(bin as usize));
|
||||
}
|
||||
hist
|
||||
}
|
||||
@@ -129,6 +127,7 @@ fn main() {
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
blocks.push(Block {
|
||||
height: h,
|
||||
values,
|
||||
output_types,
|
||||
tx_starts,
|
||||
@@ -142,7 +141,10 @@ fn main() {
|
||||
.iter()
|
||||
.map(|b| continuous.process_histogram(&build_histogram(b)))
|
||||
.collect();
|
||||
println!("Continuous oracle: {} blocks processed", continuous_bins.len());
|
||||
println!(
|
||||
"Continuous oracle: {} blocks processed",
|
||||
continuous_bins.len()
|
||||
);
|
||||
|
||||
let prev_bin = continuous_bins[restart_at - START_HEIGHT - 1];
|
||||
let seed_bin = cents_to_bin(bin_to_cents(prev_bin) as f64);
|
||||
|
||||
@@ -6,7 +6,7 @@ use std::path::PathBuf;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
Config, Histogram, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
|
||||
Config, Oracle, PRICES, HistogramRaw, START_HEIGHT, bin_to_cents, cents_to_bin,
|
||||
default_eligible_bin,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
@@ -236,7 +236,7 @@ fn main() {
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
// Drop every output of a tx carrying an OP_RETURN (protocol machinery).
|
||||
let mut hist = Histogram::zeros();
|
||||
let mut hist = HistogramRaw::zeros();
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
|
||||
@@ -0,0 +1,858 @@
|
||||
//! Generate detailed oracle accuracy report for README / documentation.
|
||||
//!
|
||||
//! Run with: cargo run -p brk_oracle --example report --release
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use brk_indexer::Indexer;
|
||||
use brk_oracle::{
|
||||
Config, HistogramEma, HistogramRaw, NUM_BINS, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
|
||||
default_eligible_bin,
|
||||
};
|
||||
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
/// Day1 1 = Jan 9, 2009 (block 1). For dates after genesis week:
|
||||
/// day1 = floor(timestamp / 86400) - 14252.
|
||||
const GENESIS_DAY: u32 = 14252;
|
||||
|
||||
const BINS_5PCT: f64 = 4.24;
|
||||
const BINS_10PCT: f64 = 8.28;
|
||||
const BINS_20PCT: f64 = 15.84;
|
||||
|
||||
/// Local copy of the oracle's 19 round-USD stencil offsets (private in lib.rs),
|
||||
/// used here only for per-block alias diagnostics.
|
||||
const STENCIL_OFFSETS: [i32; 19] = [
|
||||
-400, -340, -305, -260, -200, -165, -140, -120, -105, -60, 0, 35, 60, 95, 140, 200, 260, 340,
|
||||
400,
|
||||
];
|
||||
|
||||
/// Raw sum of EMA mass landing on the 19 stencil arms when centered at `center`.
|
||||
fn ema_stencil_sum(ema: &HistogramEma, center: i64) -> f64 {
|
||||
STENCIL_OFFSETS
|
||||
.iter()
|
||||
.map(|&off| {
|
||||
let idx = center + off as i64;
|
||||
if idx >= 0 && (idx as usize) < brk_oracle::NUM_BINS {
|
||||
ema[idx as usize]
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// log10(2) * 200 = one price octave (½× / 2×) in bins.
|
||||
const OCTAVE_BINS: i64 = 60;
|
||||
|
||||
/// Tunable octave-guard thresholds (env-overridable for sweeping).
|
||||
struct GuardCfg {
|
||||
enabled: bool,
|
||||
tau: f64, // arm "lit" if >= tau * peak arm
|
||||
raw_margin: f64, // octave neighbor raw mass must be >= raw_margin * current
|
||||
q_margin: usize, // neighbor must have >= q_margin MORE lit arms than current
|
||||
q_min: usize, // neighbor must have at least this many lit arms (looks full)
|
||||
}
|
||||
|
||||
impl GuardCfg {
|
||||
fn from_env() -> Self {
|
||||
let g = |k: &str, d: f64| -> f64 {
|
||||
std::env::var(k)
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(d)
|
||||
};
|
||||
Self {
|
||||
enabled: std::env::var("OCTAVE_GUARD")
|
||||
.ok()
|
||||
.map(|v| v != "0")
|
||||
.unwrap_or(true),
|
||||
tau: g("GUARD_TAU", 0.15),
|
||||
raw_margin: g("GUARD_RAW", 1.0),
|
||||
q_margin: g("GUARD_QMARGIN", 4.0) as usize,
|
||||
q_min: g("GUARD_QMIN", 14.0) as usize,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of stencil arms carrying real mass at `center`. The true price lights
|
||||
/// up ~all 19; a ½×/2× alias leaves ~8 structural holes (amounts with no ladder
|
||||
/// partner one octave away), so this count separates truth from alias even when
|
||||
/// the normalized score-sum cannot.
|
||||
fn arm_count(ema: &HistogramEma, center: i64, tau: f64) -> usize {
|
||||
let mut arms = [0.0f64; 19];
|
||||
let mut peak = 0.0f64;
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = center + off as i64;
|
||||
let v = if idx >= 0 && (idx as usize) < brk_oracle::NUM_BINS {
|
||||
ema[idx as usize]
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
arms[i] = v;
|
||||
if v > peak {
|
||||
peak = v;
|
||||
}
|
||||
}
|
||||
if peak <= 0.0 {
|
||||
return 0;
|
||||
}
|
||||
arms.iter().filter(|&&v| v >= tau * peak).count()
|
||||
}
|
||||
|
||||
/// 19-char lit/dark pattern of the stencil arms at `center` (arm i lit if its
|
||||
/// EMA mass >= tau * peak arm). Order: $1 $2 $3 $5 $10 $15 $20 $25 $30 $50 $100
|
||||
/// $150 $200 $300 $500 $1k $2k $5k $10k. Reveals WHICH amounts are present.
|
||||
fn arm_pattern(ema: &HistogramEma, center: i64, tau: f64) -> String {
|
||||
let mut arms = [0.0f64; 19];
|
||||
let mut peak = 0.0f64;
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = center + off as i64;
|
||||
let v = if idx >= 0 && (idx as usize) < brk_oracle::NUM_BINS {
|
||||
ema[idx as usize]
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
arms[i] = v;
|
||||
if v > peak {
|
||||
peak = v;
|
||||
}
|
||||
}
|
||||
arms.iter()
|
||||
.map(|&v| {
|
||||
if peak > 0.0 && v >= tau * peak {
|
||||
'L'
|
||||
} else {
|
||||
'.'
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// In-window stencil search (mirrors `Oracle::find_best_bin`) plus an octave
|
||||
/// guard: if the half- or double-price bin lights up strictly more stencil arms
|
||||
/// and carries comparable mass, snap to it. This escapes a ½×/2× alias lock that
|
||||
/// the ±window can never climb the 60 bins out of on its own.
|
||||
fn guarded_best_bin(
|
||||
ema: &HistogramEma,
|
||||
prev_bin: f64,
|
||||
search_below: usize,
|
||||
search_above: usize,
|
||||
guard: &GuardCfg,
|
||||
) -> f64 {
|
||||
let center = prev_bin.round() as usize;
|
||||
let search_start = center.saturating_sub(search_below);
|
||||
let search_end = (center + search_above + 1).min(brk_oracle::NUM_BINS);
|
||||
if search_start >= search_end {
|
||||
return prev_bin;
|
||||
}
|
||||
|
||||
let mut track_norm = [0.0f64; 19];
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
for bin in search_start..search_end {
|
||||
let idx = bin as i32 + off;
|
||||
if idx >= 0 && (idx as usize) < brk_oracle::NUM_BINS {
|
||||
track_norm[i] = track_norm[i].max(ema[idx as usize]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let score = |bin: usize| -> f64 {
|
||||
let mut total = 0.0;
|
||||
for (i, &off) in STENCIL_OFFSETS.iter().enumerate() {
|
||||
let idx = bin as i32 + off;
|
||||
if idx >= 0 && (idx as usize) < brk_oracle::NUM_BINS && track_norm[i] > 0.0 {
|
||||
total += ema[idx as usize] / track_norm[i];
|
||||
}
|
||||
}
|
||||
total
|
||||
};
|
||||
|
||||
let mut best_bin = search_start;
|
||||
let mut best_score = score(search_start);
|
||||
for bin in (search_start + 1)..search_end {
|
||||
let c = score(bin);
|
||||
if c > best_score {
|
||||
best_score = c;
|
||||
best_bin = bin;
|
||||
}
|
||||
}
|
||||
|
||||
if guard.enabled {
|
||||
let b = best_bin as i64;
|
||||
let qb = arm_count(ema, b, guard.tau);
|
||||
let raw_b = ema_stencil_sum(ema, b);
|
||||
let mut target = b;
|
||||
let mut best: Option<(usize, f64)> = None;
|
||||
for &delta in &[-OCTAVE_BINS, OCTAVE_BINS] {
|
||||
let n = b + delta;
|
||||
if n < 0 || n as usize >= brk_oracle::NUM_BINS {
|
||||
continue;
|
||||
}
|
||||
let qn = arm_count(ema, n, guard.tau);
|
||||
let raw_n = ema_stencil_sum(ema, n);
|
||||
if qn >= qb + guard.q_margin && qn >= guard.q_min && raw_n >= guard.raw_margin * raw_b {
|
||||
let better = best.is_none_or(|(sq, sr)| qn > sq || (qn == sq && raw_n > sr));
|
||||
if better {
|
||||
best = Some((qn, raw_n));
|
||||
target = n;
|
||||
}
|
||||
}
|
||||
}
|
||||
if target != b {
|
||||
return target as f64;
|
||||
}
|
||||
}
|
||||
|
||||
let score_center = best_score;
|
||||
let score_left = if best_bin > search_start {
|
||||
score(best_bin - 1)
|
||||
} else {
|
||||
score_center
|
||||
};
|
||||
let score_right = if best_bin + 1 < search_end {
|
||||
score(best_bin + 1)
|
||||
} else {
|
||||
score_center
|
||||
};
|
||||
let denom = score_left - 2.0 * score_center + score_right;
|
||||
let sub_bin = if denom.abs() > 1e-10 {
|
||||
(0.5 * (score_left - score_right) / denom).clamp(-0.5, 0.5)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
best_bin as f64 + sub_bin
|
||||
}
|
||||
|
||||
fn bins_to_pct(bins: f64) -> f64 {
|
||||
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
|
||||
}
|
||||
|
||||
/// Per-block EMA contribution weighting. `Off` keeps the raw count sum (a flood
|
||||
/// block dominates the window); `Unit` rescales every block to the same total
|
||||
/// mass (one block = one vote); `Cap` only scales down blocks above a ceiling.
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum NormMode {
|
||||
Off,
|
||||
Unit,
|
||||
Cap,
|
||||
}
|
||||
|
||||
/// Scale factor applied to a block's bin counts before folding into the EMA.
|
||||
fn norm_scale(total: u64, mode: NormMode, cap: f64, target: f64) -> f64 {
|
||||
if total == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
match mode {
|
||||
NormMode::Off => 1.0,
|
||||
NormMode::Unit => target / total as f64,
|
||||
NormMode::Cap => (cap / total as f64).min(1.0),
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_to_year(ts: u32) -> u16 {
|
||||
let years_since_1970 = ts as f64 / 31557600.0;
|
||||
(1970.0 + years_since_1970) as u16
|
||||
}
|
||||
|
||||
struct YearStats {
|
||||
year: u16,
|
||||
total_sq_err: f64,
|
||||
max_err: f64,
|
||||
total_blocks: u64,
|
||||
gt_5pct: u64,
|
||||
gt_10pct: u64,
|
||||
gt_20pct: u64,
|
||||
min_price: f64,
|
||||
max_price: f64,
|
||||
errors: Vec<f64>,
|
||||
}
|
||||
|
||||
impl YearStats {
|
||||
fn new(year: u16) -> Self {
|
||||
Self {
|
||||
year,
|
||||
total_sq_err: 0.0,
|
||||
max_err: 0.0,
|
||||
total_blocks: 0,
|
||||
gt_5pct: 0,
|
||||
gt_10pct: 0,
|
||||
gt_20pct: 0,
|
||||
min_price: f64::MAX,
|
||||
max_price: 0.0,
|
||||
errors: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, err: f64, exchange_high: f64, exchange_low: f64) {
|
||||
let abs_err = err.abs();
|
||||
self.total_sq_err += err * err;
|
||||
self.total_blocks += 1;
|
||||
self.errors.push(bins_to_pct(abs_err));
|
||||
if abs_err > self.max_err {
|
||||
self.max_err = abs_err;
|
||||
}
|
||||
if abs_err > BINS_5PCT {
|
||||
self.gt_5pct += 1;
|
||||
}
|
||||
if abs_err > BINS_10PCT {
|
||||
self.gt_10pct += 1;
|
||||
}
|
||||
if abs_err > BINS_20PCT {
|
||||
self.gt_20pct += 1;
|
||||
}
|
||||
if exchange_high > self.max_price {
|
||||
self.max_price = exchange_high;
|
||||
}
|
||||
if exchange_low > 0.0 && exchange_low < self.min_price {
|
||||
self.min_price = exchange_low;
|
||||
}
|
||||
}
|
||||
|
||||
fn rmse_pct(&self) -> f64 {
|
||||
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
|
||||
}
|
||||
|
||||
fn max_pct(&self) -> f64 {
|
||||
bins_to_pct(self.max_err)
|
||||
}
|
||||
|
||||
fn median_pct(&mut self) -> f64 {
|
||||
self.errors.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let n = self.errors.len();
|
||||
if n == 0 { 0.0 } else { self.errors[n / 2] }
|
||||
}
|
||||
|
||||
fn percentile(&self, p: f64) -> f64 {
|
||||
let n = self.errors.len();
|
||||
if n == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let idx = ((p / 100.0) * (n - 1) as f64).round() as usize;
|
||||
self.errors[idx.min(n - 1)]
|
||||
}
|
||||
}
|
||||
|
||||
/// Oracle OHLC for a single day, built from per-block prices.
|
||||
struct DayCandle {
|
||||
day1: usize,
|
||||
open: f64,
|
||||
high: f64,
|
||||
low: f64,
|
||||
close: f64,
|
||||
}
|
||||
|
||||
struct BlockError {
|
||||
height: usize,
|
||||
oracle_price: f64,
|
||||
exchange_low: f64,
|
||||
exchange_high: f64,
|
||||
error_pct: f64,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let data_dir = std::env::var("BRK_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
PathBuf::from(home).join(".brk")
|
||||
});
|
||||
|
||||
let start = std::env::var("ORACLE_START")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(START_HEIGHT);
|
||||
let end_override = std::env::var("ORACLE_END")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<usize>().ok());
|
||||
let trace_every: usize = std::env::var("TRACE_EVERY")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(5000);
|
||||
|
||||
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
|
||||
let total_heights = indexer.vecs.blocks.timestamp.len();
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
|
||||
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
|
||||
.expect("Failed to read height_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse height OHLC");
|
||||
|
||||
let daily_ohlc: Vec<[f64; 4]> = serde_json::from_str(
|
||||
&std::fs::read_to_string(format!("{manifest_dir}/examples/date_price_ohlc.json"))
|
||||
.expect("Failed to read date_price_ohlc.json"),
|
||||
)
|
||||
.expect("Failed to parse daily OHLC");
|
||||
|
||||
let height_bands: Vec<(f64, f64)> = height_ohlc
|
||||
.iter()
|
||||
.map(|ohlc| {
|
||||
let high = ohlc[1];
|
||||
let low = ohlc[2];
|
||||
if high > 0.0 && low > 0.0 {
|
||||
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
|
||||
} else {
|
||||
(0.0, 0.0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Read block timestamps for year + day1 mapping.
|
||||
let timestamps: Vec<brk_types::Timestamp> = indexer.vecs.blocks.timestamp.collect();
|
||||
let height_years: Vec<u16> = timestamps
|
||||
.iter()
|
||||
.map(|ts| timestamp_to_year(**ts))
|
||||
.collect();
|
||||
let height_day1s: Vec<usize> = timestamps
|
||||
.iter()
|
||||
.map(|ts| (**ts / 86400).saturating_sub(GENESIS_DAY) as usize)
|
||||
.collect();
|
||||
|
||||
let start_price: f64 = PRICES
|
||||
.lines()
|
||||
.nth(start - 1)
|
||||
.expect("prices.txt too short")
|
||||
.parse()
|
||||
.expect("Failed to parse seed price");
|
||||
|
||||
let mut config = Config::default();
|
||||
if let Some(w) = std::env::var("EMA_WINDOW")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
{
|
||||
config.window_size = w;
|
||||
}
|
||||
if let Some(a) = std::env::var("EMA_ALPHA").ok().and_then(|s| s.parse().ok()) {
|
||||
config.alpha = a;
|
||||
}
|
||||
// Investigation default: widened up-reach (9 -> 12) to survive fast rallies
|
||||
// like the 2018-04-12 candle. Kept here only; config.rs is untouched.
|
||||
config.search_below = std::env::var("SEARCH_BELOW")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(12);
|
||||
if let Some(sa) = std::env::var("SEARCH_ABOVE")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
{
|
||||
config.search_above = sa;
|
||||
}
|
||||
let guard = GuardCfg::from_env();
|
||||
let anom_thresh: f64 = std::env::var("ANOM_THRESH")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0.0);
|
||||
let norm_mode = match std::env::var("NORM_MODE").as_deref() {
|
||||
Ok("unit") => NormMode::Unit,
|
||||
Ok("cap") => NormMode::Cap,
|
||||
_ => NormMode::Off,
|
||||
};
|
||||
let norm_cap: f64 = std::env::var("NORM_CAP")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(8000.0);
|
||||
let norm_target: f64 = std::env::var("NORM_TARGET")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(4000.0);
|
||||
// Drop batch-payout txs (UTXOracle uses exactly-2-output; we cap instead).
|
||||
// 0 = disabled. A flood block's 591-output txs are dropped at 100.
|
||||
let max_outputs: usize = std::env::var("MAX_OUTPUTS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(100);
|
||||
// Apply the output-count filter only below this height (it helps the thin
|
||||
// 2018-2020 era, mildly hurts high-volume years). Default = always on.
|
||||
let max_outputs_until: usize = std::env::var("MAX_OUTPUTS_UNTIL")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(usize::MAX);
|
||||
eprintln!(
|
||||
" norm: mode={} cap={} target={} max_outputs={}",
|
||||
match norm_mode {
|
||||
NormMode::Off => "off",
|
||||
NormMode::Unit => "unit",
|
||||
NormMode::Cap => "cap",
|
||||
},
|
||||
norm_cap,
|
||||
norm_target,
|
||||
max_outputs,
|
||||
);
|
||||
eprintln!(
|
||||
" cfg: window_size={} alpha={:.5} (~{:.0}-block span) search -{}/+{} guard={} (tau={} raw={} qm={} qmin={})",
|
||||
config.window_size,
|
||||
config.alpha,
|
||||
2.0 / config.alpha - 1.0,
|
||||
config.search_below,
|
||||
config.search_above,
|
||||
guard.enabled,
|
||||
guard.tau,
|
||||
guard.raw_margin,
|
||||
guard.q_margin,
|
||||
guard.q_min,
|
||||
);
|
||||
let (sb, sa) = (config.search_below, config.search_above);
|
||||
let window_size = config.window_size;
|
||||
let alpha = config.alpha;
|
||||
let weights: Vec<f64> = (0..window_size)
|
||||
.map(|i| alpha * (1.0 - alpha).powi(i as i32))
|
||||
.collect();
|
||||
let mut ring: Vec<Vec<f64>> = vec![vec![0.0; NUM_BINS]; window_size];
|
||||
let mut ring_cursor = 0usize;
|
||||
let mut filled = 0usize;
|
||||
let mut ema = HistogramEma::zeros();
|
||||
let mut ref_bin = cents_to_bin(start_price * 100.0);
|
||||
|
||||
let total_txs = indexer.vecs.transactions.txid.len();
|
||||
let total_outputs = indexer.vecs.outputs.value.len();
|
||||
|
||||
// Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too
|
||||
// large, so the tx-indexed first_txout_index is read through a forward cursor.
|
||||
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
|
||||
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
|
||||
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
|
||||
let mut tx_starts: Vec<usize> = Vec::new();
|
||||
|
||||
let mut year_stats: Vec<YearStats> = Vec::new();
|
||||
let mut overall = YearStats::new(0);
|
||||
let mut worst_blocks: Vec<BlockError> = Vec::new();
|
||||
let mut total_bias = 0.0f64;
|
||||
|
||||
// Track oracle daily candles.
|
||||
let mut oracle_candles: Vec<DayCandle> = Vec::new();
|
||||
let mut current_di: Option<usize> = None;
|
||||
|
||||
let loop_end = end_override.unwrap_or(total_heights).min(total_heights);
|
||||
for h in start..loop_end {
|
||||
let ft = first_tx_index[h];
|
||||
let next_ft = first_tx_index
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs));
|
||||
|
||||
let block_first_tx = ft.to_usize() + 1;
|
||||
let tx_count = next_ft.to_usize() - block_first_tx;
|
||||
let out_end = out_first
|
||||
.get(h + 1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
|
||||
// First txout index of each non-coinbase tx, for per-tx grouping.
|
||||
txout_cursor.advance(block_first_tx - txout_cursor.position());
|
||||
tx_starts.clear();
|
||||
for _ in 0..tx_count {
|
||||
tx_starts.push(txout_cursor.next().unwrap().to_usize());
|
||||
}
|
||||
let out_start = tx_starts.first().copied().unwrap_or(out_end);
|
||||
|
||||
let values: Vec<Sats> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.value
|
||||
.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
// Drop every output of a tx carrying an OP_RETURN (protocol machinery).
|
||||
let mut hist = HistogramRaw::zeros();
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
if output_types[lo..hi].contains(&OutputType::OpReturn) {
|
||||
continue;
|
||||
}
|
||||
if max_outputs > 0 && h < max_outputs_until && (hi - lo) > max_outputs {
|
||||
continue;
|
||||
}
|
||||
for i in lo..hi {
|
||||
if let Some(bin) = default_eligible_bin(values[i], output_types[i]) {
|
||||
hist.increment(bin as usize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total: u64 = (0..NUM_BINS).map(|b| hist[b] as u64).sum();
|
||||
let scale = norm_scale(total, norm_mode, norm_cap, norm_target);
|
||||
{
|
||||
let slot = &mut ring[ring_cursor];
|
||||
for b in 0..NUM_BINS {
|
||||
slot[b] = hist[b] as f64 * scale;
|
||||
}
|
||||
}
|
||||
ring_cursor = (ring_cursor + 1) % window_size;
|
||||
if filled < window_size {
|
||||
filled += 1;
|
||||
}
|
||||
ema.fill(0.0);
|
||||
for age in 0..filled {
|
||||
let idx = (ring_cursor + window_size - 1 - age) % window_size;
|
||||
let w = weights[age];
|
||||
let block = &ring[idx];
|
||||
for b in 0..NUM_BINS {
|
||||
ema[b] += w * block[b];
|
||||
}
|
||||
}
|
||||
ref_bin = guarded_best_bin(&ema, ref_bin, sb, sa, &guard);
|
||||
let oracle_price = bin_to_cents(ref_bin) as f64 / 100.0;
|
||||
|
||||
let o = height_ohlc.get(h).copied().unwrap_or([0.0; 4]);
|
||||
let (ex_high, ex_low, ex_close) = (o[1], o[2], o[3]);
|
||||
let band_err = if ex_high > 0.0 && ex_low > 0.0 {
|
||||
if oracle_price > ex_high {
|
||||
(oracle_price - ex_high) / ex_high * 100.0
|
||||
} else if oracle_price < ex_low {
|
||||
(oracle_price - ex_low) / ex_low * 100.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let do_print = h % trace_every == 0 || (anom_thresh > 0.0 && band_err.abs() >= anom_thresh);
|
||||
if do_print {
|
||||
let eligible: u32 = (0..brk_oracle::NUM_BINS).map(|b| hist[b]).sum();
|
||||
// true_bin centered on exchange close; +60 bins = half price, -60 = double.
|
||||
let true_bin = if ex_close > 0.0 {
|
||||
cents_to_bin(ex_close * 100.0).round() as i64
|
||||
} else {
|
||||
ref_bin.round() as i64
|
||||
};
|
||||
let s_true = ema_stencil_sum(&ema, true_bin);
|
||||
let s_half = ema_stencil_sum(&ema, true_bin + 60);
|
||||
let s_dbl = ema_stencil_sum(&ema, true_bin - 60);
|
||||
let qt = arm_count(&ema, true_bin, guard.tau);
|
||||
let qh = arm_count(&ema, true_bin + 60, guard.tau);
|
||||
let qd = arm_count(&ema, true_bin - 60, guard.tau);
|
||||
let pat = arm_pattern(&ema, true_bin, guard.tau);
|
||||
let ts_secs: u32 = *timestamps[h];
|
||||
eprintln!(
|
||||
"{h}\t{ts_secs}\t{oracle_price:.0}\t{ex_close:.0}\t{band_err:+.2}\t{eligible}\tT={s_true:.1}\tH={s_half:.1}\tD={s_dbl:.1}\tQt={qt}\tQh={qh}\tQd={qd}\t{pat}"
|
||||
);
|
||||
}
|
||||
|
||||
// Build oracle daily candle.
|
||||
let di = height_day1s[h];
|
||||
if current_di != Some(di) {
|
||||
current_di = Some(di);
|
||||
oracle_candles.push(DayCandle {
|
||||
day1: di,
|
||||
open: oracle_price,
|
||||
high: oracle_price,
|
||||
low: oracle_price,
|
||||
close: oracle_price,
|
||||
});
|
||||
} else {
|
||||
let candle = oracle_candles.last_mut().unwrap();
|
||||
if oracle_price > candle.high {
|
||||
candle.high = oracle_price;
|
||||
}
|
||||
if oracle_price < candle.low {
|
||||
candle.low = oracle_price;
|
||||
}
|
||||
candle.close = oracle_price;
|
||||
}
|
||||
|
||||
// Per-block error stats.
|
||||
if h < height_bands.len() {
|
||||
let (high_bin, low_bin) = height_bands[h];
|
||||
if high_bin > 0.0 && low_bin > 0.0 {
|
||||
let err = if ref_bin < high_bin {
|
||||
ref_bin - high_bin
|
||||
} else if ref_bin > low_bin {
|
||||
ref_bin - low_bin
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let exchange_high = height_ohlc[h][1];
|
||||
let exchange_low = height_ohlc[h][2];
|
||||
|
||||
overall.update(err, exchange_high, exchange_low);
|
||||
total_bias += err;
|
||||
|
||||
let year = height_years[h];
|
||||
if year_stats.is_empty() || year_stats.last().unwrap().year != year {
|
||||
year_stats.push(YearStats::new(year));
|
||||
}
|
||||
year_stats
|
||||
.last_mut()
|
||||
.unwrap()
|
||||
.update(err, exchange_high, exchange_low);
|
||||
|
||||
if err.abs() > BINS_5PCT {
|
||||
worst_blocks.push(BlockError {
|
||||
height: h,
|
||||
oracle_price,
|
||||
exchange_low,
|
||||
exchange_high,
|
||||
error_pct: if err < 0.0 {
|
||||
-bins_to_pct(err.abs())
|
||||
} else {
|
||||
bins_to_pct(err.abs())
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
worst_blocks.sort_by(|a, b| b.error_pct.abs().partial_cmp(&a.error_pct.abs()).unwrap());
|
||||
overall.errors.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
|
||||
// Daily candle comparison: oracle OHLC vs exchange OHLC.
|
||||
let mut daily_open_errors: Vec<f64> = Vec::new();
|
||||
let mut daily_high_errors: Vec<f64> = Vec::new();
|
||||
let mut daily_low_errors: Vec<f64> = Vec::new();
|
||||
let mut daily_close_errors: Vec<f64> = Vec::new();
|
||||
let mut daily_days = 0u64;
|
||||
|
||||
for candle in &oracle_candles {
|
||||
let di = candle.day1;
|
||||
if di >= daily_ohlc.len() {
|
||||
continue;
|
||||
}
|
||||
let ex = &daily_ohlc[di];
|
||||
if ex[0] <= 0.0 || ex[3] <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
let ex_open = ex[0];
|
||||
let ex_high = ex[1];
|
||||
let ex_low = ex[2];
|
||||
let ex_close = ex[3];
|
||||
|
||||
// Error as percentage: (oracle - exchange) / exchange * 100
|
||||
daily_open_errors.push((candle.open - ex_open) / ex_open * 100.0);
|
||||
daily_high_errors.push((candle.high - ex_high) / ex_high * 100.0);
|
||||
daily_low_errors.push((candle.low - ex_low) / ex_low * 100.0);
|
||||
daily_close_errors.push((candle.close - ex_close) / ex_close * 100.0);
|
||||
daily_days += 1;
|
||||
}
|
||||
|
||||
fn daily_stats(errors: &mut [f64]) -> (f64, f64, f64) {
|
||||
let n = errors.len() as f64;
|
||||
let rmse = (errors.iter().map(|e| e * e).sum::<f64>() / n).sqrt();
|
||||
errors.sort_by(|a, b| a.abs().partial_cmp(&b.abs()).unwrap());
|
||||
let max = errors.last().map(|e| e.abs()).unwrap_or(0.0);
|
||||
let median = errors[errors.len() / 2].abs();
|
||||
(median, rmse, max)
|
||||
}
|
||||
|
||||
let (open_med, open_rmse, open_max) = daily_stats(&mut daily_open_errors);
|
||||
let (high_med, high_rmse, high_max) = daily_stats(&mut daily_high_errors);
|
||||
let (low_med, low_rmse, low_max) = daily_stats(&mut daily_low_errors);
|
||||
let (close_med, close_rmse, close_max) = daily_stats(&mut daily_close_errors);
|
||||
|
||||
// Print report.
|
||||
println!();
|
||||
println!(" brk_oracle accuracy report");
|
||||
println!(" ══════════════════════════");
|
||||
println!();
|
||||
println!(" Config: w12, alpha=2/7, search -9/+11, noisy/dust/round-btc filtered");
|
||||
println!(
|
||||
" Test range: height {} .. {} ({} blocks), seed ${:.2}",
|
||||
start,
|
||||
loop_end - 1,
|
||||
overall.total_blocks,
|
||||
start_price,
|
||||
);
|
||||
println!(
|
||||
" Price range: ${:.0} .. ${:.0}",
|
||||
overall.min_price, overall.max_price
|
||||
);
|
||||
|
||||
println!();
|
||||
println!(" Per-block accuracy (vs per-height exchange OHLC):");
|
||||
println!(" Median: {:.3}%", overall.percentile(50.0));
|
||||
println!(" 95th pct: {:.3}%", overall.percentile(95.0));
|
||||
println!(" 99th pct: {:.3}%", overall.percentile(99.0));
|
||||
println!(" 99.9th pct: {:.3}%", overall.percentile(99.9));
|
||||
println!(" RMSE: {:.3}%", overall.rmse_pct());
|
||||
println!(" Max: {:.1}%", overall.max_pct());
|
||||
println!(
|
||||
" Bias: {:+.2} bins",
|
||||
total_bias / overall.total_blocks as f64
|
||||
);
|
||||
println!(
|
||||
" > 5%: {} blocks ({:.3}%)",
|
||||
overall.gt_5pct,
|
||||
overall.gt_5pct as f64 / overall.total_blocks as f64 * 100.0
|
||||
);
|
||||
println!(" > 10%: {} blocks", overall.gt_10pct);
|
||||
println!(" > 20%: {} blocks", overall.gt_20pct);
|
||||
|
||||
println!();
|
||||
println!(
|
||||
" Daily candle accuracy ({} days, vs exchange daily OHLC):",
|
||||
daily_days
|
||||
);
|
||||
println!(
|
||||
" {:>8} {:>10} {:>10} {:>10}",
|
||||
"", "Median", "RMSE", "Max"
|
||||
);
|
||||
println!(
|
||||
" {:>8} {:>9.2}% {:>9.2}% {:>9.1}%",
|
||||
"Open", open_med, open_rmse, open_max
|
||||
);
|
||||
println!(
|
||||
" {:>8} {:>9.2}% {:>9.2}% {:>9.1}%",
|
||||
"High", high_med, high_rmse, high_max
|
||||
);
|
||||
println!(
|
||||
" {:>8} {:>9.2}% {:>9.2}% {:>9.1}%",
|
||||
"Low", low_med, low_rmse, low_max
|
||||
);
|
||||
println!(
|
||||
" {:>8} {:>9.2}% {:>9.2}% {:>9.1}%",
|
||||
"Close", close_med, close_rmse, close_max
|
||||
);
|
||||
|
||||
println!();
|
||||
println!(" By year:");
|
||||
println!(
|
||||
" {:<6} {:>7} {:>9} {:>9} {:>9} {:>6} {:>5} {:>5} {:>14}",
|
||||
"Year", "Blocks", "Median", "RMSE", "Max", ">5%", ">10%", ">20%", "Price range"
|
||||
);
|
||||
println!(" {}", "-".repeat(80));
|
||||
for ys in &mut year_stats {
|
||||
let median = ys.median_pct();
|
||||
println!(
|
||||
" {:<6} {:>7} {:>8.3}% {:>8.3}% {:>8.1}% {:>6} {:>5} {:>5} ${:.0}..${:.0}",
|
||||
ys.year,
|
||||
ys.total_blocks,
|
||||
median,
|
||||
ys.rmse_pct(),
|
||||
ys.max_pct(),
|
||||
ys.gt_5pct,
|
||||
ys.gt_10pct,
|
||||
ys.gt_20pct,
|
||||
ys.min_price,
|
||||
ys.max_price,
|
||||
);
|
||||
}
|
||||
|
||||
if !worst_blocks.is_empty() {
|
||||
println!();
|
||||
println!(" Worst blocks:");
|
||||
let show = worst_blocks.len().min(10);
|
||||
for wb in &worst_blocks[..show] {
|
||||
let dir = if wb.error_pct < 0.0 { "above" } else { "below" };
|
||||
println!(
|
||||
" height {:>7}: oracle ${:>9.0}, exchange ${:.0}..${:.0} ({:+.1}%, {})",
|
||||
wb.height, wb.oracle_price, wb.exchange_low, wb.exchange_high, wb.error_pct, dir
|
||||
);
|
||||
}
|
||||
if worst_blocks.len() > show {
|
||||
println!(" ... and {} more", worst_blocks.len() - show);
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
}
|
||||
@@ -29,7 +29,7 @@ impl Default for Config {
|
||||
Self {
|
||||
alpha: 2.0 / 7.0,
|
||||
window_size: 12,
|
||||
search_below: 9,
|
||||
search_below: 12,
|
||||
search_above: 11,
|
||||
min_sats: DEFAULT_MIN_SATS,
|
||||
exclude_common_round_values: true,
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
use crate::NUM_BINS;
|
||||
|
||||
/// Per-block oracle histogram: count of eligible outputs per bin. Wraps
|
||||
/// the raw `[u32; NUM_BINS]` so callers can't pass arbitrary bin-indexed
|
||||
/// arrays to `Oracle::process_histogram`. Deref to the underlying array
|
||||
/// gives indexing for read paths.
|
||||
#[derive(Clone)]
|
||||
pub struct Histogram([u32; NUM_BINS]);
|
||||
|
||||
impl Histogram {
|
||||
#[inline]
|
||||
pub fn zeros() -> Self {
|
||||
Self([0; NUM_BINS])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn increment(&mut self, bin: usize) {
|
||||
self.0[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Histogram {
|
||||
fn default() -> Self {
|
||||
Self::zeros()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for Histogram {
|
||||
type Target = [u32; NUM_BINS];
|
||||
#[inline]
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::DerefMut for Histogram {
|
||||
#[inline]
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
@@ -3,32 +3,52 @@
|
||||
//! Detects round-dollar transaction patterns ($1, $5, $10, ... $10,000) in Bitcoin
|
||||
//! block outputs to derive the current price without any exchange data.
|
||||
|
||||
use brk_types::{Cents, Dollars, OutputType, Sats};
|
||||
use brk_types::{Cents, Dollars, Histogram, OutputType, Sats};
|
||||
|
||||
mod config;
|
||||
mod histogram;
|
||||
|
||||
use config::{DEFAULT_EXCLUDED_OUTPUT_TYPES, DEFAULT_MIN_SATS};
|
||||
pub use config::Config;
|
||||
pub use histogram::Histogram;
|
||||
use config::{DEFAULT_EXCLUDED_OUTPUT_TYPES, DEFAULT_MIN_SATS};
|
||||
|
||||
/// Oracle algorithm version. Bump on any change that alters computed prices
|
||||
/// so downstream consumers can invalidate cached results.
|
||||
pub const VERSION: u32 = 2;
|
||||
pub const VERSION: u32 = 3;
|
||||
|
||||
/// Pre-oracle dollar prices, one per line, heights 0..525_000. The last
|
||||
/// entry (height 524_999) seeds the oracle's first on-chain computation
|
||||
/// at `START_HEIGHT`.
|
||||
/// Pre-oracle dollar prices, one per line, heights 0..508_000. The last entry
|
||||
/// (height `START_HEIGHT - 1`) seeds the oracle's first on-chain computation at
|
||||
/// `START_HEIGHT`.
|
||||
pub const PRICES: &str = include_str!("prices.txt");
|
||||
|
||||
/// First height where the oracle computes from on-chain data.
|
||||
pub const START_HEIGHT: usize = 525_000;
|
||||
pub const START_HEIGHT: usize = 508_000;
|
||||
|
||||
/// A transaction with more than this many outputs is a batch payout (exchange
|
||||
/// sweep, mixer fan-out), not a round-dollar payment, so it is dropped below
|
||||
/// [`MAX_OUTPUTS_UNTIL_HEIGHT`].
|
||||
pub const MAX_OUTPUTS: usize = 100;
|
||||
|
||||
/// Height below which the [`MAX_OUTPUTS`] cap applies. The thin 2018-2020
|
||||
/// signal needs batch payouts removed to stay locked onto the round-dollar
|
||||
/// pattern. Above this height on-chain volume is dense enough that the cap
|
||||
/// removes more genuine signal than noise, so it is lifted.
|
||||
pub const MAX_OUTPUTS_UNTIL_HEIGHT: usize = 630_000;
|
||||
|
||||
pub const BINS_PER_DECADE: usize = 200;
|
||||
const MIN_LOG_BTC: i32 = -8;
|
||||
const MAX_LOG_BTC: i32 = 4;
|
||||
pub const NUM_BINS: usize = BINS_PER_DECADE * (MAX_LOG_BTC - MIN_LOG_BTC) as usize;
|
||||
|
||||
/// Per-block round-dollar payment counts, one `u32` per log-scale bin: the
|
||||
/// oracle's ring-buffer element and the `histogram/raw/*` wire payload.
|
||||
pub type HistogramRaw = Histogram<u32, NUM_BINS>;
|
||||
|
||||
/// Smoothed EMA over the window, one `f64` per bin. The stencil search reads it,
|
||||
/// never serialized (projected to [`HistogramEmaCompact`] for the wire).
|
||||
pub type HistogramEma = Histogram<f64, NUM_BINS>;
|
||||
|
||||
/// Quantized `u16` projection of [`HistogramEma`] for the `histogram/ema/*` wire.
|
||||
pub type HistogramEmaCompact = Histogram<u16, NUM_BINS>;
|
||||
|
||||
/// Bin offsets for 19 round-USD amounts relative to the $100 reference (offset 0).
|
||||
/// Each offset = log10(amount / 100) * BINS_PER_DECADE.
|
||||
const STENCIL_OFFSETS: [i32; 19] = [
|
||||
@@ -95,6 +115,36 @@ pub fn default_eligible_bin(sats: Sats, output_type: OutputType) -> Option<u16>
|
||||
sats_to_bin(sats).map(|b| b as u16)
|
||||
}
|
||||
|
||||
/// The single definition of the on-chain round-dollar payment filter, shared by
|
||||
/// the indexer warm-up, per-request reconstruction, and the mempool's live
|
||||
/// histogram so every path bins identically. Calls `emit(bin)` for each eligible
|
||||
/// output, in order.
|
||||
///
|
||||
/// A whole transaction is dropped when it carries any OP_RETURN output (data
|
||||
/// carriers like consolidations and inscriptions aren't payments and would
|
||||
/// pollute the signal) or, below [`MAX_OUTPUTS_UNTIL_HEIGHT`], when it has more
|
||||
/// than [`MAX_OUTPUTS`] outputs (batch payouts). `height` is the block these
|
||||
/// outputs belong to. The mempool, always past the cap window, passes
|
||||
/// `usize::MAX`.
|
||||
#[inline]
|
||||
pub fn for_each_round_dollar_bin(
|
||||
height: usize,
|
||||
outputs: impl ExactSizeIterator<Item = (Sats, OutputType)> + Clone,
|
||||
mut emit: impl FnMut(u16),
|
||||
) {
|
||||
if height < MAX_OUTPUTS_UNTIL_HEIGHT && outputs.len() > MAX_OUTPUTS {
|
||||
return;
|
||||
}
|
||||
if outputs.clone().any(|(_, ty)| ty == OutputType::OpReturn) {
|
||||
return;
|
||||
}
|
||||
for (sats, ty) in outputs {
|
||||
if let Some(bin) = default_eligible_bin(sats, ty) {
|
||||
emit(bin);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a fractional bin to a USD price in cents.
|
||||
/// For a $D output at price P: sats = D * 1e8 / P, so P = 10^(10 - bin/200) dollars,
|
||||
/// where 10 = log10($100 reference * 1e8 sats/BTC).
|
||||
@@ -113,7 +163,7 @@ pub fn cents_to_bin(cents: f64) -> f64 {
|
||||
/// Scores each candidate bin in the search window by summing normalized stencil
|
||||
/// matches across the EMA histogram, then refines with parabolic interpolation.
|
||||
fn find_best_bin(
|
||||
ema: &[f64; NUM_BINS],
|
||||
ema: &HistogramEma,
|
||||
prev_bin: f64,
|
||||
search_below: usize,
|
||||
search_above: usize,
|
||||
@@ -182,8 +232,8 @@ fn find_best_bin(
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Oracle {
|
||||
histograms: Vec<Histogram>,
|
||||
ema: Box<[f64; NUM_BINS]>,
|
||||
histograms: Vec<HistogramRaw>,
|
||||
ema: Box<HistogramEma>,
|
||||
cursor: usize,
|
||||
filled: usize,
|
||||
ref_bin: f64,
|
||||
@@ -205,8 +255,8 @@ impl Oracle {
|
||||
.iter()
|
||||
.fold(0u16, |mask, ot| mask | (1 << *ot as u8));
|
||||
Self {
|
||||
histograms: vec![Histogram::zeros(); window_size],
|
||||
ema: Box::new([0.0; NUM_BINS]),
|
||||
histograms: vec![HistogramRaw::zeros(); window_size],
|
||||
ema: Box::new(HistogramEma::zeros()),
|
||||
cursor: 0,
|
||||
filled: 0,
|
||||
ref_bin: start_bin,
|
||||
@@ -230,7 +280,7 @@ impl Oracle {
|
||||
oracle
|
||||
}
|
||||
|
||||
pub fn process_histogram(&mut self, hist: &Histogram) -> f64 {
|
||||
pub fn process_histogram(&mut self, hist: &HistogramRaw) -> f64 {
|
||||
self.histograms[self.cursor] = hist.clone();
|
||||
self.cursor = (self.cursor + 1) % self.config.window_size;
|
||||
if self.filled < self.config.window_size {
|
||||
@@ -254,6 +304,13 @@ impl Oracle {
|
||||
self.ref_bin
|
||||
}
|
||||
|
||||
/// The current weighted EMA over the window, one value per log-scale bin.
|
||||
/// `ema()[i]` is bin `i` (see `sats_to_bin`); callers transporting it
|
||||
/// round/clamp to a smaller type.
|
||||
pub fn ema(&self) -> &HistogramEma {
|
||||
&self.ema
|
||||
}
|
||||
|
||||
pub fn price_cents(&self) -> Cents {
|
||||
bin_to_cents(self.ref_bin).into()
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@ mod block;
|
||||
mod cpfp;
|
||||
mod mempool;
|
||||
mod mining;
|
||||
mod oracle;
|
||||
mod price;
|
||||
mod series;
|
||||
mod tx;
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_computer::prices::Vecs as PricesVecs;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_indexer::Lengths;
|
||||
use brk_oracle::{
|
||||
Config, HistogramEmaCompact, HistogramRaw, Oracle, START_HEIGHT, cents_to_bin,
|
||||
for_each_round_dollar_bin,
|
||||
};
|
||||
use brk_types::{Dollars, OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
pub fn live_price(&self) -> Result<Dollars> {
|
||||
Ok(self.live_oracle()?.price_dollars())
|
||||
}
|
||||
|
||||
/// Smoothed EMA histogram at the live tip, quantized for the wire.
|
||||
pub fn live_histogram_ema(&self) -> Result<HistogramEmaCompact> {
|
||||
Ok(self.live_oracle()?.ema().to_compact())
|
||||
}
|
||||
|
||||
/// Smoothed EMA histogram for a confirmed `height`, deterministically
|
||||
/// reconstructed by replaying the window ending at `height`. EMA values are
|
||||
/// seed-independent, so the result is exact.
|
||||
pub fn confirmed_histogram_ema(&self, height: usize) -> Result<HistogramEmaCompact> {
|
||||
let safe = self.check_histogram_height(height)?;
|
||||
let ref_bin = self.seed_bin_at(height)?;
|
||||
Ok(self.warm_oracle(ref_bin, height + 1, &safe).ema().to_compact())
|
||||
}
|
||||
|
||||
/// Un-smoothed per-block round-dollar counts at the live tip: the mempool's
|
||||
/// forming-block histogram, or zeros when no mempool is configured.
|
||||
pub fn live_histogram_raw(&self) -> Result<HistogramRaw> {
|
||||
Ok(match self.mempool() {
|
||||
Some(mempool) => mempool.live_histogram(),
|
||||
None => HistogramRaw::zeros(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Un-smoothed per-block round-dollar counts for a confirmed `height`.
|
||||
pub fn confirmed_histogram_raw(&self, height: usize) -> Result<HistogramRaw> {
|
||||
let safe = self.check_histogram_height(height)?;
|
||||
Ok(self.block_raw_histogram(height, &safe))
|
||||
}
|
||||
|
||||
/// The live tip oracle: the cached committed base, with the forming block's
|
||||
/// mempool outputs blended in as a final slot when a mempool is configured.
|
||||
fn live_oracle(&self) -> Result<Oracle> {
|
||||
let mut oracle = (*self.cached_oracle()?).clone();
|
||||
if let Some(mempool) = self.mempool() {
|
||||
oracle.process_histogram(&mempool.live_histogram());
|
||||
}
|
||||
Ok(oracle)
|
||||
}
|
||||
|
||||
/// Tip oracle warmed over the last `window_size` committed blocks, seeded
|
||||
/// from the last committed price. Cached per tip height; rebuilt on advance
|
||||
/// or reorg.
|
||||
fn cached_oracle(&self) -> Result<Arc<Oracle>> {
|
||||
let safe = self.safe_lengths();
|
||||
let height = safe.height;
|
||||
|
||||
if let Some(oracle) = self
|
||||
.0
|
||||
.live_oracle
|
||||
.read()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.filter(|(h, _)| *h == height)
|
||||
.map(|(_, o)| o.clone())
|
||||
{
|
||||
return Ok(oracle);
|
||||
}
|
||||
|
||||
let last = self.computer().prices.spot.cents.height.len().saturating_sub(1);
|
||||
let seed_bin = self.seed_bin_at(last)?;
|
||||
let oracle = Arc::new(self.warm_oracle(seed_bin, height.to_usize(), &safe));
|
||||
|
||||
let mut cache = self.0.live_oracle.write().unwrap();
|
||||
if cache.as_ref().is_none_or(|(h, _)| *h != height) {
|
||||
*cache = Some((height, oracle.clone()));
|
||||
}
|
||||
Ok(oracle)
|
||||
}
|
||||
|
||||
/// An oracle seeded at `seed_bin` and warmed by replaying the `window_size`
|
||||
/// committed blocks ending just before `end`. Reads are capped at `safe` so
|
||||
/// concurrent indexer writes past the cap stay invisible.
|
||||
fn warm_oracle(&self, seed_bin: f64, end: usize, safe: &Lengths) -> Oracle {
|
||||
let config = Config::default();
|
||||
let start = end.saturating_sub(config.window_size);
|
||||
Oracle::from_checkpoint(seed_bin, config, |o| {
|
||||
PricesVecs::feed_blocks(o, self.indexer(), start..end, Some(safe));
|
||||
})
|
||||
}
|
||||
|
||||
/// Seed bin for an oracle warm-up: the stored spot price at `height` mapped
|
||||
/// `cents -> bin`. 404s when the oracle prices aren't computed that far yet,
|
||||
/// which also covers the stamp-before-write race where the vec length leads
|
||||
/// the readable data.
|
||||
fn seed_bin_at(&self, height: usize) -> Result<f64> {
|
||||
let cents = self
|
||||
.computer()
|
||||
.prices
|
||||
.spot
|
||||
.cents
|
||||
.height
|
||||
.collect_one_at(height)
|
||||
.ok_or_else(|| Error::NotFound("oracle prices not yet computed".to_string()))?;
|
||||
Ok(cents_to_bin(cents.inner() as f64))
|
||||
}
|
||||
|
||||
/// `START_HEIGHT <= height < min(spot price len, safe height)` or 404.
|
||||
/// Returns the safe lengths so callers cap reads at the same bound.
|
||||
fn check_histogram_height(&self, height: usize) -> Result<Lengths> {
|
||||
let safe = self.safe_lengths();
|
||||
let bound = self
|
||||
.computer()
|
||||
.prices
|
||||
.spot
|
||||
.cents
|
||||
.height
|
||||
.len()
|
||||
.min(safe.height.to_usize());
|
||||
if height < START_HEIGHT || height >= bound {
|
||||
return Err(Error::NotFound(format!(
|
||||
"oracle histogram unavailable for height {height}"
|
||||
)));
|
||||
}
|
||||
Ok(safe)
|
||||
}
|
||||
|
||||
/// One confirmed block's round-dollar histogram, built from batched columnar
|
||||
/// reads and the shared `for_each_round_dollar_bin` filter. Kept separate from
|
||||
/// the hot-path `feed_blocks` (cursor + reusable buffers over a block range).
|
||||
fn block_raw_histogram(&self, height: usize, safe: &Lengths) -> HistogramRaw {
|
||||
let indexer = self.indexer();
|
||||
let total_txs = safe.tx_index.to_usize();
|
||||
let total_outputs = safe.txout_index.to_usize();
|
||||
let next_height = (height + 2).min(safe.height.to_usize());
|
||||
|
||||
let first_tx_indexes: Vec<TxIndex> = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_tx_index
|
||||
.collect_range_at(height, next_height);
|
||||
let out_firsts: Vec<TxOutIndex> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.first_txout_index
|
||||
.collect_range_at(height, next_height);
|
||||
|
||||
let block_first_tx = first_tx_indexes[0].to_usize() + 1;
|
||||
let next_first_tx = first_tx_indexes
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or(TxIndex::from(total_txs))
|
||||
.to_usize();
|
||||
let tx_count = next_first_tx - block_first_tx;
|
||||
|
||||
let mut hist = HistogramRaw::zeros();
|
||||
if tx_count == 0 {
|
||||
return hist;
|
||||
}
|
||||
|
||||
let out_end = out_firsts
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or(TxOutIndex::from(total_outputs))
|
||||
.to_usize();
|
||||
let tx_starts: Vec<usize> = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_range_at(block_first_tx, next_first_tx)
|
||||
.into_iter()
|
||||
.map(|t| t.to_usize())
|
||||
.collect();
|
||||
let out_start = tx_starts.first().copied().unwrap_or(out_end);
|
||||
|
||||
let values: Vec<Sats> = indexer.vecs.outputs.value.collect_range_at(out_start, out_end);
|
||||
let output_types: Vec<OutputType> = indexer
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.collect_range_at(out_start, out_end);
|
||||
|
||||
for tx in 0..tx_count {
|
||||
let lo = tx_starts[tx] - out_start;
|
||||
let hi = tx_starts
|
||||
.get(tx + 1)
|
||||
.map(|s| s - out_start)
|
||||
.unwrap_or(out_end - out_start);
|
||||
let outputs = values[lo..hi]
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(output_types[lo..hi].iter().copied());
|
||||
for_each_round_dollar_bin(height, outputs, |bin| hist.increment(bin as usize));
|
||||
}
|
||||
hist
|
||||
}
|
||||
}
|
||||
@@ -1,70 +1,12 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_computer::prices::Vecs as PricesVecs;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_oracle::{Config, Oracle, cents_to_bin};
|
||||
use brk_error::Result;
|
||||
use brk_types::{
|
||||
Dollars, ExchangeRates, HistoricalPrice, HistoricalPriceEntry, Hour4, INDEX_EPOCH, Timestamp,
|
||||
};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
use vecdb::ReadableVec;
|
||||
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
pub fn live_price(&self) -> Result<Dollars> {
|
||||
let base = self.cached_oracle()?;
|
||||
Ok(match self.mempool() {
|
||||
Some(mempool) => {
|
||||
let mut oracle = (*base).clone();
|
||||
oracle.process_histogram(&mempool.live_histogram());
|
||||
oracle.price_dollars()
|
||||
}
|
||||
None => base.price_dollars(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Oracle warmed by the last `window_size` committed blocks, seeded from
|
||||
/// the last committed price. Cached per tip height; rebuilt on advance or
|
||||
/// reorg. Reads are capped at `safe_lengths` so concurrent indexer writes
|
||||
/// stay invisible.
|
||||
fn cached_oracle(&self) -> Result<Arc<Oracle>> {
|
||||
let safe_lengths = self.safe_lengths();
|
||||
let height = safe_lengths.height;
|
||||
|
||||
if let Some(oracle) = self
|
||||
.0
|
||||
.live_oracle
|
||||
.read()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.filter(|(h, _)| *h == height)
|
||||
.map(|(_, o)| o.clone())
|
||||
{
|
||||
return Ok(oracle);
|
||||
}
|
||||
|
||||
let cents_height = &self.computer().prices.spot.cents.height;
|
||||
let last_cents = cents_height
|
||||
.len()
|
||||
.checked_sub(1)
|
||||
.and_then(|i| cents_height.collect_one_at(i))
|
||||
.ok_or_else(|| Error::NotFound("oracle prices not yet computed".to_string()))?;
|
||||
|
||||
let config = Config::default();
|
||||
let seed_bin = cents_to_bin(last_cents.inner() as f64);
|
||||
let tip = height.to_usize();
|
||||
let warmup_range = tip.saturating_sub(config.window_size)..tip;
|
||||
let oracle = Arc::new(Oracle::from_checkpoint(seed_bin, config, |o| {
|
||||
PricesVecs::feed_blocks(o, self.indexer(), warmup_range, Some(&safe_lengths));
|
||||
}));
|
||||
|
||||
let mut cache = self.0.live_oracle.write().unwrap();
|
||||
if cache.as_ref().is_none_or(|(h, _)| *h != height) {
|
||||
*cache = Some((height, oracle.clone()));
|
||||
}
|
||||
Ok(oracle)
|
||||
}
|
||||
|
||||
pub fn historical_price(&self, timestamp: Option<Timestamp>) -> Result<HistoricalPrice> {
|
||||
let prices = match timestamp {
|
||||
Some(ts) => self.price_at(ts)?,
|
||||
|
||||
@@ -19,6 +19,7 @@ brk_computer = { workspace = true }
|
||||
brk_error = { workspace = true, features = ["jiff", "serde_json", "tokio", "vecdb"] }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_oracle = { workspace = true }
|
||||
brk_query = { workspace = true }
|
||||
brk_reader = { workspace = true }
|
||||
brk_rpc = { workspace = true }
|
||||
|
||||
@@ -25,6 +25,7 @@ mod mempool;
|
||||
mod metrics;
|
||||
mod mining;
|
||||
mod openapi;
|
||||
mod oracle;
|
||||
mod series;
|
||||
mod series_legacy;
|
||||
mod server;
|
||||
@@ -38,6 +39,7 @@ use general::GeneralRoutes;
|
||||
use mempool::MempoolRoutes;
|
||||
use mining::MiningRoutes;
|
||||
pub use openapi::*;
|
||||
use oracle::OracleRoutes;
|
||||
use transactions::TxRoutes;
|
||||
|
||||
pub trait ApiRoutes {
|
||||
@@ -57,6 +59,7 @@ impl ApiRoutes for ApiRouter<AppState> {
|
||||
.add_mining_routes()
|
||||
.add_fees_routes()
|
||||
.add_mempool_routes()
|
||||
.add_oracle_routes()
|
||||
.add_tx_routes()
|
||||
.api_route(
|
||||
"/openapi.json",
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
use aide::axum::{ApiRouter, routing::get_with};
|
||||
use axum::{
|
||||
extract::{Path, State},
|
||||
http::{HeaderMap, Uri},
|
||||
};
|
||||
use brk_oracle::{HistogramEmaCompact, HistogramRaw};
|
||||
use brk_types::{Dollars, Version};
|
||||
|
||||
use crate::{
|
||||
AppState,
|
||||
extended::TransformResponseExtended,
|
||||
params::{Empty, HeightParam},
|
||||
};
|
||||
|
||||
pub trait OracleRoutes {
|
||||
fn add_oracle_routes(self) -> Self;
|
||||
}
|
||||
|
||||
impl OracleRoutes for ApiRouter<AppState> {
|
||||
fn add_oracle_routes(self) -> Self {
|
||||
self.api_route(
|
||||
"/api/oracle/price",
|
||||
get_with(
|
||||
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
|
||||
state
|
||||
.respond_json(&headers, state.mempool_strategy(), &uri, |q| q.live_price())
|
||||
.await
|
||||
},
|
||||
|op| {
|
||||
op.id("get_oracle_price")
|
||||
.oracle_tag()
|
||||
.summary("Live BTC/USD price")
|
||||
.description(
|
||||
"Current BTC/USD price in dollars, derived purely from on-chain \
|
||||
round-dollar output patterns over the last 12 blocks plus the \
|
||||
forming mempool block. Same value as `/api/mempool/price`. \
|
||||
Confirmed per-height history is available at `/api/vecs/height-to-price`.",
|
||||
)
|
||||
.json_response::<Dollars>()
|
||||
.not_modified()
|
||||
.server_error()
|
||||
},
|
||||
),
|
||||
)
|
||||
.api_route(
|
||||
"/api/oracle/histogram/ema/live",
|
||||
get_with(
|
||||
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
|
||||
state
|
||||
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
|
||||
q.live_histogram_ema()
|
||||
})
|
||||
.await
|
||||
},
|
||||
|op| {
|
||||
op.id("get_oracle_histogram_ema_live")
|
||||
.oracle_tag()
|
||||
.summary("Live EMA histogram")
|
||||
.description(
|
||||
"Smoothed round-dollar payment histogram at the live tip: the \
|
||||
committed 12-block EMA with the forming mempool block blended in \
|
||||
as a final slot. A flat array of 2400 log-scale bins, quantized \
|
||||
to `u16` for the wire. This is the heatmap column you render.",
|
||||
)
|
||||
.json_response::<HistogramEmaCompact>()
|
||||
.not_modified()
|
||||
.server_error()
|
||||
},
|
||||
),
|
||||
)
|
||||
.api_route(
|
||||
"/api/oracle/histogram/ema/{height}",
|
||||
get_with(
|
||||
async |uri: Uri,
|
||||
headers: HeaderMap,
|
||||
Path(path): Path<HeightParam>,
|
||||
_: Empty,
|
||||
State(state): State<AppState>| {
|
||||
let strategy = state.height_strategy(Version::new(brk_oracle::VERSION), path.height);
|
||||
state
|
||||
.respond_json(&headers, strategy, &uri, move |q| {
|
||||
q.confirmed_histogram_ema(usize::from(path.height))
|
||||
})
|
||||
.await
|
||||
},
|
||||
|op| {
|
||||
op.id("get_oracle_histogram_ema")
|
||||
.oracle_tag()
|
||||
.summary("EMA histogram at height")
|
||||
.description(
|
||||
"Smoothed round-dollar payment histogram for a confirmed height, \
|
||||
deterministically reconstructed by replaying the 12-block window \
|
||||
ending at that height. Immutable once buried, so repeated requests \
|
||||
return byte-identical results. A flat array of 2400 log-scale bins, \
|
||||
quantized to `u16`.",
|
||||
)
|
||||
.json_response::<HistogramEmaCompact>()
|
||||
.not_modified()
|
||||
.bad_request()
|
||||
.not_found()
|
||||
.server_error()
|
||||
},
|
||||
),
|
||||
)
|
||||
.api_route(
|
||||
"/api/oracle/histogram/raw/live",
|
||||
get_with(
|
||||
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
|
||||
state
|
||||
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
|
||||
q.live_histogram_raw()
|
||||
})
|
||||
.await
|
||||
},
|
||||
|op| {
|
||||
op.id("get_oracle_histogram_raw_live")
|
||||
.oracle_tag()
|
||||
.summary("Live raw histogram")
|
||||
.description(
|
||||
"Un-smoothed per-block round-dollar counts for the forming mempool \
|
||||
block: the spiky primitive the EMA smooths over. A flat array of \
|
||||
2400 log-scale bins (`u32` counts), all zero when no mempool is \
|
||||
configured.",
|
||||
)
|
||||
.json_response::<HistogramRaw>()
|
||||
.not_modified()
|
||||
.server_error()
|
||||
},
|
||||
),
|
||||
)
|
||||
.api_route(
|
||||
"/api/oracle/histogram/raw/{height}",
|
||||
get_with(
|
||||
async |uri: Uri,
|
||||
headers: HeaderMap,
|
||||
Path(path): Path<HeightParam>,
|
||||
_: Empty,
|
||||
State(state): State<AppState>| {
|
||||
let strategy = state.height_strategy(Version::new(brk_oracle::VERSION), path.height);
|
||||
state
|
||||
.respond_json(&headers, strategy, &uri, move |q| {
|
||||
q.confirmed_histogram_raw(usize::from(path.height))
|
||||
})
|
||||
.await
|
||||
},
|
||||
|op| {
|
||||
op.id("get_oracle_histogram_raw")
|
||||
.oracle_tag()
|
||||
.summary("Raw histogram at height")
|
||||
.description(
|
||||
"Un-smoothed round-dollar counts for a single confirmed block. A \
|
||||
flat array of 2400 log-scale bins (`u32` counts).",
|
||||
)
|
||||
.json_response::<HistogramRaw>()
|
||||
.not_modified()
|
||||
.bad_request()
|
||||
.not_found()
|
||||
.server_error()
|
||||
},
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub trait TransformResponseExtended<'t> {
|
||||
fn mining_tag(self) -> Self;
|
||||
fn fees_tag(self) -> Self;
|
||||
fn mempool_tag(self) -> Self;
|
||||
fn oracle_tag(self) -> Self;
|
||||
fn transactions_tag(self) -> Self;
|
||||
fn server_tag(self) -> Self;
|
||||
fn series_tag(self) -> Self;
|
||||
@@ -73,6 +74,10 @@ impl<'t> TransformResponseExtended<'t> for TransformOperation<'t> {
|
||||
self.tag("Mempool")
|
||||
}
|
||||
|
||||
fn oracle_tag(self) -> Self {
|
||||
self.tag("Oracle")
|
||||
}
|
||||
|
||||
fn transactions_tag(self) -> Self {
|
||||
self.tag("Transactions")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
use std::{
|
||||
fmt,
|
||||
marker::PhantomData,
|
||||
ops::{Deref, DerefMut},
|
||||
};
|
||||
|
||||
use schemars::{JsonSchema, SchemaGenerator};
|
||||
use serde::{
|
||||
Deserialize, Deserializer, Serialize, Serializer,
|
||||
de::{SeqAccess, Visitor},
|
||||
};
|
||||
|
||||
/// Fixed-length, log-scale bin histogram generic over the per-bin counter type.
|
||||
/// Instantiated as raw counts (`u32`), the smoothed EMA buffer (`f64`), or the
|
||||
/// quantized wire projection (`u16`). Serializes as a flat JSON array of `N`
|
||||
/// values. `Deref` exposes the underlying array for indexing and iteration.
|
||||
///
|
||||
/// Backed by a fixed `[T; N]` (not a `Vec`) to keep the always-`N` invariant the
|
||||
/// callers rely on.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Histogram<T, const N: usize>([T; N]);
|
||||
|
||||
impl<T: Copy + Default, const N: usize> Histogram<T, N> {
|
||||
#[inline]
|
||||
pub fn zeros() -> Self {
|
||||
Self([T::default(); N])
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Copy + Default, const N: usize> Default for Histogram<T, N> {
|
||||
fn default() -> Self {
|
||||
Self::zeros()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, const N: usize> Deref for Histogram<T, N> {
|
||||
type Target = [T; N];
|
||||
#[inline]
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, const N: usize> DerefMut for Histogram<T, N> {
|
||||
#[inline]
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> Histogram<u32, N> {
|
||||
/// Bump the count in `bin` by one.
|
||||
#[inline]
|
||||
pub fn increment(&mut self, bin: usize) {
|
||||
self.0[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> Histogram<f64, N> {
|
||||
/// Quantize each bin to `u16` (round, then clamp into range) for the wire.
|
||||
/// Lossy by design: faint sub-0.5 bins vanish, which is invisible on a heatmap.
|
||||
pub fn to_compact(&self) -> Histogram<u16, N> {
|
||||
let mut out = [0u16; N];
|
||||
for (o, &v) in out.iter_mut().zip(self.0.iter()) {
|
||||
*o = v.round().clamp(0.0, u16::MAX as f64) as u16;
|
||||
}
|
||||
Histogram(out)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Serialize, const N: usize> Serialize for Histogram<T, N> {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
self.0.as_slice().serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de, T: Deserialize<'de> + Copy + Default, const N: usize> Deserialize<'de>
|
||||
for Histogram<T, N>
|
||||
{
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct ArrayVisitor<T, const N: usize>(PhantomData<T>);
|
||||
|
||||
impl<'de, T: Deserialize<'de> + Copy + Default, const N: usize> Visitor<'de>
|
||||
for ArrayVisitor<T, N>
|
||||
{
|
||||
type Value = Histogram<T, N>;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "an array of {N} values")
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
|
||||
let mut bins = [T::default(); N];
|
||||
for (i, bin) in bins.iter_mut().enumerate() {
|
||||
*bin = seq
|
||||
.next_element()?
|
||||
.ok_or_else(|| serde::de::Error::invalid_length(i, &self))?;
|
||||
}
|
||||
Ok(Histogram(bins))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_seq(ArrayVisitor::<T, N>(PhantomData))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: JsonSchema, const N: usize> JsonSchema for Histogram<T, N> {
|
||||
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||
format!("Histogram_{}", T::schema_name()).into()
|
||||
}
|
||||
|
||||
fn json_schema(generator: &mut SchemaGenerator) -> schemars::Schema {
|
||||
Vec::<T>::json_schema(generator)
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,7 @@ mod hashrate_summary;
|
||||
mod health;
|
||||
mod height;
|
||||
mod hex;
|
||||
mod histogram;
|
||||
mod historical_price;
|
||||
mod hour1;
|
||||
mod hour12;
|
||||
@@ -97,7 +98,6 @@ mod next_block_hash;
|
||||
mod ohlc;
|
||||
mod op_return_index;
|
||||
mod option_ext;
|
||||
mod oracle_bins;
|
||||
mod outpoint;
|
||||
mod outpoint_prefix;
|
||||
mod output;
|
||||
@@ -273,6 +273,7 @@ pub use hashrate_summary::*;
|
||||
pub use health::*;
|
||||
pub use height::*;
|
||||
pub use hex::*;
|
||||
pub use histogram::*;
|
||||
pub use historical_price::*;
|
||||
pub use hour1::*;
|
||||
pub use hour4::*;
|
||||
@@ -294,7 +295,6 @@ pub use next_block_hash::*;
|
||||
pub use ohlc::*;
|
||||
pub use op_return_index::*;
|
||||
pub use option_ext::*;
|
||||
pub use oracle_bins::*;
|
||||
pub use outpoint::*;
|
||||
pub use outpoint_prefix::*;
|
||||
pub use output::*;
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
use std::{fmt::Display, mem::size_of};
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::SeqAccess, de::Visitor};
|
||||
use vecdb::{Bytes, Formattable};
|
||||
|
||||
use crate::Sats;
|
||||
|
||||
/// Number of bins for the phase histogram
|
||||
pub const PHASE_BINS: usize = 100;
|
||||
|
||||
/// Phase histogram: counts per bin for frac(log10(sats))
|
||||
///
|
||||
/// Used for on-chain price discovery. Each bin represents 1% of the
|
||||
/// log10 fractional range [0, 1). Values are u16 (max 65535 per bin).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct OracleBins {
|
||||
pub bins: [u16; PHASE_BINS],
|
||||
}
|
||||
|
||||
impl Default for OracleBins {
|
||||
fn default() -> Self {
|
||||
Self::ZERO
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for OracleBins {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "OracleBins(peak={})", self.peak_bin())
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for OracleBins {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
self.bins.as_slice().serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for OracleBins {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct BinsVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for BinsVisitor {
|
||||
type Value = OracleBins;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
write!(formatter, "an array of {} u16 values", PHASE_BINS)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
|
||||
let mut bins = [0u16; PHASE_BINS];
|
||||
for (i, bin) in bins.iter_mut().enumerate() {
|
||||
*bin = seq
|
||||
.next_element()?
|
||||
.ok_or_else(|| serde::de::Error::invalid_length(i, &self))?;
|
||||
}
|
||||
Ok(OracleBins { bins })
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_seq(BinsVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for OracleBins {
|
||||
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||
"OracleBins".into()
|
||||
}
|
||||
|
||||
fn json_schema(_gen: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||
// Represent as array of u16 values
|
||||
Vec::<u16>::json_schema(_gen)
|
||||
}
|
||||
}
|
||||
|
||||
impl OracleBins {
|
||||
pub const ZERO: Self = Self {
|
||||
bins: [0; PHASE_BINS],
|
||||
};
|
||||
|
||||
/// Get the bin index for a sats value
|
||||
/// Filters: min 1k sats, max 100k BTC (matches Python 1e-5 to 1e5 BTC)
|
||||
#[inline]
|
||||
pub fn sats_to_bin(sats: Sats) -> Option<usize> {
|
||||
if sats < Sats::_1K || sats > Sats::_100K_BTC {
|
||||
return None;
|
||||
}
|
||||
let log_sats = f64::from(sats).log10();
|
||||
let phase = log_sats.fract();
|
||||
let phase = if phase < 0.0 { phase + 1.0 } else { phase };
|
||||
Some(((phase * PHASE_BINS as f64) as usize).min(PHASE_BINS - 1))
|
||||
}
|
||||
|
||||
/// Add a count to the bin for this sats value
|
||||
#[inline]
|
||||
pub fn add(&mut self, sats: Sats) {
|
||||
if let Some(bin) = Self::sats_to_bin(sats) {
|
||||
self.bins[bin] = self.bins[bin].saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the peak bin (index with highest count)
|
||||
pub fn peak_bin(&self) -> usize {
|
||||
self.bins
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by_key(|(_, count)| *count)
|
||||
.map(|(idx, _)| idx)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Get total count across all bins
|
||||
pub fn total_count(&self) -> u32 {
|
||||
self.bins.iter().map(|&c| c as u32).sum()
|
||||
}
|
||||
}
|
||||
|
||||
impl Bytes for OracleBins {
|
||||
type Array = [u8; size_of::<Self>()];
|
||||
|
||||
fn to_bytes(&self) -> Self::Array {
|
||||
let mut arr = [0u8; size_of::<Self>()];
|
||||
for (i, &count) in self.bins.iter().enumerate() {
|
||||
let bytes = count.to_le_bytes();
|
||||
arr[i * 2] = bytes[0];
|
||||
arr[i * 2 + 1] = bytes[1];
|
||||
}
|
||||
arr
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: &[u8]) -> vecdb::Result<Self> {
|
||||
if bytes.len() < size_of::<Self>() {
|
||||
return Err(vecdb::Error::WrongLength {
|
||||
received: bytes.len(),
|
||||
expected: size_of::<Self>(),
|
||||
});
|
||||
}
|
||||
let mut bins = [0u16; PHASE_BINS];
|
||||
for (i, bin) in bins.iter_mut().enumerate() {
|
||||
*bin = u16::from_le_bytes([bytes[i * 2], bytes[i * 2 + 1]]);
|
||||
}
|
||||
Ok(Self { bins })
|
||||
}
|
||||
}
|
||||
|
||||
impl Formattable for OracleBins {
|
||||
fn write_to(&self, buf: &mut Vec<u8>) {
|
||||
use std::fmt::Write;
|
||||
let mut s = String::new();
|
||||
write!(s, "{}", self).unwrap();
|
||||
buf.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
fn fmt_json(&self, buf: &mut Vec<u8>) {
|
||||
buf.push(b'"');
|
||||
self.write_to(buf);
|
||||
buf.push(b'"');
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// OracleBinsV2: 200-bin phase histogram for V2 phase oracle
|
||||
// ============================================================================
|
||||
|
||||
/// Number of bins for V2 phase histogram (0.5% resolution)
|
||||
pub const PHASE_BINS_V2: usize = 200;
|
||||
|
||||
/// V2 Phase histogram: counts per bin for frac(log10(sats))
|
||||
///
|
||||
/// Used for phase oracle V2 with round USD template matching.
|
||||
/// Each bin represents 0.5% of the log10 fractional range [0, 1).
|
||||
/// Values are u16 (max 65535 per bin).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct OracleBinsV2 {
|
||||
pub bins: [u16; PHASE_BINS_V2],
|
||||
}
|
||||
|
||||
impl Default for OracleBinsV2 {
|
||||
fn default() -> Self {
|
||||
Self::ZERO
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for OracleBinsV2 {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "OracleBinsV2(peak={})", self.peak_bin())
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for OracleBinsV2 {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
self.bins.as_slice().serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for OracleBinsV2 {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct BinsVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for BinsVisitor {
|
||||
type Value = OracleBinsV2;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
write!(formatter, "an array of {} u16 values", PHASE_BINS_V2)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
|
||||
let mut bins = [0u16; PHASE_BINS_V2];
|
||||
for (i, bin) in bins.iter_mut().enumerate() {
|
||||
*bin = seq
|
||||
.next_element()?
|
||||
.ok_or_else(|| serde::de::Error::invalid_length(i, &self))?;
|
||||
}
|
||||
Ok(OracleBinsV2 { bins })
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_seq(BinsVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for OracleBinsV2 {
|
||||
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||
"OracleBinsV2".into()
|
||||
}
|
||||
|
||||
fn json_schema(_gen: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||
Vec::<u16>::json_schema(_gen)
|
||||
}
|
||||
}
|
||||
|
||||
impl OracleBinsV2 {
|
||||
pub const ZERO: Self = Self {
|
||||
bins: [0; PHASE_BINS_V2],
|
||||
};
|
||||
|
||||
/// Get the bin index for a sats value
|
||||
/// Filters: min 1k sats, max 100k BTC
|
||||
#[inline]
|
||||
pub fn sats_to_bin(sats: Sats) -> Option<usize> {
|
||||
if sats < Sats::_1K || sats > Sats::_100K_BTC {
|
||||
return None;
|
||||
}
|
||||
let log_sats = f64::from(sats).log10();
|
||||
let phase = log_sats.fract();
|
||||
let phase = if phase < 0.0 { phase + 1.0 } else { phase };
|
||||
Some(((phase * PHASE_BINS_V2 as f64) as usize).min(PHASE_BINS_V2 - 1))
|
||||
}
|
||||
|
||||
/// Add a count to the bin for this sats value
|
||||
#[inline]
|
||||
pub fn add(&mut self, sats: Sats) {
|
||||
if let Some(bin) = Self::sats_to_bin(sats) {
|
||||
self.bins[bin] = self.bins[bin].saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Add another histogram to this one
|
||||
pub fn add_histogram(&mut self, other: &OracleBinsV2) {
|
||||
for (i, &count) in other.bins.iter().enumerate() {
|
||||
self.bins[i] = self.bins[i].saturating_add(count);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the peak bin (index with highest count)
|
||||
pub fn peak_bin(&self) -> usize {
|
||||
self.bins
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by_key(|(_, count)| *count)
|
||||
.map(|(idx, _)| idx)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Get total count across all bins
|
||||
pub fn total_count(&self) -> u32 {
|
||||
self.bins.iter().map(|&c| c as u32).sum()
|
||||
}
|
||||
}
|
||||
|
||||
impl Bytes for OracleBinsV2 {
|
||||
type Array = [u8; size_of::<Self>()];
|
||||
|
||||
fn to_bytes(&self) -> Self::Array {
|
||||
let mut arr = [0u8; size_of::<Self>()];
|
||||
for (i, &count) in self.bins.iter().enumerate() {
|
||||
let bytes = count.to_le_bytes();
|
||||
arr[i * 2] = bytes[0];
|
||||
arr[i * 2 + 1] = bytes[1];
|
||||
}
|
||||
arr
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: &[u8]) -> vecdb::Result<Self> {
|
||||
if bytes.len() < size_of::<Self>() {
|
||||
return Err(vecdb::Error::WrongLength {
|
||||
received: bytes.len(),
|
||||
expected: size_of::<Self>(),
|
||||
});
|
||||
}
|
||||
let mut bins = [0u16; PHASE_BINS_V2];
|
||||
for (i, bin) in bins.iter_mut().enumerate() {
|
||||
*bin = u16::from_le_bytes([bytes[i * 2], bytes[i * 2 + 1]]);
|
||||
}
|
||||
Ok(Self { bins })
|
||||
}
|
||||
}
|
||||
|
||||
impl Formattable for OracleBinsV2 {
|
||||
fn write_to(&self, buf: &mut Vec<u8>) {
|
||||
use std::fmt::Write;
|
||||
let mut s = String::new();
|
||||
write!(s, "{}", self).unwrap();
|
||||
buf.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
fn fmt_json(&self, buf: &mut Vec<u8>) {
|
||||
buf.push(b'"');
|
||||
self.write_to(buf);
|
||||
buf.push(b'"');
|
||||
}
|
||||
}
|
||||
@@ -664,6 +664,8 @@ ancestors and no descendants (matches mempool.space).
|
||||
*
|
||||
* @typedef {Dollars} High
|
||||
*/
|
||||
/** @typedef {number[]} Histogram_uint16 */
|
||||
/** @typedef {number[]} Histogram_uint32 */
|
||||
/**
|
||||
* Historical price response
|
||||
*
|
||||
@@ -11865,6 +11867,80 @@ class BrkClient extends BrkClientBase {
|
||||
return this.getJson(path, { signal, onValue });
|
||||
}
|
||||
|
||||
/**
|
||||
* Live BTC/USD price
|
||||
*
|
||||
* Current BTC/USD price in dollars, derived purely from on-chain round-dollar output patterns over the last 12 blocks plus the forming mempool block. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/price`
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Dollars) => void }} [options]
|
||||
* @returns {Promise<Dollars>}
|
||||
*/
|
||||
async getOraclePrice({ signal, onValue } = {}) {
|
||||
const path = `/api/oracle/price`;
|
||||
return this.getJson(path, { signal, onValue });
|
||||
}
|
||||
|
||||
/**
|
||||
* Live EMA histogram
|
||||
*
|
||||
* Smoothed round-dollar payment histogram at the live tip: the committed 12-block EMA with the forming mempool block blended in as a final slot. A flat array of 2400 log-scale bins, quantized to `u16` for the wire. This is the heatmap column you render.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/ema/live`
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Histogram_uint16) => void }} [options]
|
||||
* @returns {Promise<Histogram_uint16>}
|
||||
*/
|
||||
async getOracleHistogramEmaLive({ signal, onValue } = {}) {
|
||||
const path = `/api/oracle/histogram/ema/live`;
|
||||
return this.getJson(path, { signal, onValue });
|
||||
}
|
||||
|
||||
/**
|
||||
* EMA histogram at height
|
||||
*
|
||||
* Smoothed round-dollar payment histogram for a confirmed height, deterministically reconstructed by replaying the 12-block window ending at that height. Immutable once buried, so repeated requests return byte-identical results. A flat array of 2400 log-scale bins, quantized to `u16`.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/ema/{height}`
|
||||
*
|
||||
* @param {Height} height
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Histogram_uint16) => void }} [options]
|
||||
* @returns {Promise<Histogram_uint16>}
|
||||
*/
|
||||
async getOracleHistogramEma(height, { signal, onValue } = {}) {
|
||||
const path = `/api/oracle/histogram/ema/${height}`;
|
||||
return this.getJson(path, { signal, onValue });
|
||||
}
|
||||
|
||||
/**
|
||||
* Live raw histogram
|
||||
*
|
||||
* Un-smoothed per-block round-dollar counts for the forming mempool block: the spiky primitive the EMA smooths over. A flat array of 2400 log-scale bins (`u32` counts), all zero when no mempool is configured.
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/raw/live`
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Histogram_uint32) => void }} [options]
|
||||
* @returns {Promise<Histogram_uint32>}
|
||||
*/
|
||||
async getOracleHistogramRawLive({ signal, onValue } = {}) {
|
||||
const path = `/api/oracle/histogram/raw/live`;
|
||||
return this.getJson(path, { signal, onValue });
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw histogram at height
|
||||
*
|
||||
* Un-smoothed round-dollar counts for a single confirmed block. A flat array of 2400 log-scale bins (`u32` counts).
|
||||
*
|
||||
* Endpoint: `GET /api/oracle/histogram/raw/{height}`
|
||||
*
|
||||
* @param {Height} height
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: Histogram_uint32) => void }} [options]
|
||||
* @returns {Promise<Histogram_uint32>}
|
||||
*/
|
||||
async getOracleHistogramRaw(height, { signal, onValue } = {}) {
|
||||
const path = `/api/oracle/histogram/raw/${height}`;
|
||||
return this.getJson(path, { signal, onValue });
|
||||
}
|
||||
|
||||
/**
|
||||
* Txid by index
|
||||
*
|
||||
|
||||
@@ -174,6 +174,8 @@ Halving = int
|
||||
Hex = str
|
||||
# Highest price value for a time period
|
||||
High = Dollars
|
||||
Histogram_uint16 = List[int]
|
||||
Histogram_uint32 = List[int]
|
||||
Hour1 = int
|
||||
Hour12 = int
|
||||
Hour4 = int
|
||||
@@ -8658,6 +8660,46 @@ class BrkClient(BrkClientBase):
|
||||
Endpoint: `GET /api/mempool/price`"""
|
||||
return self.get_json('/api/mempool/price')
|
||||
|
||||
def get_oracle_price(self) -> Dollars:
|
||||
"""Live BTC/USD price.
|
||||
|
||||
Current BTC/USD price in dollars, derived purely from on-chain round-dollar output patterns over the last 12 blocks plus the forming mempool block. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
|
||||
|
||||
Endpoint: `GET /api/oracle/price`"""
|
||||
return self.get_json('/api/oracle/price')
|
||||
|
||||
def get_oracle_histogram_ema_live(self) -> Histogram_uint16:
|
||||
"""Live EMA histogram.
|
||||
|
||||
Smoothed round-dollar payment histogram at the live tip: the committed 12-block EMA with the forming mempool block blended in as a final slot. A flat array of 2400 log-scale bins, quantized to `u16` for the wire. This is the heatmap column you render.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/ema/live`"""
|
||||
return self.get_json('/api/oracle/histogram/ema/live')
|
||||
|
||||
def get_oracle_histogram_ema(self, height: Height) -> Histogram_uint16:
|
||||
"""EMA histogram at height.
|
||||
|
||||
Smoothed round-dollar payment histogram for a confirmed height, deterministically reconstructed by replaying the 12-block window ending at that height. Immutable once buried, so repeated requests return byte-identical results. A flat array of 2400 log-scale bins, quantized to `u16`.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/ema/{height}`"""
|
||||
return self.get_json(f'/api/oracle/histogram/ema/{height}')
|
||||
|
||||
def get_oracle_histogram_raw_live(self) -> Histogram_uint32:
|
||||
"""Live raw histogram.
|
||||
|
||||
Un-smoothed per-block round-dollar counts for the forming mempool block: the spiky primitive the EMA smooths over. A flat array of 2400 log-scale bins (`u32` counts), all zero when no mempool is configured.
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/raw/live`"""
|
||||
return self.get_json('/api/oracle/histogram/raw/live')
|
||||
|
||||
def get_oracle_histogram_raw(self, height: Height) -> Histogram_uint32:
|
||||
"""Raw histogram at height.
|
||||
|
||||
Un-smoothed round-dollar counts for a single confirmed block. A flat array of 2400 log-scale bins (`u32` counts).
|
||||
|
||||
Endpoint: `GET /api/oracle/histogram/raw/{height}`"""
|
||||
return self.get_json(f'/api/oracle/histogram/raw/{height}')
|
||||
|
||||
def get_tx_by_index(self, index: TxIndex) -> Txid:
|
||||
"""Txid by index.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user