diff --git a/crates/brk_client/src/lib.rs b/crates/brk_client/src/lib.rs index a14aed23a..3e0dc428d 100644 --- a/crates/brk_client/src/lib.rs +++ b/crates/brk_client/src/lib.rs @@ -5145,7 +5145,9 @@ impl MetricsTree_Price_Usd { pub struct MetricsTree_Price_Oracle { pub price_cents: MetricPattern11, pub ohlc_cents: MetricPattern6, - pub ohlc_dollars: MetricPattern6, + pub split: CloseHighLowOpenPattern2, + pub ohlc: MetricPattern1, + pub ohlc_dollars: MetricPattern1, } impl MetricsTree_Price_Oracle { @@ -5153,7 +5155,9 @@ impl MetricsTree_Price_Oracle { Self { price_cents: MetricPattern11::new(client.clone(), "oracle_price_cents".to_string()), ohlc_cents: MetricPattern6::new(client.clone(), "oracle_ohlc_cents".to_string()), - ohlc_dollars: MetricPattern6::new(client.clone(), "oracle_ohlc_dollars".to_string()), + split: CloseHighLowOpenPattern2::new(client.clone(), "oracle_price".to_string()), + ohlc: MetricPattern1::new(client.clone(), "oracle_price_ohlc".to_string()), + ohlc_dollars: MetricPattern1::new(client.clone(), "oracle_ohlc_dollars".to_string()), } } } @@ -6318,10 +6322,10 @@ impl BrkClient { /// Live BTC/USD price /// - /// Returns the current BTC/USD price in cents, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool. + /// Returns the current BTC/USD price in dollars, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool. /// /// Endpoint: `GET /api/mempool/price` - pub fn get_live_price(&self) -> Result { + pub fn get_live_price(&self) -> Result { self.base.get_json(&format!("/api/mempool/price")) } diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs b/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs index 49a06542c..e920c2484 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs @@ -249,8 +249,10 @@ impl UTXOCohorts { .try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))?; // 2. Compute net_sentiment.height for separate cohorts (greed - pain) - self.par_iter_separate_mut() - .try_for_each(|v| v.metrics.compute_net_sentiment_height(starting_indexes, exit))?; + self.par_iter_separate_mut().try_for_each(|v| { + v.metrics + .compute_net_sentiment_height(starting_indexes, exit) + })?; // 3. Compute net_sentiment.height for aggregate cohorts (weighted average) self.for_each_aggregate(|vecs, sources| { @@ -260,8 +262,10 @@ impl UTXOCohorts { })?; // 4. Compute net_sentiment dateindex for ALL cohorts - self.par_iter_mut() - .try_for_each(|v| v.metrics.compute_net_sentiment_rest(indexes, starting_indexes, exit)) + self.par_iter_mut().try_for_each(|v| { + v.metrics + .compute_net_sentiment_rest(indexes, starting_indexes, exit) + }) } /// Second phase of post-processing: compute relative metrics. @@ -468,7 +472,8 @@ impl UTXOCohorts { // Collect merged entries during the merge (already in sorted order) // Pre-allocate with max possible unique prices (actual count likely lower due to dedup) let max_unique_prices = relevant.iter().map(|e| e.len()).max().unwrap_or(0); - let mut merged: Vec<(CentsUnsignedCompact, Sats)> = Vec::with_capacity(max_unique_prices); + let mut merged: Vec<(CentsUnsignedCompact, Sats)> = + Vec::with_capacity(max_unique_prices); // Finalize a price point: compute percentiles and accumulate for merged vec let mut finalize_price = |price: CentsUnsigned, sats: u64, usd: u128| { @@ -489,7 +494,8 @@ impl UTXOCohorts { } // Round to nearest dollar with N significant digits for storage - let rounded: CentsUnsignedCompact = price.round_to_dollar(COST_BASIS_PRICE_DIGITS).into(); + let rounded: CentsUnsignedCompact = + price.round_to_dollar(COST_BASIS_PRICE_DIGITS).into(); // Merge entries with same rounded price using last_mut if let Some((last_price, last_sats)) = merged.last_mut() @@ -562,7 +568,10 @@ impl UTXOCohorts { let dir = states_path.join(format!("utxo_{cohort_name}_cost_basis/by_date")); fs::create_dir_all(&dir)?; let path = dir.join(date.to_string()); - fs::write(path, CostBasisDistribution::serialize_iter(merged.into_iter())?)?; + fs::write( + path, + CostBasisDistribution::serialize_iter(merged.into_iter())?, + )?; Ok(()) }) diff --git a/crates/brk_computer/src/price/mod.rs b/crates/brk_computer/src/price/mod.rs index 839514408..5b160c0fb 100644 --- a/crates/brk_computer/src/price/mod.rs +++ b/crates/brk_computer/src/price/mod.rs @@ -67,7 +67,7 @@ impl Vecs { let cents = CentsVecs::forced_import(db, version)?; let usd = UsdVecs::forced_import(db, version, indexes)?; let sats = SatsVecs::forced_import(db, version, indexes)?; - let oracle = OracleVecs::forced_import(db, version)?; + let oracle = OracleVecs::forced_import(db, version, indexes)?; Ok(Self { db: db.clone(), diff --git a/crates/brk_computer/src/price/oracle/compute.rs b/crates/brk_computer/src/price/oracle/compute.rs index 985c68bcf..9a4f3da02 100644 --- a/crates/brk_computer/src/price/oracle/compute.rs +++ b/crates/brk_computer/src/price/oracle/compute.rs @@ -4,8 +4,8 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_oracle::{Config, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin}; use brk_types::{ - CentsUnsigned, Close, DateIndex, Height, High, Low, OHLCCentsUnsigned, Open, OutputType, Sats, - TxIndex, TxOutIndex, + CentsUnsigned, Close, DateIndex, Height, High, Low, OHLCCentsUnsigned, OHLCDollars, Open, + OutputType, Sats, TxIndex, TxOutIndex, }; use tracing::info; use vecdb::{ @@ -26,6 +26,224 @@ impl Vecs { ) -> Result<()> { self.compute_prices(indexer, starting_indexes, exit)?; self.compute_daily_ohlc(indexes, starting_indexes, exit)?; + self.compute_split_and_ohlc(starting_indexes, exit)?; + Ok(()) + } + + fn compute_split_and_ohlc( + &mut self, + starting_indexes: &ComputeIndexes, + exit: &Exit, + ) -> Result<()> { + // Destructure to allow simultaneous borrows of different fields + let Self { + price_cents, + ohlc_cents, + split, + ohlc, + ohlc_dollars, + } = self; + + // Open: first-value aggregation + split.open.height.compute_transform( + starting_indexes.height, + &*price_cents, + |(h, price, ..)| (h, Open::new(price)), + exit, + )?; + split.open.compute_rest(starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + &*ohlc_cents, + |(di, ohlc_val, ..)| (di, ohlc_val.open), + exit, + )?; + Ok(()) + })?; + + // High: max-value aggregation + split.high.height.compute_transform( + starting_indexes.height, + &*price_cents, + |(h, price, ..)| (h, High::new(price)), + exit, + )?; + split.high.compute_rest(starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + &*ohlc_cents, + |(di, ohlc_val, ..)| (di, ohlc_val.high), + exit, + )?; + Ok(()) + })?; + + // Low: min-value aggregation + split.low.height.compute_transform( + starting_indexes.height, + &*price_cents, + |(h, price, ..)| (h, Low::new(price)), + exit, + )?; + split.low.compute_rest(starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + &*ohlc_cents, + |(di, ohlc_val, ..)| (di, ohlc_val.low), + exit, + )?; + Ok(()) + })?; + + // Close: last-value aggregation + split.close.height.compute_transform( + starting_indexes.height, + &*price_cents, + |(h, price, ..)| (h, Close::new(price)), + exit, + )?; + split.close.compute_rest(starting_indexes, exit, |v| { + v.compute_transform( + starting_indexes.dateindex, + &*ohlc_cents, + |(di, ohlc_val, ..)| (di, ohlc_val.close), + exit, + )?; + Ok(()) + })?; + + // Period OHLC aggregates - time based + ohlc.dateindex.compute_transform4( + starting_indexes.dateindex, + &split.open.dateindex, + &split.high.dateindex, + &split.low.dateindex, + &split.close.dateindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.week.compute_transform4( + starting_indexes.weekindex, + &*split.open.weekindex, + &*split.high.weekindex, + &*split.low.weekindex, + &*split.close.weekindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.month.compute_transform4( + starting_indexes.monthindex, + &*split.open.monthindex, + &*split.high.monthindex, + &*split.low.monthindex, + &*split.close.monthindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.quarter.compute_transform4( + starting_indexes.quarterindex, + &*split.open.quarterindex, + &*split.high.quarterindex, + &*split.low.quarterindex, + &*split.close.quarterindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.semester.compute_transform4( + starting_indexes.semesterindex, + &*split.open.semesterindex, + &*split.high.semesterindex, + &*split.low.semesterindex, + &*split.close.semesterindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.year.compute_transform4( + starting_indexes.yearindex, + &*split.open.yearindex, + &*split.high.yearindex, + &*split.low.yearindex, + &*split.close.yearindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.decade.compute_transform4( + starting_indexes.decadeindex, + &*split.open.decadeindex, + &*split.high.decadeindex, + &*split.low.decadeindex, + &*split.close.decadeindex, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + // Period OHLC aggregates - chain based + ohlc.height.compute_transform4( + starting_indexes.height, + &split.open.height, + &split.high.height, + &split.low.height, + &split.close.height, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + ohlc.difficultyepoch.compute_transform4( + starting_indexes.difficultyepoch, + &*split.open.difficultyepoch, + &*split.high.difficultyepoch, + &*split.low.difficultyepoch, + &*split.close.difficultyepoch, + |(i, open, high, low, close, _)| { + (i, OHLCCentsUnsigned { open, high, low, close }) + }, + exit, + )?; + + // OHLC dollars - transform cents to dollars at every period level + macro_rules! cents_to_dollars { + ($field:ident, $idx:expr) => { + ohlc_dollars.$field.compute_transform( + $idx, + &ohlc.$field, + |(i, c, ..)| (i, OHLCDollars::from(c)), + exit, + )?; + }; + } + + cents_to_dollars!(dateindex, starting_indexes.dateindex); + cents_to_dollars!(week, starting_indexes.weekindex); + cents_to_dollars!(month, starting_indexes.monthindex); + cents_to_dollars!(quarter, starting_indexes.quarterindex); + cents_to_dollars!(semester, starting_indexes.semesterindex); + cents_to_dollars!(year, starting_indexes.yearindex); + cents_to_dollars!(decade, starting_indexes.decadeindex); + cents_to_dollars!(height, starting_indexes.height); + cents_to_dollars!(difficultyepoch, starting_indexes.difficultyepoch); + Ok(()) } diff --git a/crates/brk_computer/src/price/oracle/import.rs b/crates/brk_computer/src/price/oracle/import.rs index fa71630b4..e946666e3 100644 --- a/crates/brk_computer/src/price/oracle/import.rs +++ b/crates/brk_computer/src/price/oracle/import.rs @@ -1,29 +1,53 @@ use brk_error::Result; -use brk_types::{DateIndex, OHLCCentsUnsigned, OHLCDollars, Version}; -use vecdb::{BytesVec, Database, ImportableVec, IterableCloneableVec, LazyVecFrom1, PcoVec}; +use brk_types::Version; +use vecdb::{BytesVec, Database, EagerVec, ImportableVec, PcoVec}; use super::Vecs; +use crate::indexes; +use crate::internal::{ComputedOHLC, LazyFromHeightAndDateOHLC}; impl Vecs { - pub fn forced_import(db: &Database, parent_version: Version) -> Result { - let version = parent_version + Version::new(10); + pub fn forced_import( + db: &Database, + parent_version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let version = parent_version + Version::new(11); let price_cents = PcoVec::forced_import(db, "oracle_price_cents", version)?; let ohlc_cents = BytesVec::forced_import(db, "oracle_ohlc_cents", version)?; - let ohlc_dollars = LazyVecFrom1::init( - "oracle_ohlc_dollars", - version, - ohlc_cents.boxed_clone(), - |di: DateIndex, iter| { - iter.get(di) - .map(|o: OHLCCentsUnsigned| OHLCDollars::from(o)) - }, - ); + let split = ComputedOHLC::forced_import(db, "oracle_price", version, indexes)?; + + let ohlc = LazyFromHeightAndDateOHLC { + dateindex: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + week: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + month: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + quarter: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + semester: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + year: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + decade: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + height: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + difficultyepoch: EagerVec::forced_import(db, "oracle_price_ohlc", version)?, + }; + + let ohlc_dollars = LazyFromHeightAndDateOHLC { + dateindex: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + week: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + month: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + quarter: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + semester: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + year: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + decade: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + height: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + difficultyepoch: EagerVec::forced_import(db, "oracle_ohlc_dollars", version)?, + }; Ok(Self { price_cents, ohlc_cents, + split, + ohlc, ohlc_dollars, }) } diff --git a/crates/brk_computer/src/price/oracle/vecs.rs b/crates/brk_computer/src/price/oracle/vecs.rs index df228305f..3cf1fbeeb 100644 --- a/crates/brk_computer/src/price/oracle/vecs.rs +++ b/crates/brk_computer/src/price/oracle/vecs.rs @@ -1,10 +1,14 @@ use brk_traversable::Traversable; use brk_types::{CentsUnsigned, DateIndex, Height, OHLCCentsUnsigned, OHLCDollars}; -use vecdb::{BytesVec, LazyVecFrom1, PcoVec}; +use vecdb::{BytesVec, PcoVec}; + +use crate::internal::{ComputedOHLC, LazyFromHeightAndDateOHLC}; #[derive(Clone, Traversable)] pub struct Vecs { pub price_cents: PcoVec, pub ohlc_cents: BytesVec, - pub ohlc_dollars: LazyVecFrom1, + pub split: ComputedOHLC, + pub ohlc: LazyFromHeightAndDateOHLC, + pub ohlc_dollars: LazyFromHeightAndDateOHLC, } diff --git a/crates/brk_computer/src/supply/compute.rs b/crates/brk_computer/src/supply/compute.rs index ba6ed6847..1e4a569bd 100644 --- a/crates/brk_computer/src/supply/compute.rs +++ b/crates/brk_computer/src/supply/compute.rs @@ -52,7 +52,7 @@ impl Vecs { vec.compute_percentage_change( starting_indexes.dateindex, mcap_dateindex, - 30, + 365, exit, )?; Ok(()) @@ -66,7 +66,7 @@ impl Vecs { vec.compute_percentage_change( starting_indexes.dateindex, rcap_dateindex, - 30, + 365, exit, )?; Ok(()) diff --git a/crates/brk_computer/src/supply/import.rs b/crates/brk_computer/src/supply/import.rs index f5a5de6ba..bc0c1a1a4 100644 --- a/crates/brk_computer/src/supply/import.rs +++ b/crates/brk_computer/src/supply/import.rs @@ -7,11 +7,12 @@ use vecdb::{Database, IterableCloneableVec, LazyVecFrom2, PAGE_SIZE}; use super::Vecs; use crate::{ - distribution, indexes, price, + distribution, indexes, internal::{ ComputedFromDateAverage, ComputedFromDateLast, DifferenceF32, DollarsIdentity, LazyFromHeightLast, LazyValueFromHeightLast, SatsIdentity, }, + price, }; const VERSION: Version = Version::ONE; @@ -61,10 +62,18 @@ impl Vecs { }); // Growth rates - let market_cap_growth_rate = - ComputedFromDateLast::forced_import(&db, "market_cap_growth_rate", version, indexes)?; - let realized_cap_growth_rate = - ComputedFromDateLast::forced_import(&db, "realized_cap_growth_rate", version, indexes)?; + let market_cap_growth_rate = ComputedFromDateLast::forced_import( + &db, + "market_cap_growth_rate", + version + Version::ONE, + indexes, + )?; + let realized_cap_growth_rate = ComputedFromDateLast::forced_import( + &db, + "realized_cap_growth_rate", + version + Version::ONE, + indexes, + )?; let cap_growth_rate_diff = LazyVecFrom2::transformed::( "cap_growth_rate_diff", version, diff --git a/crates/brk_mempool/src/projected_blocks/snapshot.rs b/crates/brk_mempool/src/projected_blocks/snapshot.rs index dc8c55a19..5f83c5b2a 100644 --- a/crates/brk_mempool/src/projected_blocks/snapshot.rs +++ b/crates/brk_mempool/src/projected_blocks/snapshot.rs @@ -1,3 +1,5 @@ +use std::hash::{DefaultHasher, Hash, Hasher}; + use brk_types::RecommendedFees; use super::{fees, stats::{self, BlockStats}}; @@ -36,4 +38,14 @@ impl Snapshot { fees, } } + + /// Hash of the first projected block (the one about to be mined). + pub fn next_block_hash(&self) -> u64 { + let Some(block) = self.blocks.first() else { + return 0; + }; + let mut hasher = DefaultHasher::new(); + block.hash(&mut hasher); + hasher.finish() + } } diff --git a/crates/brk_mempool/src/sync.rs b/crates/brk_mempool/src/sync.rs index 7079a8d16..adcaf246b 100644 --- a/crates/brk_mempool/src/sync.rs +++ b/crates/brk_mempool/src/sync.rs @@ -1,4 +1,5 @@ use std::{ + hash::{DefaultHasher, Hash, Hasher}, sync::{ Arc, atomic::{AtomicBool, AtomicU64, Ordering}, @@ -9,7 +10,7 @@ use std::{ use brk_error::Result; use brk_rpc::Client; -use brk_types::{MempoolEntryInfo, MempoolInfo, TxWithHex, Txid, TxidPrefix}; +use brk_types::{AddressBytes, MempoolEntryInfo, MempoolInfo, TxWithHex, Txid, TxidPrefix}; use derive_more::Deref; use parking_lot::{RwLock, RwLockReadGuard}; use rustc_hash::FxHashMap; @@ -87,6 +88,20 @@ impl MempoolInner { self.snapshot.read().block_stats.clone() } + pub fn next_block_hash(&self) -> u64 { + self.snapshot.read().next_block_hash() + } + + pub fn address_hash(&self, address: &AddressBytes) -> u64 { + let addresses = self.addresses.read(); + let Some((stats, _)) = addresses.get(address) else { + return 0; + }; + let mut hasher = DefaultHasher::new(); + stats.hash(&mut hasher); + hasher.finish() + } + pub fn get_txs(&self) -> RwLockReadGuard<'_, TxStore> { self.txs.read() } diff --git a/crates/brk_oracle/README.md b/crates/brk_oracle/README.md index 1ab010434..d7d4235fc 100644 --- a/crates/brk_oracle/README.md +++ b/crates/brk_oracle/README.md @@ -36,7 +36,7 @@ All parameters are exposed via Config with sensible defaults: - **window_size** (12): number of block histograms in the ring buffer - **search_below / search_above** (9 / 11): how far to search around the previous estimate, in bins - **min_sats** (1,000): minimum output value, filters dust -- **exclude_round_btc** (true): exclude round BTC amounts that create false stencil matches +- **exclude_common_round_values** (true): exclude common round values (d × 10^n, d ∈ {1,2,3,5,6}) that create false stencil matches - **excluded_output_types** (P2TR, P2WSH): script types dominated by protocol activity, not round-dollar purchases ## Inspiration @@ -69,11 +69,11 @@ Tested over 361,245 blocks (heights 575,000 to 936,244) against exchange OHLC da | 95th percentile | 0.55% | | 99th percentile | 1.4% | | 99.9th percentile | 4.4% | -| RMSE | 0.39% | -| Max error | 18.2% | +| RMSE | 0.38% | +| Max error | 18.1% | | Bias | +0.04 bins (essentially zero) | -| Blocks > 5% error | 261 (0.07%) | -| Blocks > 10% error | 40 (0.01%) | +| Blocks > 5% error | 237 (0.07%) | +| Blocks > 10% error | 22 (0.006%) | | Blocks > 20% error | 0 | ### Daily candles diff --git a/crates/brk_oracle/examples/compare_digits.rs b/crates/brk_oracle/examples/compare_digits.rs new file mode 100644 index 000000000..7fd44e206 --- /dev/null +++ b/crates/brk_oracle/examples/compare_digits.rs @@ -0,0 +1,286 @@ +//! Compare specific digit filter configurations across multiple start heights. +//! +//! Run with: cargo run -p brk_oracle --example compare_digits --release + +use std::path::PathBuf; +use std::time::Instant; + +use brk_indexer::Indexer; +use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin}; +use brk_types::{OutputType, Sats, TxIndex, TxOutIndex}; +use vecdb::{AnyVec, VecIndex, VecIterator}; + +const BINS_5PCT: f64 = 4.24; +const BINS_10PCT: f64 = 8.28; +const BINS_20PCT: f64 = 15.84; + +fn bins_to_pct(bins: f64) -> f64 { + (10.0_f64.powf(bins / 200.0) - 1.0) * 100.0 +} + +fn seed_bin(start_height: usize) -> f64 { + let price: f64 = PRICES + .lines() + .nth(start_height - 1) + .expect("prices.txt too short") + .parse() + .expect("Failed to parse seed price"); + cents_to_bin(price * 100.0) +} + +fn leading_digit(sats: u64) -> u8 { + let log = (sats as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let d = (sats as f64 / magnitude).round() as u8; + if d >= 10 { 1 } else { d } +} + +fn is_round(sats: u64) -> bool { + let log = (sats as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let leading = (sats as f64 / magnitude).round(); + let round_val = leading * magnitude; + (sats as f64 - round_val).abs() <= round_val * 0.001 +} + +struct Stats { + total_sq_err: f64, + total_bias: f64, + max_err: f64, + total_blocks: u64, + gt_5pct: u64, + gt_10pct: u64, + gt_20pct: u64, +} + +impl Stats { + fn new() -> Self { + Self { + total_sq_err: 0.0, + total_bias: 0.0, + max_err: 0.0, + total_blocks: 0, + gt_5pct: 0, + gt_10pct: 0, + gt_20pct: 0, + } + } + + fn update(&mut self, err: f64) { + self.total_sq_err += err * err; + self.total_bias += err; + self.total_blocks += 1; + let abs_err = err.abs(); + if abs_err > self.max_err { + self.max_err = abs_err; + } + if abs_err > BINS_5PCT { + self.gt_5pct += 1; + } + if abs_err > BINS_10PCT { + self.gt_10pct += 1; + } + if abs_err > BINS_20PCT { + self.gt_20pct += 1; + } + } + + fn rmse_pct(&self) -> f64 { + bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt()) + } + + fn max_pct(&self) -> f64 { + bins_to_pct(self.max_err) + } + + fn bias(&self) -> f64 { + self.total_bias / self.total_blocks as f64 + } +} + +fn main() { + let t0 = Instant::now(); + + let data_dir = std::env::var("BRK_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| { + let home = std::env::var("HOME").unwrap(); + PathBuf::from(home).join(".brk") + }); + + let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer"); + let total_heights = indexer.vecs.blocks.timestamp.len(); + + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + + let height_ohlc: Vec<[f64; 4]> = serde_json::from_str( + &std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json")) + .expect("Failed to read height_price_ohlc.json"), + ) + .expect("Failed to parse height OHLC"); + + let height_bands: Vec<(f64, f64)> = height_ohlc + .iter() + .map(|ohlc| { + let high = ohlc[1]; + let low = ohlc[2]; + if high > 0.0 && low > 0.0 { + (cents_to_bin(high * 100.0), cents_to_bin(low * 100.0)) + } else { + (0.0, 0.0) + } + }) + .collect(); + + // Configs to compare. + // 987654321 + let masks: &[(u16, &str)] = &[ + (0b0_0111_0111, "{1,2,3,5,6,7}"), + (0b0_0011_0111, "{1,2,3,5,6}"), + (0b0_0001_1111, "{1,2,3,4,5}"), + (0b0_0001_0111, "{1,2,3,5}"), + ]; + + let start_heights: &[usize] = &[575_000, 600_000, 630_000]; + + // (mask_idx, start_idx) -> (Oracle, Stats) + let n = masks.len() * start_heights.len(); + let mut oracles: Vec> = (0..n).map(|_| None).collect(); + let mut stats: Vec = (0..n).map(|_| Stats::new()).collect(); + + let idx = |m: usize, s: usize| -> usize { m * start_heights.len() + s }; + + let total_txs = indexer.vecs.transactions.height.len(); + let total_outputs = indexer.vecs.outputs.value.len(); + + let mut first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter(); + let mut first_txoutindex_iter = indexer.vecs.transactions.first_txoutindex.into_iter(); + let mut out_first_iter = indexer.vecs.outputs.first_txoutindex.into_iter(); + let mut value_iter = indexer.vecs.outputs.value.into_iter(); + let mut outputtype_iter = indexer.vecs.outputs.outputtype.into_iter(); + + let ref_config = Config::default(); + let earliest_start = *start_heights.iter().min().unwrap(); + + for h in START_HEIGHT..total_heights { + let first_txindex: TxIndex = first_txindex_iter.get_at_unwrap(h); + let next_first_txindex = first_txindex_iter + .get_at(h + 1) + .unwrap_or(TxIndex::from(total_txs)); + + let out_start = if first_txindex.to_usize() + 1 < next_first_txindex.to_usize() { + first_txoutindex_iter + .get_at_unwrap(first_txindex.to_usize() + 1) + .to_usize() + } else { + out_first_iter + .get_at(h + 1) + .unwrap_or(TxOutIndex::from(total_outputs)) + .to_usize() + }; + let out_end = out_first_iter + .get_at(h + 1) + .unwrap_or(TxOutIndex::from(total_outputs)) + .to_usize(); + + if h < earliest_start { + continue; + } + + // Build full histogram and per-digit histograms. + let mut full_hist = [0u32; NUM_BINS]; + let mut digit_hist = [[0u32; NUM_BINS]; 9]; + + for i in out_start..out_end { + let sats: Sats = value_iter.get_at_unwrap(i); + let output_type: OutputType = outputtype_iter.get_at_unwrap(i); + if ref_config.excluded_output_types.contains(&output_type) { + continue; + } + if *sats < ref_config.min_sats { + continue; + } + if let Some(bin) = sats_to_bin(sats) { + full_hist[bin] += 1; + if is_round(*sats) { + let d = leading_digit(*sats); + if (1..=9).contains(&d) { + digit_hist[(d - 1) as usize][bin] += 1; + } + } + } + } + + // Feed each (mask, start_height) combo. + for (mi, &(mask, _)) in masks.iter().enumerate() { + // Build filtered histogram for this mask. + let mut hist = full_hist; + (0..9usize).for_each(|d| { + if mask & (1 << d) != 0 { + for bin in 0..NUM_BINS { + hist[bin] -= digit_hist[d][bin]; + } + } + }); + + for (si, &sh) in start_heights.iter().enumerate() { + if h < sh { + continue; + } + let i = idx(mi, si); + if oracles[i].is_none() { + oracles[i] = Some(Oracle::new( + seed_bin(sh), + Config { + exclude_common_round_values: false, + ..Default::default() + }, + )); + } + + let ref_bin = oracles[i].as_mut().unwrap().process_histogram(&hist); + + if h < height_bands.len() { + let (high_bin, low_bin) = height_bands[h]; + if high_bin > 0.0 && low_bin > 0.0 { + let err = if ref_bin < high_bin { + ref_bin - high_bin + } else if ref_bin > low_bin { + ref_bin - low_bin + } else { + 0.0 + }; + stats[i].update(err); + } + } + } + } + } + + // Print results grouped by start height. + for (si, &sh) in start_heights.iter().enumerate() { + println!(); + println!("@ {}k:", sh / 1000); + println!( + " {:<16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}", + "Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias" + ); + println!(" {}", "-".repeat(72)); + for (mi, &(_, label)) in masks.iter().enumerate() { + let s = &stats[idx(mi, si)]; + println!( + " {:<16} {:>8} {:>7.3}% {:>7.1}% {:>6} {:>6} {:>6} {:>+8.2}", + label, + s.total_blocks, + s.rmse_pct(), + s.max_pct(), + s.gt_5pct, + s.gt_10pct, + s.gt_20pct, + s.bias() + ); + } + } + + println!("\nDone in {:.1}s", t0.elapsed().as_secs_f64()); +} diff --git a/crates/brk_oracle/examples/report.rs b/crates/brk_oracle/examples/report.rs index a8a105bf6..b26e7fb3a 100644 --- a/crates/brk_oracle/examples/report.rs +++ b/crates/brk_oracle/examples/report.rs @@ -229,7 +229,8 @@ fn main() { if ref_config.excluded_output_types.contains(&output_type) { continue; } - if *sats < ref_config.min_sats || (ref_config.exclude_round_btc && sats.is_round_btc()) + if *sats < ref_config.min_sats + || (ref_config.exclude_common_round_values && sats.is_common_round_value()) { continue; } @@ -339,7 +340,7 @@ fn main() { daily_days += 1; } - fn daily_stats(errors: &mut Vec) -> (f64, f64, f64) { + fn daily_stats(errors: &mut [f64]) -> (f64, f64, f64) { let n = errors.len() as f64; let rmse = (errors.iter().map(|e| e * e).sum::() / n).sqrt(); errors.sort_by(|a, b| a.abs().partial_cmp(&b.abs()).unwrap()); diff --git a/crates/brk_oracle/examples/sweep_digits.rs b/crates/brk_oracle/examples/sweep_digits.rs new file mode 100644 index 000000000..2c86f7612 --- /dev/null +++ b/crates/brk_oracle/examples/sweep_digits.rs @@ -0,0 +1,407 @@ +//! Sweep round-value digit filter to find optimal configuration. +//! +//! Tests all 512 subsets of leading digits {1,...,9} to find which +//! digits to filter out for best oracle accuracy. +//! +//! Phase 1: single pass over indexer, precompute per-block histograms. +//! Phase 2: run 512 configs in parallel across CPU cores. +//! +//! Run with: cargo run -p brk_oracle --example sweep_digits --release + +use std::path::PathBuf; +use std::time::Instant; + +use brk_indexer::Indexer; +use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin}; +use brk_types::{OutputType, Sats, TxIndex, TxOutIndex}; +use vecdb::{AnyVec, VecIndex, VecIterator}; + +const BINS_5PCT: f64 = 4.24; +const BINS_10PCT: f64 = 8.28; +const BINS_20PCT: f64 = 15.84; + +fn bins_to_pct(bins: f64) -> f64 { + (10.0_f64.powf(bins / 200.0) - 1.0) * 100.0 +} + +fn seed_bin(start_height: usize) -> f64 { + let price: f64 = PRICES + .lines() + .nth(start_height - 1) + .expect("prices.txt too short") + .parse() + .expect("Failed to parse seed price"); + cents_to_bin(price * 100.0) +} + +fn leading_digit(sats: u64) -> u8 { + let log = (sats as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let d = (sats as f64 / magnitude).round() as u8; + if d >= 10 { 1 } else { d } +} + +fn is_round(sats: u64) -> bool { + let log = (sats as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let leading = (sats as f64 / magnitude).round(); + let round_val = leading * magnitude; + (sats as f64 - round_val).abs() <= round_val * 0.001 +} + +fn mask_label(mask: u16) -> String { + let digits: String = (1..=9u8) + .filter(|&d| mask & (1 << (d - 1)) != 0) + .map(|d| char::from_digit(d as u32, 10).unwrap()) + .collect(); + if digits.is_empty() { + "none".to_string() + } else { + digits + } +} + +struct Stats { + total_sq_err: f64, + total_bias: f64, + max_err: f64, + total_blocks: u64, + gt_5pct: u64, + gt_10pct: u64, + gt_20pct: u64, +} + +impl Stats { + fn new() -> Self { + Self { + total_sq_err: 0.0, + total_bias: 0.0, + max_err: 0.0, + total_blocks: 0, + gt_5pct: 0, + gt_10pct: 0, + gt_20pct: 0, + } + } + + fn update(&mut self, err: f64) { + self.total_sq_err += err * err; + self.total_bias += err; + self.total_blocks += 1; + let abs_err = err.abs(); + if abs_err > self.max_err { + self.max_err = abs_err; + } + if abs_err > BINS_5PCT { + self.gt_5pct += 1; + } + if abs_err > BINS_10PCT { + self.gt_10pct += 1; + } + if abs_err > BINS_20PCT { + self.gt_20pct += 1; + } + } + + fn rmse_pct(&self) -> f64 { + bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt()) + } + + fn max_pct(&self) -> f64 { + bins_to_pct(self.max_err) + } + + fn bias(&self) -> f64 { + self.total_bias / self.total_blocks as f64 + } +} + +struct BlockData { + full_hist: Box<[u32; NUM_BINS]>, + /// (bin_index, leading_digit) for outputs that are round values. + round_outputs: Vec<(u16, u8)>, + high_bin: f64, + low_bin: f64, +} + +fn main() { + let t0 = Instant::now(); + + let data_dir = std::env::var("BRK_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| { + let home = std::env::var("HOME").unwrap(); + PathBuf::from(home).join(".brk") + }); + + let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer"); + let total_heights = indexer.vecs.blocks.timestamp.len(); + + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + + let height_ohlc: Vec<[f64; 4]> = serde_json::from_str( + &std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json")) + .expect("Failed to read height_price_ohlc.json"), + ) + .expect("Failed to parse height OHLC"); + + let height_bands: Vec<(f64, f64)> = height_ohlc + .iter() + .map(|ohlc| { + let high = ohlc[1]; + let low = ohlc[2]; + if high > 0.0 && low > 0.0 { + (cents_to_bin(high * 100.0), cents_to_bin(low * 100.0)) + } else { + (0.0, 0.0) + } + }) + .collect(); + + let sweep_start: usize = 575_000; + + // Phase 1: precompute per-block data in a single pass over the indexer. + eprintln!("Phase 1: precomputing block data..."); + + let total_txs = indexer.vecs.transactions.height.len(); + let total_outputs = indexer.vecs.outputs.value.len(); + + let mut first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter(); + let mut first_txoutindex_iter = indexer.vecs.transactions.first_txoutindex.into_iter(); + let mut out_first_iter = indexer.vecs.outputs.first_txoutindex.into_iter(); + let mut value_iter = indexer.vecs.outputs.value.into_iter(); + let mut outputtype_iter = indexer.vecs.outputs.outputtype.into_iter(); + + let ref_config = Config::default(); + let total_blocks = total_heights - sweep_start; + let mut blocks: Vec = Vec::with_capacity(total_blocks); + + for h in START_HEIGHT..total_heights { + let first_txindex: TxIndex = first_txindex_iter.get_at_unwrap(h); + let next_first_txindex = first_txindex_iter + .get_at(h + 1) + .unwrap_or(TxIndex::from(total_txs)); + + let out_start = if first_txindex.to_usize() + 1 < next_first_txindex.to_usize() { + first_txoutindex_iter + .get_at_unwrap(first_txindex.to_usize() + 1) + .to_usize() + } else { + out_first_iter + .get_at(h + 1) + .unwrap_or(TxOutIndex::from(total_outputs)) + .to_usize() + }; + let out_end = out_first_iter + .get_at(h + 1) + .unwrap_or(TxOutIndex::from(total_outputs)) + .to_usize(); + + if h < sweep_start { + continue; + } + + let mut full_hist = Box::new([0u32; NUM_BINS]); + let mut round_outputs = Vec::new(); + + for i in out_start..out_end { + let sats: Sats = value_iter.get_at_unwrap(i); + let output_type: OutputType = outputtype_iter.get_at_unwrap(i); + if ref_config.excluded_output_types.contains(&output_type) { + continue; + } + if *sats < ref_config.min_sats { + continue; + } + if let Some(bin) = sats_to_bin(sats) { + full_hist[bin] += 1; + if is_round(*sats) { + let d = leading_digit(*sats); + if (1..=9).contains(&d) { + round_outputs.push((bin as u16, d)); + } + } + } + } + + let (high_bin, low_bin) = if h < height_bands.len() { + height_bands[h] + } else { + (0.0, 0.0) + }; + + blocks.push(BlockData { + full_hist, + round_outputs, + high_bin, + low_bin, + }); + + if (h - sweep_start).is_multiple_of(50_000) { + eprint!( + "\r {}/{} ({:.0}%)", + h - sweep_start, + total_blocks, + (h - sweep_start) as f64 / total_blocks as f64 * 100.0 + ); + } + } + + let mem_hists = blocks.len() * std::mem::size_of::<[u32; NUM_BINS]>(); + let mem_rounds: usize = blocks.iter().map(|b| b.round_outputs.len() * 3).sum(); + eprintln!( + "\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s", + blocks.len(), + mem_hists as f64 / 1e9, + mem_rounds as f64 / 1e6, + t0.elapsed().as_secs_f64() + ); + + // Phase 2: sweep digit masks in parallel. + // Always filter digit 1 (powers of 10), sweep digits 2-9. + let base_mask: u16 = 1 << 0; // digit 1 always on + let num_masks: usize = 256; // 2^8 subsets of {2,...,9} + let num_threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8); + eprintln!( + "Phase 2: sweeping {} masks across {} threads...", + num_masks, num_threads + ); + + let t1 = Instant::now(); + let blocks = &blocks; // shared reference for threads + + let all_results: Vec<(u16, Stats)> = std::thread::scope(|s| { + let masks_per_thread = num_masks.div_ceil(num_threads); + + let handles: Vec<_> = (0..num_threads) + .map(|t| { + s.spawn(move || { + let mask_start = t * masks_per_thread; + let mask_end = ((t + 1) * masks_per_thread).min(num_masks); + let mut results = Vec::with_capacity(mask_end - mask_start); + + for idx in mask_start..mask_end { + // Shift idx bits into positions 1-8 (digits 2-9) and add base_mask (digit 1). + let mask = base_mask | ((idx as u16) << 1); + let mut oracle = Oracle::new( + seed_bin(sweep_start), + Config { + exclude_common_round_values: false, + ..Default::default() + }, + ); + let mut stats = Stats::new(); + + for bd in blocks.iter() { + let mut hist = *bd.full_hist; + for &(bin, digit) in &bd.round_outputs { + if mask & (1 << (digit - 1)) != 0 { + hist[bin as usize] -= 1; + } + } + + let ref_bin = oracle.process_histogram(&hist); + + if bd.high_bin > 0.0 && bd.low_bin > 0.0 { + let err = if ref_bin < bd.high_bin { + ref_bin - bd.high_bin + } else if ref_bin > bd.low_bin { + ref_bin - bd.low_bin + } else { + 0.0 + }; + stats.update(err); + } + } + + results.push((mask, stats)); + } + + results + }) + }) + .collect(); + + handles + .into_iter() + .flat_map(|h| h.join().unwrap()) + .collect() + }); + + eprintln!(" Done in {:.1}s.", t1.elapsed().as_secs_f64()); + + // Sort by RMSE. + let mut results: Vec<&(u16, Stats)> = all_results.iter().collect(); + results.sort_by(|a, b| a.1.rmse_pct().partial_cmp(&b.1.rmse_pct()).unwrap()); + + // Print top 20. + println!(); + println!("Top 20 (by RMSE):"); + println!( + "{:>4} {:>12} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}", + "#", "Digits", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias" + ); + println!("{}", "-".repeat(70)); + for (rank, (mask, s)) in results.iter().take(20).enumerate() { + println!( + "{:>4} {:>12} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}", + rank + 1, + mask_label(*mask), + s.rmse_pct(), + s.max_pct(), + s.gt_5pct, + s.gt_10pct, + s.gt_20pct, + s.bias() + ); + } + + // Print bottom 5. + println!(); + println!("Bottom 5 (worst):"); + println!( + "{:>4} {:>12} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}", + "#", "Digits", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias" + ); + println!("{}", "-".repeat(70)); + for (mask, s) in results.iter().rev().take(5) { + println!( + "{:>4} {:>12} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}", + "", + mask_label(*mask), + s.rmse_pct(), + s.max_pct(), + s.gt_5pct, + s.gt_10pct, + s.gt_20pct, + s.bias() + ); + } + + // Print current config {1,2,3,5} for reference. + let current_mask: u16 = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 4); // digits 1,2,3,5 + let current_stats = all_results + .iter() + .find(|(m, _)| *m == current_mask) + .map(|(_, s)| s) + .unwrap(); + let current_rank = results + .iter() + .position(|(m, _)| *m == current_mask) + .unwrap(); + println!(); + println!( + "Current {{1,2,3,5}} = rank {}/{}: RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}", + current_rank + 1, + num_masks, + current_stats.rmse_pct(), + current_stats.max_pct(), + current_stats.gt_5pct, + current_stats.gt_10pct, + current_stats.gt_20pct, + ); + + println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64()); +} diff --git a/crates/brk_oracle/examples/validate.rs b/crates/brk_oracle/examples/validate.rs index a2310f69b..36a9f70f2 100644 --- a/crates/brk_oracle/examples/validate.rs +++ b/crates/brk_oracle/examples/validate.rs @@ -174,7 +174,7 @@ fn main() { continue; } if *sats < ref_config.min_sats - || (ref_config.exclude_round_btc && sats.is_round_btc()) + || (ref_config.exclude_common_round_values && sats.is_common_round_value()) { continue; } @@ -237,8 +237,8 @@ fn main() { // trunc w12 @ 600k: 174 >5%, 31 >10%, 0 >20% // trunc w12 @ 630k: 84 >5%, 9 >10%, 0 >20% let expected: &[(&str, u64, u64, u64)] = &[ - ("w12 @ 575k", 261, 40, 0), - ("w12 @ 600k", 174, 31, 0), + ("w12 @ 575k", 237, 22, 0), + ("w12 @ 600k", 152, 15, 0), ("w12 @ 630k", 84, 9, 0), ]; diff --git a/crates/brk_oracle/src/lib.rs b/crates/brk_oracle/src/lib.rs index b4b9165ab..c67fc5917 100644 --- a/crates/brk_oracle/src/lib.rs +++ b/crates/brk_oracle/src/lib.rs @@ -143,7 +143,7 @@ pub struct Config { /// Minimum output value in sats (dust filter). pub min_sats: u64, /// Exclude round BTC amounts that create false stencil matches. - pub exclude_round_btc: bool, + pub exclude_common_round_values: bool, /// Output types to ignore (e.g. P2TR, P2WSH are noisy). pub excluded_output_types: Vec, } @@ -156,7 +156,7 @@ impl Default for Config { search_below: 9, search_above: 11, min_sats: 1000, - exclude_round_btc: true, + exclude_common_round_values: true, excluded_output_types: vec![OutputType::P2TR, OutputType::P2WSH], } } @@ -241,7 +241,7 @@ impl Oracle { if self.config.excluded_output_types.contains(&output_type) { return None; } - if *sats < self.config.min_sats || (self.config.exclude_round_btc && sats.is_round_btc()) { + if *sats < self.config.min_sats || (self.config.exclude_common_round_values && sats.is_common_round_value()) { return None; } sats_to_bin(sats) diff --git a/crates/brk_query/src/impl/address.rs b/crates/brk_query/src/impl/address.rs index a8695682d..f5123d3ff 100644 --- a/crates/brk_query/src/impl/address.rs +++ b/crates/brk_query/src/impl/address.rs @@ -201,6 +201,16 @@ impl Query { Ok(utxos) } + pub fn address_mempool_hash(&self, address: &Address) -> u64 { + let Some(mempool) = self.mempool() else { + return 0; + }; + let Ok(bytes) = AddressBytes::from_str(address) else { + return 0; + }; + mempool.address_hash(&bytes) + } + pub fn address_mempool_txids(&self, address: Address) -> Result> { let mempool = self.mempool().ok_or(Error::MempoolNotAvailable)?; diff --git a/crates/brk_server/src/api/addresses/mod.rs b/crates/brk_server/src/api/addresses/mod.rs index 2421b7567..b3afe34ad 100644 --- a/crates/brk_server/src/api/addresses/mod.rs +++ b/crates/brk_server/src/api/addresses/mod.rs @@ -91,8 +91,8 @@ impl AddressRoutes for ApiRouter { Path(path): Path, State(state): State | { - // Mempool txs for an address - use MaxAge since it's volatile - state.cached_json(&headers, CacheStrategy::MaxAge(5), move |q| q.address_mempool_txids(path.address)).await + let hash = state.sync(|q| q.address_mempool_hash(&path.address)); + state.cached_json(&headers, CacheStrategy::MempoolHash(hash), move |q| q.address_mempool_txids(path.address)).await }, |op| op .id("get_address_mempool_txs") .addresses_tag() diff --git a/crates/brk_server/src/api/mempool/mod.rs b/crates/brk_server/src/api/mempool/mod.rs index bfb27a261..937e1b413 100644 --- a/crates/brk_server/src/api/mempool/mod.rs +++ b/crates/brk_server/src/api/mempool/mod.rs @@ -1,8 +1,8 @@ use aide::axum::{ApiRouter, routing::get_with}; use axum::{extract::State, http::HeaderMap, response::Redirect, routing::get}; -use brk_types::{MempoolBlock, MempoolInfo, RecommendedFees, Txid}; +use brk_types::{Dollars, MempoolBlock, MempoolInfo, RecommendedFees, Txid}; -use crate::{CacheStrategy, extended::TransformResponseExtended}; +use crate::extended::TransformResponseExtended; use super::AppState; @@ -18,7 +18,7 @@ impl MempoolRoutes for ApiRouter { "/api/mempool/info", get_with( async |headers: HeaderMap, State(state): State| { - state.cached_json(&headers, CacheStrategy::MaxAge(5), |q| q.mempool_info()).await + state.cached_json(&headers, state.mempool_cache(), |q| q.mempool_info()).await }, |op| { op.id("get_mempool") @@ -34,7 +34,7 @@ impl MempoolRoutes for ApiRouter { "/api/mempool/txids", get_with( async |headers: HeaderMap, State(state): State| { - state.cached_json(&headers, CacheStrategy::MaxAge(5), |q| q.mempool_txids()).await + state.cached_json(&headers, state.mempool_cache(), |q| q.mempool_txids()).await }, |op| { op.id("get_mempool_txids") @@ -50,7 +50,7 @@ impl MempoolRoutes for ApiRouter { "/api/v1/fees/recommended", get_with( async |headers: HeaderMap, State(state): State| { - state.cached_json(&headers, CacheStrategy::MaxAge(3), |q| q.recommended_fees()).await + state.cached_json(&headers, state.mempool_cache(), |q| q.recommended_fees()).await }, |op| { op.id("get_recommended_fees") @@ -67,7 +67,7 @@ impl MempoolRoutes for ApiRouter { get_with( async |headers: HeaderMap, State(state): State| { state - .cached_json(&headers, CacheStrategy::MaxAge(5), |q| q.live_price()) + .cached_json(&headers, state.mempool_cache(), |q| q.live_price()) .await }, |op| { @@ -75,11 +75,11 @@ impl MempoolRoutes for ApiRouter { .mempool_tag() .summary("Live BTC/USD price") .description( - "Returns the current BTC/USD price in cents, derived from \ + "Returns the current BTC/USD price in dollars, derived from \ on-chain round-dollar output patterns in the last 12 blocks \ plus mempool.", ) - .ok_response::() + .ok_response::() .server_error() }, ), @@ -88,7 +88,7 @@ impl MempoolRoutes for ApiRouter { "/api/v1/fees/mempool-blocks", get_with( async |headers: HeaderMap, State(state): State| { - state.cached_json(&headers, CacheStrategy::MaxAge(5), |q| q.mempool_blocks()).await + state.cached_json(&headers, state.mempool_cache(), |q| q.mempool_blocks()).await }, |op| { op.id("get_mempool_blocks") diff --git a/crates/brk_server/src/cache.rs b/crates/brk_server/src/cache.rs index 4dfb34a22..bbeb8172f 100644 --- a/crates/brk_server/src/cache.rs +++ b/crates/brk_server/src/cache.rs @@ -12,9 +12,9 @@ pub enum CacheStrategy { /// Etag = VERSION only, Cache-Control: must-revalidate Static, - /// Volatile data (mempool) - no etag, just max-age - /// Cache-Control: max-age={seconds} - MaxAge(u64), + /// Mempool data - etag from next projected block hash + short max-age + /// Etag = VERSION-m{hash:x}, Cache-Control: max-age=1, must-revalidate + MempoolHash(u64), } /// Resolved cache parameters @@ -50,9 +50,9 @@ impl CacheParams { etag: Some(VERSION.to_string()), cache_control: "public, max-age=1, must-revalidate".into(), }, - MaxAge(secs) => Self { - etag: None, - cache_control: format!("public, max-age={secs}"), + MempoolHash(hash) => Self { + etag: Some(format!("{VERSION}-m{hash:x}")), + cache_control: "public, max-age=1, must-revalidate".into(), }, } } diff --git a/crates/brk_server/src/state.rs b/crates/brk_server/src/state.rs index 3460b91c2..8a3e3b084 100644 --- a/crates/brk_server/src/state.rs +++ b/crates/brk_server/src/state.rs @@ -30,6 +30,11 @@ pub struct AppState { } impl AppState { + pub fn mempool_cache(&self) -> CacheStrategy { + let hash = self.sync(|q| q.mempool().map(|m| m.next_block_hash()).unwrap_or(0)); + CacheStrategy::MempoolHash(hash) + } + /// JSON response with caching pub async fn cached_json( &self, diff --git a/crates/brk_types/src/addressmempoolstats.rs b/crates/brk_types/src/addressmempoolstats.rs index 1f5d1dd82..4b390aa35 100644 --- a/crates/brk_types/src/addressmempoolstats.rs +++ b/crates/brk_types/src/addressmempoolstats.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// /// Based on mempool.space's format. /// -#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Default, Clone, Hash, Serialize, Deserialize, JsonSchema)] pub struct AddressMempoolStats { /// Number of unconfirmed transaction outputs funding this address #[schemars(example = 0)] diff --git a/crates/brk_types/src/sats.rs b/crates/brk_types/src/sats.rs index 1edd78ecf..a7cf8927a 100644 --- a/crates/brk_types/src/sats.rs +++ b/crates/brk_types/src/sats.rs @@ -26,6 +26,7 @@ use super::{Bitcoin, CentsUnsigned, Dollars, Height}; Default, Serialize, Deserialize, + Hash, Pco, JsonSchema, )] @@ -76,39 +77,51 @@ impl Sats { *self == Self::MAX } - /// Check if value is a "round" BTC amount (±0.1% of common round values). + /// Check if value is a "round" BTC amount (±0.1% of d × 10^n, d ∈ {1,2,3,5,6}). /// Used to filter out non-price-related transactions. - /// Round amounts: 1k, 10k, 20k, 30k, 50k, 100k, 200k, 300k, 500k sats, - /// 0.01, 0.02, 0.03, 0.05, 0.1, 0.2, 0.3, 0.5, 1, 10 BTC - pub fn is_round_btc(&self) -> bool { - const ROUND_SATS: [u64; 19] = [ - 1_000, // 1k sats - 10_000, // 10k sats - 20_000, // 20k sats - 30_000, // 30k sats - 50_000, // 50k sats - 100_000, // 100k sats (0.001 BTC) - 200_000, // 200k sats - 300_000, // 300k sats - 500_000, // 500k sats - 1_000_000, // 0.01 BTC - 2_000_000, // 0.02 BTC - 3_000_000, // 0.03 BTC - 5_000_000, // 0.05 BTC - 10_000_000, // 0.1 BTC - 20_000_000, // 0.2 BTC - 30_000_000, // 0.3 BTC - 50_000_000, // 0.5 BTC - 100_000_000, // 1 BTC - 1_000_000_000, // 10 BTC - ]; - const TOLERANCE: f64 = 0.001; // 0.1% - - let v = self.0 as f64; - ROUND_SATS - .iter() - .any(|&r| (v - r as f64).abs() <= r as f64 * TOLERANCE) + pub fn is_common_round_value(&self) -> bool { + if self.0 == 0 { + return false; + } + let log = (self.0 as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let leading = (self.0 as f64 / magnitude).round() as u64; + if !matches!(leading, 1 | 2 | 3 | 5 | 6 | 10) { + return false; + } + let round_val = leading as f64 * magnitude; + (self.0 as f64 - round_val).abs() <= round_val * 0.001 } + + // pub fn is_common_round_value(&self) -> bool { + // const ROUND_SATS: [u64; 19] = [ + // 1_000, // 1k sats + // 10_000, // 10k sats + // 20_000, // 20k sats + // 30_000, // 30k sats + // 50_000, // 50k sats + // 100_000, // 100k sats (0.001 BTC) + // 200_000, // 200k sats + // 300_000, // 300k sats + // 500_000, // 500k sats + // 1_000_000, // 0.01 BTC + // 2_000_000, // 0.02 BTC + // 3_000_000, // 0.03 BTC + // 5_000_000, // 0.05 BTC + // 10_000_000, // 0.1 BTC + // 20_000_000, // 0.2 BTC + // 30_000_000, // 0.3 BTC + // 50_000_000, // 0.5 BTC + // 100_000_000, // 1 BTC + // 1_000_000_000, // 10 BTC + // ]; + // const TOLERANCE: f64 = 0.001; // 0.1% + // + // let v = self.0 as f64; + // ROUND_SATS + // .iter() + // .any(|&r| (v - r as f64).abs() <= r as f64 * TOLERANCE) + // } } impl Add for Sats { diff --git a/modules/brk-client/index.js b/modules/brk-client/index.js index b532a7d17..96e6c50b6 100644 --- a/modules/brk-client/index.js +++ b/modules/brk-client/index.js @@ -4577,7 +4577,9 @@ function createRatioPattern2(client, acc) { * @typedef {Object} MetricsTree_Price_Oracle * @property {MetricPattern11} priceCents * @property {MetricPattern6} ohlcCents - * @property {MetricPattern6} ohlcDollars + * @property {CloseHighLowOpenPattern2} split + * @property {MetricPattern1} ohlc + * @property {MetricPattern1} ohlcDollars */ /** @@ -6744,7 +6746,9 @@ class BrkClient extends BrkClientBase { oracle: { priceCents: createMetricPattern11(this, 'oracle_price_cents'), ohlcCents: createMetricPattern6(this, 'oracle_ohlc_cents'), - ohlcDollars: createMetricPattern6(this, 'oracle_ohlc_dollars'), + split: createCloseHighLowOpenPattern2(this, 'oracle_price'), + ohlc: createMetricPattern1(this, 'oracle_price_ohlc'), + ohlcDollars: createMetricPattern1(this, 'oracle_ohlc_dollars'), }, }, distribution: { @@ -7361,10 +7365,10 @@ class BrkClient extends BrkClientBase { /** * Live BTC/USD price * - * Returns the current BTC/USD price in cents, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool. + * Returns the current BTC/USD price in dollars, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool. * * Endpoint: `GET /api/mempool/price` - * @returns {Promise} + * @returns {Promise} */ async getLivePrice() { return this.getJson(`/api/mempool/price`); diff --git a/packages/brk_client/brk_client/__init__.py b/packages/brk_client/brk_client/__init__.py index 36a15a4d7..47b43b31d 100644 --- a/packages/brk_client/brk_client/__init__.py +++ b/packages/brk_client/brk_client/__init__.py @@ -3939,7 +3939,9 @@ class MetricsTree_Price_Oracle: def __init__(self, client: BrkClientBase, base_path: str = ''): self.price_cents: MetricPattern11[CentsUnsigned] = MetricPattern11(client, 'oracle_price_cents') self.ohlc_cents: MetricPattern6[OHLCCentsUnsigned] = MetricPattern6(client, 'oracle_ohlc_cents') - self.ohlc_dollars: MetricPattern6[OHLCDollars] = MetricPattern6(client, 'oracle_ohlc_dollars') + self.split: CloseHighLowOpenPattern2[CentsUnsigned] = CloseHighLowOpenPattern2(client, 'oracle_price') + self.ohlc: MetricPattern1[OHLCCentsUnsigned] = MetricPattern1(client, 'oracle_price_ohlc') + self.ohlc_dollars: MetricPattern1[OHLCDollars] = MetricPattern1(client, 'oracle_ohlc_dollars') class MetricsTree_Price: """Metrics tree node.""" @@ -5503,10 +5505,10 @@ class BrkClient(BrkClientBase): Endpoint: `GET /api/mempool/info`""" return self.get_json('/api/mempool/info') - def get_live_price(self) -> float: + def get_live_price(self) -> Dollars: """Live BTC/USD price. - Returns the current BTC/USD price in cents, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool. + Returns the current BTC/USD price in dollars, derived from on-chain round-dollar output patterns in the last 12 blocks plus mempool. Endpoint: `GET /api/mempool/price`""" return self.get_json('/api/mempool/price') diff --git a/website/index.html b/website/index.html index 6f5a719d5..99fd4ce68 100644 --- a/website/index.html +++ b/website/index.html @@ -17,6 +17,10 @@ + + + + @@ -83,8 +87,6 @@
-
-
@@ -107,8 +109,6 @@