From d18c872072acce0feed853986b3287422c6eec7e Mon Sep 17 00:00:00 2001 From: nym21 Date: Fri, 13 Feb 2026 15:25:13 +0100 Subject: [PATCH] global: snapshot --- Cargo.lock | 4 +- crates/brk_oracle/examples/sweep_tolerance.rs | 447 ++++++++++++++++++ crates/brk_server/src/api/mempool/mod.rs | 4 +- crates/brk_server/src/api/metrics/bulk.rs | 62 +-- crates/brk_server/src/api/metrics/data.rs | 62 +-- crates/brk_server/src/api/metrics/legacy.rs | 63 +-- crates/brk_server/src/state.rs | 68 ++- rust-toolchain.toml | 2 +- website/scripts/entry.js | 2 +- website/scripts/main.js | 66 +-- website/scripts/options/full.js | 86 ++-- website/scripts/options/types.js | 4 +- website/scripts/options/unused.js | 96 ++-- website/scripts/panes/chart.js | 8 +- 14 files changed, 707 insertions(+), 267 deletions(-) create mode 100644 crates/brk_oracle/examples/sweep_tolerance.rs diff --git a/Cargo.lock b/Cargo.lock index b9214c607..b38f99287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -784,9 +784,9 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" [[package]] name = "cc" -version = "1.2.55" +version = "1.2.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", "jobserver", diff --git a/crates/brk_oracle/examples/sweep_tolerance.rs b/crates/brk_oracle/examples/sweep_tolerance.rs new file mode 100644 index 000000000..90671ffbc --- /dev/null +++ b/crates/brk_oracle/examples/sweep_tolerance.rs @@ -0,0 +1,447 @@ +//! Sweep round-value tolerance to find optimal rounding threshold. +//! +//! Tests different tolerance percentages (0%, 0.01%, 0.1%, 1%, etc.) for +//! detecting round BTC amounts, combined with several digit filter masks. +//! +//! Phase 1: single pass over indexer, store per-output relative errors. +//! Phase 2: sweep tolerance × mask combos across CPU cores. +//! +//! Run with: cargo run -p brk_oracle --example sweep_tolerance --release + +use std::path::PathBuf; +use std::time::Instant; + +use brk_indexer::Indexer; +use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin}; +use brk_types::{OutputType, Sats, TxIndex, TxOutIndex}; +use vecdb::{AnyVec, VecIndex, VecIterator}; + +const BINS_5PCT: f64 = 4.24; +const BINS_10PCT: f64 = 8.28; +const BINS_20PCT: f64 = 15.84; + +fn bins_to_pct(bins: f64) -> f64 { + (10.0_f64.powf(bins / 200.0) - 1.0) * 100.0 +} + +fn seed_bin(start_height: usize) -> f64 { + let price: f64 = PRICES + .lines() + .nth(start_height - 1) + .expect("prices.txt too short") + .parse() + .expect("Failed to parse seed price"); + cents_to_bin(price * 100.0) +} + +fn leading_digit(sats: u64) -> u8 { + let log = (sats as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let d = (sats as f64 / magnitude).round() as u8; + if d >= 10 { 1 } else { d } +} + +/// Returns the relative error of `sats` from its nearest round value (d × 10^n). +/// e.g. 10_050 → leading=1, round_val=10_000, rel_err = 50/10000 = 0.005 +fn relative_roundness(sats: u64) -> f64 { + let log = (sats as f64).log10(); + let magnitude = 10.0_f64.powf(log.floor()); + let leading = (sats as f64 / magnitude).round(); + let round_val = leading * magnitude; + (sats as f64 - round_val).abs() / round_val +} + +struct Stats { + total_sq_err: f64, + total_bias: f64, + max_err: f64, + total_blocks: u64, + gt_5pct: u64, + gt_10pct: u64, + gt_20pct: u64, +} + +impl Stats { + fn new() -> Self { + Self { + total_sq_err: 0.0, + total_bias: 0.0, + max_err: 0.0, + total_blocks: 0, + gt_5pct: 0, + gt_10pct: 0, + gt_20pct: 0, + } + } + + fn update(&mut self, err: f64) { + self.total_sq_err += err * err; + self.total_bias += err; + self.total_blocks += 1; + let abs_err = err.abs(); + if abs_err > self.max_err { + self.max_err = abs_err; + } + if abs_err > BINS_5PCT { + self.gt_5pct += 1; + } + if abs_err > BINS_10PCT { + self.gt_10pct += 1; + } + if abs_err > BINS_20PCT { + self.gt_20pct += 1; + } + } + + fn rmse_pct(&self) -> f64 { + bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt()) + } + + fn max_pct(&self) -> f64 { + bins_to_pct(self.max_err) + } + + fn bias(&self) -> f64 { + self.total_bias / self.total_blocks as f64 + } +} + +/// Per-output data: bin index, leading digit, relative error from round value. +struct RoundOutput { + bin: u16, + digit: u8, + rel_err: f32, // f32 is plenty of precision, saves memory +} + +struct BlockData { + full_hist: Box<[u32; NUM_BINS]>, + round_outputs: Vec, + high_bin: f64, + low_bin: f64, +} + +fn main() { + let t0 = Instant::now(); + + let data_dir = std::env::var("BRK_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| { + let home = std::env::var("HOME").unwrap(); + PathBuf::from(home).join(".brk") + }); + + let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer"); + let total_heights = indexer.vecs.blocks.timestamp.len(); + + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + + let height_ohlc: Vec<[f64; 4]> = serde_json::from_str( + &std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json")) + .expect("Failed to read height_price_ohlc.json"), + ) + .expect("Failed to parse height OHLC"); + + let height_bands: Vec<(f64, f64)> = height_ohlc + .iter() + .map(|ohlc| { + let high = ohlc[1]; + let low = ohlc[2]; + if high > 0.0 && low > 0.0 { + (cents_to_bin(high * 100.0), cents_to_bin(low * 100.0)) + } else { + (0.0, 0.0) + } + }) + .collect(); + + let sweep_start: usize = 575_000; + + // Phase 1: precompute per-block data. + // Store all potentially-round outputs with their relative error so we can + // filter at different tolerance thresholds in Phase 2. + eprintln!("Phase 1: precomputing block data..."); + + let total_txs = indexer.vecs.transactions.height.len(); + let total_outputs = indexer.vecs.outputs.value.len(); + + let mut first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter(); + let mut first_txoutindex_iter = indexer.vecs.transactions.first_txoutindex.into_iter(); + let mut out_first_iter = indexer.vecs.outputs.first_txoutindex.into_iter(); + let mut value_iter = indexer.vecs.outputs.value.into_iter(); + let mut outputtype_iter = indexer.vecs.outputs.outputtype.into_iter(); + + let ref_config = Config::default(); + let total_blocks = total_heights - sweep_start; + let mut blocks: Vec = Vec::with_capacity(total_blocks); + + // Use the widest tolerance we'll test (5%) to decide what to store. + // Outputs beyond 5% relative error will never be filtered at any tolerance. + let max_tolerance: f64 = 0.05; + + for h in START_HEIGHT..total_heights { + let first_txindex: TxIndex = first_txindex_iter.get_at_unwrap(h); + let next_first_txindex = first_txindex_iter + .get_at(h + 1) + .unwrap_or(TxIndex::from(total_txs)); + + let out_start = if first_txindex.to_usize() + 1 < next_first_txindex.to_usize() { + first_txoutindex_iter + .get_at_unwrap(first_txindex.to_usize() + 1) + .to_usize() + } else { + out_first_iter + .get_at(h + 1) + .unwrap_or(TxOutIndex::from(total_outputs)) + .to_usize() + }; + let out_end = out_first_iter + .get_at(h + 1) + .unwrap_or(TxOutIndex::from(total_outputs)) + .to_usize(); + + if h < sweep_start { + continue; + } + + let mut full_hist = Box::new([0u32; NUM_BINS]); + let mut round_outputs = Vec::new(); + + for i in out_start..out_end { + let sats: Sats = value_iter.get_at_unwrap(i); + let output_type: OutputType = outputtype_iter.get_at_unwrap(i); + if ref_config.excluded_output_types.contains(&output_type) { + continue; + } + if *sats < ref_config.min_sats { + continue; + } + if let Some(bin) = sats_to_bin(sats) { + full_hist[bin] += 1; + let d = leading_digit(*sats); + if (1..=9).contains(&d) { + let rel_err = relative_roundness(*sats); + if rel_err <= max_tolerance { + round_outputs.push(RoundOutput { + bin: bin as u16, + digit: d, + rel_err: rel_err as f32, + }); + } + } + } + } + + let (high_bin, low_bin) = if h < height_bands.len() { + height_bands[h] + } else { + (0.0, 0.0) + }; + + blocks.push(BlockData { + full_hist, + round_outputs, + high_bin, + low_bin, + }); + + if (h - sweep_start).is_multiple_of(50_000) { + eprint!( + "\r {}/{} ({:.0}%)", + h - sweep_start, + total_blocks, + (h - sweep_start) as f64 / total_blocks as f64 * 100.0 + ); + } + } + + let mem_hists = blocks.len() * std::mem::size_of::<[u32; NUM_BINS]>(); + let mem_rounds: usize = blocks + .iter() + .map(|b| b.round_outputs.len() * std::mem::size_of::()) + .sum(); + eprintln!( + "\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s", + blocks.len(), + mem_hists as f64 / 1e9, + mem_rounds as f64 / 1e6, + t0.elapsed().as_secs_f64() + ); + + // Phase 2: sweep tolerance × mask combos. + // Tolerances as fractions (not percentages). + let tolerances: &[(f64, &str)] = &[ + (0.0, "0%"), + (0.0001, "0.01%"), + (0.0005, "0.05%"), + (0.001, "0.1%"), + (0.002, "0.2%"), + (0.005, "0.5%"), + (0.01, "1%"), + (0.02, "2%"), + (0.05, "5%"), + ]; + + // 987654321 + let masks: &[(u16, &str)] = &[ + (0b0_0000_0000, "none"), + (0b0_0001_0111, "{1,2,3,5}"), + (0b0_0001_1111, "{1,2,3,4,5}"), + (0b0_0011_0111, "{1,2,3,5,6}"), + (0b0_0111_0111, "{1,2,3,5,6,7}"), + (0b1_1111_1111, "{1-9}"), + ]; + + let num_configs = tolerances.len() * masks.len(); + let num_threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8); + eprintln!( + "Phase 2: sweeping {} configs ({} tolerances × {} masks) across {} threads...", + num_configs, + tolerances.len(), + masks.len(), + num_threads + ); + + let t1 = Instant::now(); + let blocks = &blocks; + let tolerances_ref = tolerances; + let masks_ref = masks; + + let all_results: Vec<(usize, usize, Stats)> = std::thread::scope(|s| { + let configs_per_thread = num_configs.div_ceil(num_threads); + + let handles: Vec<_> = (0..num_threads) + .map(|t| { + s.spawn(move || { + let cfg_start = t * configs_per_thread; + let cfg_end = ((t + 1) * configs_per_thread).min(num_configs); + if cfg_start >= cfg_end { + return vec![]; + } + let mut results = Vec::with_capacity(cfg_end - cfg_start); + + for cfg_idx in cfg_start..cfg_end { + let ti = cfg_idx / masks_ref.len(); + let mi = cfg_idx % masks_ref.len(); + let (tolerance, _) = tolerances_ref[ti]; + let (mask, _) = masks_ref[mi]; + + let mut oracle = Oracle::new( + seed_bin(sweep_start), + Config { + exclude_common_round_values: false, + ..Default::default() + }, + ); + let mut stats = Stats::new(); + + for bd in blocks.iter() { + let mut hist = *bd.full_hist; + + // Remove outputs matching this tolerance + mask. + let tol_f32 = tolerance as f32; + for ro in &bd.round_outputs { + if mask & (1 << (ro.digit - 1)) != 0 + && ro.rel_err <= tol_f32 + { + hist[ro.bin as usize] -= 1; + } + } + + let ref_bin = oracle.process_histogram(&hist); + + if bd.high_bin > 0.0 && bd.low_bin > 0.0 { + let err = if ref_bin < bd.high_bin { + ref_bin - bd.high_bin + } else if ref_bin > bd.low_bin { + ref_bin - bd.low_bin + } else { + 0.0 + }; + stats.update(err); + } + } + + results.push((ti, mi, stats)); + } + + results + }) + }) + .collect(); + + handles + .into_iter() + .flat_map(|h| h.join().unwrap()) + .collect() + }); + + eprintln!(" Done in {:.1}s.", t1.elapsed().as_secs_f64()); + + // Print results grouped by tolerance. + println!(); + println!( + "{:>8} {:>16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}", + "Tol", "Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias" + ); + println!("{}", "-".repeat(88)); + + for (ti, &(_, tol_label)) in tolerances.iter().enumerate() { + for (mi, &(_, mask_label)) in masks.iter().enumerate() { + let (_, _, stats) = all_results + .iter() + .find(|(t, m, _)| *t == ti && *m == mi) + .unwrap(); + println!( + "{:>8} {:>16} {:>8} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}", + tol_label, + mask_label, + stats.total_blocks, + stats.rmse_pct(), + stats.max_pct(), + stats.gt_5pct, + stats.gt_10pct, + stats.gt_20pct, + stats.bias() + ); + } + println!(); + } + + // Find overall best config by RMSE. + let best = all_results + .iter() + .min_by(|a, b| a.2.rmse_pct().partial_cmp(&b.2.rmse_pct()).unwrap()) + .unwrap(); + let (bti, bmi, bs) = best; + println!( + "Best: tolerance={}, digits={} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}", + tolerances[*bti].1, + masks[*bmi].1, + bs.rmse_pct(), + bs.max_pct(), + bs.gt_5pct, + bs.gt_10pct, + bs.gt_20pct, + ); + + // Show current config for reference. + let current = all_results + .iter() + .find(|(t, m, _)| { + tolerances[*t].0 == 0.001 && masks[*m].0 == 0b0_0011_0111 + }) + .unwrap(); + let (_, _, cs) = current; + println!( + "Current: tolerance=0.1%, digits={{1,2,3,5,6}} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}", + cs.rmse_pct(), + cs.max_pct(), + cs.gt_5pct, + cs.gt_10pct, + cs.gt_20pct, + ); + + println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64()); +} diff --git a/crates/brk_server/src/api/mempool/mod.rs b/crates/brk_server/src/api/mempool/mod.rs index 937e1b413..c553afcd2 100644 --- a/crates/brk_server/src/api/mempool/mod.rs +++ b/crates/brk_server/src/api/mempool/mod.rs @@ -67,7 +67,9 @@ impl MempoolRoutes for ApiRouter { get_with( async |headers: HeaderMap, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), |q| q.live_price()) + .server_cached_json(&headers, state.mempool_cache(), "price", |q| { + q.live_price() + }) .await }, |op| { diff --git a/crates/brk_server/src/api/metrics/bulk.rs b/crates/brk_server/src/api/metrics/bulk.rs index b4359ab18..af976f0a4 100644 --- a/crates/brk_server/src/api/metrics/bulk.rs +++ b/crates/brk_server/src/api/metrics/bulk.rs @@ -1,14 +1,13 @@ -use std::{net::SocketAddr, time::Duration}; +use std::net::SocketAddr; use axum::{ Extension, - body::Body, + body::{Body, Bytes}, extract::{Query, State}, http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, }; use brk_types::{Format, MetricSelection, Output}; -use quick_cache::sync::GuardResult; use crate::{ Result, @@ -23,56 +22,41 @@ pub async fn handler( headers: HeaderMap, Extension(addr): Extension, Query(params): Query, - State(AppState { query, cache, .. }): State, + State(state): State, ) -> Result { // Phase 1: Search and resolve metadata (cheap) - let resolved = query.run(move |q| q.resolve(params, max_weight(&addr))).await?; + let resolved = state.run(move |q| q.resolve(params, max_weight(&addr))).await?; let format = resolved.format(); let etag = resolved.etag(); - // Check if client has fresh cache if headers.has_etag(etag.as_str()) { - let response = (StatusCode::NOT_MODIFIED, "").into_response(); - return Ok(response); + return Ok((StatusCode::NOT_MODIFIED, "").into_response()); } - // Check server-side cache + // Phase 2: Format (expensive, server-side cached) let cache_key = format!("bulk-{}{}{}", uri.path(), uri.query().unwrap_or(""), etag); - let guard_res = cache.get_value_or_guard(&cache_key, Some(Duration::from_millis(50))); - - let mut response = if let GuardResult::Value(v) = guard_res { - Response::new(Body::from(v)) - } else { - // Phase 2: Format (expensive, only on cache miss) - let metric_output = query.run(move |q| q.format(resolved)).await?; - - match metric_output.output { - Output::CSV(s) => { - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(s.clone().into()); - } - s.into_response() - } - Output::Json(v) => { - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(v.clone().into()); - } - Response::new(Body::from(v)) - } - } - }; - - let headers = response.headers_mut(); - headers.insert_etag(etag.as_str()); - headers.insert_cache_control(CACHE_CONTROL); + let query = &state; + let bytes = state + .get_or_insert(&cache_key, async move { + let out = query.run(move |q| q.format(resolved)).await?; + Ok(match out.output { + Output::CSV(s) => Bytes::from(s), + Output::Json(v) => Bytes::from(v), + }) + }) + .await?; + let mut response = Response::new(Body::from(bytes)); + let h = response.headers_mut(); + h.insert_etag(etag.as_str()); + h.insert_cache_control(CACHE_CONTROL); match format { Format::CSV => { - headers.insert_content_disposition_attachment(); - headers.insert_content_type_text_csv() + h.insert_content_disposition_attachment(); + h.insert_content_type_text_csv() } - Format::JSON => headers.insert_content_type_application_json(), + Format::JSON => h.insert_content_type_application_json(), } Ok(response) diff --git a/crates/brk_server/src/api/metrics/data.rs b/crates/brk_server/src/api/metrics/data.rs index eed715b58..3a4e5282e 100644 --- a/crates/brk_server/src/api/metrics/data.rs +++ b/crates/brk_server/src/api/metrics/data.rs @@ -1,14 +1,13 @@ -use std::{net::SocketAddr, time::Duration}; +use std::net::SocketAddr; use axum::{ Extension, - body::Body, + body::{Body, Bytes}, extract::{Query, State}, http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, }; use brk_types::{Format, MetricSelection, Output}; -use quick_cache::sync::GuardResult; use crate::{ Result, @@ -23,56 +22,41 @@ pub async fn handler( headers: HeaderMap, Extension(addr): Extension, Query(params): Query, - State(AppState { query, cache, .. }): State, + State(state): State, ) -> Result { // Phase 1: Search and resolve metadata (cheap) - let resolved = query.run(move |q| q.resolve(params, max_weight(&addr))).await?; + let resolved = state.run(move |q| q.resolve(params, max_weight(&addr))).await?; let format = resolved.format(); let etag = resolved.etag(); - // Check if client has fresh cache if headers.has_etag(etag.as_str()) { - let response = (StatusCode::NOT_MODIFIED, "").into_response(); - return Ok(response); + return Ok((StatusCode::NOT_MODIFIED, "").into_response()); } - // Check server-side cache + // Phase 2: Format (expensive, server-side cached) let cache_key = format!("single-{}{}{}", uri.path(), uri.query().unwrap_or(""), etag); - let guard_res = cache.get_value_or_guard(&cache_key, Some(Duration::from_millis(50))); - - let mut response = if let GuardResult::Value(v) = guard_res { - Response::new(Body::from(v)) - } else { - // Phase 2: Format (expensive, only on cache miss) - let metric_output = query.run(move |q| q.format(resolved)).await?; - - match metric_output.output { - Output::CSV(s) => { - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(s.clone().into()); - } - s.into_response() - } - Output::Json(v) => { - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(v.clone().into()); - } - Response::new(Body::from(v)) - } - } - }; - - let headers = response.headers_mut(); - headers.insert_etag(etag.as_str()); - headers.insert_cache_control(CACHE_CONTROL); + let query = &state; + let bytes = state + .get_or_insert(&cache_key, async move { + let out = query.run(move |q| q.format(resolved)).await?; + Ok(match out.output { + Output::CSV(s) => Bytes::from(s), + Output::Json(v) => Bytes::from(v), + }) + }) + .await?; + let mut response = Response::new(Body::from(bytes)); + let h = response.headers_mut(); + h.insert_etag(etag.as_str()); + h.insert_cache_control(CACHE_CONTROL); match format { Format::CSV => { - headers.insert_content_disposition_attachment(); - headers.insert_content_type_text_csv() + h.insert_content_disposition_attachment(); + h.insert_content_type_text_csv() } - Format::JSON => headers.insert_content_type_application_json(), + Format::JSON => h.insert_content_type_application_json(), } Ok(response) diff --git a/crates/brk_server/src/api/metrics/legacy.rs b/crates/brk_server/src/api/metrics/legacy.rs index 8bd5c038b..c190b132c 100644 --- a/crates/brk_server/src/api/metrics/legacy.rs +++ b/crates/brk_server/src/api/metrics/legacy.rs @@ -1,14 +1,13 @@ -use std::{net::SocketAddr, time::Duration}; +use std::net::SocketAddr; use axum::{ Extension, - body::Body, + body::{Body, Bytes}, extract::{Query, State}, http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, }; use brk_types::{Format, MetricSelection, OutputLegacy}; -use quick_cache::sync::GuardResult; use crate::{ Result, @@ -23,57 +22,41 @@ pub async fn handler( headers: HeaderMap, Extension(addr): Extension, Query(params): Query, - State(AppState { query, cache, .. }): State, + State(state): State, ) -> Result { // Phase 1: Search and resolve metadata (cheap) - let resolved = query.run(move |q| q.resolve(params, max_weight(&addr))).await?; + let resolved = state.run(move |q| q.resolve(params, max_weight(&addr))).await?; let format = resolved.format(); let etag = resolved.etag(); - // Check if client has fresh cache if headers.has_etag(etag.as_str()) { - let response = (StatusCode::NOT_MODIFIED, "").into_response(); - return Ok(response); + return Ok((StatusCode::NOT_MODIFIED, "").into_response()); } - // Check server-side cache + // Phase 2: Format (expensive, server-side cached) let cache_key = format!("legacy-{}{}{}", uri.path(), uri.query().unwrap_or(""), etag); - let guard_res = cache.get_value_or_guard(&cache_key, Some(Duration::from_millis(50))); - - let mut response = if let GuardResult::Value(v) = guard_res { - Response::new(Body::from(v)) - } else { - // Phase 2: Format (expensive, only on cache miss) - let metric_output = query.run(move |q| q.format_legacy(resolved)).await?; - - match metric_output.output { - OutputLegacy::CSV(s) => { - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(s.clone().into()); - } - s.into_response() - } - OutputLegacy::Json(v) => { - let json = v.to_vec(); - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(json.clone().into()); - } - json.into_response() - } - } - }; - - let headers = response.headers_mut(); - headers.insert_etag(etag.as_str()); - headers.insert_cache_control(CACHE_CONTROL); + let query = &state; + let bytes = state + .get_or_insert(&cache_key, async move { + let out = query.run(move |q| q.format_legacy(resolved)).await?; + Ok(match out.output { + OutputLegacy::CSV(s) => Bytes::from(s), + OutputLegacy::Json(v) => Bytes::from(v.to_vec()), + }) + }) + .await?; + let mut response = Response::new(Body::from(bytes)); + let h = response.headers_mut(); + h.insert_etag(etag.as_str()); + h.insert_cache_control(CACHE_CONTROL); match format { Format::CSV => { - headers.insert_content_disposition_attachment(); - headers.insert_content_type_text_csv() + h.insert_content_disposition_attachment(); + h.insert_content_type_text_csv() } - Format::JSON => headers.insert_content_type_application_json(), + Format::JSON => h.insert_content_type_application_json(), } Ok(response) diff --git a/crates/brk_server/src/state.rs b/crates/brk_server/src/state.rs index 8a3e3b084..80216ca10 100644 --- a/crates/brk_server/src/state.rs +++ b/crates/brk_server/src/state.rs @@ -14,7 +14,7 @@ use serde::Serialize; use crate::{ CacheParams, CacheStrategy, Website, - extended::{ResponseExtended, ResultExtended}, + extended::{HeaderMapExtended, ResponseExtended, ResultExtended}, }; #[derive(Clone, Deref)] @@ -56,6 +56,72 @@ impl AppState { } } + /// JSON response with HTTP caching + server-side cache + pub async fn server_cached_json( + &self, + headers: &HeaderMap, + strategy: CacheStrategy, + cache_prefix: &str, + f: F, + ) -> Response + where + T: Serialize + Send + 'static, + F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, + { + let params = CacheParams::resolve(&strategy, || self.sync(|q| q.height().into())); + if params.matches_etag(headers) { + return ResponseExtended::new_not_modified(); + } + + let cache_key = format!("{cache_prefix}-{}", params.etag_str()); + let result = self + .get_or_insert(&cache_key, async move { + let value = self.run(f).await?; + Ok(serde_json::to_vec(&value).unwrap().into()) + }) + .await; + + match result { + Ok(bytes) => { + let mut response = Response::new(Body::from(bytes)); + let h = response.headers_mut(); + h.insert_content_type_application_json(); + h.insert_cache_control(¶ms.cache_control); + if let Some(etag) = ¶ms.etag { + h.insert_etag(etag); + } + response + } + Err(e) => ResultExtended::::to_json_response(Err(e), params.etag_str()), + } + } + + /// Check server-side cache, compute on miss + pub async fn get_or_insert( + &self, + cache_key: &str, + compute: impl std::future::Future>, + ) -> brk_error::Result { + use quick_cache::sync::GuardResult; + + let guard_res = self.cache.get_value_or_guard( + cache_key, + Some(std::time::Duration::from_millis(50)), + ); + + if let GuardResult::Value(bytes) = guard_res { + return Ok(bytes); + } + + let bytes = compute.await?; + + if let GuardResult::Guard(g) = guard_res { + let _ = g.insert(bytes.clone()); + } + + Ok(bytes) + } + /// Text response with caching pub async fn cached_text( &self, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 075062e5e..535513300 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.93.0" +channel = "1.93.1" diff --git a/website/scripts/entry.js b/website/scripts/entry.js index a8d78fab0..3ef381e09 100644 --- a/website/scripts/entry.js +++ b/website/scripts/entry.js @@ -1 +1 @@ -import("./main.js"); +import "./main.js"; diff --git a/website/scripts/main.js b/website/scripts/main.js index d4db5df84..6be9df045 100644 --- a/website/scripts/main.js +++ b/website/scripts/main.js @@ -8,7 +8,6 @@ import { setOption as setChartOption, } from "./panes/chart.js"; import { initSearch } from "./panes/search.js"; -import { next } from "./utils/timing.js"; import { replaceHistory } from "./utils/url.js"; import { removeStored, writeToStorage } from "./utils/storage.js"; import { @@ -195,68 +194,13 @@ function initSelected() { } initSelected(); -onFirstIntersection(navElement, async () => { +requestIdleCallback(() => options.setParent(navElement)); + +onFirstIntersection(navElement, () => { options.setParent(navElement); - const option = options.selected.value; - if (!option) throw "Selected should be set by now"; - const path = [...option.path]; - - /** @type {HTMLUListElement | null} */ - let ul = /** @type {any} */ (null); - async function getFirstChild() { - try { - ul = /** @type {HTMLUListElement} */ (navElement.firstElementChild); - await next(); - if (!ul) { - await getFirstChild(); - } - } catch (_) { - await next(); - await getFirstChild(); - } - } - await getFirstChild(); - if (!ul) throw Error("Unreachable"); - - while (path.length > 1) { - const name = path.shift(); - if (!name) throw "Unreachable"; - /** @type {HTMLDetailsElement[]} */ - let detailsList = []; - while (!detailsList.length) { - detailsList = Array.from(ul.querySelectorAll(":scope > li > details")); - if (!detailsList.length) { - await next(); - } - } - const details = detailsList.find((s) => s.dataset.name == name); - if (!details) return; - details.open = true; - ul = null; - while (!ul) { - const uls = /** @type {HTMLUListElement[]} */ ( - Array.from(details.querySelectorAll(":scope > ul")) - ); - if (!uls.length) { - await next(); - } else if (uls.length > 1) { - throw "Shouldn't be possible"; - } else { - ul = /** @type {HTMLUListElement} */ (uls.pop()); - } - } - } - /** @type {HTMLAnchorElement[]} */ - let anchors = []; - while (!anchors.length) { - anchors = Array.from(ul.querySelectorAll(":scope > li > a")); - if (!anchors.length) { - await next(); - } - } - anchors - .find((a) => a.getAttribute("href") == window.document.location.pathname) + navElement + .querySelector(`a[href="${window.document.location.pathname}"]`) ?.scrollIntoView({ behavior: "instant", block: "center", diff --git a/website/scripts/options/full.js b/website/scripts/options/full.js index 4a8b33b69..fae11f490 100644 --- a/website/scripts/options/full.js +++ b/website/scripts/options/full.js @@ -3,13 +3,7 @@ import { createButtonElement, createAnchorElement } from "../utils/dom.js"; import { pushHistory, resetParams } from "../utils/url.js"; import { readStored, writeToStorage } from "../utils/storage.js"; import { stringToId } from "../utils/format.js"; -import { - collect, - markUsed, - logUnused, - extractTreeStructure, -} from "./unused.js"; -import { localhost } from "../utils/env.js"; +import { logUnused } from "./unused.js"; import { setQr } from "../panes/share.js"; import { getConstant } from "./constants.js"; import { colors } from "../utils/colors.js"; @@ -17,8 +11,6 @@ import { Unit } from "../utils/units.js"; import { brk } from "../client.js"; export function initOptions() { - collect(brk.metrics); - const LS_SELECTED_KEY = `selected_path`; const urlPath_ = window.document.location.pathname @@ -31,11 +23,6 @@ export function initOptions() { const partialOptions = createPartialOptions(); - // Log tree structure for analysis (localhost only) - if (localhost) { - console.log(extractTreeStructure(partialOptions)); - } - /** @type {Option[]} */ const list = []; @@ -45,18 +32,26 @@ export function initOptions() { /** @type {Set<(option: Option) => void>} */ const selectedListeners = new Set(); + /** @type {HTMLLIElement[]} */ + let highlightedLis = []; + /** * @param {Option | undefined} sel */ function updateHighlight(sel) { if (!sel) return; - liByPath.forEach((li) => { + for (const li of highlightedLis) { delete li.dataset.highlight; - }); - for (let i = 1; i <= sel.path.length; i++) { - const pathKey = sel.path.slice(0, i).join("/"); + } + highlightedLis = []; + let pathKey = ""; + for (const segment of sel.path) { + pathKey = pathKey ? `${pathKey}/${segment}` : segment; const li = liByPath.get(pathKey); - if (li) li.dataset.highlight = ""; + if (li) { + li.dataset.highlight = ""; + highlightedLis.push(li); + } } } @@ -89,6 +84,24 @@ export function initOptions() { ); } + /** + * @template T + * @param {() => T} fn + * @returns {() => T} + */ + function lazy(fn) { + /** @type {T | undefined} */ + let cached; + let computed = false; + return () => { + if (!computed) { + computed = true; + cached = fn(); + } + return /** @type {T} */ (cached); + }; + } + /** * @param {(AnyFetchedSeriesBlueprint | FetchedPriceSeriesBlueprint)[]} [arr] */ @@ -122,11 +135,9 @@ export function initOptions() { ); if (maybePriceMetric.dollars?.by && maybePriceMetric.sats?.by) { const { dollars, sats } = maybePriceMetric; - markUsed(dollars); if (!usdArr) map.set(Unit.usd, (usdArr = [])); usdArr.push({ ...blueprint, metric: dollars, unit: Unit.usd }); - markUsed(sats); if (!satsArr) map.set(Unit.sats, (satsArr = [])); satsArr.push({ ...blueprint, metric: sats, unit: Unit.sats }); continue; @@ -140,7 +151,6 @@ export function initOptions() { const unit = regularBlueprint.unit; if (!unit) continue; - markUsed(metric); let unitArr = map.get(unit); if (!unitArr) map.set(unit, (unitArr = [])); unitArr.push(regularBlueprint); @@ -169,7 +179,6 @@ export function initOptions() { if (!arr) continue; for (const baseValue of values) { const metric = getConstant(brk.metrics.constants, baseValue); - markUsed(metric); arr.push({ metric, title: `${baseValue}`, @@ -240,8 +249,8 @@ export function initOptions() { let savedOption; /** - * @typedef {{ type: "group"; name: string; serName: string; path: string[]; count: number; children: ProcessedNode[] }} ProcessedGroup - * @typedef {{ type: "option"; option: Option; path: string[] }} ProcessedOption + * @typedef {{ type: "group"; name: string; serName: string; path: string[]; pathKey: string; count: number; children: ProcessedNode[] }} ProcessedGroup + * @typedef {{ type: "option"; option: Option; path: string[]; pathKey: string }} ProcessedOption * @typedef {ProcessedGroup | ProcessedOption} ProcessedNode */ @@ -285,6 +294,7 @@ export function initOptions() { name: anyPartial.name, serName, path, + pathKey: pathStr, count, children, }); @@ -322,6 +332,10 @@ export function initOptions() { ); } else { const title = option.title || name; + const topArr = anyPartial.top; + const bottomArr = anyPartial.bottom; + const topFn = lazy(() => arrayToMap(topArr)); + const bottomFn = lazy(() => arrayToMap(bottomArr)); Object.assign( option, /** @satisfies {ChartOption} */ ({ @@ -329,8 +343,8 @@ export function initOptions() { name, title, path, - top: arrayToMap(anyPartial.top), - bottom: arrayToMap(anyPartial.bottom), + top: topFn, + bottom: bottomFn, }), ); } @@ -349,6 +363,7 @@ export function initOptions() { type: "option", option, path, + pathKey: pathStr, }); } } @@ -356,8 +371,8 @@ export function initOptions() { return { nodes, count: totalCount }; } + logUnused(brk.metrics, partialOptions); const { nodes: processedTree } = processPartialTree(partialOptions); - logUnused(); /** * @param {ProcessedNode[]} nodes @@ -365,16 +380,16 @@ export function initOptions() { */ function buildTreeDOM(nodes, parentEl) { const ul = window.document.createElement("ul"); - parentEl.append(ul); for (const node of nodes) { const li = window.document.createElement("li"); ul.append(li); - const pathKey = node.path.join("/"); - liByPath.set(pathKey, li); + liByPath.set(node.pathKey, li); - if (isOnSelectedPath(node.path)) { + const onSelectedPath = isOnSelectedPath(node.path); + + if (onSelectedPath) { li.dataset.highlight = ""; } @@ -392,6 +407,11 @@ export function initOptions() { summary.append(count); let built = false; + if (onSelectedPath) { + built = true; + details.open = true; + buildTreeDOM(node.children, details); + } details.addEventListener("toggle", () => { if (details.open && !built) { built = true; @@ -405,6 +425,8 @@ export function initOptions() { li.append(element); } } + + parentEl.append(ul); } /** @type {HTMLElement | null} */ diff --git a/website/scripts/options/types.js b/website/scripts/options/types.js index fdde56bc2..408f3fd71 100644 --- a/website/scripts/options/types.js +++ b/website/scripts/options/types.js @@ -90,8 +90,8 @@ * @typedef {PartialOption & PartialChartOptionSpecific} PartialChartOption * * @typedef {Object} ProcessedChartOptionAddons - * @property {Map} top - * @property {Map} bottom + * @property {() => Map} top + * @property {() => Map} bottom * * @typedef {Required> & ProcessedChartOptionAddons & ProcessedOptionAddons} ChartOption * diff --git a/website/scripts/options/unused.js b/website/scripts/options/unused.js index e4c9eb801..cdba0e8e2 100644 --- a/website/scripts/options/unused.js +++ b/website/scripts/options/unused.js @@ -1,9 +1,6 @@ import { localhost } from "../utils/env.js"; import { serdeChartableIndex } from "../utils/serde.js"; -/** @type {Map | null} */ -export const unused = localhost ? new Map() : null; - /** * Check if a metric pattern has at least one chartable index * @param {AnyMetricPattern} node @@ -15,11 +12,12 @@ function hasChartableIndex(node) { } /** + * Walk a metrics tree and collect all chartable metric patterns * @param {TreeNode | null | undefined} node * @param {Map} map * @param {string[]} path */ -function walk(node, map, path) { +function walkMetrics(node, map, path) { if (node && "by" in node) { const metricNode = /** @type {AnyMetricPattern} */ (node); if (!hasChartableIndex(metricNode)) return; @@ -33,32 +31,22 @@ function walk(node, map, path) { key.endsWith("State") || key.endsWith("Start") || kn === "mvrv" || - // kn === "time" || - // kn === "height" || kn === "constants" || kn === "blockhash" || kn === "date" || - // kn === "oracle" || kn === "split" || - // kn === "ohlc" || kn === "outpoint" || kn === "positions" || - // kn === "outputtype" || kn === "heighttopool" || kn === "txid" || kn.startsWith("timestamp") || kn.startsWith("satdays") || kn.startsWith("satblocks") || - // kn.endsWith("state") || - // kn.endsWith("cents") || kn.endsWith("index") || kn.endsWith("indexes") - // kn.endsWith("raw") || - // kn.endsWith("bytes") || - // (kn.startsWith("_") && kn.endsWith("start")) ) continue; - walk(/** @type {TreeNode | null | undefined} */ (value), map, [ + walkMetrics(/** @type {TreeNode | null | undefined} */ (value), map, [ ...path, key, ]); @@ -67,29 +55,64 @@ function walk(node, map, path) { } /** - * Collect all AnyMetricPatterns from tree - * @param {TreeNode} tree + * Walk partial options tree and delete referenced metrics from the map + * @param {PartialOptionsTree} options + * @param {Map} map */ -export function collect(tree) { - if (unused) walk(tree, unused, []); +function walkOptions(options, map) { + for (const node of options) { + if ("tree" in node && node.tree) { + walkOptions(node.tree, map); + } else if ("top" in node || "bottom" in node) { + const chartNode = /** @type {PartialChartOption} */ (node); + markUsedBlueprints(map, chartNode.top); + markUsedBlueprints(map, chartNode.bottom); + } + } } /** - * Mark a metric as used - * @param {AnyMetricPattern} metric + * @param {Map} map + * @param {(AnyFetchedSeriesBlueprint | FetchedPriceSeriesBlueprint)[]} [arr] */ -export function markUsed(metric) { - unused?.delete(metric); +function markUsedBlueprints(map, arr) { + if (!arr) return; + for (let i = 0; i < arr.length; i++) { + const metric = arr[i].metric; + if (!metric) continue; + const maybePriceMetric = + /** @type {{ dollars?: AnyMetricPattern, sats?: AnyMetricPattern }} */ ( + /** @type {unknown} */ (metric) + ); + if (maybePriceMetric.dollars?.by && maybePriceMetric.sats?.by) { + map.delete(maybePriceMetric.dollars); + map.delete(maybePriceMetric.sats); + } else { + map.delete(/** @type {AnyMetricPattern} */ (metric)); + } + } } -/** Log unused metrics to console */ -export function logUnused() { - if (!unused?.size) return; +/** + * Log unused metrics to console (localhost only) + * @param {TreeNode} metricsTree + * @param {PartialOptionsTree} partialOptions + */ +export function logUnused(metricsTree, partialOptions) { + if (!localhost) return; + + console.log(extractTreeStructure(partialOptions)); + + /** @type {Map} */ + const all = new Map(); + walkMetrics(metricsTree, all, []); + walkOptions(partialOptions, all); + + if (!all.size) return; /** @type {Record} */ const tree = {}; - - for (const path of unused.values()) { + for (const path of all.values()) { let current = tree; for (let i = 0; i < path.length; i++) { const part = path[i]; @@ -102,7 +125,7 @@ export function logUnused() { } } - console.log("Unused metrics:", { count: unused.size, tree }); + console.log("Unused metrics:", { count: all.size, tree }); } /** @@ -121,7 +144,6 @@ export function extractTreeStructure(options) { /** @type {Record} */ const grouped = {}; for (const s of series) { - // Price patterns in top pane have dollars/sats sub-metrics const metric = /** @type {any} */ (s.metric); if (isTop && metric?.dollars && metric?.sats) { const title = s.title || s.key || "unnamed"; @@ -142,14 +164,12 @@ export function extractTreeStructure(options) { * @returns {object} */ function processNode(node) { - // Group with children if ("tree" in node && node.tree) { return { name: node.name, children: node.tree.map(processNode), }; } - // Chart option if ("top" in node || "bottom" in node) { const chartNode = /** @type {PartialChartOption} */ (node); const top = chartNode.top ? groupByUnit(chartNode.top, true) : undefined; @@ -163,23 +183,11 @@ export function extractTreeStructure(options) { ...(bottom && Object.keys(bottom).length > 0 ? { bottom } : {}), }; } - // URL option if ("url" in node) { return { name: node.name, url: true }; } - // Other options (explorer, table, simulation) return { name: node.name }; } return options.map(processNode); } - -/** - * Log the options tree structure to console (localhost only) - * @param {PartialOptionsTree} options - */ -export function logTreeStructure(options) { - if (!localhost) return; - const structure = extractTreeStructure(options); - console.log("Options tree structure:", JSON.stringify(structure, null, 2)); -} diff --git a/website/scripts/panes/chart.js b/website/scripts/panes/chart.js index 50371d835..4a0c4cd02 100644 --- a/website/scripts/panes/chart.js +++ b/website/scripts/panes/chart.js @@ -91,8 +91,8 @@ export function init() { // Set blueprints first so storageId is correct before any index change chart.setBlueprints({ name: opt.title, - top: buildTopBlueprints(opt.top), - bottom: opt.bottom, + top: buildTopBlueprints(opt.top()), + bottom: opt.bottom(), onDataLoaded: updatePriceWithLatest, }); @@ -120,11 +120,11 @@ const ALL_CHOICES = /** @satisfies {ChartableIndexName[]} */ ([ * @returns {ChartableIndexName[]} */ function computeChoices(opt) { - if (!opt.top.size && !opt.bottom.size) { + if (!opt.top().size && !opt.bottom().size) { return [...ALL_CHOICES]; } const rawIndexes = new Set( - [Array.from(opt.top.values()), Array.from(opt.bottom.values())] + [Array.from(opt.top().values()), Array.from(opt.bottom().values())] .flat(2) .filter((blueprint) => { const path = Object.values(blueprint.metric.by)[0]?.path ?? "";