global: snapshot

Cargo.lock

@@ -784,9 +784,9 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9"
 
 [[package]]
 name = "cc"
-version = "1.2.55"
+version = "1.2.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29"
+checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
 dependencies = [
  "find-msvc-tools",
  "jobserver",

crates/brk_oracle/examples/sweep_tolerance.rs

@@ -0,0 +1,447 @@
+//! Sweep round-value tolerance to find optimal rounding threshold.
+//!
+//! Tests different tolerance percentages (0%, 0.01%, 0.1%, 1%, etc.) for
+//! detecting round BTC amounts, combined with several digit filter masks.
+//!
+//! Phase 1: single pass over indexer, store per-output relative errors.
+//! Phase 2: sweep tolerance × mask combos across CPU cores.
+//!
+//! Run with: cargo run -p brk_oracle --example sweep_tolerance --release
+
+use std::path::PathBuf;
+use std::time::Instant;
+
+use brk_indexer::Indexer;
+use brk_oracle::{Config, NUM_BINS, Oracle, PRICES, START_HEIGHT, cents_to_bin, sats_to_bin};
+use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
+use vecdb::{AnyVec, VecIndex, VecIterator};
+
+const BINS_5PCT: f64 = 4.24;
+const BINS_10PCT: f64 = 8.28;
+const BINS_20PCT: f64 = 15.84;
+
+fn bins_to_pct(bins: f64) -> f64 {
+    (10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
+}
+
+fn seed_bin(start_height: usize) -> f64 {
+    let price: f64 = PRICES
+        .lines()
+        .nth(start_height - 1)
+        .expect("prices.txt too short")
+        .parse()
+        .expect("Failed to parse seed price");
+    cents_to_bin(price * 100.0)
+}
+
+fn leading_digit(sats: u64) -> u8 {
+    let log = (sats as f64).log10();
+    let magnitude = 10.0_f64.powf(log.floor());
+    let d = (sats as f64 / magnitude).round() as u8;
+    if d >= 10 { 1 } else { d }
+}
+
+/// Returns the relative error of `sats` from its nearest round value (d × 10^n).
+/// e.g. 10_050 → leading=1, round_val=10_000, rel_err = 50/10000 = 0.005
+fn relative_roundness(sats: u64) -> f64 {
+    let log = (sats as f64).log10();
+    let magnitude = 10.0_f64.powf(log.floor());
+    let leading = (sats as f64 / magnitude).round();
+    let round_val = leading * magnitude;
+    (sats as f64 - round_val).abs() / round_val
+}
+
+struct Stats {
+    total_sq_err: f64,
+    total_bias: f64,
+    max_err: f64,
+    total_blocks: u64,
+    gt_5pct: u64,
+    gt_10pct: u64,
+    gt_20pct: u64,
+}
+
+impl Stats {
+    fn new() -> Self {
+        Self {
+            total_sq_err: 0.0,
+            total_bias: 0.0,
+            max_err: 0.0,
+            total_blocks: 0,
+            gt_5pct: 0,
+            gt_10pct: 0,
+            gt_20pct: 0,
+        }
+    }
+
+    fn update(&mut self, err: f64) {
+        self.total_sq_err += err * err;
+        self.total_bias += err;
+        self.total_blocks += 1;
+        let abs_err = err.abs();
+        if abs_err > self.max_err {
+            self.max_err = abs_err;
+        }
+        if abs_err > BINS_5PCT {
+            self.gt_5pct += 1;
+        }
+        if abs_err > BINS_10PCT {
+            self.gt_10pct += 1;
+        }
+        if abs_err > BINS_20PCT {
+            self.gt_20pct += 1;
+        }
+    }
+
+    fn rmse_pct(&self) -> f64 {
+        bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
+    }
+
+    fn max_pct(&self) -> f64 {
+        bins_to_pct(self.max_err)
+    }
+
+    fn bias(&self) -> f64 {
+        self.total_bias / self.total_blocks as f64
+    }
+}
+
+/// Per-output data: bin index, leading digit, relative error from round value.
+struct RoundOutput {
+    bin: u16,
+    digit: u8,
+    rel_err: f32, // f32 is plenty of precision, saves memory
+}
+
+struct BlockData {
+    full_hist: Box<[u32; NUM_BINS]>,
+    round_outputs: Vec<RoundOutput>,
+    high_bin: f64,
+    low_bin: f64,
+}
+
+fn main() {
+    let t0 = Instant::now();
+
+    let data_dir = std::env::var("BRK_DIR")
+        .map(PathBuf::from)
+        .unwrap_or_else(|_| {
+            let home = std::env::var("HOME").unwrap();
+            PathBuf::from(home).join(".brk")
+        });
+
+    let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
+    let total_heights = indexer.vecs.blocks.timestamp.len();
+
+    let manifest_dir = env!("CARGO_MANIFEST_DIR");
+
+    let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
+        &std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
+            .expect("Failed to read height_price_ohlc.json"),
+    )
+    .expect("Failed to parse height OHLC");
+
+    let height_bands: Vec<(f64, f64)> = height_ohlc
+        .iter()
+        .map(|ohlc| {
+            let high = ohlc[1];
+            let low = ohlc[2];
+            if high > 0.0 && low > 0.0 {
+                (cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
+            } else {
+                (0.0, 0.0)
+            }
+        })
+        .collect();
+
+    let sweep_start: usize = 575_000;
+
+    // Phase 1: precompute per-block data.
+    // Store all potentially-round outputs with their relative error so we can
+    // filter at different tolerance thresholds in Phase 2.
+    eprintln!("Phase 1: precomputing block data...");
+
+    let total_txs = indexer.vecs.transactions.height.len();
+    let total_outputs = indexer.vecs.outputs.value.len();
+
+    let mut first_txindex_iter = indexer.vecs.transactions.first_txindex.into_iter();
+    let mut first_txoutindex_iter = indexer.vecs.transactions.first_txoutindex.into_iter();
+    let mut out_first_iter = indexer.vecs.outputs.first_txoutindex.into_iter();
+    let mut value_iter = indexer.vecs.outputs.value.into_iter();
+    let mut outputtype_iter = indexer.vecs.outputs.outputtype.into_iter();
+
+    let ref_config = Config::default();
+    let total_blocks = total_heights - sweep_start;
+    let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
+
+    // Use the widest tolerance we'll test (5%) to decide what to store.
+    // Outputs beyond 5% relative error will never be filtered at any tolerance.
+    let max_tolerance: f64 = 0.05;
+
+    for h in START_HEIGHT..total_heights {
+        let first_txindex: TxIndex = first_txindex_iter.get_at_unwrap(h);
+        let next_first_txindex = first_txindex_iter
+            .get_at(h + 1)
+            .unwrap_or(TxIndex::from(total_txs));
+
+        let out_start = if first_txindex.to_usize() + 1 < next_first_txindex.to_usize() {
+            first_txoutindex_iter
+                .get_at_unwrap(first_txindex.to_usize() + 1)
+                .to_usize()
+        } else {
+            out_first_iter
+                .get_at(h + 1)
+                .unwrap_or(TxOutIndex::from(total_outputs))
+                .to_usize()
+        };
+        let out_end = out_first_iter
+            .get_at(h + 1)
+            .unwrap_or(TxOutIndex::from(total_outputs))
+            .to_usize();
+
+        if h < sweep_start {
+            continue;
+        }
+
+        let mut full_hist = Box::new([0u32; NUM_BINS]);
+        let mut round_outputs = Vec::new();
+
+        for i in out_start..out_end {
+            let sats: Sats = value_iter.get_at_unwrap(i);
+            let output_type: OutputType = outputtype_iter.get_at_unwrap(i);
+            if ref_config.excluded_output_types.contains(&output_type) {
+                continue;
+            }
+            if *sats < ref_config.min_sats {
+                continue;
+            }
+            if let Some(bin) = sats_to_bin(sats) {
+                full_hist[bin] += 1;
+                let d = leading_digit(*sats);
+                if (1..=9).contains(&d) {
+                    let rel_err = relative_roundness(*sats);
+                    if rel_err <= max_tolerance {
+                        round_outputs.push(RoundOutput {
+                            bin: bin as u16,
+                            digit: d,
+                            rel_err: rel_err as f32,
+                        });
+                    }
+                }
+            }
+        }
+
+        let (high_bin, low_bin) = if h < height_bands.len() {
+            height_bands[h]
+        } else {
+            (0.0, 0.0)
+        };
+
+        blocks.push(BlockData {
+            full_hist,
+            round_outputs,
+            high_bin,
+            low_bin,
+        });
+
+        if (h - sweep_start).is_multiple_of(50_000) {
+            eprint!(
+                "\r  {}/{} ({:.0}%)",
+                h - sweep_start,
+                total_blocks,
+                (h - sweep_start) as f64 / total_blocks as f64 * 100.0
+            );
+        }
+    }
+
+    let mem_hists = blocks.len() * std::mem::size_of::<[u32; NUM_BINS]>();
+    let mem_rounds: usize = blocks
+        .iter()
+        .map(|b| b.round_outputs.len() * std::mem::size_of::<RoundOutput>())
+        .sum();
+    eprintln!(
+        "\r  {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s",
+        blocks.len(),
+        mem_hists as f64 / 1e9,
+        mem_rounds as f64 / 1e6,
+        t0.elapsed().as_secs_f64()
+    );
+
+    // Phase 2: sweep tolerance × mask combos.
+    // Tolerances as fractions (not percentages).
+    let tolerances: &[(f64, &str)] = &[
+        (0.0, "0%"),
+        (0.0001, "0.01%"),
+        (0.0005, "0.05%"),
+        (0.001, "0.1%"),
+        (0.002, "0.2%"),
+        (0.005, "0.5%"),
+        (0.01, "1%"),
+        (0.02, "2%"),
+        (0.05, "5%"),
+    ];
+
+    //                              987654321
+    let masks: &[(u16, &str)] = &[
+        (0b0_0000_0000, "none"),
+        (0b0_0001_0111, "{1,2,3,5}"),
+        (0b0_0001_1111, "{1,2,3,4,5}"),
+        (0b0_0011_0111, "{1,2,3,5,6}"),
+        (0b0_0111_0111, "{1,2,3,5,6,7}"),
+        (0b1_1111_1111, "{1-9}"),
+    ];
+
+    let num_configs = tolerances.len() * masks.len();
+    let num_threads = std::thread::available_parallelism()
+        .map(|n| n.get())
+        .unwrap_or(8);
+    eprintln!(
+        "Phase 2: sweeping {} configs ({} tolerances × {} masks) across {} threads...",
+        num_configs,
+        tolerances.len(),
+        masks.len(),
+        num_threads
+    );
+
+    let t1 = Instant::now();
+    let blocks = &blocks;
+    let tolerances_ref = tolerances;
+    let masks_ref = masks;
+
+    let all_results: Vec<(usize, usize, Stats)> = std::thread::scope(|s| {
+        let configs_per_thread = num_configs.div_ceil(num_threads);
+
+        let handles: Vec<_> = (0..num_threads)
+            .map(|t| {
+                s.spawn(move || {
+                    let cfg_start = t * configs_per_thread;
+                    let cfg_end = ((t + 1) * configs_per_thread).min(num_configs);
+                    if cfg_start >= cfg_end {
+                        return vec![];
+                    }
+                    let mut results = Vec::with_capacity(cfg_end - cfg_start);
+
+                    for cfg_idx in cfg_start..cfg_end {
+                        let ti = cfg_idx / masks_ref.len();
+                        let mi = cfg_idx % masks_ref.len();
+                        let (tolerance, _) = tolerances_ref[ti];
+                        let (mask, _) = masks_ref[mi];
+
+                        let mut oracle = Oracle::new(
+                            seed_bin(sweep_start),
+                            Config {
+                                exclude_common_round_values: false,
+                                ..Default::default()
+                            },
+                        );
+                        let mut stats = Stats::new();
+
+                        for bd in blocks.iter() {
+                            let mut hist = *bd.full_hist;
+
+                            // Remove outputs matching this tolerance + mask.
+                            let tol_f32 = tolerance as f32;
+                            for ro in &bd.round_outputs {
+                                if mask & (1 << (ro.digit - 1)) != 0
+                                    && ro.rel_err <= tol_f32
+                                {
+                                    hist[ro.bin as usize] -= 1;
+                                }
+                            }
+
+                            let ref_bin = oracle.process_histogram(&hist);
+
+                            if bd.high_bin > 0.0 && bd.low_bin > 0.0 {
+                                let err = if ref_bin < bd.high_bin {
+                                    ref_bin - bd.high_bin
+                                } else if ref_bin > bd.low_bin {
+                                    ref_bin - bd.low_bin
+                                } else {
+                                    0.0
+                                };
+                                stats.update(err);
+                            }
+                        }
+
+                        results.push((ti, mi, stats));
+                    }
+
+                    results
+                })
+            })
+            .collect();
+
+        handles
+            .into_iter()
+            .flat_map(|h| h.join().unwrap())
+            .collect()
+    });
+
+    eprintln!("  Done in {:.1}s.", t1.elapsed().as_secs_f64());
+
+    // Print results grouped by tolerance.
+    println!();
+    println!(
+        "{:>8} {:>16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
+        "Tol", "Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
+    );
+    println!("{}", "-".repeat(88));
+
+    for (ti, &(_, tol_label)) in tolerances.iter().enumerate() {
+        for (mi, &(_, mask_label)) in masks.iter().enumerate() {
+            let (_, _, stats) = all_results
+                .iter()
+                .find(|(t, m, _)| *t == ti && *m == mi)
+                .unwrap();
+            println!(
+                "{:>8} {:>16} {:>8} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
+                tol_label,
+                mask_label,
+                stats.total_blocks,
+                stats.rmse_pct(),
+                stats.max_pct(),
+                stats.gt_5pct,
+                stats.gt_10pct,
+                stats.gt_20pct,
+                stats.bias()
+            );
+        }
+        println!();
+    }
+
+    // Find overall best config by RMSE.
+    let best = all_results
+        .iter()
+        .min_by(|a, b| a.2.rmse_pct().partial_cmp(&b.2.rmse_pct()).unwrap())
+        .unwrap();
+    let (bti, bmi, bs) = best;
+    println!(
+        "Best: tolerance={}, digits={} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
+        tolerances[*bti].1,
+        masks[*bmi].1,
+        bs.rmse_pct(),
+        bs.max_pct(),
+        bs.gt_5pct,
+        bs.gt_10pct,
+        bs.gt_20pct,
+    );
+
+    // Show current config for reference.
+    let current = all_results
+        .iter()
+        .find(|(t, m, _)| {
+            tolerances[*t].0 == 0.001 && masks[*m].0 == 0b0_0011_0111
+        })
+        .unwrap();
+    let (_, _, cs) = current;
+    println!(
+        "Current: tolerance=0.1%, digits={{1,2,3,5,6}} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
+        cs.rmse_pct(),
+        cs.max_pct(),
+        cs.gt_5pct,
+        cs.gt_10pct,
+        cs.gt_20pct,
+    );
+
+    println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
+}

crates/brk_server/src/api/mempool/mod.rs

@@ -67,7 +67,9 @@ impl MempoolRoutes for ApiRouter<AppState> {
                 get_with(
                     async |headers: HeaderMap, State(state): State<AppState>| {
                         state
-                            .cached_json(&headers, state.mempool_cache(), |q| q.live_price())
+                            .server_cached_json(&headers, state.mempool_cache(), "price", |q| {
+                                q.live_price()
+                            })
                             .await
                     },
                     |op| {

crates/brk_server/src/api/metrics/bulk.rs

@@ -1,14 +1,13 @@
-use std::{net::SocketAddr, time::Duration};
+use std::net::SocketAddr;
 
 use axum::{
     Extension,
-    body::Body,
+    body::{Body, Bytes},
     extract::{Query, State},
     http::{HeaderMap, StatusCode, Uri},
     response::{IntoResponse, Response},
 };
 use brk_types::{Format, MetricSelection, Output};
-use quick_cache::sync::GuardResult;
 
 use crate::{
     Result,
@@ -23,56 +22,41 @@ pub async fn handler(
     headers: HeaderMap,
     Extension(addr): Extension<SocketAddr>,
     Query(params): Query<MetricSelection>,
-    State(AppState { query, cache, .. }): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Response> {
     // Phase 1: Search and resolve metadata (cheap)
-    let resolved = query.run(move |q| q.resolve(params, max_weight(&addr))).await?;
+    let resolved = state.run(move |q| q.resolve(params, max_weight(&addr))).await?;
 
     let format = resolved.format();
     let etag = resolved.etag();
 
-    // Check if client has fresh cache
     if headers.has_etag(etag.as_str()) {
-        let response = (StatusCode::NOT_MODIFIED, "").into_response();
-        return Ok(response);
+        return Ok((StatusCode::NOT_MODIFIED, "").into_response());
     }
 
-    // Check server-side cache
+    // Phase 2: Format (expensive, server-side cached)
     let cache_key = format!("bulk-{}{}{}", uri.path(), uri.query().unwrap_or(""), etag);
-    let guard_res = cache.get_value_or_guard(&cache_key, Some(Duration::from_millis(50)));
-
-    let mut response = if let GuardResult::Value(v) = guard_res {
-        Response::new(Body::from(v))
-    } else {
-        // Phase 2: Format (expensive, only on cache miss)
-        let metric_output = query.run(move |q| q.format(resolved)).await?;
-
-        match metric_output.output {
-            Output::CSV(s) => {
-                if let GuardResult::Guard(g) = guard_res {
-                    let _ = g.insert(s.clone().into());
-                }
-                s.into_response()
-            }
-            Output::Json(v) => {
-                if let GuardResult::Guard(g) = guard_res {
-                    let _ = g.insert(v.clone().into());
-                }
-                Response::new(Body::from(v))
-            }
-        }
-    };
-
-    let headers = response.headers_mut();
-    headers.insert_etag(etag.as_str());
-    headers.insert_cache_control(CACHE_CONTROL);
+    let query = &state;
+    let bytes = state
+        .get_or_insert(&cache_key, async move {
+            let out = query.run(move |q| q.format(resolved)).await?;
+            Ok(match out.output {
+                Output::CSV(s) => Bytes::from(s),
+                Output::Json(v) => Bytes::from(v),
+            })
+        })
+        .await?;
 
+    let mut response = Response::new(Body::from(bytes));
+    let h = response.headers_mut();
+    h.insert_etag(etag.as_str());
+    h.insert_cache_control(CACHE_CONTROL);
     match format {
         Format::CSV => {
-            headers.insert_content_disposition_attachment();
-            headers.insert_content_type_text_csv()
+            h.insert_content_disposition_attachment();
+            h.insert_content_type_text_csv()
         }
-        Format::JSON => headers.insert_content_type_application_json(),
+        Format::JSON => h.insert_content_type_application_json(),
     }
 
     Ok(response)

crates/brk_server/src/api/metrics/data.rs

@@ -1,14 +1,13 @@
-use std::{net::SocketAddr, time::Duration};
+use std::net::SocketAddr;
 
 use axum::{
     Extension,
-    body::Body,
+    body::{Body, Bytes},
     extract::{Query, State},
     http::{HeaderMap, StatusCode, Uri},
     response::{IntoResponse, Response},
 };
 use brk_types::{Format, MetricSelection, Output};
-use quick_cache::sync::GuardResult;
 
 use crate::{
     Result,
@@ -23,56 +22,41 @@ pub async fn handler(
     headers: HeaderMap,
     Extension(addr): Extension<SocketAddr>,
     Query(params): Query<MetricSelection>,
-    State(AppState { query, cache, .. }): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Response> {
     // Phase 1: Search and resolve metadata (cheap)
-    let resolved = query.run(move |q| q.resolve(params, max_weight(&addr))).await?;
+    let resolved = state.run(move |q| q.resolve(params, max_weight(&addr))).await?;
 
     let format = resolved.format();
     let etag = resolved.etag();
 
-    // Check if client has fresh cache
     if headers.has_etag(etag.as_str()) {
-        let response = (StatusCode::NOT_MODIFIED, "").into_response();
-        return Ok(response);
+        return Ok((StatusCode::NOT_MODIFIED, "").into_response());
     }
 
-    // Check server-side cache
+    // Phase 2: Format (expensive, server-side cached)
     let cache_key = format!("single-{}{}{}", uri.path(), uri.query().unwrap_or(""), etag);
-    let guard_res = cache.get_value_or_guard(&cache_key, Some(Duration::from_millis(50)));
-
-    let mut response = if let GuardResult::Value(v) = guard_res {
-        Response::new(Body::from(v))
-    } else {
-        // Phase 2: Format (expensive, only on cache miss)
-        let metric_output = query.run(move |q| q.format(resolved)).await?;
-
-        match metric_output.output {
-            Output::CSV(s) => {
-                if let GuardResult::Guard(g) = guard_res {
-                    let _ = g.insert(s.clone().into());
-                }
-                s.into_response()
-            }
-            Output::Json(v) => {
-                if let GuardResult::Guard(g) = guard_res {
-                    let _ = g.insert(v.clone().into());
-                }
-                Response::new(Body::from(v))
-            }
-        }
-    };
-
-    let headers = response.headers_mut();
-    headers.insert_etag(etag.as_str());
-    headers.insert_cache_control(CACHE_CONTROL);
+    let query = &state;
+    let bytes = state
+        .get_or_insert(&cache_key, async move {
+            let out = query.run(move |q| q.format(resolved)).await?;
+            Ok(match out.output {
+                Output::CSV(s) => Bytes::from(s),
+                Output::Json(v) => Bytes::from(v),
+            })
+        })
+        .await?;
 
+    let mut response = Response::new(Body::from(bytes));
+    let h = response.headers_mut();
+    h.insert_etag(etag.as_str());
+    h.insert_cache_control(CACHE_CONTROL);
     match format {
         Format::CSV => {
-            headers.insert_content_disposition_attachment();
-            headers.insert_content_type_text_csv()
+            h.insert_content_disposition_attachment();
+            h.insert_content_type_text_csv()
         }
-        Format::JSON => headers.insert_content_type_application_json(),
+        Format::JSON => h.insert_content_type_application_json(),
     }
 
     Ok(response)

crates/brk_server/src/api/metrics/legacy.rs

@@ -1,14 +1,13 @@
-use std::{net::SocketAddr, time::Duration};
+use std::net::SocketAddr;
 
 use axum::{
     Extension,
-    body::Body,
+    body::{Body, Bytes},
     extract::{Query, State},
     http::{HeaderMap, StatusCode, Uri},
     response::{IntoResponse, Response},
 };
 use brk_types::{Format, MetricSelection, OutputLegacy};
-use quick_cache::sync::GuardResult;
 
 use crate::{
     Result,
@@ -23,57 +22,41 @@ pub async fn handler(
     headers: HeaderMap,
     Extension(addr): Extension<SocketAddr>,
     Query(params): Query<MetricSelection>,
-    State(AppState { query, cache, .. }): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Response> {
     // Phase 1: Search and resolve metadata (cheap)
-    let resolved = query.run(move |q| q.resolve(params, max_weight(&addr))).await?;
+    let resolved = state.run(move |q| q.resolve(params, max_weight(&addr))).await?;
 
     let format = resolved.format();
     let etag = resolved.etag();
 
-    // Check if client has fresh cache
     if headers.has_etag(etag.as_str()) {
-        let response = (StatusCode::NOT_MODIFIED, "").into_response();
-        return Ok(response);
+        return Ok((StatusCode::NOT_MODIFIED, "").into_response());
     }
 
-    // Check server-side cache
+    // Phase 2: Format (expensive, server-side cached)
     let cache_key = format!("legacy-{}{}{}", uri.path(), uri.query().unwrap_or(""), etag);
-    let guard_res = cache.get_value_or_guard(&cache_key, Some(Duration::from_millis(50)));
-
-    let mut response = if let GuardResult::Value(v) = guard_res {
-        Response::new(Body::from(v))
-    } else {
-        // Phase 2: Format (expensive, only on cache miss)
-        let metric_output = query.run(move |q| q.format_legacy(resolved)).await?;
-
-        match metric_output.output {
-            OutputLegacy::CSV(s) => {
-                if let GuardResult::Guard(g) = guard_res {
-                    let _ = g.insert(s.clone().into());
-                }
-                s.into_response()
-            }
-            OutputLegacy::Json(v) => {
-                let json = v.to_vec();
-                if let GuardResult::Guard(g) = guard_res {
-                    let _ = g.insert(json.clone().into());
-                }
-                json.into_response()
-            }
-        }
-    };
-
-    let headers = response.headers_mut();
-    headers.insert_etag(etag.as_str());
-    headers.insert_cache_control(CACHE_CONTROL);
+    let query = &state;
+    let bytes = state
+        .get_or_insert(&cache_key, async move {
+            let out = query.run(move |q| q.format_legacy(resolved)).await?;
+            Ok(match out.output {
+                OutputLegacy::CSV(s) => Bytes::from(s),
+                OutputLegacy::Json(v) => Bytes::from(v.to_vec()),
+            })
+        })
+        .await?;
 
+    let mut response = Response::new(Body::from(bytes));
+    let h = response.headers_mut();
+    h.insert_etag(etag.as_str());
+    h.insert_cache_control(CACHE_CONTROL);
     match format {
         Format::CSV => {
-            headers.insert_content_disposition_attachment();
-            headers.insert_content_type_text_csv()
+            h.insert_content_disposition_attachment();
+            h.insert_content_type_text_csv()
         }
-        Format::JSON => headers.insert_content_type_application_json(),
+        Format::JSON => h.insert_content_type_application_json(),
     }
 
     Ok(response)

crates/brk_server/src/state.rs

@@ -14,7 +14,7 @@ use serde::Serialize;
 
 use crate::{
     CacheParams, CacheStrategy, Website,
-    extended::{ResponseExtended, ResultExtended},
+    extended::{HeaderMapExtended, ResponseExtended, ResultExtended},
 };
 
 #[derive(Clone, Deref)]
@@ -56,6 +56,72 @@ impl AppState {
         }
     }
 
+    /// JSON response with HTTP caching + server-side cache
+    pub async fn server_cached_json<T, F>(
+        &self,
+        headers: &HeaderMap,
+        strategy: CacheStrategy,
+        cache_prefix: &str,
+        f: F,
+    ) -> Response<Body>
+    where
+        T: Serialize + Send + 'static,
+        F: FnOnce(&brk_query::Query) -> brk_error::Result<T> + Send + 'static,
+    {
+        let params = CacheParams::resolve(&strategy, || self.sync(|q| q.height().into()));
+        if params.matches_etag(headers) {
+            return ResponseExtended::new_not_modified();
+        }
+
+        let cache_key = format!("{cache_prefix}-{}", params.etag_str());
+        let result = self
+            .get_or_insert(&cache_key, async move {
+                let value = self.run(f).await?;
+                Ok(serde_json::to_vec(&value).unwrap().into())
+            })
+            .await;
+
+        match result {
+            Ok(bytes) => {
+                let mut response = Response::new(Body::from(bytes));
+                let h = response.headers_mut();
+                h.insert_content_type_application_json();
+                h.insert_cache_control(&params.cache_control);
+                if let Some(etag) = &params.etag {
+                    h.insert_etag(etag);
+                }
+                response
+            }
+            Err(e) => ResultExtended::<T>::to_json_response(Err(e), params.etag_str()),
+        }
+    }
+
+    /// Check server-side cache, compute on miss
+    pub async fn get_or_insert(
+        &self,
+        cache_key: &str,
+        compute: impl std::future::Future<Output = brk_error::Result<Bytes>>,
+    ) -> brk_error::Result<Bytes> {
+        use quick_cache::sync::GuardResult;
+
+        let guard_res = self.cache.get_value_or_guard(
+            cache_key,
+            Some(std::time::Duration::from_millis(50)),
+        );
+
+        if let GuardResult::Value(bytes) = guard_res {
+            return Ok(bytes);
+        }
+
+        let bytes = compute.await?;
+
+        if let GuardResult::Guard(g) = guard_res {
+            let _ = g.insert(bytes.clone());
+        }
+
+        Ok(bytes)
+    }
+
     /// Text response with caching
     pub async fn cached_text<T, F>(
         &self,

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.93.0"
+channel = "1.93.1"

website/scripts/entry.js

@@ -1 +1 @@
-import("./main.js");
+import "./main.js";

website/scripts/main.js

@@ -8,7 +8,6 @@ import {
   setOption as setChartOption,
 } from "./panes/chart.js";
 import { initSearch } from "./panes/search.js";
-import { next } from "./utils/timing.js";
 import { replaceHistory } from "./utils/url.js";
 import { removeStored, writeToStorage } from "./utils/storage.js";
 import {
@@ -195,68 +194,13 @@ function initSelected() {
 }
 initSelected();
 
-onFirstIntersection(navElement, async () => {
+requestIdleCallback(() => options.setParent(navElement));
+
+onFirstIntersection(navElement, () => {
   options.setParent(navElement);
 
-  const option = options.selected.value;
-  if (!option) throw "Selected should be set by now";
-  const path = [...option.path];
-
-  /** @type {HTMLUListElement | null} */
-  let ul = /** @type {any} */ (null);
-  async function getFirstChild() {
-    try {
-      ul = /** @type {HTMLUListElement} */ (navElement.firstElementChild);
-      await next();
-      if (!ul) {
-        await getFirstChild();
-      }
-    } catch (_) {
-      await next();
-      await getFirstChild();
-    }
-  }
-  await getFirstChild();
-  if (!ul) throw Error("Unreachable");
-
-  while (path.length > 1) {
-    const name = path.shift();
-    if (!name) throw "Unreachable";
-    /** @type {HTMLDetailsElement[]} */
-    let detailsList = [];
-    while (!detailsList.length) {
-      detailsList = Array.from(ul.querySelectorAll(":scope > li > details"));
-      if (!detailsList.length) {
-        await next();
-      }
-    }
-    const details = detailsList.find((s) => s.dataset.name == name);
-    if (!details) return;
-    details.open = true;
-    ul = null;
-    while (!ul) {
-      const uls = /** @type {HTMLUListElement[]} */ (
-        Array.from(details.querySelectorAll(":scope > ul"))
-      );
-      if (!uls.length) {
-        await next();
-      } else if (uls.length > 1) {
-        throw "Shouldn't be possible";
-      } else {
-        ul = /** @type {HTMLUListElement} */ (uls.pop());
-      }
-    }
-  }
-  /** @type {HTMLAnchorElement[]} */
-  let anchors = [];
-  while (!anchors.length) {
-    anchors = Array.from(ul.querySelectorAll(":scope > li > a"));
-    if (!anchors.length) {
-      await next();
-    }
-  }
-  anchors
-    .find((a) => a.getAttribute("href") == window.document.location.pathname)
+  navElement
+    .querySelector(`a[href="${window.document.location.pathname}"]`)
     ?.scrollIntoView({
       behavior: "instant",
       block: "center",

website/scripts/options/full.js

@@ -3,13 +3,7 @@ import { createButtonElement, createAnchorElement } from "../utils/dom.js";
 import { pushHistory, resetParams } from "../utils/url.js";
 import { readStored, writeToStorage } from "../utils/storage.js";
 import { stringToId } from "../utils/format.js";
-import {
-  collect,
-  markUsed,
-  logUnused,
-  extractTreeStructure,
-} from "./unused.js";
-import { localhost } from "../utils/env.js";
+import { logUnused } from "./unused.js";
 import { setQr } from "../panes/share.js";
 import { getConstant } from "./constants.js";
 import { colors } from "../utils/colors.js";
@@ -17,8 +11,6 @@ import { Unit } from "../utils/units.js";
 import { brk } from "../client.js";
 
 export function initOptions() {
-  collect(brk.metrics);
-
   const LS_SELECTED_KEY = `selected_path`;
 
   const urlPath_ = window.document.location.pathname
@@ -31,11 +23,6 @@ export function initOptions() {
 
   const partialOptions = createPartialOptions();
 
-  // Log tree structure for analysis (localhost only)
-  if (localhost) {
-    console.log(extractTreeStructure(partialOptions));
-  }
-
   /** @type {Option[]} */
   const list = [];
 
@@ -45,18 +32,26 @@ export function initOptions() {
   /** @type {Set<(option: Option) => void>} */
   const selectedListeners = new Set();
 
+  /** @type {HTMLLIElement[]} */
+  let highlightedLis = [];
+
   /**
    * @param {Option | undefined} sel
    */
   function updateHighlight(sel) {
     if (!sel) return;
-    liByPath.forEach((li) => {
+    for (const li of highlightedLis) {
       delete li.dataset.highlight;
-    });
-    for (let i = 1; i <= sel.path.length; i++) {
-      const pathKey = sel.path.slice(0, i).join("/");
+    }
+    highlightedLis = [];
+    let pathKey = "";
+    for (const segment of sel.path) {
+      pathKey = pathKey ? `${pathKey}/${segment}` : segment;
       const li = liByPath.get(pathKey);
-      if (li) li.dataset.highlight = "";
+      if (li) {
+        li.dataset.highlight = "";
+        highlightedLis.push(li);
+      }
     }
   }
 
@@ -89,6 +84,24 @@ export function initOptions() {
     );
   }
 
+  /**
+   * @template T
+   * @param {() => T} fn
+   * @returns {() => T}
+   */
+  function lazy(fn) {
+    /** @type {T | undefined} */
+    let cached;
+    let computed = false;
+    return () => {
+      if (!computed) {
+        computed = true;
+        cached = fn();
+      }
+      return /** @type {T} */ (cached);
+    };
+  }
+
   /**
    * @param {(AnyFetchedSeriesBlueprint | FetchedPriceSeriesBlueprint)[]} [arr]
    */
@@ -122,11 +135,9 @@ export function initOptions() {
         );
       if (maybePriceMetric.dollars?.by && maybePriceMetric.sats?.by) {
         const { dollars, sats } = maybePriceMetric;
-        markUsed(dollars);
         if (!usdArr) map.set(Unit.usd, (usdArr = []));
         usdArr.push({ ...blueprint, metric: dollars, unit: Unit.usd });
 
-        markUsed(sats);
         if (!satsArr) map.set(Unit.sats, (satsArr = []));
         satsArr.push({ ...blueprint, metric: sats, unit: Unit.sats });
         continue;
@@ -140,7 +151,6 @@ export function initOptions() {
       const unit = regularBlueprint.unit;
       if (!unit) continue;
 
-      markUsed(metric);
       let unitArr = map.get(unit);
       if (!unitArr) map.set(unit, (unitArr = []));
       unitArr.push(regularBlueprint);
@@ -169,7 +179,6 @@ export function initOptions() {
       if (!arr) continue;
       for (const baseValue of values) {
         const metric = getConstant(brk.metrics.constants, baseValue);
-        markUsed(metric);
         arr.push({
           metric,
           title: `${baseValue}`,
@@ -240,8 +249,8 @@ export function initOptions() {
   let savedOption;
 
   /**
-   * @typedef {{ type: "group"; name: string; serName: string; path: string[]; count: number; children: ProcessedNode[] }} ProcessedGroup
-   * @typedef {{ type: "option"; option: Option; path: string[] }} ProcessedOption
+   * @typedef {{ type: "group"; name: string; serName: string; path: string[]; pathKey: string; count: number; children: ProcessedNode[] }} ProcessedGroup
+   * @typedef {{ type: "option"; option: Option; path: string[]; pathKey: string }} ProcessedOption
    * @typedef {ProcessedGroup | ProcessedOption} ProcessedNode
    */
 
@@ -285,6 +294,7 @@ export function initOptions() {
           name: anyPartial.name,
           serName,
           path,
+          pathKey: pathStr,
           count,
           children,
         });
@@ -322,6 +332,10 @@ export function initOptions() {
           );
         } else {
           const title = option.title || name;
+          const topArr = anyPartial.top;
+          const bottomArr = anyPartial.bottom;
+          const topFn = lazy(() => arrayToMap(topArr));
+          const bottomFn = lazy(() => arrayToMap(bottomArr));
           Object.assign(
             option,
             /** @satisfies {ChartOption} */ ({
@@ -329,8 +343,8 @@ export function initOptions() {
               name,
               title,
               path,
-              top: arrayToMap(anyPartial.top),
-              bottom: arrayToMap(anyPartial.bottom),
+              top: topFn,
+              bottom: bottomFn,
             }),
           );
         }
@@ -349,6 +363,7 @@ export function initOptions() {
           type: "option",
           option,
           path,
+          pathKey: pathStr,
         });
       }
     }
@@ -356,8 +371,8 @@ export function initOptions() {
     return { nodes, count: totalCount };
   }
 
+  logUnused(brk.metrics, partialOptions);
   const { nodes: processedTree } = processPartialTree(partialOptions);
-  logUnused();
 
   /**
    * @param {ProcessedNode[]} nodes
@@ -365,16 +380,16 @@ export function initOptions() {
    */
   function buildTreeDOM(nodes, parentEl) {
     const ul = window.document.createElement("ul");
-    parentEl.append(ul);
 
     for (const node of nodes) {
       const li = window.document.createElement("li");
       ul.append(li);
 
-      const pathKey = node.path.join("/");
-      liByPath.set(pathKey, li);
+      liByPath.set(node.pathKey, li);
 
-      if (isOnSelectedPath(node.path)) {
+      const onSelectedPath = isOnSelectedPath(node.path);
+
+      if (onSelectedPath) {
         li.dataset.highlight = "";
       }
 
@@ -392,6 +407,11 @@ export function initOptions() {
         summary.append(count);
 
         let built = false;
+        if (onSelectedPath) {
+          built = true;
+          details.open = true;
+          buildTreeDOM(node.children, details);
+        }
         details.addEventListener("toggle", () => {
           if (details.open && !built) {
             built = true;
@@ -405,6 +425,8 @@ export function initOptions() {
         li.append(element);
       }
     }
+
+    parentEl.append(ul);
   }
 
   /** @type {HTMLElement | null} */

website/scripts/options/types.js

@@ -90,8 +90,8 @@
  * @typedef {PartialOption & PartialChartOptionSpecific} PartialChartOption
  *
  * @typedef {Object} ProcessedChartOptionAddons
- * @property {Map<Unit, AnyFetchedSeriesBlueprint[]>} top
- * @property {Map<Unit, AnyFetchedSeriesBlueprint[]>} bottom
+ * @property {() => Map<Unit, AnyFetchedSeriesBlueprint[]>} top
+ * @property {() => Map<Unit, AnyFetchedSeriesBlueprint[]>} bottom
  *
  * @typedef {Required<Omit<PartialChartOption, "top" | "bottom">> & ProcessedChartOptionAddons & ProcessedOptionAddons} ChartOption
  *

website/scripts/options/unused.js

@@ -1,9 +1,6 @@
 import { localhost } from "../utils/env.js";
 import { serdeChartableIndex } from "../utils/serde.js";
 
-/** @type {Map<AnyMetricPattern, string[]> | null} */
-export const unused = localhost ? new Map() : null;
-
 /**
  * Check if a metric pattern has at least one chartable index
  * @param {AnyMetricPattern} node
@@ -15,11 +12,12 @@ function hasChartableIndex(node) {
 }
 
 /**
+ * Walk a metrics tree and collect all chartable metric patterns
  * @param {TreeNode | null | undefined} node
  * @param {Map<AnyMetricPattern, string[]>} map
  * @param {string[]} path
  */
-function walk(node, map, path) {
+function walkMetrics(node, map, path) {
   if (node && "by" in node) {
     const metricNode = /** @type {AnyMetricPattern} */ (node);
     if (!hasChartableIndex(metricNode)) return;
@@ -33,32 +31,22 @@ function walk(node, map, path) {
         key.endsWith("State") ||
         key.endsWith("Start") ||
         kn === "mvrv" ||
-        //   kn === "time" ||
-        //   kn === "height" ||
         kn === "constants" ||
         kn === "blockhash" ||
         kn === "date" ||
-        //   kn === "oracle" ||
         kn === "split" ||
-        //   kn === "ohlc" ||
         kn === "outpoint" ||
         kn === "positions" ||
-        //   kn === "outputtype" ||
         kn === "heighttopool" ||
         kn === "txid" ||
         kn.startsWith("timestamp") ||
         kn.startsWith("satdays") ||
         kn.startsWith("satblocks") ||
-        //   kn.endsWith("state") ||
-        //   kn.endsWith("cents") ||
         kn.endsWith("index") ||
         kn.endsWith("indexes")
-        //   kn.endsWith("raw") ||
-        //   kn.endsWith("bytes") ||
-        //   (kn.startsWith("_") && kn.endsWith("start"))
       )
         continue;
-      walk(/** @type {TreeNode | null | undefined} */ (value), map, [
+      walkMetrics(/** @type {TreeNode | null | undefined} */ (value), map, [
         ...path,
         key,
       ]);
@@ -67,29 +55,64 @@ function walk(node, map, path) {
 }
 
 /**
- * Collect all AnyMetricPatterns from tree
- * @param {TreeNode} tree
+ * Walk partial options tree and delete referenced metrics from the map
+ * @param {PartialOptionsTree} options
+ * @param {Map<AnyMetricPattern, string[]>} map
  */
-export function collect(tree) {
-  if (unused) walk(tree, unused, []);
+function walkOptions(options, map) {
+  for (const node of options) {
+    if ("tree" in node && node.tree) {
+      walkOptions(node.tree, map);
+    } else if ("top" in node || "bottom" in node) {
+      const chartNode = /** @type {PartialChartOption} */ (node);
+      markUsedBlueprints(map, chartNode.top);
+      markUsedBlueprints(map, chartNode.bottom);
+    }
+  }
 }
 
 /**
- * Mark a metric as used
- * @param {AnyMetricPattern} metric
+ * @param {Map<AnyMetricPattern, string[]>} map
+ * @param {(AnyFetchedSeriesBlueprint | FetchedPriceSeriesBlueprint)[]} [arr]
  */
-export function markUsed(metric) {
-  unused?.delete(metric);
+function markUsedBlueprints(map, arr) {
+  if (!arr) return;
+  for (let i = 0; i < arr.length; i++) {
+    const metric = arr[i].metric;
+    if (!metric) continue;
+    const maybePriceMetric =
+      /** @type {{ dollars?: AnyMetricPattern, sats?: AnyMetricPattern }} */ (
+        /** @type {unknown} */ (metric)
+      );
+    if (maybePriceMetric.dollars?.by && maybePriceMetric.sats?.by) {
+      map.delete(maybePriceMetric.dollars);
+      map.delete(maybePriceMetric.sats);
+    } else {
+      map.delete(/** @type {AnyMetricPattern} */ (metric));
+    }
+  }
 }
 
-/** Log unused metrics to console */
-export function logUnused() {
-  if (!unused?.size) return;
+/**
+ * Log unused metrics to console (localhost only)
+ * @param {TreeNode} metricsTree
+ * @param {PartialOptionsTree} partialOptions
+ */
+export function logUnused(metricsTree, partialOptions) {
+  if (!localhost) return;
+
+  console.log(extractTreeStructure(partialOptions));
+
+  /** @type {Map<AnyMetricPattern, string[]>} */
+  const all = new Map();
+  walkMetrics(metricsTree, all, []);
+  walkOptions(partialOptions, all);
+
+  if (!all.size) return;
 
   /** @type {Record<string, any>} */
   const tree = {};
-
-  for (const path of unused.values()) {
+  for (const path of all.values()) {
     let current = tree;
     for (let i = 0; i < path.length; i++) {
       const part = path[i];
@@ -102,7 +125,7 @@ export function logUnused() {
     }
   }
 
-  console.log("Unused metrics:", { count: unused.size, tree });
+  console.log("Unused metrics:", { count: all.size, tree });
 }
 
 /**
@@ -121,7 +144,6 @@ export function extractTreeStructure(options) {
     /** @type {Record<string, string[]>} */
     const grouped = {};
     for (const s of series) {
-      // Price patterns in top pane have dollars/sats sub-metrics
       const metric = /** @type {any} */ (s.metric);
       if (isTop && metric?.dollars && metric?.sats) {
         const title = s.title || s.key || "unnamed";
@@ -142,14 +164,12 @@ export function extractTreeStructure(options) {
    * @returns {object}
    */
   function processNode(node) {
-    // Group with children
     if ("tree" in node && node.tree) {
       return {
         name: node.name,
         children: node.tree.map(processNode),
       };
     }
-    // Chart option
     if ("top" in node || "bottom" in node) {
       const chartNode = /** @type {PartialChartOption} */ (node);
       const top = chartNode.top ? groupByUnit(chartNode.top, true) : undefined;
@@ -163,23 +183,11 @@ export function extractTreeStructure(options) {
         ...(bottom && Object.keys(bottom).length > 0 ? { bottom } : {}),
       };
     }
-    // URL option
     if ("url" in node) {
       return { name: node.name, url: true };
     }
-    // Other options (explorer, table, simulation)
     return { name: node.name };
   }
 
   return options.map(processNode);
 }
-
-/**
- * Log the options tree structure to console (localhost only)
- * @param {PartialOptionsTree} options
- */
-export function logTreeStructure(options) {
-  if (!localhost) return;
-  const structure = extractTreeStructure(options);
-  console.log("Options tree structure:", JSON.stringify(structure, null, 2));
-}

website/scripts/panes/chart.js

@@ -91,8 +91,8 @@ export function init() {
     // Set blueprints first so storageId is correct before any index change
     chart.setBlueprints({
       name: opt.title,
-      top: buildTopBlueprints(opt.top),
-      bottom: opt.bottom,
+      top: buildTopBlueprints(opt.top()),
+      bottom: opt.bottom(),
       onDataLoaded: updatePriceWithLatest,
     });
 
@@ -120,11 +120,11 @@ const ALL_CHOICES = /** @satisfies {ChartableIndexName[]} */ ([
  * @returns {ChartableIndexName[]}
  */
 function computeChoices(opt) {
-  if (!opt.top.size && !opt.bottom.size) {
+  if (!opt.top().size && !opt.bottom().size) {
     return [...ALL_CHOICES];
   }
   const rawIndexes = new Set(
-    [Array.from(opt.top.values()), Array.from(opt.bottom.values())]
+    [Array.from(opt.top().values()), Array.from(opt.bottom().values())]
       .flat(2)
       .filter((blueprint) => {
         const path = Object.values(blueprint.metric.by)[0]?.path ?? "";