global: snapshot

crates/brk_bindgen/src/generators/javascript/client.rs

@@ -51,9 +51,11 @@ class BrkError extends Error {{
 /**
  * @template T
  * @typedef {{Object}} MetricData
+ * @property {{number}} version - Version of the metric data
  * @property {{number}} total - Total number of data points
  * @property {{number}} start - Start index (inclusive)
  * @property {{number}} end - End index (exclusive)
+ * @property {{string}} stamp - ISO 8601 timestamp of when the response was generated
  * @property {{T[]}} data - The metric data
  */
 /** @typedef {{MetricData<any>}} AnyMetricData */

crates/brk_bindgen/src/generators/python/client.rs

@@ -133,9 +133,11 @@ pub fn generate_endpoint_class(output: &mut String) {
         output,
         r#"class MetricData(TypedDict, Generic[T]):
     """Metric data with range information."""
+    version: int
     total: int
     start: int
     end: int
+    stamp: str
     data: List[T]
 
 

crates/brk_cli/src/main.rs

@@ -51,6 +51,22 @@ pub fn run() -> anyhow::Result<()> {
 
     let mut indexer = Indexer::forced_import(&config.brkdir())?;
 
+    #[cfg(not(debug_assertions))]
+    {
+        // Pre-run indexer if too far behind, then drop and reimport to reduce memory
+        let chain_height = client.get_last_height()?;
+        let indexed_height = indexer.vecs.starting_height();
+        let blocks_behind = chain_height.saturating_sub(*indexed_height);
+        if blocks_behind > 1000 {
+            info!("Indexing {blocks_behind} blocks before starting server...");
+            sleep(Duration::from_secs(3));
+            indexer.index(&blocks, &client, &exit)?;
+            drop(indexer);
+            Mimalloc::collect();
+            indexer = Indexer::forced_import(&config.brkdir())?;
+        }
+    }
+
     let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
 
     let mempool = Mempool::new(&client);

crates/brk_computer/Cargo.toml

@@ -32,6 +32,5 @@ vecdb = { workspace = true }
 
 [dev-dependencies]
 brk_alloc = { workspace = true }
-plotters = "0.3"
 brk_bencher = { workspace = true }
 color-eyre = { workspace = true }

crates/brk_computer/src/internal/compute.rs

@@ -70,7 +70,9 @@ where
         }};
     }
 
-    let index = validate_vec!(first, last, min, max, average, sum, cumulative, median, pct10, pct25, pct75, pct90);
+    let index = validate_vec!(
+        first, last, min, max, average, sum, cumulative, median, pct10, pct25, pct75, pct90
+    );
 
     let needs_first = first.is_some();
     let needs_last = last.is_some();
@@ -298,7 +300,9 @@ where
         };
     }
 
-    write_vec!(first, last, min, max, average, sum, cumulative, median, pct10, pct25, pct75, pct90);
+    write_vec!(
+        first, last, min, max, average, sum, cumulative, median, pct10, pct25, pct75, pct90
+    );
 
     Ok(())
 }
@@ -306,7 +310,7 @@ where
 /// Compute cumulative extension from a source vec.
 ///
 /// Used when only cumulative needs to be extended from an existing source.
-pub fn compute_cumulative_extend<I, T>(
+pub fn compute_cumulative<I, T>(
     max_from: I,
     source: &impl IterableVec<I, T>,
     cumulative: &mut EagerVec<PcoVec<I, T>>,
@@ -340,6 +344,47 @@ where
     Ok(())
 }
 
+/// Compute cumulative from binary transform of two source vecs.
+pub fn compute_cumulative_transform2<I, T, S1, S2, F>(
+    max_from: I,
+    source1: &impl IterableVec<I, S1>,
+    source2: &impl IterableVec<I, S2>,
+    cumulative: &mut EagerVec<PcoVec<I, T>>,
+    transform: F,
+    exit: &Exit,
+) -> Result<()>
+where
+    I: VecIndex,
+    T: ComputedVecValue + JsonSchema,
+    S1: VecValue,
+    S2: VecValue,
+    F: Fn(S1, S2) -> T,
+{
+    let combined_version = source1.version() + source2.version();
+    cumulative.validate_computed_version_or_reset(combined_version)?;
+
+    let index = max_from.min(I::from(cumulative.len()));
+    let target_len = source1.len().min(source2.len());
+
+    let mut cumulative_val = index
+        .decremented()
+        .map_or(T::from(0_usize), |idx| cumulative.read_unwrap_once(idx));
+
+    let mut iter1 = source1.iter();
+    let mut iter2 = source2.iter();
+
+    for i in index.to_usize()..target_len {
+        let idx = I::from(i);
+        cumulative_val += transform(iter1.get_unwrap(idx), iter2.get_unwrap(idx));
+        cumulative.truncate_push_at(i, cumulative_val)?;
+    }
+
+    let _lock = exit.lock();
+    cumulative.write()?;
+
+    Ok(())
+}
+
 /// Compute coarser aggregations from already-aggregated source data.
 ///
 /// This is used for dateindex → weekindex, monthindex, etc. where we derive

crates/brk_computer/src/internal/multi/from_tx/value_dollars.rs

@@ -12,15 +12,14 @@ use brk_types::{
 };
 use derive_more::{Deref, DerefMut};
 use vecdb::{
-    AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
-    IterableBoxedVec, IterableCloneableVec, IterableVec, LazyVecFrom3,
+    Database, EagerVec, Exit, ImportableVec, IterableBoxedVec, IterableCloneableVec, LazyVecFrom3,
 };
 
 use crate::{
     ComputeIndexes, indexes,
     internal::{
         CumulativeVec, Full, LazyBinaryTransformFull, LazyDateDerivedFull, LazyFull,
-        SatsTimesClosePrice, Stats,
+        SatsTimesClosePrice, Stats, compute_cumulative,
     },
 };
 
@@ -137,7 +136,12 @@ impl ValueDollarsFromTxFull {
         exit: &Exit,
     ) -> Result<()> {
         // Compute height cumulative by summing lazy height.sum values
-        self.compute_height_cumulative(starting_indexes.height, exit)?;
+        compute_cumulative(
+            starting_indexes.height,
+            &self.height.sum,
+            &mut self.height_cumulative.0,
+            exit,
+        )?;
 
         // Compute dateindex stats by aggregating lazy height stats
         self.dateindex.compute(
@@ -150,30 +154,6 @@ impl ValueDollarsFromTxFull {
 
         Ok(())
     }
-
-    /// Compute cumulative USD by summing `sum_sats[h] * price[h]` for all heights.
-    fn compute_height_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
-        let starting_height = max_from.min(Height::from(self.height_cumulative.0.len()));
-
-        let mut cumulative = starting_height.decremented().map_or(Dollars::ZERO, |h| {
-            self.height_cumulative.0.iter().get_unwrap(h)
-        });
-
-        let mut sum_iter = self.height.sum.iter();
-        let start_idx = *starting_height as usize;
-        let end_idx = sum_iter.len();
-
-        for h in start_idx..end_idx {
-            let sum_usd = sum_iter.get_unwrap(Height::from(h));
-            cumulative += sum_usd;
-            self.height_cumulative.0.truncate_push_at(h, cumulative)?;
-        }
-
-        let _lock = exit.lock();
-        self.height_cumulative.0.write()?;
-
-        Ok(())
-    }
 }
 
 fn create_lazy_txindex(

crates/brk_computer/src/internal/multi/height_derived/full.rs

@@ -11,8 +11,8 @@ use vecdb::{Database, Exit, IterableBoxedVec, IterableCloneableVec, IterableVec}
 use crate::{
     ComputeIndexes, indexes,
     internal::{
-        ComputedVecValue, CumulativeVec, LazyDateDerivedFull, Full, LazyFull, NumericValue,
-        compute_cumulative_extend,
+        ComputedVecValue, CumulativeVec, Full, LazyDateDerivedFull, LazyFull, NumericValue,
+        compute_cumulative,
     },
 };
 
@@ -102,6 +102,6 @@ where
         height_source: &impl IterableVec<Height, T>,
         exit: &Exit,
     ) -> Result<()> {
-        compute_cumulative_extend(max_from, height_source, &mut self.height_cumulative.0, exit)
+        compute_cumulative(max_from, height_source, &mut self.height_cumulative.0, exit)
     }
 }

crates/brk_computer/src/internal/multi/height_derived/sum_cum.rs

@@ -15,7 +15,7 @@ use crate::{
     ComputeIndexes, indexes,
     internal::{
         ComputedVecValue, CumulativeVec, LazyDateDerivedSumCum, LazySumCum, NumericValue, SumCum,
-        compute_cumulative_extend,
+        compute_cumulative,
     },
 };
 
@@ -99,7 +99,7 @@ where
         source: &impl IterableVec<Height, T>,
         exit: &Exit,
     ) -> Result<()> {
-        compute_cumulative_extend(max_from, source, &mut self.height_cumulative.0, exit)
+        compute_cumulative(max_from, source, &mut self.height_cumulative.0, exit)
     }
 
     fn compute_dateindex_sum_cum(

crates/brk_computer/src/internal/single/group/sum_cum.rs

@@ -1,7 +1,10 @@
 use brk_error::Result;
 use brk_traversable::Traversable;
 use schemars::JsonSchema;
-use vecdb::{AnyVec, Database, Exit, IterableBoxedVec, IterableCloneableVec, IterableVec, VecIndex, VecValue, Version};
+use vecdb::{
+    AnyVec, Database, Exit, IterableBoxedVec, IterableCloneableVec, IterableVec, VecIndex,
+    VecValue, Version,
+};
 
 use crate::internal::{ComputedVecValue, CumulativeVec, SumVec};
 
@@ -48,7 +51,7 @@ impl<I: VecIndex, T: ComputedVecValue + JsonSchema> SumCum<I, T> {
             first_indexes,
             count_indexes,
             exit,
-            0, // min_skip_count
+            0,    // min_skip_count
             None, // first
             None, // last
             None, // min
@@ -64,16 +67,6 @@ impl<I: VecIndex, T: ComputedVecValue + JsonSchema> SumCum<I, T> {
         )
     }
 
-    /// Extend cumulative from an existing source vec.
-    pub fn extend_cumulative(
-        &mut self,
-        max_from: I,
-        source: &impl IterableVec<I, T>,
-        exit: &Exit,
-    ) -> Result<()> {
-        crate::internal::compute_cumulative_extend(max_from, source, &mut self.cumulative.0, exit)
-    }
-
     pub fn len(&self) -> usize {
         self.sum.0.len().min(self.cumulative.0.len())
     }

crates/brk_logger/Cargo.toml

@@ -9,9 +9,7 @@ repository.workspace = true
 
 [dependencies]
 jiff = { workspace = true }
-logroller = "0.1"
 owo-colors = "4.2.3"
 tracing = { workspace = true }
-tracing-appender = "0.2"
 tracing-log = "0.2"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "std"] }

crates/brk_logger/src/format.rs

@@ -0,0 +1,168 @@
+use std::fmt::Write;
+
+use jiff::{Timestamp, tz};
+use owo_colors::OwoColorize;
+use tracing::{Event, Level, Subscriber, field::Field};
+use tracing_subscriber::{
+    fmt::{FmtContext, FormatEvent, FormatFields, format::Writer},
+    registry::LookupSpan,
+};
+
+// Don't remove, used to know the target of unwanted logs
+const WITH_TARGET: bool = false;
+// const WITH_TARGET: bool = true;
+
+const fn level_str(level: Level) -> &'static str {
+    match level {
+        Level::ERROR => "error",
+        Level::WARN => "warn ",
+        Level::INFO => "info ",
+        Level::DEBUG => "debug",
+        Level::TRACE => "trace",
+    }
+}
+
+pub struct Formatter<const ANSI: bool>;
+
+impl<S, N, const ANSI: bool> FormatEvent<S, N> for Formatter<ANSI>
+where
+    S: Subscriber + for<'a> LookupSpan<'a>,
+    N: for<'a> FormatFields<'a> + 'static,
+{
+    fn format_event(
+        &self,
+        _ctx: &FmtContext<'_, S, N>,
+        mut writer: Writer<'_>,
+        event: &Event<'_>,
+    ) -> std::fmt::Result {
+        let ts = Timestamp::now()
+            .to_zoned(tz::TimeZone::system())
+            .strftime("%Y-%m-%d %H:%M:%S")
+            .to_string();
+
+        let level = *event.metadata().level();
+        let level_str = level_str(level);
+
+        if ANSI {
+            let level_colored = match level {
+                Level::ERROR => level_str.red().to_string(),
+                Level::WARN => level_str.yellow().to_string(),
+                Level::INFO => level_str.green().to_string(),
+                Level::DEBUG => level_str.blue().to_string(),
+                Level::TRACE => level_str.cyan().to_string(),
+            };
+            if WITH_TARGET {
+                write!(
+                    writer,
+                    "{} {} {} {level_colored} ",
+                    ts.bright_black(),
+                    event.metadata().target(),
+                    "-".bright_black(),
+                )?;
+            } else {
+                write!(
+                    writer,
+                    "{} {} {level_colored} ",
+                    ts.bright_black(),
+                    "-".bright_black()
+                )?;
+            }
+        } else if WITH_TARGET {
+            write!(writer, "{ts} {} - {level_str} ", event.metadata().target())?;
+        } else {
+            write!(writer, "{ts} - {level_str} ")?;
+        }
+
+        let mut visitor = FieldVisitor::<ANSI>::new();
+        event.record(&mut visitor);
+        write!(writer, "{}", visitor.finish())?;
+        writeln!(writer)
+    }
+}
+
+struct FieldVisitor<const ANSI: bool> {
+    result: String,
+    status: Option<u64>,
+    uri: Option<String>,
+    latency: Option<String>,
+}
+
+impl<const ANSI: bool> FieldVisitor<ANSI> {
+    fn new() -> Self {
+        Self {
+            result: String::new(),
+            status: None,
+            uri: None,
+            latency: None,
+        }
+    }
+
+    fn finish(self) -> String {
+        if let Some(status) = self.status {
+            let status_str = if ANSI {
+                match status {
+                    200..=299 => status.green().to_string(),
+                    300..=399 => status.bright_black().to_string(),
+                    _ => status.red().to_string(),
+                }
+            } else {
+                status.to_string()
+            };
+
+            let uri = self.uri.as_deref().unwrap_or("");
+            let latency = self.latency.as_deref().unwrap_or("");
+
+            if ANSI {
+                format!("{status_str} {uri} {}", latency.bright_black())
+            } else {
+                format!("{status_str} {uri} {latency}")
+            }
+        } else {
+            self.result
+        }
+    }
+}
+
+impl<const ANSI: bool> tracing::field::Visit for FieldVisitor<ANSI> {
+    fn record_u64(&mut self, field: &Field, value: u64) {
+        let name = field.name();
+        if name == "status" {
+            self.status = Some(value);
+        } else if !name.starts_with("log.") {
+            let _ = write!(self.result, "{}={} ", name, value);
+        }
+    }
+
+    fn record_i64(&mut self, field: &Field, value: i64) {
+        let name = field.name();
+        if !name.starts_with("log.") {
+            let _ = write!(self.result, "{}={} ", name, value);
+        }
+    }
+
+    fn record_str(&mut self, field: &Field, value: &str) {
+        let name = field.name();
+        if name == "uri" {
+            self.uri = Some(value.to_string());
+        } else if name == "message" {
+            let _ = write!(self.result, "{value}");
+        } else if !name.starts_with("log.") {
+            let _ = write!(self.result, "{}={} ", name, value);
+        }
+    }
+
+    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+        let name = field.name();
+        match name {
+            "uri" => self.uri = Some(format!("{value:?}")),
+            "latency" => self.latency = Some(format!("{value:?}")),
+            "message" => {
+                let _ = write!(self.result, "{value:?}");
+            }
+            _ if name.starts_with("log.") => {}
+            _ => {
+                let _ = write!(self.result, "{}={:?} ", name, value);
+            }
+        }
+    }
+}

crates/brk_logger/src/hook.rs

@@ -0,0 +1,30 @@
+use std::{fmt::Write, sync::OnceLock};
+
+use tracing::{Event, Subscriber, field::Field};
+
+type LogHook = Box<dyn Fn(&str) + Send + Sync>;
+
+pub static LOG_HOOK: OnceLock<LogHook> = OnceLock::new();
+
+pub struct HookLayer;
+
+impl<S: Subscriber> tracing_subscriber::Layer<S> for HookLayer {
+    fn on_event(&self, event: &Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) {
+        if let Some(hook) = LOG_HOOK.get() {
+            let mut msg = String::new();
+            event.record(&mut MessageVisitor(&mut msg));
+            hook(&msg);
+        }
+    }
+}
+
+struct MessageVisitor<'a>(&'a mut String);
+
+impl tracing::field::Visit for MessageVisitor<'_> {
+    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+        if field.name() == "message" {
+            self.0.clear();
+            let _ = write!(self.0, "{value:?}");
+        }
+    }
+}

crates/brk_logger/src/lib.rs

@@ -1,215 +1,21 @@
 #![doc = include_str!("../README.md")]
 
-use std::{fmt::Write as _, io, path::Path, sync::OnceLock};
+mod format;
+mod hook;
+mod rate_limit;
 
-use jiff::{Timestamp, tz};
-use logroller::{LogRollerBuilder, Rotation, RotationSize};
-use owo_colors::OwoColorize;
-use tracing::{Event, Level, Subscriber, field::Field};
-use tracing_appender::non_blocking::WorkerGuard;
-use tracing_subscriber::{
-    EnvFilter,
-    fmt::{self, FmtContext, FormatEvent, FormatFields, format::Writer},
-    layer::SubscriberExt,
-    registry::LookupSpan,
-    util::SubscriberInitExt,
-};
+use std::{io, path::Path, time::Duration};
 
-type LogHook = Box<dyn Fn(&str) + Send + Sync>;
+use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};
 
-static GUARD: OnceLock<WorkerGuard> = OnceLock::new();
-static LOG_HOOK: OnceLock<LogHook> = OnceLock::new();
+use format::Formatter;
+use hook::{HookLayer, LOG_HOOK};
+use rate_limit::RateLimitedFile;
 
-const MAX_LOG_FILES: u64 = 5;
-const MAX_FILE_SIZE_MB: u64 = 42;
-
-// Don't remove, used to know the target of unwanted logs
-const WITH_TARGET: bool = false;
-// const WITH_TARGET: bool = true;
-
-const fn level_str(level: Level) -> &'static str {
-    match level {
-        Level::ERROR => "error",
-        Level::WARN => "warn ",
-        Level::INFO => "info ",
-        Level::DEBUG => "debug",
-        Level::TRACE => "trace",
-    }
-}
-
-struct Formatter<const ANSI: bool>;
-
-/// Visitor that collects structured fields for colored formatting
-struct FieldVisitor<const ANSI: bool> {
-    result: String,
-    status: Option<u64>,
-    uri: Option<String>,
-    latency: Option<String>,
-}
-
-impl<const ANSI: bool> FieldVisitor<ANSI> {
-    fn new() -> Self {
-        Self {
-            result: String::new(),
-            status: None,
-            uri: None,
-            latency: None,
-        }
-    }
-
-    fn finish(self) -> String {
-        // Format HTTP-style log if we have status
-        if let Some(status) = self.status {
-            let status_str = if ANSI {
-                match status {
-                    200..=299 => status.green().to_string(),
-                    300..=399 => status.bright_black().to_string(),
-                    _ => status.red().to_string(),
-                }
-            } else {
-                status.to_string()
-            };
-
-            let uri = self.uri.as_deref().unwrap_or("");
-            let latency = self.latency.as_deref().unwrap_or("");
-
-            if ANSI {
-                format!("{status_str} {uri} {}", latency.bright_black())
-            } else {
-                format!("{status_str} {uri} {latency}")
-            }
-        } else {
-            self.result
-        }
-    }
-}
-
-impl<const ANSI: bool> tracing::field::Visit for FieldVisitor<ANSI> {
-    fn record_u64(&mut self, field: &Field, value: u64) {
-        let name = field.name();
-        if name == "status" {
-            self.status = Some(value);
-        } else if !name.starts_with("log.") {
-            let _ = write!(self.result, "{}={} ", name, value);
-        }
-    }
-
-    fn record_i64(&mut self, field: &Field, value: i64) {
-        let name = field.name();
-        if !name.starts_with("log.") {
-            let _ = write!(self.result, "{}={} ", name, value);
-        }
-    }
-
-    fn record_str(&mut self, field: &Field, value: &str) {
-        let name = field.name();
-        if name == "uri" {
-            self.uri = Some(value.to_string());
-        } else if name == "message" {
-            let _ = write!(self.result, "{value}");
-        } else if !name.starts_with("log.") {
-            let _ = write!(self.result, "{}={} ", name, value);
-        }
-    }
-
-    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
-        let name = field.name();
-        match name {
-            "uri" => self.uri = Some(format!("{value:?}")),
-            "latency" => self.latency = Some(format!("{value:?}")),
-            "message" => {
-                let _ = write!(self.result, "{value:?}");
-            }
-            _ if name.starts_with("log.") => {}
-            _ => {
-                let _ = write!(self.result, "{}={:?} ", name, value);
-            }
-        }
-    }
-}
-
-impl<S, N, const ANSI: bool> FormatEvent<S, N> for Formatter<ANSI>
-where
-    S: Subscriber + for<'a> LookupSpan<'a>,
-    N: for<'a> FormatFields<'a> + 'static,
-{
-    fn format_event(
-        &self,
-        _ctx: &FmtContext<'_, S, N>,
-        mut writer: Writer<'_>,
-        event: &Event<'_>,
-    ) -> std::fmt::Result {
-        let ts = Timestamp::now()
-            .to_zoned(tz::TimeZone::system())
-            .strftime("%Y-%m-%d %H:%M:%S")
-            .to_string();
-
-        let level = *event.metadata().level();
-        let level_str = level_str(level);
-
-        if ANSI {
-            let level_colored = match level {
-                Level::ERROR => level_str.red().to_string(),
-                Level::WARN => level_str.yellow().to_string(),
-                Level::INFO => level_str.green().to_string(),
-                Level::DEBUG => level_str.blue().to_string(),
-                Level::TRACE => level_str.cyan().to_string(),
-            };
-            if WITH_TARGET {
-                write!(
-                    writer,
-                    "{} {} {} {level_colored} ",
-                    ts.bright_black(),
-                    event.metadata().target(),
-                    "-".bright_black(),
-                )?;
-            } else {
-                write!(
-                    writer,
-                    "{} {} {level_colored} ",
-                    ts.bright_black(),
-                    "-".bright_black()
-                )?;
-            }
-        } else if WITH_TARGET {
-            write!(writer, "{ts} {} - {level_str} ", event.metadata().target())?;
-        } else {
-            write!(writer, "{ts} - {level_str} ")?;
-        }
-
-        let mut visitor = FieldVisitor::<ANSI>::new();
-        event.record(&mut visitor);
-        write!(writer, "{}", visitor.finish())?;
-        writeln!(writer)
-    }
-}
-
-struct HookLayer;
-
-impl<S: Subscriber> tracing_subscriber::Layer<S> for HookLayer {
-    fn on_event(&self, event: &Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) {
-        if let Some(hook) = LOG_HOOK.get() {
-            let mut msg = String::new();
-            event.record(&mut MessageVisitor(&mut msg));
-            hook(&msg);
-        }
-    }
-}
-
-struct MessageVisitor<'a>(&'a mut String);
-
-impl tracing::field::Visit for MessageVisitor<'_> {
-    fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
-        use std::fmt::Write;
-        if field.name() == "message" {
-            self.0.clear();
-            let _ = write!(self.0, "{value:?}");
-        }
-    }
-}
+/// Days to keep log files before cleanup
+const MAX_LOG_AGE_DAYS: u64 = 7;
 
 pub fn init(path: Option<&Path>) -> io::Result<()> {
-    // Bridge log crate to tracing (for vecdb and other log-based crates)
     tracing_log::LogTracer::init().ok();
 
     #[cfg(debug_assertions)]
@@ -217,12 +23,11 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
     #[cfg(not(debug_assertions))]
     const DEFAULT_LEVEL: &str = "info";
 
-    let default_filter = format!(
-        "{DEFAULT_LEVEL},bitcoin=off,bitcoincore-rpc=off,fjall=off,brk_fjall=off,lsm_tree=off,brk_rolldown=off,rolldown=off,tracing=off,aide=off,rustls=off,notify=off,oxc_resolver=off,tower_http=off"
-    );
-
-    let filter =
-        EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_filter));
+    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
+        EnvFilter::new(format!(
+            "{DEFAULT_LEVEL},bitcoin=off,bitcoincore-rpc=off,fjall=off,brk_fjall=off,lsm_tree=off,brk_rolldown=off,rolldown=off,tracing=off,aide=off,rustls=off,notify=off,oxc_resolver=off,tower_http=off"
+        ))
+    });
 
     let registry = tracing_subscriber::registry()
         .with(filter)
@@ -231,25 +36,20 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
 
     if let Some(path) = path {
         let dir = path.parent().unwrap_or(Path::new("."));
-        let filename = path
+        let prefix = path
             .file_name()
             .and_then(|s| s.to_str())
             .unwrap_or("app.log");
 
-        let roller = LogRollerBuilder::new(dir, Path::new(filename))
-            .rotation(Rotation::SizeBased(RotationSize::MB(MAX_FILE_SIZE_MB)))
-            .max_keep_files(MAX_LOG_FILES)
-            .build()
-            .map_err(io::Error::other)?;
+        cleanup_old_logs(dir, prefix);
 
-        let (non_blocking, guard) = tracing_appender::non_blocking(roller);
-        GUARD.set(guard).ok();
+        let writer = RateLimitedFile::new(dir, prefix);
 
         registry
             .with(
                 fmt::layer()
                     .event_format(Formatter::<false>)
-                    .with_writer(non_blocking),
+                    .with_writer(writer),
             )
             .init();
     } else {
@@ -260,7 +60,6 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
 }
 
 /// Register a hook that gets called for every log message.
-/// Can only be called once.
 pub fn register_hook<F>(hook: F) -> Result<(), &'static str>
 where
     F: Fn(&str) + Send + Sync + 'static,
@@ -269,3 +68,29 @@ where
         .set(Box::new(hook))
         .map_err(|_| "Hook already registered")
 }
+
+fn cleanup_old_logs(dir: &Path, prefix: &str) {
+    let max_age = Duration::from_secs(MAX_LOG_AGE_DAYS * 24 * 60 * 60);
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+
+    for entry in entries.flatten() {
+        let path = entry.path();
+        let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
+            continue;
+        };
+
+        if !name.starts_with(prefix) || name == prefix {
+            continue;
+        }
+
+        if let Ok(meta) = path.metadata()
+            && let Ok(modified) = meta.modified()
+            && let Ok(age) = modified.elapsed()
+            && age > max_age
+        {
+            let _ = std::fs::remove_file(&path);
+        }
+    }
+}

crates/brk_logger/src/rate_limit.rs

@@ -0,0 +1,90 @@
+use std::{
+    fs::OpenOptions,
+    io::{self, Write},
+    path::PathBuf,
+    sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    },
+    time::{SystemTime, UNIX_EPOCH},
+};
+
+use jiff::{Timestamp, tz};
+use tracing_subscriber::fmt::MakeWriter;
+
+const MAX_WRITES_PER_SEC: u64 = 100;
+
+struct Inner {
+    dir: PathBuf,
+    prefix: String,
+    count: AtomicU64,
+    last_second: AtomicU64,
+}
+
+impl Inner {
+    fn can_write(&self) -> bool {
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+
+        let last = self.last_second.load(Ordering::Relaxed);
+        if now != last {
+            self.last_second.store(now, Ordering::Relaxed);
+            self.count.store(1, Ordering::Relaxed);
+            true
+        } else {
+            self.count.fetch_add(1, Ordering::Relaxed) < MAX_WRITES_PER_SEC
+        }
+    }
+
+    fn path(&self) -> PathBuf {
+        let date = Timestamp::now()
+            .to_zoned(tz::TimeZone::system())
+            .strftime("%Y-%m-%d")
+            .to_string();
+        self.dir.join(format!("{}.{}", self.prefix, date))
+    }
+}
+
+#[derive(Clone)]
+pub struct RateLimitedFile(Arc<Inner>);
+
+impl RateLimitedFile {
+    pub fn new(dir: &std::path::Path, prefix: &str) -> Self {
+        Self(Arc::new(Inner {
+            dir: dir.to_path_buf(),
+            prefix: prefix.to_string(),
+            count: AtomicU64::new(0),
+            last_second: AtomicU64::new(0),
+        }))
+    }
+}
+
+pub struct FileWriter(Arc<Inner>);
+
+impl Write for FileWriter {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        if !self.0.can_write() {
+            return Ok(buf.len());
+        }
+
+        OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(self.0.path())?
+            .write(buf)
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+impl<'a> MakeWriter<'a> for RateLimitedFile {
+    type Writer = FileWriter;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        FileWriter(Arc::clone(&self.0))
+    }
+}

crates/brk_playground/src/lib.rs

@@ -20,7 +20,9 @@ pub use constants::{HeatmapFilter, NUM_BINS, ROUND_USD_AMOUNTS};
 pub use filters::FILTERS;
 pub use oracle::{
     derive_daily_ohlc, derive_daily_ohlc_with_confidence, derive_height_price,
-    derive_ohlc_from_height_prices, derive_price_from_histogram, OracleConfig, OracleResult,
+    derive_height_price_with_confidence, derive_ohlc_from_height_prices,
+    derive_ohlc_from_height_prices_with_confidence, derive_price_from_histogram,
+    HeightPriceResult, OracleConfig, OracleResult,
 };
 pub use signal::{compute_expected_bins_per_day, usd_to_bin};
 pub use histogram::load_or_compute_output_conditions;

crates/brk_server/Cargo.toml

@@ -7,10 +7,13 @@ license.workspace = true
 homepage.workspace = true
 repository.workspace = true
 
+[features]
+bindgen = ["dep:brk_bindgen"]
+
 [dependencies]
 aide = { workspace = true }
 axum = { workspace = true }
-brk_bindgen = { workspace = true }
+brk_bindgen = { workspace = true, optional = true }
 brk_computer = { workspace = true }
 brk_error = { workspace = true, features = ["jiff", "serde_json", "tokio", "vecdb"] }
 brk_fetcher = { workspace = true }

crates/brk_server/src/lib.rs

@@ -1,7 +1,6 @@
 #![doc = include_str!("../README.md")]
 
 use std::{
-    panic,
     path::PathBuf,
     sync::Arc,
     time::{Duration, Instant},
@@ -62,6 +61,9 @@ impl Server {
     pub async fn serve(self, port: Option<Port>) -> brk_error::Result<()> {
         let state = self.0;
 
+        #[cfg(feature = "bindgen")]
+        let vecs = state.query.inner().vecs();
+
         let compression_layer = CompressionLayer::new().br(true).gzip(true).zstd(true);
 
         let response_uri_layer = axum::middleware::from_fn(
@@ -96,8 +98,6 @@ impl Server {
             )
             .on_eos(());
 
-        let vecs = state.query.inner().vecs();
-
         let website_router = brk_website::router(state.website.clone());
         let mut router = ApiRouter::new().add_api_routes();
         if !state.website.is_enabled() {
@@ -141,28 +141,33 @@ impl Server {
         let mut openapi = create_openapi();
         let router = router.finish_api(&mut openapi);
 
-        let workspace_root: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-            .parent()
-            .and_then(|p| p.parent())
-            .unwrap()
-            .into();
-        let output_paths = brk_bindgen::ClientOutputPaths::new()
-            .rust(workspace_root.join("crates/brk_client/src/lib.rs"))
-            .javascript(workspace_root.join("modules/brk-client/index.js"))
-            .python(workspace_root.join("packages/brk_client/brk_client/__init__.py"));
+        #[cfg(feature = "bindgen")]
+        {
+            let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+                .parent()
+                .and_then(|p| p.parent())
+                .unwrap()
+                .to_path_buf();
+
+            let output_paths = brk_bindgen::ClientOutputPaths::new()
+                .rust(workspace_root.join("crates/brk_client/src/lib.rs"))
+                .javascript(workspace_root.join("modules/brk-client/index.js"))
+                .python(workspace_root.join("packages/brk_client/brk_client/__init__.py"));
+
+            let openapi_json = serde_json::to_string(&openapi).unwrap();
+
+            let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                brk_bindgen::generate_clients(vecs, &openapi_json, &output_paths)
+            }));
+
+            match result {
+                Ok(Ok(())) => info!("Generated clients"),
+                Ok(Err(e)) => error!("Failed to generate clients: {e}"),
+                Err(_) => error!("Client generation panicked"),
+            }
+        }
 
         let api_json = Arc::new(ApiJson::new(&openapi));
-        let openapi_json = serde_json::to_string(&openapi).unwrap();
-
-        let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
-            brk_bindgen::generate_clients(vecs, &openapi_json, &output_paths)
-        }));
-
-        match result {
-            Ok(Ok(())) => info!("Generated clients"),
-            Ok(Err(e)) => error!("Failed to generate clients: {e}"),
-            Err(_) => error!("Client generation panicked"),
-        }
 
         let router = router
             .layer(Extension(Arc::new(openapi)))