From 1d2c927d94a4cee0ded9454f8a0e5462e5d79b41 Mon Sep 17 00:00:00 2001 From: nym21 Date: Tue, 11 Nov 2025 19:21:43 +0100 Subject: [PATCH] global: snapshot --- Cargo.lock | 2 +- Cargo.toml | 4 +- crates/brk_bencher_visualizer/src/lib.rs | 55 +++++++++++++++----- crates/brk_indexer/examples/indexer.rs | 4 +- crates/brk_indexer/examples/indexer_bench.rs | 4 +- crates/brk_indexer/src/lib.rs | 20 +++---- crates/brk_logger/src/lib.rs | 2 +- 7 files changed, 61 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ce679ba8..fafe9c35a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1273,7 +1273,7 @@ version = "0.0.111" dependencies = [ "bitcoin", "brk_error", - "byteview 0.8.0", + "byteview 0.6.1", "derive_deref", "itoa", "jiff", diff --git a/Cargo.toml b/Cargo.toml index 7887533d6..ec5ca2ca5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,8 +58,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" } brk_types = { version = "0.0.111", path = "crates/brk_types" } brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] } brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" } -# byteview = "=0.6.1" -byteview = "~0.8.0" +byteview = "=0.6.1" +# byteview = "~0.8.0" derive_deref = "1.1.1" # fjall2 = { version = "2.11.5", package = "brk_fjall" } fjall2 = { path = "../fjall2", package = "brk_fjall" } diff --git a/crates/brk_bencher_visualizer/src/lib.rs b/crates/brk_bencher_visualizer/src/lib.rs index 8673956f2..d6329328f 100644 --- a/crates/brk_bencher_visualizer/src/lib.rs +++ b/crates/brk_bencher_visualizer/src/lib.rs @@ -2,11 +2,15 @@ use plotters::prelude::*; use std::{ fs, path::{Path, PathBuf}, + slice, }; type Result = std::result::Result>; const FONT: &str = "monospace"; +const FONT_SIZE: i32 = 20; +const FONT_SIZE_BIG: i32 = 30; +const SIZE: (u32, u32) = (2000, 1000); macro_rules! configure_chart_mesh { ($chart:expr, $x_desc:expr, $y_desc:expr, $y_formatter:expr) => { @@ -19,8 +23,8 @@ macro_rules! configure_chart_mesh { .y_label_formatter(&$y_formatter) .x_labels(12) .y_labels(10) - .x_label_style((FONT, 16).into_font().color(&TEXT_COLOR.mix(0.7))) - .y_label_style((FONT, 16).into_font().color(&TEXT_COLOR.mix(0.7))) + .x_label_style((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.7))) + .y_label_style((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.7))) .axis_style(TEXT_COLOR.mix(0.3)) .draw()? }; @@ -32,7 +36,7 @@ struct DataPoint { value: f64, } -#[derive(Debug)] +#[derive(Debug, Clone)] struct BenchmarkRun { run_id: String, data: Vec, @@ -103,6 +107,7 @@ impl Visualizer { let memory_runs = self.read_benchmark_runs(crate_path, "memory.csv")?; let progress_runs = self.read_benchmark_runs(crate_path, "progress.csv")?; + // Generate combined charts (all runs together) if !disk_runs.is_empty() { self.generate_disk_chart(crate_path, crate_name, &disk_runs)?; } @@ -115,6 +120,22 @@ impl Visualizer { self.generate_progress_chart(crate_path, crate_name, &progress_runs)?; } + // Generate individual charts for each run + for run in &disk_runs { + let run_path = crate_path.join(&run.run_id); + self.generate_disk_chart(&run_path, crate_name, slice::from_ref(run))?; + } + + for run in &memory_runs { + let run_path = crate_path.join(&run.run_id); + self.generate_memory_chart(&run_path, crate_name, slice::from_ref(run))?; + } + + for run in &progress_runs { + let run_path = crate_path.join(&run.run_id); + self.generate_progress_chart(&run_path, crate_name, slice::from_ref(run))?; + } + Ok(()) } @@ -291,7 +312,7 @@ impl Visualizer { chart .configure_series_labels() .position(SeriesLabelPosition::UpperLeft) - .label_font((FONT, 16).into_font().color(&TEXT_COLOR.mix(0.9))) + .label_font((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.9))) .background_style(BG_COLOR.mix(0.98)) .border_style(BG_COLOR) .margin(10) @@ -308,7 +329,7 @@ impl Visualizer { runs: &[BenchmarkRun], ) -> Result<()> { let output_path = crate_path.join("disk_chart.svg"); - let root = SVGBackend::new(&output_path, (1200, 700)).into_drawing_area(); + let root = SVGBackend::new(&output_path, SIZE).into_drawing_area(); root.fill(&BG_COLOR)?; // Calculate time window based on shortest run + buffer @@ -328,7 +349,7 @@ impl Visualizer { let mut chart = ChartBuilder::on(&root) .caption( format!("{} — Disk Usage", crate_name), - (FONT, 24).into_font().color(&TEXT_COLOR), + (FONT, FONT_SIZE_BIG).into_font().color(&TEXT_COLOR), ) .margin(20) .x_label_area_size(50) @@ -372,7 +393,7 @@ impl Visualizer { runs: &[BenchmarkRun], ) -> Result<()> { let output_path = crate_path.join("memory_chart.svg"); - let root = SVGBackend::new(&output_path, (1200, 700)).into_drawing_area(); + let root = SVGBackend::new(&output_path, SIZE).into_drawing_area(); root.fill(&BG_COLOR)?; // Calculate time window based on shortest run + buffer @@ -412,7 +433,7 @@ impl Visualizer { let mut chart = ChartBuilder::on(&root) .caption( format!("{} — Memory", crate_name), - (FONT, 24).into_font().color(&TEXT_COLOR), + (FONT, FONT_SIZE_BIG).into_font().color(&TEXT_COLOR), ) .margin(20) .x_label_area_size(50) @@ -492,7 +513,15 @@ impl Visualizer { let mut enhanced_runs = Vec::new(); for run in runs { - let csv_path = crate_path.join(&run.run_id).join("memory.csv"); + // For individual charts, crate_path is already the run folder + // For combined charts, we need to append run_id + let direct_path = crate_path.join("memory.csv"); + let nested_path = crate_path.join(&run.run_id).join("memory.csv"); + let csv_path = if direct_path.exists() { + direct_path + } else { + nested_path + }; if let Ok(content) = fs::read_to_string(&csv_path) { let mut footprint_data = Vec::new(); let mut peak_data = Vec::new(); @@ -535,7 +564,7 @@ impl Visualizer { runs: &[BenchmarkRun], ) -> Result<()> { let output_path = crate_path.join("progress_chart.svg"); - let root = SVGBackend::new(&output_path, (1200, 700)).into_drawing_area(); + let root = SVGBackend::new(&output_path, SIZE).into_drawing_area(); root.fill(&BG_COLOR)?; // Calculate time window based on shortest run + buffer @@ -553,7 +582,7 @@ impl Visualizer { let mut chart = ChartBuilder::on(&root) .caption( format!("{} — Progress", crate_name), - (FONT, 24).into_font().color(&TEXT_COLOR), + (FONT, FONT_SIZE_BIG).into_font().color(&TEXT_COLOR), ) .margin(20) .x_label_area_size(50) @@ -570,8 +599,8 @@ impl Visualizer { .y_label_formatter(&|y| Self::format_axis_number(*y)) .x_labels(12) .y_labels(10) - .x_label_style((FONT, 16).into_font().color(&TEXT_COLOR.mix(0.7))) - .y_label_style((FONT, 16).into_font().color(&TEXT_COLOR.mix(0.7))) + .x_label_style((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.7))) + .y_label_style((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.7))) .axis_style(TEXT_COLOR.mix(0.3)) .draw()?; diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 47dbacb63..f43c514ce 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -10,7 +10,7 @@ use brk_indexer::Indexer; use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; -use log::debug; +use log::{debug, info}; use vecdb::Exit; fn main() -> Result<()> { @@ -45,7 +45,7 @@ fn main() -> Result<()> { loop { let i = Instant::now(); indexer.checked_index(&blocks, &client, &exit)?; - dbg!(i.elapsed()); + info!("Done in {:?}", i.elapsed()); sleep(Duration::from_secs(60)); } diff --git a/crates/brk_indexer/examples/indexer_bench.rs b/crates/brk_indexer/examples/indexer_bench.rs index 8ae95ce12..28dabe5cb 100644 --- a/crates/brk_indexer/examples/indexer_bench.rs +++ b/crates/brk_indexer/examples/indexer_bench.rs @@ -1,4 +1,4 @@ -use std::{env, fs, io::Write, path::Path, time::Instant}; +use std::{env, fs, path::Path, time::Instant}; use brk_bencher::Bencher; use brk_error::Result; @@ -15,7 +15,7 @@ fn main() -> Result<()> { let bitcoin_dir = Client::default_bitcoin_path(); // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); - let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk"); + let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk/benches"); fs::create_dir_all(&outputs_dir)?; // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index b31483320..ac0f150ed 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -17,13 +17,13 @@ use rayon::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, TypedVecIterator}; mod indexes; -// mod stores_v2; -mod stores_v3; +mod stores_v2; +// mod stores_v3; mod vecs; pub use indexes::*; -// pub use stores_v2::*; -pub use stores_v3::*; +pub use stores_v2::*; +// pub use stores_v3::*; pub use vecs::*; // One version for all data sources @@ -46,13 +46,15 @@ impl Indexer { let (vecs, stores) = thread::scope(|s| -> Result<_> { let vecs = s.spawn(|| -> Result<_> { + let i = Instant::now(); let vecs = Vecs::forced_import(&path, VERSION)?; - info!("Imported vecs"); + info!("Imported vecs in {:?}", i.elapsed()); Ok(vecs) }); + let i = Instant::now(); let stores = Stores::forced_import(&path, VERSION)?; - info!("Imported stores"); + info!("Imported stores in {:?}", i.elapsed()); Ok((vecs.join().unwrap()?, stores)) })?; @@ -97,7 +99,7 @@ impl Indexer { } else { (Indexes::default(), None) }; - info!("Starting indexes set."); + debug!("Starting indexes set."); let lock = exit.lock(); self.stores @@ -121,10 +123,10 @@ impl Indexer { let _lock = exit.lock(); let i = Instant::now(); stores.commit(height).unwrap(); - info!("Commited stores in {}s", i.elapsed().as_secs()); + debug!("Commited stores in {}s", i.elapsed().as_secs()); let i = Instant::now(); vecs.flush(height)?; - info!("Flushed vecs in {}s", i.elapsed().as_secs()); + debug!("Flushed vecs in {}s", i.elapsed().as_secs()); Ok(()) }; diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index cafce99ab..cebea6c36 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -28,7 +28,7 @@ pub fn init(path: Option<&Path>) -> io::Result<()> { } Builder::from_env(Env::default().default_filter_or( - "debug,bitcoin=off,bitcoincore-rpc=off,fjall=off,brk_fjall=off,lsm-tree=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off", + "debug,bitcoin=off,bitcoincore-rpc=off,fjall=off,brk_fjall=off,lsm_tree=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off", // "debug,fjall=trace,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off", )) .format(move |buf, record| {