global: snapshot

This commit is contained in:
nym21
2025-12-07 19:13:41 +01:00
parent f23907768f
commit b88f0bab56
17 changed files with 388 additions and 377 deletions
Generated
+29 -25
View File
@@ -181,9 +181,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "async-compression"
version = "0.4.34"
version = "0.4.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e86f6d3dc9dc4352edeea6b8e499e13e3f5dc3b964d7ca5fd411415a3498473"
checksum = "07a926debf178f2d355197f9caddb08e54a9329d44748034bba349c5848cb519"
dependencies = [
"compression-codecs",
"compression-core",
@@ -398,9 +398,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bitcoin"
version = "0.32.7"
version = "0.32.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda569d741b895131a88ee5589a467e73e9c4718e958ac9308e4f7dc44b6945"
checksum = "1e499f9fc0407f50fe98af744ab44fa67d409f76b6772e1689ec8485eb0c0f66"
dependencies = [
"base58ck",
"bech32",
@@ -441,9 +441,9 @@ dependencies = [
[[package]]
name = "bitcoin_hashes"
version = "0.14.0"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b"
dependencies = [
"bitcoin-io",
"hex-conservative",
@@ -602,6 +602,7 @@ dependencies = [
"color-eyre",
"log",
"minreq",
"rlimit",
"serde",
"tokio",
"toml",
@@ -1204,7 +1205,7 @@ dependencies = [
"brk_fjall",
"brk_types",
"byteview 0.6.1",
"byteview 0.8.0",
"byteview 0.9.1",
"fjall",
"rustc-hash",
]
@@ -1320,12 +1321,6 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "byteview"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6b0e42e210b794e14b152c6fe1a55831e30ef4a0f5dc39d73d714fb5f1906c"
[[package]]
name = "byteview"
version = "0.9.1"
@@ -1343,9 +1338,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.48"
version = "1.2.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a"
checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1521,9 +1516,9 @@ checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
[[package]]
name = "compression-codecs"
version = "0.4.33"
version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "302266479cb963552d11bd042013a58ef1adc56768016c8b82b4199488f2d4ad"
checksum = "34a3cbbb8b6eca96f3a5c4bf6938d5b27ced3675d69f95bb51948722870bc323"
dependencies = [
"brotli",
"compression-core",
@@ -2077,9 +2072,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "fjall"
version = "3.0.0-rc.4"
version = "3.0.0-rc.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b79d92323d8e942a28890c99ccfe93a75efdebf41304d8922a8251cd16562e42"
checksum = "c91b735d557d6636f69dedbea4d7a465e6095a00409b2b146a64cf4c136ab833"
dependencies = [
"byteorder-lite",
"byteview 0.9.1",
@@ -3070,9 +3065,9 @@ dependencies = [
[[package]]
name = "mio"
version = "1.1.0"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873"
checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
dependencies = [
"libc",
"log",
@@ -3596,9 +3591,9 @@ dependencies = [
[[package]]
name = "oxc_resolver"
version = "11.14.2"
version = "11.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a22ff4e8307f0392031c2221a3c14f448c446f7ca6312eac62e3181d01e93fe5"
checksum = "630a9355dd8fab35d4731f99e1d303499d70b55f7c5a996466050a0a29f4bc76"
dependencies = [
"cfg-if",
"fast-glob",
@@ -4336,6 +4331,15 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rlimit"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
dependencies = [
"libc",
]
[[package]]
name = "rolldown-ariadne"
version = "0.5.3"
@@ -5061,9 +5065,9 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.23.7"
version = "0.23.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832"
dependencies = [
"indexmap",
"toml_datetime",
+3 -6
View File
@@ -34,7 +34,7 @@ inherits = "release"
[workspace.dependencies]
aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] }
axum = "0.8.7"
bitcoin = { version = "0.32.7", features = ["serde"] }
bitcoin = { version = "0.32.8", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_bencher = { version = "0.0.111", path = "crates/brk_bencher" }
brk_binder = { version = "0.0.111", path = "crates/brk_binder" }
@@ -58,15 +58,12 @@ brk_types = { version = "0.0.111", path = "crates/brk_types" }
brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
byteview = "=0.6.1"
# byteview = "~0.8.0"
# byteview = "0.9.1"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall2 = { version = "2.11.8", package = "brk_fjall" }
# fjall2 = { path = "../fjall2", package = "brk_fjall" }
# fjall2 = { version = "2.11.2", package = "fjall" }
fjall3 = { version = "3.0.0-rc.4", package = "fjall" }
# fjall3 = { path = "../fjall3", package = "fjall" }
# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "f0bf96c2017b3543eb176012b8eff69c639dff1d", package = "fjall" }
fjall3 = { version = "3.0.0-rc.5", package = "fjall" }
jiff = "0.2.16"
log = "0.4.29"
minreq = { version = "2.14.1", features = ["https", "serde_json"] }
+199 -62
View File
@@ -108,13 +108,13 @@ impl Visualizer {
let progress_runs = self.read_benchmark_runs(crate_path, "progress.csv")?;
let io_runs = self.read_benchmark_runs(crate_path, "io.csv")?;
// Generate combined charts (all runs together)
// Generate combined charts (all runs together) - use progress-based cutoffs
if !disk_runs.is_empty() {
self.generate_disk_chart(crate_path, crate_name, &disk_runs)?;
self.generate_disk_chart(crate_path, crate_name, &disk_runs, &progress_runs)?;
}
if !memory_runs.is_empty() {
self.generate_memory_chart(crate_path, crate_name, &memory_runs)?;
self.generate_memory_chart(crate_path, crate_name, &memory_runs, &progress_runs)?;
}
if !progress_runs.is_empty() {
@@ -122,19 +122,19 @@ impl Visualizer {
}
if !io_runs.is_empty() {
self.generate_io_read_chart(crate_path, crate_name, &io_runs)?;
self.generate_io_write_chart(crate_path, crate_name, &io_runs)?;
self.generate_io_read_chart(crate_path, crate_name, &io_runs, &progress_runs)?;
self.generate_io_write_chart(crate_path, crate_name, &io_runs, &progress_runs)?;
}
// Generate individual charts for each run
// Generate individual charts for each run (no progress-based cutoffs for single runs)
for run in &disk_runs {
let run_path = crate_path.join(&run.run_id);
self.generate_disk_chart(&run_path, crate_name, slice::from_ref(run))?;
self.generate_disk_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
}
for run in &memory_runs {
let run_path = crate_path.join(&run.run_id);
self.generate_memory_chart(&run_path, crate_name, slice::from_ref(run))?;
self.generate_memory_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
}
for run in &progress_runs {
@@ -144,8 +144,8 @@ impl Visualizer {
for run in &io_runs {
let run_path = crate_path.join(&run.run_id);
self.generate_io_read_chart(&run_path, crate_name, slice::from_ref(run))?;
self.generate_io_write_chart(&run_path, crate_name, slice::from_ref(run))?;
self.generate_io_read_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
self.generate_io_write_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
}
Ok(())
@@ -262,6 +262,69 @@ impl Visualizer {
.unwrap_or(1000)
}
/// Find the minimum of the max progress values across all runs,
/// then return the cutoff timestamp for each run when that progress was reached.
fn calculate_progress_based_cutoffs(
runs: &[BenchmarkRun],
progress_runs: &[BenchmarkRun],
) -> Option<Vec<u64>> {
// Find the minimum of max progress across all runs
let min_max_progress = progress_runs
.iter()
.filter_map(|r| r.data.iter().map(|d| d.value).fold(None, |acc, v| {
Some(acc.map_or(v, |a: f64| a.max(v)))
}))
.fold(f64::MAX, f64::min);
if min_max_progress == f64::MAX {
return None;
}
// For each run, find the timestamp when this progress was first reached
let cutoffs: Vec<u64> = runs
.iter()
.map(|run| {
// Find the matching progress run
let progress_run = progress_runs.iter().find(|pr| pr.run_id == run.run_id);
if let Some(pr) = progress_run {
// Find the timestamp when min_max_progress was reached
pr.data
.iter()
.find(|d| d.value >= min_max_progress)
.map(|d| d.timestamp_ms)
.unwrap_or_else(|| {
// Fallback to max timestamp if progress not found
run.data.iter().map(|d| d.timestamp_ms).max().unwrap_or(1000)
})
} else {
// No matching progress run, use max timestamp
run.data.iter().map(|d| d.timestamp_ms).max().unwrap_or(1000)
}
})
.collect();
Some(cutoffs)
}
fn trim_runs_with_individual_cutoffs(
runs: &[BenchmarkRun],
cutoffs: &[u64],
) -> Vec<BenchmarkRun> {
runs.iter()
.zip(cutoffs.iter())
.map(|(run, &cutoff)| BenchmarkRun {
run_id: run.run_id.clone(),
data: run
.data
.iter()
.filter(|d| d.timestamp_ms <= cutoff)
.cloned()
.collect(),
})
.collect()
}
fn calculate_max_value(runs: &[BenchmarkRun]) -> f64 {
runs.iter()
.flat_map(|r| r.data.iter().map(|d| d.value))
@@ -339,17 +402,26 @@ impl Visualizer {
crate_path: &Path,
crate_name: &str,
runs: &[BenchmarkRun],
progress_runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("disk.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
// Calculate time window based on shortest run + buffer
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let max_time_s = (min_max_time_ms as f64) / 1000.0;
// Try progress-based cutoffs first, fall back to time-based
let (trimmed_runs, max_time_ms) =
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000);
(
Self::trim_runs_with_individual_cutoffs(runs, &cutoffs),
max_cutoff + TIME_BUFFER_MS,
)
} else {
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
(Self::trim_runs_to_time_window(runs, min_max_time_ms), min_max_time_ms)
};
// Trim all runs to the same time window
let trimmed_runs = Self::trim_runs_to_time_window(runs, min_max_time_ms);
let max_time_s = (max_time_ms as f64) / 1000.0;
let max_value = Self::calculate_max_value(&trimmed_runs);
let (max_value_scaled, unit) = Self::format_bytes(max_value);
@@ -403,33 +475,55 @@ impl Visualizer {
crate_path: &Path,
crate_name: &str,
runs: &[BenchmarkRun],
progress_runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("memory.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
// Calculate time window based on shortest run + buffer
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let max_time_s = (min_max_time_ms as f64) / 1000.0;
// Read memory CSV files which have 3 columns: timestamp, footprint, peak
let enhanced_runs = self.read_memory_data(crate_path, runs)?;
// Trim enhanced runs to the same time window
let trimmed_enhanced_runs: Vec<_> = enhanced_runs
.into_iter()
.map(|(run_id, footprint, peak)| {
let trimmed_footprint: Vec<_> = footprint
// Try progress-based cutoffs first, fall back to time-based
let (trimmed_enhanced_runs, max_time_ms) =
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
let trimmed: Vec<_> = enhanced_runs
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.zip(cutoffs.iter())
.map(|((run_id, footprint, peak), &cutoff)| {
let trimmed_footprint: Vec<_> = footprint
.into_iter()
.filter(|d| d.timestamp_ms <= cutoff)
.collect();
let trimmed_peak: Vec<_> = peak
.into_iter()
.filter(|d| d.timestamp_ms <= cutoff)
.collect();
(run_id, trimmed_footprint, trimmed_peak)
})
.collect();
let trimmed_peak: Vec<_> = peak
(trimmed, max_cutoff)
} else {
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let trimmed: Vec<_> = enhanced_runs
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.map(|(run_id, footprint, peak)| {
let trimmed_footprint: Vec<_> = footprint
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.collect();
let trimmed_peak: Vec<_> = peak
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.collect();
(run_id, trimmed_footprint, trimmed_peak)
})
.collect();
(run_id, trimmed_footprint, trimmed_peak)
})
.collect();
(trimmed, min_max_time_ms)
};
let max_time_s = (max_time_ms as f64) / 1000.0;
let max_value = trimmed_enhanced_runs
.iter()
@@ -627,29 +721,47 @@ impl Visualizer {
crate_path: &Path,
crate_name: &str,
runs: &[BenchmarkRun],
progress_runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("io_read.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
// Calculate time window based on shortest run + buffer
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let max_time_s = (min_max_time_ms as f64) / 1000.0;
// Read I/O CSV files which have 3 columns: timestamp, bytes_read, bytes_written
let io_runs = self.read_io_data(crate_path, runs)?;
// Trim I/O runs to the same time window and extract only read data
let trimmed_io_runs: Vec<_> = io_runs
.into_iter()
.map(|(run_id, read_data, _write_data)| {
let trimmed_read: Vec<_> = read_data
// Try progress-based cutoffs first, fall back to time-based
let (trimmed_io_runs, max_time_ms) =
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
let trimmed: Vec<_> = io_runs
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.zip(cutoffs.iter())
.map(|((run_id, read_data, _write_data), &cutoff)| {
let trimmed_read: Vec<_> = read_data
.into_iter()
.filter(|d| d.timestamp_ms <= cutoff)
.collect();
(run_id, trimmed_read)
})
.collect();
(run_id, trimmed_read)
})
.collect();
(trimmed, max_cutoff)
} else {
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let trimmed: Vec<_> = io_runs
.into_iter()
.map(|(run_id, read_data, _write_data)| {
let trimmed_read: Vec<_> = read_data
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.collect();
(run_id, trimmed_read)
})
.collect();
(trimmed, min_max_time_ms)
};
let max_time_s = (max_time_ms as f64) / 1000.0;
let max_value = trimmed_io_runs
.iter()
@@ -709,29 +821,47 @@ impl Visualizer {
crate_path: &Path,
crate_name: &str,
runs: &[BenchmarkRun],
progress_runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("io_write.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
// Calculate time window based on shortest run + buffer
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let max_time_s = (min_max_time_ms as f64) / 1000.0;
// Read I/O CSV files which have 3 columns: timestamp, bytes_read, bytes_written
let io_runs = self.read_io_data(crate_path, runs)?;
// Trim I/O runs to the same time window and extract only write data
let trimmed_io_runs: Vec<_> = io_runs
.into_iter()
.map(|(run_id, _read_data, write_data)| {
let trimmed_write: Vec<_> = write_data
// Try progress-based cutoffs first, fall back to time-based
let (trimmed_io_runs, max_time_ms) =
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
let trimmed: Vec<_> = io_runs
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.zip(cutoffs.iter())
.map(|((run_id, _read_data, write_data), &cutoff)| {
let trimmed_write: Vec<_> = write_data
.into_iter()
.filter(|d| d.timestamp_ms <= cutoff)
.collect();
(run_id, trimmed_write)
})
.collect();
(run_id, trimmed_write)
})
.collect();
(trimmed, max_cutoff)
} else {
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let trimmed: Vec<_> = io_runs
.into_iter()
.map(|(run_id, _read_data, write_data)| {
let trimmed_write: Vec<_> = write_data
.into_iter()
.filter(|d| d.timestamp_ms <= min_max_time_ms)
.collect();
(run_id, trimmed_write)
})
.collect();
(trimmed, min_max_time_ms)
};
let max_time_s = (max_time_ms as f64) / 1000.0;
let max_value = trimmed_io_runs
.iter()
@@ -796,13 +926,20 @@ impl Visualizer {
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
// Calculate time window based on shortest run + buffer
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
let max_time_s = (min_max_time_ms as f64) / 1000.0;
// Trim all runs to the same time window
let trimmed_runs = Self::trim_runs_to_time_window(runs, min_max_time_ms);
// Try progress-based cutoffs first, fall back to time-based
let (trimmed_runs, max_time_ms) =
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, runs) {
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
(
Self::trim_runs_with_individual_cutoffs(runs, &cutoffs),
max_cutoff,
)
} else {
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
(Self::trim_runs_to_time_window(runs, min_max_time_ms), min_max_time_ms)
};
let max_time_s = (max_time_ms as f64) / 1000.0;
let max_block = Self::calculate_max_value(&trimmed_runs);
// Format time based on duration
+1
View File
@@ -31,6 +31,7 @@ serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.8"
zip = { version = "6.0.0", default-features = false, features = ["deflate"] }
rlimit = "0.10.2"
[[bin]]
name = "brk"
+7
View File
@@ -28,6 +28,13 @@ mod website;
use crate::{config::Config, paths::*};
pub fn main() -> color_eyre::Result<()> {
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit(
rlimit::Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
// Can't increase main thread's stack size, thus we need to use another thread
thread::Builder::new()
.stack_size(512 * 1024 * 1024)
+7 -7
View File
@@ -75,7 +75,7 @@ pub struct Vecs {
pub indexes_to_fee_rate: ComputedVecsFromTxindex<FeeRate>,
/// Value == 0 when Coinbase
pub txindex_to_input_value: EagerVec<PcoVec<TxIndex, Sats>>,
pub indexes_to_sent: ComputedValueVecsFromHeight,
pub indexes_to_sent_sum: ComputedValueVecsFromHeight,
// pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU64>,
pub txindex_to_output_value: EagerVec<PcoVec<TxIndex, Sats>>,
@@ -376,7 +376,7 @@ impl Vecs {
indexes_to_tx_v1: computed_h!("tx_v1", Source::Compute, sum_cum()),
indexes_to_tx_v2: computed_h!("tx_v2", Source::Compute, sum_cum()),
indexes_to_tx_v3: computed_h!("tx_v3", Source::Compute, sum_cum()),
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
indexes_to_sent_sum: ComputedValueVecsFromHeight::forced_import(
&db,
"sent_sum",
Source::Compute,
@@ -915,7 +915,7 @@ impl Vecs {
exit,
)?;
self.indexes_to_sent
self.indexes_to_sent_sum
.compute_all(indexes, price, starting_indexes, exit, |v| {
v.compute_filtered_sum_from_indexes(
starting_indexes.height,
@@ -1608,7 +1608,7 @@ impl Vecs {
.compute_all(starting_indexes, exit, |v| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_sent.sats.dateindex.unwrap_sum(),
self.indexes_to_sent_sum.sats.dateindex.unwrap_sum(),
365,
exit,
)?;
@@ -1619,7 +1619,7 @@ impl Vecs {
.compute_all(starting_indexes, exit, |v| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_sent.bitcoin.dateindex.unwrap_sum(),
self.indexes_to_sent_sum.bitcoin.dateindex.unwrap_sum(),
365,
exit,
)?;
@@ -1643,12 +1643,12 @@ impl Vecs {
Ok(())
})?;
if let Some(indexes_to_sent) = self.indexes_to_sent.dollars.as_ref() {
if let Some(indexes_to_sent_sum) = self.indexes_to_sent_sum.dollars.as_ref() {
self.indexes_to_annualized_volume_usd
.compute_all(starting_indexes, exit, |v| {
v.compute_sum(
starting_indexes.dateindex,
indexes_to_sent.dateindex.unwrap_sum(),
indexes_to_sent_sum.dateindex.unwrap_sum(),
365,
exit,
)?;
@@ -151,7 +151,7 @@ impl Vecs {
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
db,
&suffix("sent"),
Source::Compute,
Source::None,
version + v0,
sum(),
compute_dollars,
+6
View File
@@ -1080,6 +1080,12 @@ impl Vecs {
.get_mut(&typeindex)
{
addressdata.deref_mut().tx_count += tx_count;
} else if let Some(addressdata) =
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
.get_mut_unwrap(address_type)
.get_mut(&typeindex)
{
addressdata.deref_mut().tx_count += tx_count;
}
}
+7 -6
View File
@@ -165,7 +165,7 @@ impl Stores {
self.addresstype_to_addressindex_and_unspentoutpoint
.par_values_mut()
.map(|s| s as &mut dyn AnyStore),
) // Changed from par_iter_mut()
)
.map(|store| {
let items = store.take_all_f2();
store.export_meta_if_needed(height)?;
@@ -249,11 +249,12 @@ impl Stores {
let txindex = TxIndex::from(txindex);
let txidprefix = TxidPrefix::from(&txid);
let is_known_dup = crate::DUPLICATE_TXID_PREFIXES
.iter()
.any(|(dup_prefix, dup_txindex)| {
txindex == *dup_txindex && txidprefix == *dup_prefix
});
let is_known_dup =
crate::DUPLICATE_TXID_PREFIXES
.iter()
.any(|(dup_prefix, dup_txindex)| {
txindex == *dup_txindex && txidprefix == *dup_prefix
});
if !is_known_dup {
self.txidprefix_to_txindex.remove(txidprefix);
+57 -219
View File
@@ -10,14 +10,13 @@ use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_store::{AnyStore, Kind3, Mode3, StoreFjallV3 as Store};
use brk_types::{
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height,
OutPoint, OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version,
Vout,
AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height, OutPoint,
OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout,
};
use fjall3::{AbstractTree, Database, PersistMode};
use fjall3::{Database, PersistMode};
use log::info;
use rayon::prelude::*;
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, VecIndex, VecIterator};
use vecdb::{AnyVec, TypedVecIterator, VecIndex, VecIterator};
use crate::Indexes;
@@ -130,24 +129,35 @@ impl Stores {
}
pub fn starting_height(&self) -> Height {
self.iter_any_store()
.map(|store| {
// let height =
store.height().map(Height::incremented).unwrap_or_default()
// dbg!((height, store.name()));
})
.min()
.unwrap()
[
&self.blockhashprefix_to_height as &dyn AnyStore,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
]
.into_iter()
.chain(
self.addresstype_to_addresshash_to_addressindex
.values()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_txindex
.values()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_unspentoutpoint
.values()
.map(|s| s as &dyn AnyStore),
)
.map(|store| store.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn commit(&mut self, height: Height) -> Result<()> {
info!(
"self.db.config.cache.size = {}",
self.db.config.cache.size()
);
let i = Instant::now();
let tuples = [
[
&mut self.blockhashprefix_to_height as &mut dyn AnyStore,
&mut self.height_to_coinbase_tag,
&mut self.txidprefix_to_txindex,
@@ -155,40 +165,21 @@ impl Stores {
.into_par_iter()
.chain(
self.addresstype_to_addresshash_to_addressindex
.par_iter_mut()
.par_values_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_txindex
.par_iter_mut()
.par_values_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_unspentoutpoint
.par_iter_mut()
.par_values_mut()
.map(|s| s as &mut dyn AnyStore),
) // Changed from par_iter_mut()
.map(|store| {
let items = store.take_all_f3();
store.export_meta_if_needed(height)?;
Ok((store.keyspace(), items))
})
.collect::<Result<Vec<_>>>()?;
info!("Store items collected in {:?}", i.elapsed());
let version_memtable_size_sum = tuples
.iter()
.map(|(keyspace, _)| keyspace.tree.version_memtable_size_sum())
.collect::<Vec<_>>();
// let sum = version_memtable_size_sum.iter().sum::<usize>();
println!(
"version_memtable_size_sum = {:?} = ",
version_memtable_size_sum
);
let i = Instant::now();
self.db.batch().commit_keyspaces(tuples)?;
info!("Batch done in {:?}", i.elapsed());
.try_for_each(|store| store.commit_f3(height))?;
info!("Commits done in {:?}", i.elapsed());
let i = Instant::now();
self.db.persist(PersistMode::SyncData)?;
@@ -202,30 +193,6 @@ impl Stores {
Ok(())
}
fn iter_any_store(&self) -> impl Iterator<Item = &dyn AnyStore> {
[
&self.blockhashprefix_to_height as &dyn AnyStore,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
]
.into_iter()
.chain(
self.addresstype_to_addresshash_to_addressindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_txindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_addressindex_and_unspentoutpoint
.iter()
.map(|s| s as &dyn AnyStore),
)
}
pub fn rollback_if_needed(
&mut self,
vecs: &mut Vecs,
@@ -236,15 +203,15 @@ impl Stores {
&& self.height_to_coinbase_tag.is_empty()?
&& self
.addresstype_to_addresshash_to_addressindex
.iter()
.values()
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
&& self
.addresstype_to_addressindex_and_txindex
.iter()
.values()
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
&& self
.addresstype_to_addressindex_and_unspentoutpoint
.iter()
.values()
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
{
return Ok(());
@@ -265,145 +232,25 @@ impl Stores {
self.height_to_coinbase_tag.remove(h);
});
if let Ok(mut index) = vecs
.height_to_first_p2pk65addressindex
.read_once(starting_indexes.height)
{
let mut p2pk65addressindex_to_p2pk65bytes_iter =
vecs.p2pk65addressindex_to_p2pk65bytes.iter()?;
while let Some(typedbytes) = p2pk65addressindex_to_p2pk65bytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
// Remove address hashes for all address types starting from rollback height
for address_type in [
OutputType::P2PK65,
OutputType::P2PK33,
OutputType::P2PKH,
OutputType::P2SH,
OutputType::P2WPKH,
OutputType::P2WSH,
OutputType::P2TR,
OutputType::P2A,
] {
for hash in vecs.iter_address_hashes_from(address_type, starting_indexes.height)? {
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2PK65)
.get_mut_unwrap(address_type)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2pk33addressindex
.read_once(starting_indexes.height)
{
let mut p2pk33addressindex_to_p2pk33bytes_iter =
vecs.p2pk33addressindex_to_p2pk33bytes.iter()?;
while let Some(typedbytes) = p2pk33addressindex_to_p2pk33bytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2PK33)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2pkhaddressindex
.read_once(starting_indexes.height)
{
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?;
while let Some(typedbytes) = p2pkhaddressindex_to_p2pkhbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2PKH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2shaddressindex
.read_once(starting_indexes.height)
{
let mut p2shaddressindex_to_p2shbytes_iter =
vecs.p2shaddressindex_to_p2shbytes.iter()?;
while let Some(typedbytes) = p2shaddressindex_to_p2shbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2SH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2wpkhaddressindex
.read_once(starting_indexes.height)
{
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?;
while let Some(typedbytes) = p2wpkhaddressindex_to_p2wpkhbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2WPKH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2wshaddressindex
.read_once(starting_indexes.height)
{
let mut p2wshaddressindex_to_p2wshbytes_iter =
vecs.p2wshaddressindex_to_p2wshbytes.iter()?;
while let Some(typedbytes) = p2wshaddressindex_to_p2wshbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2WSH)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2traddressindex
.read_once(starting_indexes.height)
{
let mut p2traddressindex_to_p2trbytes_iter =
vecs.p2traddressindex_to_p2trbytes.iter()?;
while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2TR)
.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2aaddressindex
.read_once(starting_indexes.height)
{
let mut p2aaddressindex_to_p2abytes_iter =
vecs.p2aaddressindex_to_p2abytes.iter()?;
while let Some(typedbytes) = p2aaddressindex_to_p2abytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressHash::from(&bytes);
self.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(OutputType::P2A)
.remove(hash);
index.increment();
}
}
} else {
unreachable!();
// self.blockhashprefix_to_height.reset()?;
// self.addresshash_to_typeindex.reset()?;
}
if starting_indexes.txindex != TxIndex::ZERO {
@@ -413,24 +260,21 @@ impl Stores {
.skip(starting_indexes.txindex.to_usize())
.for_each(|(txindex, txid)| {
let txindex = TxIndex::from(txindex);
let txidprefix = TxidPrefix::from(&txid);
// "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"
let is_not_first_dup = txindex != TxIndex::new(142783)
|| txidprefix != TxidPrefix::from([153, 133, 216, 41, 84, 225, 15, 34]);
let is_known_dup =
crate::DUPLICATE_TXID_PREFIXES
.iter()
.any(|(dup_prefix, dup_txindex)| {
txindex == *dup_txindex && txidprefix == *dup_prefix
});
// "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"
let is_not_second_dup = txindex != TxIndex::new(142841)
|| txidprefix != TxidPrefix::from([104, 180, 95, 88, 182, 116, 233, 78]);
if is_not_first_dup && is_not_second_dup {
if !is_known_dup {
self.txidprefix_to_txindex.remove(txidprefix);
}
});
} else {
unreachable!();
// self.txidprefix_to_txindex.reset()?;
}
if starting_indexes.txoutindex != TxOutIndex::ZERO {
@@ -504,12 +348,6 @@ impl Stores {
});
} else {
unreachable!();
// self.addresstype_to_typeindex_and_txindex
// .iter_mut()
// .try_for_each(|s| s.reset())?;
// self.addresstype_to_typeindex_and_unspentoutpoint
// .iter_mut()
// .try_for_each(|s| s.reset())?;
}
self.commit(starting_indexes.height.decremented().unwrap_or_default())?;
+4 -2
View File
@@ -75,11 +75,13 @@ pub fn get_address(Address { address }: Address, query: &Query) -> Result<Addres
spent_txo_sum: address_data.sent,
tx_count: address_data.tx_count,
},
mempool_stats: query.mempool().and_then(|mempool| {
mempool_stats: query.mempool().map(|mempool| {
mempool
.get_addresses()
.get(&bytes)
.map(|(stats, ..)| stats.clone())
.map(|(stats, _)| stats)
.cloned()
.unwrap_or_default()
}),
})
}
+1 -1
View File
@@ -110,7 +110,7 @@ impl<'a> Vecs<'a> {
// Not the most performant or type safe but only built once so that's okay
fn insert(&mut self, vec: &'a dyn AnyExportableVec) {
let name = vec.name();
dbg!(vec.region_name());
// dbg!(vec.region_name());
let serialized_index = vec.index_type_to_string();
let index = Index::try_from(serialized_index)
.inspect_err(|_| {
+2 -2
View File
@@ -13,8 +13,8 @@ build = "build.rs"
[dependencies]
brk_error = { workspace = true }
brk_types = { workspace = true }
byteview6 = { version = "=0.6.1", package = "byteview" }
byteview8 = { version = "~0.8.0", package = "byteview" }
byteview_f2 = { version = "=0.6.1", package = "byteview" }
byteview_f3 = { version = "0.9.1", package = "byteview" }
fjall2 = { workspace = true }
fjall3 = { workspace = true }
rustc-hash = { workspace = true }
+1
View File
@@ -11,6 +11,7 @@ pub trait AnyStore: Send + Sync {
fn keyspace(&self) -> &fjall3::Keyspace;
fn partition(&self) -> &fjall2::PartitionHandle;
fn take_all_f2(&mut self) -> Vec<fjall2::InnerItem>;
fn commit_f3(&mut self, height: Height) -> Result<()>;
// fn take_all_f3(&mut self) -> Vec<fjall3::InnerItem>;
// fn take_all_f3(&mut self) -> Box<dyn Iterator<Item = Item>>;
}
+5 -1
View File
@@ -2,7 +2,7 @@ use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path};
use brk_error::Result;
use brk_types::{Height, Version};
use byteview6::ByteView;
use byteview_f2::ByteView;
use fjall2::{
CompressionType, InnerItem, PartitionCreateOptions, TransactionalKeyspace,
TransactionalPartitionHandle, ValueType,
@@ -228,6 +228,10 @@ where
fn version(&self) -> Version {
self.meta.version()
}
fn commit_f3(&mut self, _: Height) -> Result<()> {
Ok(())
}
}
enum Item<K, V> {
+56 -43
View File
@@ -1,10 +1,10 @@
use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path};
use std::{borrow::Cow, cmp::Ordering, fmt::Debug, fs, hash::Hash, mem, path::Path};
use brk_error::Result;
use brk_types::{Height, Version};
use byteview8::ByteView;
use byteview_f3::ByteView;
use fjall3::{
Database, Keyspace, KeyspaceCreateOptions, ValueType,
Database, Keyspace, KeyspaceCreateOptions,
config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy},
};
@@ -38,9 +38,10 @@ where
K: Debug + Clone + From<ByteView> + Ord + Eq + Hash,
V: Debug + Clone + From<ByteView>,
ByteView: From<K> + From<V>,
Self: Send + Sync,
{
pub fn import(
database: &Database,
db: &Database,
path: &Path,
name: &str,
version: Version,
@@ -50,11 +51,11 @@ where
fs::create_dir_all(path)?;
let (meta, keyspace) = StoreMeta::checked_open(
database,
db,
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
|| {
Self::open_keyspace(database, name, mode, kind).inspect_err(|e| {
Self::open_keyspace(db, name, mode, kind).inspect_err(|e| {
eprintln!("{e}");
eprintln!("Delete {path:?} and try again");
})
@@ -79,7 +80,7 @@ where
let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true);
if kind.is_not_vec() {
options = options.filter_policy(FilterPolicy::new(&[
options = options.filter_policy(FilterPolicy::new([
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(8.0)),
@@ -97,7 +98,7 @@ where
.index_block_pinning_policy(PinningPolicy::all(false));
}
database.keyspace(name, options).map_err(|e| e.into())
database.keyspace(name, || options).map_err(|e| e.into())
}
#[inline]
@@ -188,20 +189,6 @@ where
panic!()
}
fn take_all_f3(&mut self) -> Vec<fjall3::InnerItem> {
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
.chain(
mem::take(&mut self.dels)
.into_iter()
.map(|key| Item::Tomb(key)),
)
.collect::<Vec<_>>();
items.sort_unstable();
items.into_iter().map(|v| v.fjalled()).collect()
}
fn export_meta_if_needed(&mut self, height: Height) -> Result<()> {
if self.has(height) {
return Ok(());
@@ -229,6 +216,51 @@ where
fn version(&self) -> Version {
self.meta.version()
}
fn commit_f3(&mut self, height: Height) -> Result<()> {
self.export_meta_if_needed(height)?;
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
.chain(
mem::take(&mut self.dels)
.into_iter()
.map(|key| Item::Tomb(key)),
)
.collect::<Vec<_>>();
if items.is_empty() {
return Ok(());
}
items.sort_unstable();
// let mut batch = OwnedWriteBatch::with_capacity(self.db.clone(), items.len());
// let p = self.keyspace();
// for item in items {
// match item {
// Item::Value { key, value } => {
// batch.insert(p, ByteView::from(key), ByteView::from(value))
// }
// Item::Tomb(key) => batch.remove(p, ByteView::from(key)),
// }
// }
// batch.commit()?;
let mut ingestion = self.keyspace.start_ingestion()?;
for item in items {
match item {
Item::Value { key, value } => {
ingestion.write(ByteView::from(key), ByteView::from(value))
}
Item::Tomb(key) => ingestion.write_tombstone(ByteView::from(key)),
}?
}
ingestion.finish()?;
Ok(())
}
}
pub enum Item<K, V> {
@@ -237,13 +269,13 @@ pub enum Item<K, V> {
}
impl<K: Ord, V> Ord for Item<K, V> {
fn cmp(&self, other: &Self) -> cmp::Ordering {
fn cmp(&self, other: &Self) -> Ordering {
self.key().cmp(other.key())
}
}
impl<K: Ord, V> PartialOrd for Item<K, V> {
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
@@ -262,25 +294,6 @@ impl<K, V> Item<K, V> {
Self::Value { key, .. } | Self::Tomb(key) => key,
}
}
pub fn fjalled(self) -> fjall3::InnerItem
where
K: Into<ByteView>,
V: Into<ByteView>,
{
match self {
Item::Value { key, value } => fjall3::InnerItem {
key: key.into().into(),
value: value.into().into(),
value_type: ValueType::Value,
},
Item::Tomb(key) => fjall3::InnerItem {
key: key.into().into(),
value: [].into(),
value_type: ValueType::Tombstone,
},
}
}
}
#[derive(Debug, Clone, Copy)]
+2 -2
View File
@@ -2,8 +2,8 @@
mod any;
mod fjall_v2;
// mod fjall_v3;
mod fjall_v3;
pub use any::*;
pub use fjall_v2::*;
// pub use fjall_v3::*;
pub use fjall_v3::*;