mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 06:01:57 -07:00
global: snapshot
This commit is contained in:
Generated
+29
-25
@@ -181,9 +181,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.34"
|
||||
version = "0.4.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e86f6d3dc9dc4352edeea6b8e499e13e3f5dc3b964d7ca5fd411415a3498473"
|
||||
checksum = "07a926debf178f2d355197f9caddb08e54a9329d44748034bba349c5848cb519"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
@@ -398,9 +398,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin"
|
||||
version = "0.32.7"
|
||||
version = "0.32.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda569d741b895131a88ee5589a467e73e9c4718e958ac9308e4f7dc44b6945"
|
||||
checksum = "1e499f9fc0407f50fe98af744ab44fa67d409f76b6772e1689ec8485eb0c0f66"
|
||||
dependencies = [
|
||||
"base58ck",
|
||||
"bech32",
|
||||
@@ -441,9 +441,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bitcoin_hashes"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
|
||||
checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b"
|
||||
dependencies = [
|
||||
"bitcoin-io",
|
||||
"hex-conservative",
|
||||
@@ -602,6 +602,7 @@ dependencies = [
|
||||
"color-eyre",
|
||||
"log",
|
||||
"minreq",
|
||||
"rlimit",
|
||||
"serde",
|
||||
"tokio",
|
||||
"toml",
|
||||
@@ -1204,7 +1205,7 @@ dependencies = [
|
||||
"brk_fjall",
|
||||
"brk_types",
|
||||
"byteview 0.6.1",
|
||||
"byteview 0.8.0",
|
||||
"byteview 0.9.1",
|
||||
"fjall",
|
||||
"rustc-hash",
|
||||
]
|
||||
@@ -1320,12 +1321,6 @@ version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
|
||||
|
||||
[[package]]
|
||||
name = "byteview"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e6b0e42e210b794e14b152c6fe1a55831e30ef4a0f5dc39d73d714fb5f1906c"
|
||||
|
||||
[[package]]
|
||||
name = "byteview"
|
||||
version = "0.9.1"
|
||||
@@ -1343,9 +1338,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.48"
|
||||
version = "1.2.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a"
|
||||
checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -1521,9 +1516,9 @@ checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.33"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "302266479cb963552d11bd042013a58ef1adc56768016c8b82b4199488f2d4ad"
|
||||
checksum = "34a3cbbb8b6eca96f3a5c4bf6938d5b27ced3675d69f95bb51948722870bc323"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"compression-core",
|
||||
@@ -2077,9 +2072,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
||||
|
||||
[[package]]
|
||||
name = "fjall"
|
||||
version = "3.0.0-rc.4"
|
||||
version = "3.0.0-rc.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b79d92323d8e942a28890c99ccfe93a75efdebf41304d8922a8251cd16562e42"
|
||||
checksum = "c91b735d557d6636f69dedbea4d7a465e6095a00409b2b146a64cf4c136ab833"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"byteview 0.9.1",
|
||||
@@ -3070,9 +3065,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.1.0"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873"
|
||||
checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
@@ -3596,9 +3591,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_resolver"
|
||||
version = "11.14.2"
|
||||
version = "11.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a22ff4e8307f0392031c2221a3c14f448c446f7ca6312eac62e3181d01e93fe5"
|
||||
checksum = "630a9355dd8fab35d4731f99e1d303499d70b55f7c5a996466050a0a29f4bc76"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"fast-glob",
|
||||
@@ -4336,6 +4331,15 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rlimit"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rolldown-ariadne"
|
||||
version = "0.5.3"
|
||||
@@ -5061,9 +5065,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.23.7"
|
||||
version = "0.23.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
|
||||
checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"toml_datetime",
|
||||
|
||||
+3
-6
@@ -34,7 +34,7 @@ inherits = "release"
|
||||
[workspace.dependencies]
|
||||
aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] }
|
||||
axum = "0.8.7"
|
||||
bitcoin = { version = "0.32.7", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.8", features = ["serde"] }
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
brk_bencher = { version = "0.0.111", path = "crates/brk_bencher" }
|
||||
brk_binder = { version = "0.0.111", path = "crates/brk_binder" }
|
||||
@@ -58,15 +58,12 @@ brk_types = { version = "0.0.111", path = "crates/brk_types" }
|
||||
brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["pco", "derive"] }
|
||||
brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
|
||||
byteview = "=0.6.1"
|
||||
# byteview = "~0.8.0"
|
||||
# byteview = "0.9.1"
|
||||
color-eyre = "0.6.5"
|
||||
derive_deref = "1.1.1"
|
||||
fjall2 = { version = "2.11.8", package = "brk_fjall" }
|
||||
# fjall2 = { path = "../fjall2", package = "brk_fjall" }
|
||||
# fjall2 = { version = "2.11.2", package = "fjall" }
|
||||
fjall3 = { version = "3.0.0-rc.4", package = "fjall" }
|
||||
# fjall3 = { path = "../fjall3", package = "fjall" }
|
||||
# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "f0bf96c2017b3543eb176012b8eff69c639dff1d", package = "fjall" }
|
||||
fjall3 = { version = "3.0.0-rc.5", package = "fjall" }
|
||||
jiff = "0.2.16"
|
||||
log = "0.4.29"
|
||||
minreq = { version = "2.14.1", features = ["https", "serde_json"] }
|
||||
|
||||
@@ -108,13 +108,13 @@ impl Visualizer {
|
||||
let progress_runs = self.read_benchmark_runs(crate_path, "progress.csv")?;
|
||||
let io_runs = self.read_benchmark_runs(crate_path, "io.csv")?;
|
||||
|
||||
// Generate combined charts (all runs together)
|
||||
// Generate combined charts (all runs together) - use progress-based cutoffs
|
||||
if !disk_runs.is_empty() {
|
||||
self.generate_disk_chart(crate_path, crate_name, &disk_runs)?;
|
||||
self.generate_disk_chart(crate_path, crate_name, &disk_runs, &progress_runs)?;
|
||||
}
|
||||
|
||||
if !memory_runs.is_empty() {
|
||||
self.generate_memory_chart(crate_path, crate_name, &memory_runs)?;
|
||||
self.generate_memory_chart(crate_path, crate_name, &memory_runs, &progress_runs)?;
|
||||
}
|
||||
|
||||
if !progress_runs.is_empty() {
|
||||
@@ -122,19 +122,19 @@ impl Visualizer {
|
||||
}
|
||||
|
||||
if !io_runs.is_empty() {
|
||||
self.generate_io_read_chart(crate_path, crate_name, &io_runs)?;
|
||||
self.generate_io_write_chart(crate_path, crate_name, &io_runs)?;
|
||||
self.generate_io_read_chart(crate_path, crate_name, &io_runs, &progress_runs)?;
|
||||
self.generate_io_write_chart(crate_path, crate_name, &io_runs, &progress_runs)?;
|
||||
}
|
||||
|
||||
// Generate individual charts for each run
|
||||
// Generate individual charts for each run (no progress-based cutoffs for single runs)
|
||||
for run in &disk_runs {
|
||||
let run_path = crate_path.join(&run.run_id);
|
||||
self.generate_disk_chart(&run_path, crate_name, slice::from_ref(run))?;
|
||||
self.generate_disk_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
|
||||
}
|
||||
|
||||
for run in &memory_runs {
|
||||
let run_path = crate_path.join(&run.run_id);
|
||||
self.generate_memory_chart(&run_path, crate_name, slice::from_ref(run))?;
|
||||
self.generate_memory_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
|
||||
}
|
||||
|
||||
for run in &progress_runs {
|
||||
@@ -144,8 +144,8 @@ impl Visualizer {
|
||||
|
||||
for run in &io_runs {
|
||||
let run_path = crate_path.join(&run.run_id);
|
||||
self.generate_io_read_chart(&run_path, crate_name, slice::from_ref(run))?;
|
||||
self.generate_io_write_chart(&run_path, crate_name, slice::from_ref(run))?;
|
||||
self.generate_io_read_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
|
||||
self.generate_io_write_chart(&run_path, crate_name, slice::from_ref(run), &[])?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -262,6 +262,69 @@ impl Visualizer {
|
||||
.unwrap_or(1000)
|
||||
}
|
||||
|
||||
/// Find the minimum of the max progress values across all runs,
|
||||
/// then return the cutoff timestamp for each run when that progress was reached.
|
||||
fn calculate_progress_based_cutoffs(
|
||||
runs: &[BenchmarkRun],
|
||||
progress_runs: &[BenchmarkRun],
|
||||
) -> Option<Vec<u64>> {
|
||||
// Find the minimum of max progress across all runs
|
||||
let min_max_progress = progress_runs
|
||||
.iter()
|
||||
.filter_map(|r| r.data.iter().map(|d| d.value).fold(None, |acc, v| {
|
||||
Some(acc.map_or(v, |a: f64| a.max(v)))
|
||||
}))
|
||||
.fold(f64::MAX, f64::min);
|
||||
|
||||
if min_max_progress == f64::MAX {
|
||||
return None;
|
||||
}
|
||||
|
||||
// For each run, find the timestamp when this progress was first reached
|
||||
let cutoffs: Vec<u64> = runs
|
||||
.iter()
|
||||
.map(|run| {
|
||||
// Find the matching progress run
|
||||
let progress_run = progress_runs.iter().find(|pr| pr.run_id == run.run_id);
|
||||
|
||||
if let Some(pr) = progress_run {
|
||||
// Find the timestamp when min_max_progress was reached
|
||||
pr.data
|
||||
.iter()
|
||||
.find(|d| d.value >= min_max_progress)
|
||||
.map(|d| d.timestamp_ms)
|
||||
.unwrap_or_else(|| {
|
||||
// Fallback to max timestamp if progress not found
|
||||
run.data.iter().map(|d| d.timestamp_ms).max().unwrap_or(1000)
|
||||
})
|
||||
} else {
|
||||
// No matching progress run, use max timestamp
|
||||
run.data.iter().map(|d| d.timestamp_ms).max().unwrap_or(1000)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some(cutoffs)
|
||||
}
|
||||
|
||||
fn trim_runs_with_individual_cutoffs(
|
||||
runs: &[BenchmarkRun],
|
||||
cutoffs: &[u64],
|
||||
) -> Vec<BenchmarkRun> {
|
||||
runs.iter()
|
||||
.zip(cutoffs.iter())
|
||||
.map(|(run, &cutoff)| BenchmarkRun {
|
||||
run_id: run.run_id.clone(),
|
||||
data: run
|
||||
.data
|
||||
.iter()
|
||||
.filter(|d| d.timestamp_ms <= cutoff)
|
||||
.cloned()
|
||||
.collect(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn calculate_max_value(runs: &[BenchmarkRun]) -> f64 {
|
||||
runs.iter()
|
||||
.flat_map(|r| r.data.iter().map(|d| d.value))
|
||||
@@ -339,17 +402,26 @@ impl Visualizer {
|
||||
crate_path: &Path,
|
||||
crate_name: &str,
|
||||
runs: &[BenchmarkRun],
|
||||
progress_runs: &[BenchmarkRun],
|
||||
) -> Result<()> {
|
||||
let output_path = crate_path.join("disk.svg");
|
||||
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
// Calculate time window based on shortest run + buffer
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let max_time_s = (min_max_time_ms as f64) / 1000.0;
|
||||
// Try progress-based cutoffs first, fall back to time-based
|
||||
let (trimmed_runs, max_time_ms) =
|
||||
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
|
||||
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000);
|
||||
(
|
||||
Self::trim_runs_with_individual_cutoffs(runs, &cutoffs),
|
||||
max_cutoff + TIME_BUFFER_MS,
|
||||
)
|
||||
} else {
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
(Self::trim_runs_to_time_window(runs, min_max_time_ms), min_max_time_ms)
|
||||
};
|
||||
|
||||
// Trim all runs to the same time window
|
||||
let trimmed_runs = Self::trim_runs_to_time_window(runs, min_max_time_ms);
|
||||
let max_time_s = (max_time_ms as f64) / 1000.0;
|
||||
|
||||
let max_value = Self::calculate_max_value(&trimmed_runs);
|
||||
let (max_value_scaled, unit) = Self::format_bytes(max_value);
|
||||
@@ -403,33 +475,55 @@ impl Visualizer {
|
||||
crate_path: &Path,
|
||||
crate_name: &str,
|
||||
runs: &[BenchmarkRun],
|
||||
progress_runs: &[BenchmarkRun],
|
||||
) -> Result<()> {
|
||||
let output_path = crate_path.join("memory.svg");
|
||||
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
// Calculate time window based on shortest run + buffer
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let max_time_s = (min_max_time_ms as f64) / 1000.0;
|
||||
|
||||
// Read memory CSV files which have 3 columns: timestamp, footprint, peak
|
||||
let enhanced_runs = self.read_memory_data(crate_path, runs)?;
|
||||
|
||||
// Trim enhanced runs to the same time window
|
||||
let trimmed_enhanced_runs: Vec<_> = enhanced_runs
|
||||
.into_iter()
|
||||
.map(|(run_id, footprint, peak)| {
|
||||
let trimmed_footprint: Vec<_> = footprint
|
||||
// Try progress-based cutoffs first, fall back to time-based
|
||||
let (trimmed_enhanced_runs, max_time_ms) =
|
||||
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
|
||||
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
|
||||
let trimmed: Vec<_> = enhanced_runs
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.zip(cutoffs.iter())
|
||||
.map(|((run_id, footprint, peak), &cutoff)| {
|
||||
let trimmed_footprint: Vec<_> = footprint
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= cutoff)
|
||||
.collect();
|
||||
let trimmed_peak: Vec<_> = peak
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= cutoff)
|
||||
.collect();
|
||||
(run_id, trimmed_footprint, trimmed_peak)
|
||||
})
|
||||
.collect();
|
||||
let trimmed_peak: Vec<_> = peak
|
||||
(trimmed, max_cutoff)
|
||||
} else {
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let trimmed: Vec<_> = enhanced_runs
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.map(|(run_id, footprint, peak)| {
|
||||
let trimmed_footprint: Vec<_> = footprint
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.collect();
|
||||
let trimmed_peak: Vec<_> = peak
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.collect();
|
||||
(run_id, trimmed_footprint, trimmed_peak)
|
||||
})
|
||||
.collect();
|
||||
(run_id, trimmed_footprint, trimmed_peak)
|
||||
})
|
||||
.collect();
|
||||
(trimmed, min_max_time_ms)
|
||||
};
|
||||
|
||||
let max_time_s = (max_time_ms as f64) / 1000.0;
|
||||
|
||||
let max_value = trimmed_enhanced_runs
|
||||
.iter()
|
||||
@@ -627,29 +721,47 @@ impl Visualizer {
|
||||
crate_path: &Path,
|
||||
crate_name: &str,
|
||||
runs: &[BenchmarkRun],
|
||||
progress_runs: &[BenchmarkRun],
|
||||
) -> Result<()> {
|
||||
let output_path = crate_path.join("io_read.svg");
|
||||
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
// Calculate time window based on shortest run + buffer
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let max_time_s = (min_max_time_ms as f64) / 1000.0;
|
||||
|
||||
// Read I/O CSV files which have 3 columns: timestamp, bytes_read, bytes_written
|
||||
let io_runs = self.read_io_data(crate_path, runs)?;
|
||||
|
||||
// Trim I/O runs to the same time window and extract only read data
|
||||
let trimmed_io_runs: Vec<_> = io_runs
|
||||
.into_iter()
|
||||
.map(|(run_id, read_data, _write_data)| {
|
||||
let trimmed_read: Vec<_> = read_data
|
||||
// Try progress-based cutoffs first, fall back to time-based
|
||||
let (trimmed_io_runs, max_time_ms) =
|
||||
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
|
||||
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
|
||||
let trimmed: Vec<_> = io_runs
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.zip(cutoffs.iter())
|
||||
.map(|((run_id, read_data, _write_data), &cutoff)| {
|
||||
let trimmed_read: Vec<_> = read_data
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= cutoff)
|
||||
.collect();
|
||||
(run_id, trimmed_read)
|
||||
})
|
||||
.collect();
|
||||
(run_id, trimmed_read)
|
||||
})
|
||||
.collect();
|
||||
(trimmed, max_cutoff)
|
||||
} else {
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let trimmed: Vec<_> = io_runs
|
||||
.into_iter()
|
||||
.map(|(run_id, read_data, _write_data)| {
|
||||
let trimmed_read: Vec<_> = read_data
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.collect();
|
||||
(run_id, trimmed_read)
|
||||
})
|
||||
.collect();
|
||||
(trimmed, min_max_time_ms)
|
||||
};
|
||||
|
||||
let max_time_s = (max_time_ms as f64) / 1000.0;
|
||||
|
||||
let max_value = trimmed_io_runs
|
||||
.iter()
|
||||
@@ -709,29 +821,47 @@ impl Visualizer {
|
||||
crate_path: &Path,
|
||||
crate_name: &str,
|
||||
runs: &[BenchmarkRun],
|
||||
progress_runs: &[BenchmarkRun],
|
||||
) -> Result<()> {
|
||||
let output_path = crate_path.join("io_write.svg");
|
||||
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
// Calculate time window based on shortest run + buffer
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let max_time_s = (min_max_time_ms as f64) / 1000.0;
|
||||
|
||||
// Read I/O CSV files which have 3 columns: timestamp, bytes_read, bytes_written
|
||||
let io_runs = self.read_io_data(crate_path, runs)?;
|
||||
|
||||
// Trim I/O runs to the same time window and extract only write data
|
||||
let trimmed_io_runs: Vec<_> = io_runs
|
||||
.into_iter()
|
||||
.map(|(run_id, _read_data, write_data)| {
|
||||
let trimmed_write: Vec<_> = write_data
|
||||
// Try progress-based cutoffs first, fall back to time-based
|
||||
let (trimmed_io_runs, max_time_ms) =
|
||||
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, progress_runs) {
|
||||
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
|
||||
let trimmed: Vec<_> = io_runs
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.zip(cutoffs.iter())
|
||||
.map(|((run_id, _read_data, write_data), &cutoff)| {
|
||||
let trimmed_write: Vec<_> = write_data
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= cutoff)
|
||||
.collect();
|
||||
(run_id, trimmed_write)
|
||||
})
|
||||
.collect();
|
||||
(run_id, trimmed_write)
|
||||
})
|
||||
.collect();
|
||||
(trimmed, max_cutoff)
|
||||
} else {
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let trimmed: Vec<_> = io_runs
|
||||
.into_iter()
|
||||
.map(|(run_id, _read_data, write_data)| {
|
||||
let trimmed_write: Vec<_> = write_data
|
||||
.into_iter()
|
||||
.filter(|d| d.timestamp_ms <= min_max_time_ms)
|
||||
.collect();
|
||||
(run_id, trimmed_write)
|
||||
})
|
||||
.collect();
|
||||
(trimmed, min_max_time_ms)
|
||||
};
|
||||
|
||||
let max_time_s = (max_time_ms as f64) / 1000.0;
|
||||
|
||||
let max_value = trimmed_io_runs
|
||||
.iter()
|
||||
@@ -796,13 +926,20 @@ impl Visualizer {
|
||||
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
// Calculate time window based on shortest run + buffer
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
let max_time_s = (min_max_time_ms as f64) / 1000.0;
|
||||
|
||||
// Trim all runs to the same time window
|
||||
let trimmed_runs = Self::trim_runs_to_time_window(runs, min_max_time_ms);
|
||||
// Try progress-based cutoffs first, fall back to time-based
|
||||
let (trimmed_runs, max_time_ms) =
|
||||
if let Some(cutoffs) = Self::calculate_progress_based_cutoffs(runs, runs) {
|
||||
let max_cutoff = cutoffs.iter().copied().max().unwrap_or(1000) + TIME_BUFFER_MS;
|
||||
(
|
||||
Self::trim_runs_with_individual_cutoffs(runs, &cutoffs),
|
||||
max_cutoff,
|
||||
)
|
||||
} else {
|
||||
let min_max_time_ms = Self::calculate_min_max_time(runs) + TIME_BUFFER_MS;
|
||||
(Self::trim_runs_to_time_window(runs, min_max_time_ms), min_max_time_ms)
|
||||
};
|
||||
|
||||
let max_time_s = (max_time_ms as f64) / 1000.0;
|
||||
let max_block = Self::calculate_max_value(&trimmed_runs);
|
||||
|
||||
// Format time based on duration
|
||||
|
||||
@@ -31,6 +31,7 @@ serde = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
toml = "0.9.8"
|
||||
zip = { version = "6.0.0", default-features = false, features = ["deflate"] }
|
||||
rlimit = "0.10.2"
|
||||
|
||||
[[bin]]
|
||||
name = "brk"
|
||||
|
||||
@@ -28,6 +28,13 @@ mod website;
|
||||
use crate::{config::Config, paths::*};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
|
||||
rlimit::setrlimit(
|
||||
rlimit::Resource::NOFILE,
|
||||
no_file_limit.0.max(10_000),
|
||||
no_file_limit.1,
|
||||
)?;
|
||||
|
||||
// Can't increase main thread's stack size, thus we need to use another thread
|
||||
thread::Builder::new()
|
||||
.stack_size(512 * 1024 * 1024)
|
||||
|
||||
@@ -75,7 +75,7 @@ pub struct Vecs {
|
||||
pub indexes_to_fee_rate: ComputedVecsFromTxindex<FeeRate>,
|
||||
/// Value == 0 when Coinbase
|
||||
pub txindex_to_input_value: EagerVec<PcoVec<TxIndex, Sats>>,
|
||||
pub indexes_to_sent: ComputedValueVecsFromHeight,
|
||||
pub indexes_to_sent_sum: ComputedValueVecsFromHeight,
|
||||
// pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
|
||||
pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU64>,
|
||||
pub txindex_to_output_value: EagerVec<PcoVec<TxIndex, Sats>>,
|
||||
@@ -376,7 +376,7 @@ impl Vecs {
|
||||
indexes_to_tx_v1: computed_h!("tx_v1", Source::Compute, sum_cum()),
|
||||
indexes_to_tx_v2: computed_h!("tx_v2", Source::Compute, sum_cum()),
|
||||
indexes_to_tx_v3: computed_h!("tx_v3", Source::Compute, sum_cum()),
|
||||
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
|
||||
indexes_to_sent_sum: ComputedValueVecsFromHeight::forced_import(
|
||||
&db,
|
||||
"sent_sum",
|
||||
Source::Compute,
|
||||
@@ -915,7 +915,7 @@ impl Vecs {
|
||||
exit,
|
||||
)?;
|
||||
|
||||
self.indexes_to_sent
|
||||
self.indexes_to_sent_sum
|
||||
.compute_all(indexes, price, starting_indexes, exit, |v| {
|
||||
v.compute_filtered_sum_from_indexes(
|
||||
starting_indexes.height,
|
||||
@@ -1608,7 +1608,7 @@ impl Vecs {
|
||||
.compute_all(starting_indexes, exit, |v| {
|
||||
v.compute_sum(
|
||||
starting_indexes.dateindex,
|
||||
self.indexes_to_sent.sats.dateindex.unwrap_sum(),
|
||||
self.indexes_to_sent_sum.sats.dateindex.unwrap_sum(),
|
||||
365,
|
||||
exit,
|
||||
)?;
|
||||
@@ -1619,7 +1619,7 @@ impl Vecs {
|
||||
.compute_all(starting_indexes, exit, |v| {
|
||||
v.compute_sum(
|
||||
starting_indexes.dateindex,
|
||||
self.indexes_to_sent.bitcoin.dateindex.unwrap_sum(),
|
||||
self.indexes_to_sent_sum.bitcoin.dateindex.unwrap_sum(),
|
||||
365,
|
||||
exit,
|
||||
)?;
|
||||
@@ -1643,12 +1643,12 @@ impl Vecs {
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
if let Some(indexes_to_sent) = self.indexes_to_sent.dollars.as_ref() {
|
||||
if let Some(indexes_to_sent_sum) = self.indexes_to_sent_sum.dollars.as_ref() {
|
||||
self.indexes_to_annualized_volume_usd
|
||||
.compute_all(starting_indexes, exit, |v| {
|
||||
v.compute_sum(
|
||||
starting_indexes.dateindex,
|
||||
indexes_to_sent.dateindex.unwrap_sum(),
|
||||
indexes_to_sent_sum.dateindex.unwrap_sum(),
|
||||
365,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
@@ -151,7 +151,7 @@ impl Vecs {
|
||||
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
|
||||
db,
|
||||
&suffix("sent"),
|
||||
Source::Compute,
|
||||
Source::None,
|
||||
version + v0,
|
||||
sum(),
|
||||
compute_dollars,
|
||||
|
||||
@@ -1080,6 +1080,12 @@ impl Vecs {
|
||||
.get_mut(&typeindex)
|
||||
{
|
||||
addressdata.deref_mut().tx_count += tx_count;
|
||||
} else if let Some(addressdata) =
|
||||
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
|
||||
.get_mut_unwrap(address_type)
|
||||
.get_mut(&typeindex)
|
||||
{
|
||||
addressdata.deref_mut().tx_count += tx_count;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@ impl Stores {
|
||||
self.addresstype_to_addressindex_and_unspentoutpoint
|
||||
.par_values_mut()
|
||||
.map(|s| s as &mut dyn AnyStore),
|
||||
) // Changed from par_iter_mut()
|
||||
)
|
||||
.map(|store| {
|
||||
let items = store.take_all_f2();
|
||||
store.export_meta_if_needed(height)?;
|
||||
@@ -249,11 +249,12 @@ impl Stores {
|
||||
let txindex = TxIndex::from(txindex);
|
||||
let txidprefix = TxidPrefix::from(&txid);
|
||||
|
||||
let is_known_dup = crate::DUPLICATE_TXID_PREFIXES
|
||||
.iter()
|
||||
.any(|(dup_prefix, dup_txindex)| {
|
||||
txindex == *dup_txindex && txidprefix == *dup_prefix
|
||||
});
|
||||
let is_known_dup =
|
||||
crate::DUPLICATE_TXID_PREFIXES
|
||||
.iter()
|
||||
.any(|(dup_prefix, dup_txindex)| {
|
||||
txindex == *dup_txindex && txidprefix == *dup_prefix
|
||||
});
|
||||
|
||||
if !is_known_dup {
|
||||
self.txidprefix_to_txindex.remove(txidprefix);
|
||||
|
||||
@@ -10,14 +10,13 @@ use brk_error::Result;
|
||||
use brk_grouper::ByAddressType;
|
||||
use brk_store::{AnyStore, Kind3, Mode3, StoreFjallV3 as Store};
|
||||
use brk_types::{
|
||||
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height,
|
||||
OutPoint, OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version,
|
||||
Vout,
|
||||
AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height, OutPoint,
|
||||
OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout,
|
||||
};
|
||||
use fjall3::{AbstractTree, Database, PersistMode};
|
||||
use fjall3::{Database, PersistMode};
|
||||
use log::info;
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, VecIndex, VecIterator};
|
||||
use vecdb::{AnyVec, TypedVecIterator, VecIndex, VecIterator};
|
||||
|
||||
use crate::Indexes;
|
||||
|
||||
@@ -130,24 +129,35 @@ impl Stores {
|
||||
}
|
||||
|
||||
pub fn starting_height(&self) -> Height {
|
||||
self.iter_any_store()
|
||||
.map(|store| {
|
||||
// let height =
|
||||
store.height().map(Height::incremented).unwrap_or_default()
|
||||
// dbg!((height, store.name()));
|
||||
})
|
||||
.min()
|
||||
.unwrap()
|
||||
[
|
||||
&self.blockhashprefix_to_height as &dyn AnyStore,
|
||||
&self.height_to_coinbase_tag,
|
||||
&self.txidprefix_to_txindex,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.values()
|
||||
.map(|s| s as &dyn AnyStore),
|
||||
)
|
||||
.chain(
|
||||
self.addresstype_to_addressindex_and_txindex
|
||||
.values()
|
||||
.map(|s| s as &dyn AnyStore),
|
||||
)
|
||||
.chain(
|
||||
self.addresstype_to_addressindex_and_unspentoutpoint
|
||||
.values()
|
||||
.map(|s| s as &dyn AnyStore),
|
||||
)
|
||||
.map(|store| store.height().map(Height::incremented).unwrap_or_default())
|
||||
.min()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn commit(&mut self, height: Height) -> Result<()> {
|
||||
info!(
|
||||
"self.db.config.cache.size = {}",
|
||||
self.db.config.cache.size()
|
||||
);
|
||||
|
||||
let i = Instant::now();
|
||||
let tuples = [
|
||||
[
|
||||
&mut self.blockhashprefix_to_height as &mut dyn AnyStore,
|
||||
&mut self.height_to_coinbase_tag,
|
||||
&mut self.txidprefix_to_txindex,
|
||||
@@ -155,40 +165,21 @@ impl Stores {
|
||||
.into_par_iter()
|
||||
.chain(
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.par_iter_mut()
|
||||
.par_values_mut()
|
||||
.map(|s| s as &mut dyn AnyStore),
|
||||
)
|
||||
.chain(
|
||||
self.addresstype_to_addressindex_and_txindex
|
||||
.par_iter_mut()
|
||||
.par_values_mut()
|
||||
.map(|s| s as &mut dyn AnyStore),
|
||||
)
|
||||
.chain(
|
||||
self.addresstype_to_addressindex_and_unspentoutpoint
|
||||
.par_iter_mut()
|
||||
.par_values_mut()
|
||||
.map(|s| s as &mut dyn AnyStore),
|
||||
) // Changed from par_iter_mut()
|
||||
.map(|store| {
|
||||
let items = store.take_all_f3();
|
||||
store.export_meta_if_needed(height)?;
|
||||
Ok((store.keyspace(), items))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
info!("Store items collected in {:?}", i.elapsed());
|
||||
|
||||
let version_memtable_size_sum = tuples
|
||||
.iter()
|
||||
.map(|(keyspace, _)| keyspace.tree.version_memtable_size_sum())
|
||||
.collect::<Vec<_>>();
|
||||
// let sum = version_memtable_size_sum.iter().sum::<usize>();
|
||||
println!(
|
||||
"version_memtable_size_sum = {:?} = ",
|
||||
version_memtable_size_sum
|
||||
);
|
||||
|
||||
let i = Instant::now();
|
||||
self.db.batch().commit_keyspaces(tuples)?;
|
||||
info!("Batch done in {:?}", i.elapsed());
|
||||
.try_for_each(|store| store.commit_f3(height))?;
|
||||
info!("Commits done in {:?}", i.elapsed());
|
||||
|
||||
let i = Instant::now();
|
||||
self.db.persist(PersistMode::SyncData)?;
|
||||
@@ -202,30 +193,6 @@ impl Stores {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn iter_any_store(&self) -> impl Iterator<Item = &dyn AnyStore> {
|
||||
[
|
||||
&self.blockhashprefix_to_height as &dyn AnyStore,
|
||||
&self.height_to_coinbase_tag,
|
||||
&self.txidprefix_to_txindex,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.iter()
|
||||
.map(|s| s as &dyn AnyStore),
|
||||
)
|
||||
.chain(
|
||||
self.addresstype_to_addressindex_and_txindex
|
||||
.iter()
|
||||
.map(|s| s as &dyn AnyStore),
|
||||
)
|
||||
.chain(
|
||||
self.addresstype_to_addressindex_and_unspentoutpoint
|
||||
.iter()
|
||||
.map(|s| s as &dyn AnyStore),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn rollback_if_needed(
|
||||
&mut self,
|
||||
vecs: &mut Vecs,
|
||||
@@ -236,15 +203,15 @@ impl Stores {
|
||||
&& self.height_to_coinbase_tag.is_empty()?
|
||||
&& self
|
||||
.addresstype_to_addresshash_to_addressindex
|
||||
.iter()
|
||||
.values()
|
||||
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
|
||||
&& self
|
||||
.addresstype_to_addressindex_and_txindex
|
||||
.iter()
|
||||
.values()
|
||||
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
|
||||
&& self
|
||||
.addresstype_to_addressindex_and_unspentoutpoint
|
||||
.iter()
|
||||
.values()
|
||||
.try_fold(true, |acc, s| s.is_empty().map(|empty| acc && empty))?
|
||||
{
|
||||
return Ok(());
|
||||
@@ -265,145 +232,25 @@ impl Stores {
|
||||
self.height_to_coinbase_tag.remove(h);
|
||||
});
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2pk65addressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2pk65addressindex_to_p2pk65bytes_iter =
|
||||
vecs.p2pk65addressindex_to_p2pk65bytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2pk65addressindex_to_p2pk65bytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
// Remove address hashes for all address types starting from rollback height
|
||||
for address_type in [
|
||||
OutputType::P2PK65,
|
||||
OutputType::P2PK33,
|
||||
OutputType::P2PKH,
|
||||
OutputType::P2SH,
|
||||
OutputType::P2WPKH,
|
||||
OutputType::P2WSH,
|
||||
OutputType::P2TR,
|
||||
OutputType::P2A,
|
||||
] {
|
||||
for hash in vecs.iter_address_hashes_from(address_type, starting_indexes.height)? {
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2PK65)
|
||||
.get_mut_unwrap(address_type)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2pk33addressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2pk33addressindex_to_p2pk33bytes_iter =
|
||||
vecs.p2pk33addressindex_to_p2pk33bytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2pk33addressindex_to_p2pk33bytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2PK33)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2pkhaddressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
|
||||
vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2pkhaddressindex_to_p2pkhbytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2PKH)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2shaddressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2shaddressindex_to_p2shbytes_iter =
|
||||
vecs.p2shaddressindex_to_p2shbytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2shaddressindex_to_p2shbytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2SH)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2wpkhaddressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
|
||||
vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2wpkhaddressindex_to_p2wpkhbytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2WPKH)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2wshaddressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2wshaddressindex_to_p2wshbytes_iter =
|
||||
vecs.p2wshaddressindex_to_p2wshbytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2wshaddressindex_to_p2wshbytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2WSH)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2traddressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2traddressindex_to_p2trbytes_iter =
|
||||
vecs.p2traddressindex_to_p2trbytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2TR)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut index) = vecs
|
||||
.height_to_first_p2aaddressindex
|
||||
.read_once(starting_indexes.height)
|
||||
{
|
||||
let mut p2aaddressindex_to_p2abytes_iter =
|
||||
vecs.p2aaddressindex_to_p2abytes.iter()?;
|
||||
|
||||
while let Some(typedbytes) = p2aaddressindex_to_p2abytes_iter.get(index) {
|
||||
let bytes = AddressBytes::from(typedbytes);
|
||||
let hash = AddressHash::from(&bytes);
|
||||
self.addresstype_to_addresshash_to_addressindex
|
||||
.get_mut_unwrap(OutputType::P2A)
|
||||
.remove(hash);
|
||||
index.increment();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
unreachable!();
|
||||
// self.blockhashprefix_to_height.reset()?;
|
||||
// self.addresshash_to_typeindex.reset()?;
|
||||
}
|
||||
|
||||
if starting_indexes.txindex != TxIndex::ZERO {
|
||||
@@ -413,24 +260,21 @@ impl Stores {
|
||||
.skip(starting_indexes.txindex.to_usize())
|
||||
.for_each(|(txindex, txid)| {
|
||||
let txindex = TxIndex::from(txindex);
|
||||
|
||||
let txidprefix = TxidPrefix::from(&txid);
|
||||
|
||||
// "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"
|
||||
let is_not_first_dup = txindex != TxIndex::new(142783)
|
||||
|| txidprefix != TxidPrefix::from([153, 133, 216, 41, 84, 225, 15, 34]);
|
||||
let is_known_dup =
|
||||
crate::DUPLICATE_TXID_PREFIXES
|
||||
.iter()
|
||||
.any(|(dup_prefix, dup_txindex)| {
|
||||
txindex == *dup_txindex && txidprefix == *dup_prefix
|
||||
});
|
||||
|
||||
// "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"
|
||||
let is_not_second_dup = txindex != TxIndex::new(142841)
|
||||
|| txidprefix != TxidPrefix::from([104, 180, 95, 88, 182, 116, 233, 78]);
|
||||
|
||||
if is_not_first_dup && is_not_second_dup {
|
||||
if !is_known_dup {
|
||||
self.txidprefix_to_txindex.remove(txidprefix);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
unreachable!();
|
||||
// self.txidprefix_to_txindex.reset()?;
|
||||
}
|
||||
|
||||
if starting_indexes.txoutindex != TxOutIndex::ZERO {
|
||||
@@ -504,12 +348,6 @@ impl Stores {
|
||||
});
|
||||
} else {
|
||||
unreachable!();
|
||||
// self.addresstype_to_typeindex_and_txindex
|
||||
// .iter_mut()
|
||||
// .try_for_each(|s| s.reset())?;
|
||||
// self.addresstype_to_typeindex_and_unspentoutpoint
|
||||
// .iter_mut()
|
||||
// .try_for_each(|s| s.reset())?;
|
||||
}
|
||||
|
||||
self.commit(starting_indexes.height.decremented().unwrap_or_default())?;
|
||||
|
||||
@@ -75,11 +75,13 @@ pub fn get_address(Address { address }: Address, query: &Query) -> Result<Addres
|
||||
spent_txo_sum: address_data.sent,
|
||||
tx_count: address_data.tx_count,
|
||||
},
|
||||
mempool_stats: query.mempool().and_then(|mempool| {
|
||||
mempool_stats: query.mempool().map(|mempool| {
|
||||
mempool
|
||||
.get_addresses()
|
||||
.get(&bytes)
|
||||
.map(|(stats, ..)| stats.clone())
|
||||
.map(|(stats, _)| stats)
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ impl<'a> Vecs<'a> {
|
||||
// Not the most performant or type safe but only built once so that's okay
|
||||
fn insert(&mut self, vec: &'a dyn AnyExportableVec) {
|
||||
let name = vec.name();
|
||||
dbg!(vec.region_name());
|
||||
// dbg!(vec.region_name());
|
||||
let serialized_index = vec.index_type_to_string();
|
||||
let index = Index::try_from(serialized_index)
|
||||
.inspect_err(|_| {
|
||||
|
||||
@@ -13,8 +13,8 @@ build = "build.rs"
|
||||
[dependencies]
|
||||
brk_error = { workspace = true }
|
||||
brk_types = { workspace = true }
|
||||
byteview6 = { version = "=0.6.1", package = "byteview" }
|
||||
byteview8 = { version = "~0.8.0", package = "byteview" }
|
||||
byteview_f2 = { version = "=0.6.1", package = "byteview" }
|
||||
byteview_f3 = { version = "0.9.1", package = "byteview" }
|
||||
fjall2 = { workspace = true }
|
||||
fjall3 = { workspace = true }
|
||||
rustc-hash = { workspace = true }
|
||||
|
||||
@@ -11,6 +11,7 @@ pub trait AnyStore: Send + Sync {
|
||||
fn keyspace(&self) -> &fjall3::Keyspace;
|
||||
fn partition(&self) -> &fjall2::PartitionHandle;
|
||||
fn take_all_f2(&mut self) -> Vec<fjall2::InnerItem>;
|
||||
fn commit_f3(&mut self, height: Height) -> Result<()>;
|
||||
// fn take_all_f3(&mut self) -> Vec<fjall3::InnerItem>;
|
||||
// fn take_all_f3(&mut self) -> Box<dyn Iterator<Item = Item>>;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path};
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{Height, Version};
|
||||
use byteview6::ByteView;
|
||||
use byteview_f2::ByteView;
|
||||
use fjall2::{
|
||||
CompressionType, InnerItem, PartitionCreateOptions, TransactionalKeyspace,
|
||||
TransactionalPartitionHandle, ValueType,
|
||||
@@ -228,6 +228,10 @@ where
|
||||
fn version(&self) -> Version {
|
||||
self.meta.version()
|
||||
}
|
||||
|
||||
fn commit_f3(&mut self, _: Height) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
enum Item<K, V> {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path};
|
||||
use std::{borrow::Cow, cmp::Ordering, fmt::Debug, fs, hash::Hash, mem, path::Path};
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{Height, Version};
|
||||
use byteview8::ByteView;
|
||||
use byteview_f3::ByteView;
|
||||
use fjall3::{
|
||||
Database, Keyspace, KeyspaceCreateOptions, ValueType,
|
||||
Database, Keyspace, KeyspaceCreateOptions,
|
||||
config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy},
|
||||
};
|
||||
|
||||
@@ -38,9 +38,10 @@ where
|
||||
K: Debug + Clone + From<ByteView> + Ord + Eq + Hash,
|
||||
V: Debug + Clone + From<ByteView>,
|
||||
ByteView: From<K> + From<V>,
|
||||
Self: Send + Sync,
|
||||
{
|
||||
pub fn import(
|
||||
database: &Database,
|
||||
db: &Database,
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
@@ -50,11 +51,11 @@ where
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
let (meta, keyspace) = StoreMeta::checked_open(
|
||||
database,
|
||||
db,
|
||||
&path.join(format!("meta/{name}")),
|
||||
MAJOR_FJALL_VERSION + version,
|
||||
|| {
|
||||
Self::open_keyspace(database, name, mode, kind).inspect_err(|e| {
|
||||
Self::open_keyspace(db, name, mode, kind).inspect_err(|e| {
|
||||
eprintln!("{e}");
|
||||
eprintln!("Delete {path:?} and try again");
|
||||
})
|
||||
@@ -79,7 +80,7 @@ where
|
||||
let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true);
|
||||
|
||||
if kind.is_not_vec() {
|
||||
options = options.filter_policy(FilterPolicy::new(&[
|
||||
options = options.filter_policy(FilterPolicy::new([
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(8.0)),
|
||||
@@ -97,7 +98,7 @@ where
|
||||
.index_block_pinning_policy(PinningPolicy::all(false));
|
||||
}
|
||||
|
||||
database.keyspace(name, options).map_err(|e| e.into())
|
||||
database.keyspace(name, || options).map_err(|e| e.into())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -188,20 +189,6 @@ where
|
||||
panic!()
|
||||
}
|
||||
|
||||
fn take_all_f3(&mut self) -> Vec<fjall3::InnerItem> {
|
||||
let mut items = mem::take(&mut self.puts)
|
||||
.into_iter()
|
||||
.map(|(key, value)| Item::Value { key, value })
|
||||
.chain(
|
||||
mem::take(&mut self.dels)
|
||||
.into_iter()
|
||||
.map(|key| Item::Tomb(key)),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
items.sort_unstable();
|
||||
items.into_iter().map(|v| v.fjalled()).collect()
|
||||
}
|
||||
|
||||
fn export_meta_if_needed(&mut self, height: Height) -> Result<()> {
|
||||
if self.has(height) {
|
||||
return Ok(());
|
||||
@@ -229,6 +216,51 @@ where
|
||||
fn version(&self) -> Version {
|
||||
self.meta.version()
|
||||
}
|
||||
|
||||
fn commit_f3(&mut self, height: Height) -> Result<()> {
|
||||
self.export_meta_if_needed(height)?;
|
||||
|
||||
let mut items = mem::take(&mut self.puts)
|
||||
.into_iter()
|
||||
.map(|(key, value)| Item::Value { key, value })
|
||||
.chain(
|
||||
mem::take(&mut self.dels)
|
||||
.into_iter()
|
||||
.map(|key| Item::Tomb(key)),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if items.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
items.sort_unstable();
|
||||
|
||||
// let mut batch = OwnedWriteBatch::with_capacity(self.db.clone(), items.len());
|
||||
// let p = self.keyspace();
|
||||
// for item in items {
|
||||
// match item {
|
||||
// Item::Value { key, value } => {
|
||||
// batch.insert(p, ByteView::from(key), ByteView::from(value))
|
||||
// }
|
||||
// Item::Tomb(key) => batch.remove(p, ByteView::from(key)),
|
||||
// }
|
||||
// }
|
||||
// batch.commit()?;
|
||||
|
||||
let mut ingestion = self.keyspace.start_ingestion()?;
|
||||
for item in items {
|
||||
match item {
|
||||
Item::Value { key, value } => {
|
||||
ingestion.write(ByteView::from(key), ByteView::from(value))
|
||||
}
|
||||
Item::Tomb(key) => ingestion.write_tombstone(ByteView::from(key)),
|
||||
}?
|
||||
}
|
||||
ingestion.finish()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub enum Item<K, V> {
|
||||
@@ -237,13 +269,13 @@ pub enum Item<K, V> {
|
||||
}
|
||||
|
||||
impl<K: Ord, V> Ord for Item<K, V> {
|
||||
fn cmp(&self, other: &Self) -> cmp::Ordering {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.key().cmp(other.key())
|
||||
}
|
||||
}
|
||||
|
||||
impl<K: Ord, V> PartialOrd for Item<K, V> {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
@@ -262,25 +294,6 @@ impl<K, V> Item<K, V> {
|
||||
Self::Value { key, .. } | Self::Tomb(key) => key,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fjalled(self) -> fjall3::InnerItem
|
||||
where
|
||||
K: Into<ByteView>,
|
||||
V: Into<ByteView>,
|
||||
{
|
||||
match self {
|
||||
Item::Value { key, value } => fjall3::InnerItem {
|
||||
key: key.into().into(),
|
||||
value: value.into().into(),
|
||||
value_type: ValueType::Value,
|
||||
},
|
||||
Item::Tomb(key) => fjall3::InnerItem {
|
||||
key: key.into().into(),
|
||||
value: [].into(),
|
||||
value_type: ValueType::Tombstone,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
mod any;
|
||||
mod fjall_v2;
|
||||
// mod fjall_v3;
|
||||
mod fjall_v3;
|
||||
|
||||
pub use any::*;
|
||||
pub use fjall_v2::*;
|
||||
// pub use fjall_v3::*;
|
||||
pub use fjall_v3::*;
|
||||
|
||||
Reference in New Issue
Block a user