release: v0.0.75

computer: reduce even more the number of par threads for compute_rest_part2
release: v0.0.74
2026-06-26 22:24:30 -07:00 · 2025-07-09 12:33:06 +02:00 · 2025-07-09 12:32:50 +02:00 · 2025-07-09 11:51:24 +02:00 · 2025-07-09 11:48:41 +02:00
4 changed files with 100 additions and 88 deletions
@@ -477,7 +477,7 @@ dependencies = [

 [[package]]
 name = "brk"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "brk_bundler",
 "brk_cli",
@@ -497,7 +497,7 @@ dependencies = [

 [[package]]
 name = "brk_bundler"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "brk_rolldown",
 "log",
@@ -508,7 +508,7 @@ dependencies = [

 [[package]]
 name = "brk_cli"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "bitcoincore-rpc",
 "brk_computer",
@@ -531,7 +531,7 @@ dependencies = [

 [[package]]
 name = "brk_computer"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "bincode",
 "bitcoin",
@@ -546,6 +546,7 @@ dependencies = [
 "brk_vec",
 "color-eyre",
 "derive_deref",
+ "either",
 "fjall",
 "jiff",
 "log",
@@ -557,7 +558,7 @@ dependencies = [

 [[package]]
 name = "brk_core"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "bincode",
 "bitcoin",
@@ -578,7 +579,7 @@ dependencies = [

 [[package]]
 name = "brk_exit"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "brk_logger",
 "ctrlc",
@@ -587,7 +588,7 @@ dependencies = [

 [[package]]
 name = "brk_fetcher"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "brk_core",
 "brk_logger",
@@ -599,7 +600,7 @@ dependencies = [

 [[package]]
 name = "brk_indexer"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
@@ -617,7 +618,7 @@ dependencies = [

 [[package]]
 name = "brk_interface"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "brk_computer",
 "brk_core",
@@ -635,7 +636,7 @@ dependencies = [

 [[package]]
 name = "brk_logger"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "color-eyre",
 "env_logger",
@@ -645,7 +646,7 @@ dependencies = [

 [[package]]
 name = "brk_mcp"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "axum",
 "brk_interface",
@@ -655,7 +656,7 @@ dependencies = [

 [[package]]
 name = "brk_parser"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
@@ -1024,7 +1025,7 @@ dependencies = [

 [[package]]
 name = "brk_server"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "axum",
 "bitcoincore-rpc",
@@ -1054,7 +1055,7 @@ dependencies = [

 [[package]]
 name = "brk_store"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "brk_core",
 "byteview",
@@ -1076,7 +1077,7 @@ dependencies = [

 [[package]]
 name = "brk_vec"
-version = "0.0.73"
+version = "0.0.75"
 dependencies = [
 "arc-swap",
 "brk_core",
@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.73"
+package.version = "0.0.75"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"

@@ -26,22 +26,22 @@ axum = "0.8.4"
 bincode = { version = "2.0.1", features = ["serde"] }
 bitcoin = { version = "0.32.6", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
-brk_bundler = { version = "0.0.73", path = "crates/brk_bundler" }
-brk_cli = { version = "0.0.73", path = "crates/brk_cli" }
-brk_computer = { version = "0.0.73", path = "crates/brk_computer" }
-brk_core = { version = "0.0.73", path = "crates/brk_core" }
-brk_exit = { version = "0.0.73", path = "crates/brk_exit" }
-brk_fetcher = { version = "0.0.73", path = "crates/brk_fetcher" }
-brk_indexer = { version = "0.0.73", path = "crates/brk_indexer" }
-brk_interface = { version = "0.0.73", path = "crates/brk_interface" }
-brk_logger = { version = "0.0.73", path = "crates/brk_logger" }
-brk_mcp = { version = "0.0.73", path = "crates/brk_mcp" }
-brk_parser = { version = "0.0.73", path = "crates/brk_parser" }
+brk_bundler = { version = "0.0.75", path = "crates/brk_bundler" }
+brk_cli = { version = "0.0.75", path = "crates/brk_cli" }
+brk_computer = { version = "0.0.75", path = "crates/brk_computer" }
+brk_core = { version = "0.0.75", path = "crates/brk_core" }
+brk_exit = { version = "0.0.75", path = "crates/brk_exit" }
+brk_fetcher = { version = "0.0.75", path = "crates/brk_fetcher" }
+brk_indexer = { version = "0.0.75", path = "crates/brk_indexer" }
+brk_interface = { version = "0.0.75", path = "crates/brk_interface" }
+brk_logger = { version = "0.0.75", path = "crates/brk_logger" }
+brk_mcp = { version = "0.0.75", path = "crates/brk_mcp" }
+brk_parser = { version = "0.0.75", path = "crates/brk_parser" }
 brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]}
 # brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]}
-brk_server = { version = "0.0.73", path = "crates/brk_server" }
-brk_store = { version = "0.0.73", path = "crates/brk_store" }
-brk_vec = { version = "0.0.73", path = "crates/brk_vec" }
+brk_server = { version = "0.0.75", path = "crates/brk_server" }
+brk_store = { version = "0.0.75", path = "crates/brk_store" }
+brk_vec = { version = "0.0.75", path = "crates/brk_vec" }
 byteview = "=0.6.1"
 clap = { version = "4.5.40", features = ["string"] }
 clap_derive = "4.5.40"
@@ -21,6 +21,7 @@ brk_store = { workspace = true }
 brk_vec = { workspace = true }
 color-eyre = { workspace = true }
 derive_deref = { workspace = true }
+either = "1.15.0"
 fjall = { workspace = true }
 jiff = { workspace = true }
 log = { workspace = true }
@@ -10,6 +10,7 @@ use brk_vec::{
    AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, GenericStoredVec,
    StoredIndex, StoredVec, UnsafeSlice, VecIterator,
 };
+use either::Either;
 use log::info;
 use rayon::prelude::*;

@@ -1238,26 +1239,28 @@ impl Vecs {
            &self.addresstype_to_height_to_empty_address_count,
        )?;

-        thread::scope(|scope| {
-            scope.spawn(|| {
-                self.utxo_vecs
-                    .as_mut_vecs()
-                    .par_iter_mut()
-                    .try_for_each(|(_, v)| {
-                        v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
-                    })
-                    .unwrap();
-            });
-            scope.spawn(|| {
+        self.utxo_vecs
+            .as_mut_vecs()
+            .into_iter()
+            .map(|(_, v)| v)
+            .map(Either::Left)
+            .chain(
                self.address_vecs
                    .as_mut_vecs()
-                    .par_iter_mut()
-                    .try_for_each(|(_, v)| {
-                        v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
-                    })
-                    .unwrap();
-            });
-        });
+                    .into_iter()
+                    .map(|(_, v)| v)
+                    .map(Either::Right),
+            )
+            .collect::<Vec<Either<&mut utxo_cohort::Vecs, &mut address_cohort::Vecs>>>()
+            .into_par_iter()
+            .try_for_each(|either| match either {
+                Either::Left(v) => {
+                    v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
+                }
+                Either::Right(v) => {
+                    v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
+                }
+            })?;

        info!("Computing rest part 2...");

@@ -1278,49 +1281,56 @@ impl Vecs {
            .indexes_to_realized_cap
            .as_ref()
            .map(|v| v.dateindex.unwrap_last().clone());
+        let dateindex_to_supply_ref = dateindex_to_supply.as_ref().unwrap();
+        let height_to_realized_cap_ref = height_to_realized_cap.as_ref();
+        let dateindex_to_realized_cap_ref = dateindex_to_realized_cap.as_ref();

-        thread::scope(|scope| {
-            scope.spawn(|| {
-                self.utxo_vecs
-                    .as_mut_vecs()
-                    .par_iter_mut()
-                    .try_for_each(|(_, v)| {
-                        v.compute_rest_part2(
-                            indexer,
-                            indexes,
-                            fetched,
-                            starting_indexes,
-                            market,
-                            &height_to_supply,
-                            dateindex_to_supply.as_ref().unwrap(),
-                            height_to_realized_cap.as_ref(),
-                            dateindex_to_realized_cap.as_ref(),
-                            exit,
-                        )
-                    })
-                    .unwrap();
-            });
-            scope.spawn(|| {
+        let vecs = self
+            .utxo_vecs
+            .as_mut_vecs()
+            .into_iter()
+            .map(|(_, v)| v)
+            .map(Either::Left)
+            .chain(
                self.address_vecs
                    .as_mut_vecs()
-                    .par_iter_mut()
-                    .try_for_each(|(_, v)| {
-                        v.compute_rest_part2(
-                            indexer,
-                            indexes,
-                            fetched,
-                            starting_indexes,
-                            market,
-                            &height_to_supply,
-                            dateindex_to_supply.as_ref().unwrap(),
-                            height_to_realized_cap.as_ref(),
-                            dateindex_to_realized_cap.as_ref(),
-                            exit,
-                        )
-                    })
-                    .unwrap();
-            });
-        });
+                    .into_iter()
+                    .map(|(_, v)| v)
+                    .map(Either::Right),
+            )
+            .collect::<Vec<Either<&mut utxo_cohort::Vecs, &mut address_cohort::Vecs>>>();
+
+        // Capped as external drives (even thunderbolt 4 SSDs) can be overwhelmed
+        let chunk_size = (vecs.len() as f64 / 3.0).ceil() as usize;
+        vecs.into_par_iter().chunks(chunk_size).try_for_each(|v| {
+            v.into_iter().try_for_each(|either| match either {
+                Either::Left(v) => v.compute_rest_part2(
+                    indexer,
+                    indexes,
+                    fetched,
+                    starting_indexes,
+                    market,
+                    &height_to_supply,
+                    dateindex_to_supply_ref,
+                    height_to_realized_cap_ref,
+                    dateindex_to_realized_cap_ref,
+                    exit,
+                ),
+                Either::Right(v) => v.compute_rest_part2(
+                    indexer,
+                    indexes,
+                    fetched,
+                    starting_indexes,
+                    market,
+                    &height_to_supply,
+                    dateindex_to_supply_ref,
+                    height_to_realized_cap_ref,
+                    dateindex_to_realized_cap_ref,
+                    exit,
+                ),
+            })
+        })?;
+
        self.indexes_to_unspendable_supply.compute_rest(
            indexer,
            indexes,
Author	SHA1	Message	Date
nym21	50ff6e2745	release: v0.0.75	2025-07-09 12:33:06 +02:00
nym21	811dec713b	computer: reduce even more the number of par threads for compute_rest_part2	2025-07-09 12:32:50 +02:00
nym21	617d6f4bd7	release: v0.0.74	2025-07-09 11:51:24 +02:00
nym21	57cd2d6252	computer: fix par compute_rest_part2 crashing external drives	2025-07-09 11:48:41 +02:00