Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: improve par writes

Browse Source
This commit is contained in:
nym21
2025-12-21 16:22:25 +01:00
parent 26c6c92bb8
commit 6e0ac138d8
29 changed files with 378 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Cargo.lock generated
View File

@@ -4218,8 +4218,6 @@ dependencies = [
[[package]]
name = "rawdb"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48b018c8650c492e984d9f6bda52fff0742e1b91ebd97a8ea7046bc12cf1e321"
dependencies = [
"libc",
"log",
@@ -5410,8 +5408,6 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vecdb"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc5945ac4513223cdff7cb954725adfd8941c4dddae3e8765ad2faad26d11903"
dependencies = [
"ctrlc",
"log",
@@ -5419,6 +5415,7 @@ dependencies = [
"parking_lot",
"pco",
"rawdb",
"schemars",
"serde",
"serde_json",
"thiserror 2.0.17",
@@ -5430,8 +5427,6 @@ dependencies = [
[[package]]
name = "vecdb_derive"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f901bd1eab67e00047ed19fcc1e0550b0073716c972866ce06bcecee74f0096"
dependencies = [
"quote",
"syn 2.0.111",

4
Cargo.toml
View File

@@ -79,8 +79,8 @@ serde_derive = "1.0.228"
serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
smallvec = "1.15.1"
tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
vecdb = { version = "0.4.4", features = ["derive", "serde_json", "pco"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { version = "0.4.4", features = ["derive", "serde_json", "pco"] }
vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { git = "https://github.com/anydb-rs/anydb", features = ["derive", "serde_json", "pco"] }
[workspace.metadata.release]

16
crates/brk_computer/src/grouped/price_percentiles.rs
View File

@@ -1,7 +1,10 @@
use brk_error::Result;
use brk_traversable::{Traversable, TreeNode};
use brk_types::{DateIndex, Dollars, Version};
use vecdb::{AnyExportableVec, AnyStoredVec, Database, EagerVec, Exit, GenericStoredVec, PcoVec};
use rayon::prelude::*;
use vecdb::{
AnyExportableVec, AnyStoredVec, Database, EagerVec, Exit, GenericStoredVec, PcoVec,
};
use crate::{Indexes, indexes};
@@ -91,6 +94,17 @@ impl PricePercentiles {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.vecs
.iter_mut()
.flatten()
.filter_map(|v| v.dateindex.as_mut())
.map(|v| v as &mut dyn AnyStoredVec)
.collect::<Vec<_>>()
.into_par_iter()
}
/// Validate computed versions or reset if mismatched.
pub fn validate_computed_version_or_reset(&mut self, version: Version) -> Result<()> {
for vec in self.vecs.iter_mut().flatten() {

17
crates/brk_computer/src/stateful/address/address_count.rs
View File

@@ -5,6 +5,7 @@ use brk_grouper::ByAddressType;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec,
TypedVecIterator,
@@ -88,6 +89,22 @@ impl AddressTypeToHeightToAddressCount {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let inner = &mut self.0;
[
&mut inner.p2pk65 as &mut dyn AnyStoredVec,
&mut inner.p2pk33 as &mut dyn AnyStoredVec,
&mut inner.p2pkh as &mut dyn AnyStoredVec,
&mut inner.p2sh as &mut dyn AnyStoredVec,
&mut inner.p2wpkh as &mut dyn AnyStoredVec,
&mut inner.p2wsh as &mut dyn AnyStoredVec,
&mut inner.p2tr as &mut dyn AnyStoredVec,
&mut inner.p2a as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
pub fn safe_write(&mut self, exit: &Exit) -> Result<()> {
self.p2pk65.safe_write(exit)?;
self.p2pk33.safe_write(exit)?;

6
crates/brk_computer/src/stateful/address/any_address_indexes.rs
View File

@@ -7,6 +7,7 @@ use brk_types::{
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
TypeIndex, Version,
};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Reader, Stamp,
};
@@ -81,6 +82,11 @@ macro_rules! define_any_address_indexes_vecs {
$(self.$field.stamped_write_maybe_with_changes(stamp, with_changes)?;)*
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![$(&mut self.$field as &mut dyn AnyStoredVec),*].into_par_iter()
}
}
};
}

10
crates/brk_computer/src/stateful/address/data.rs
View File

@@ -5,6 +5,7 @@ use brk_traversable::Traversable;
use brk_types::{
EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, Version,
};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportOptions, ImportableVec, Stamp,
};
@@ -63,4 +64,13 @@ impl AddressesDataVecs {
.stamped_write_maybe_with_changes(stamp, with_changes)?;
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.loaded as &mut dyn AnyStoredVec,
&mut self.empty as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
}

26
crates/brk_computer/src/stateful/cohorts/address.rs
View File

@@ -6,6 +6,7 @@ use brk_error::Result;
use brk_grouper::{CohortContext, Filter, Filtered};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec,
PcoVec,
@@ -113,6 +114,20 @@ impl AddressCohortVecs {
.min(self.metrics.supply.min_len())
.min(self.metrics.activity.min_len())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
rayon::iter::once(&mut self.height_to_addr_count as &mut dyn AnyStoredVec)
.chain(self.metrics.par_iter_mut())
}
/// Commit state to disk (separate from vec writes for parallelization).
pub fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.inner.write(height, cleanup)?;
}
Ok(())
}
}
impl Filtered for AddressCohortVecs {
@@ -224,17 +239,6 @@ impl DynCohortVecs for AddressCohortVecs {
Ok(())
}
fn write_stateful_vecs(&mut self, height: Height) -> Result<()> {
self.height_to_addr_count.write()?;
self.metrics.write()?;
if let Some(state) = self.state.as_mut() {
state.inner.commit(height)?;
}
Ok(())
}
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,

18
crates/brk_computer/src/stateful/cohorts/address_cohorts.rs
View File

@@ -11,7 +11,7 @@ use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{Database, Exit, IterableVec};
use vecdb::{AnyStoredVec, Database, Exit, IterableVec};
use crate::{Indexes, indexes, price, stateful::DynCohortVecs};
@@ -222,10 +222,20 @@ impl AddressCohorts {
})
}
/// Write stateful vectors for separate cohorts.
pub fn write_stateful_vecs(&mut self, height: Height) -> Result<()> {
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
// Collect all vecs from all cohorts
self.0
.iter_mut()
.flat_map(|v| v.par_iter_vecs_mut().collect::<Vec<_>>())
.collect::<Vec<_>>()
.into_par_iter()
}
/// Commit all states to disk (separate from vec writes for parallelization).
pub fn commit_all_states(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.write_stateful_vecs(height))
.try_for_each(|v| v.write_state(height, cleanup))
}
/// Get minimum height from all separate cohorts' height-indexed vectors.

3
crates/brk_computer/src/stateful/cohorts/traits.rs
View File

@@ -34,9 +34,6 @@ pub trait DynCohortVecs: Send + Sync {
date_price: Option<Option<Dollars>>,
) -> Result<()>;
/// Write stateful vectors to disk.
fn write_stateful_vecs(&mut self, height: Height) -> Result<()>;
/// First phase of post-processing computations.
#[allow(clippy::too_many_arguments)]
fn compute_rest_part1(

26
crates/brk_computer/src/stateful/cohorts/utxo.rs
View File

@@ -6,7 +6,8 @@ use brk_error::Result;
use brk_grouper::{CohortContext, Filter, Filtered, StateLevel};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Database, Exit, IterableVec};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, IterableVec};
use crate::{
Indexes, indexes, price,
@@ -87,6 +88,19 @@ impl UTXOCohortVecs {
state.reset();
}
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.metrics.par_iter_mut()
}
/// Commit state to disk (separate from vec writes for parallelization).
pub fn write_state(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(state) = self.state.as_mut() {
state.write(height, cleanup)?;
}
Ok(())
}
}
impl Filtered for UTXOCohortVecs {
@@ -189,16 +203,6 @@ impl DynCohortVecs for UTXOCohortVecs {
Ok(())
}
fn write_stateful_vecs(&mut self, height: Height) -> Result<()> {
self.metrics.write()?;
if let Some(state) = self.state.as_mut() {
state.commit(height)?;
}
Ok(())
}
fn compute_rest_part1(
&mut self,
indexes: &indexes::Vecs,

36
crates/brk_computer/src/stateful/cohorts/utxo_cohorts/mod.rs
View File

@@ -9,16 +9,18 @@ use std::path::Path;
use brk_error::Result;
use brk_grouper::{
AmountFilter, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount,
ByMaxAge, ByMinAge, BySpendableType, ByTerm, ByYear, Filter, Filtered, StateLevel, Term,
TimeFilter, UTXOGroups, DAYS_10Y, DAYS_12Y, DAYS_15Y, DAYS_1D, DAYS_1M, DAYS_1W, DAYS_1Y,
ByMaxAge, ByMinAge, BySpendableType, ByTerm, ByYear, DAYS_1D, DAYS_1M, DAYS_1W, DAYS_1Y,
DAYS_2M, DAYS_2Y, DAYS_3M, DAYS_3Y, DAYS_4M, DAYS_4Y, DAYS_5M, DAYS_5Y, DAYS_6M, DAYS_6Y,
DAYS_7Y, DAYS_8Y,
DAYS_7Y, DAYS_8Y, DAYS_10Y, DAYS_12Y, DAYS_15Y, Filter, Filtered, StateLevel, Term, TimeFilter,
UTXOGroups,
};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, HalvingEpoch, Height, OutputType, Sats, Version, Year};
use brk_types::{
Bitcoin, DateIndex, Dollars, HalvingEpoch, Height, OutputType, Sats, Version, Year,
};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{Database, Exit, IterableVec};
use vecdb::{AnyStoredVec, Database, Exit, IterableVec};
use crate::{
Indexes,
@@ -372,18 +374,20 @@ impl UTXOCohorts {
})
}
/// Write stateful vectors for separate and aggregate cohorts.
pub fn write_stateful_vecs(&mut self, height: Height) -> Result<()> {
// Flush separate cohorts (includes metrics + state)
self.par_iter_separate_mut()
.try_for_each(|v| v.write_stateful_vecs(height))?;
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_vecs_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
// Collect all vecs from all cohorts (separate + aggregate)
self.0
.iter_mut()
.flat_map(|v| v.par_iter_vecs_mut().collect::<Vec<_>>())
.collect::<Vec<_>>()
.into_par_iter()
}
// Write aggregate cohorts' metrics (including price_percentiles)
// Note: aggregate cohorts no longer maintain price_to_amount state
for v in self.0.iter_aggregate_mut() {
v.metrics.write()?;
}
Ok(())
/// Commit all states to disk (separate from vec writes for parallelization).
pub fn commit_all_states(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.par_iter_separate_mut()
.try_for_each(|v| v.write_state(height, cleanup))
}
/// Get minimum height from all separate cohorts' height-indexed vectors.

69
crates/brk_computer/src/stateful/compute/write.rs
View File

@@ -53,11 +53,11 @@ pub fn process_address_updates(
/// Flush checkpoint to disk (pure I/O, no processing).
///
/// Writes all accumulated data:
/// - Cohort stateful vectors
/// Writes all accumulated data in parallel:
/// - Cohort stateful vectors (parallel internally)
/// - Height-indexed vectors
/// - Address indexes and data (parallel)
/// - Transaction output index mappings (parallel)
/// - Address indexes and data
/// - Transaction output index mappings
/// - Chain state
///
/// Set `with_changes=true` near chain tip to enable rollback support.
@@ -67,43 +67,48 @@ pub fn write(
chain_state: &[BlockState],
with_changes: bool,
) -> Result<()> {
use rayon::prelude::*;
info!("Writing to disk...");
let i = Instant::now();
// Flush cohort states (separate + aggregate)
vecs.utxo_cohorts.write_stateful_vecs(height)?;
vecs.address_cohorts.write_stateful_vecs(height)?;
// Flush height-indexed vectors
vecs.height_to_unspendable_supply.write()?;
vecs.height_to_opreturn_supply.write()?;
vecs.addresstype_to_height_to_addr_count.write()?;
vecs.addresstype_to_height_to_empty_addr_count.write()?;
// Flush large vecs in parallel
let stamp = Stamp::from(height);
let any_address_indexes = &mut vecs.any_address_indexes;
let addresses_data = &mut vecs.addresses_data;
let txoutindex_to_txinindex = &mut vecs.txoutindex_to_txinindex;
let (addr_result, txout_result) = rayon::join(
|| {
any_address_indexes
.write(stamp, with_changes)
.and(addresses_data.write(stamp, with_changes))
},
|| txoutindex_to_txinindex.stamped_write_maybe_with_changes(stamp, with_changes),
);
addr_result?;
txout_result?;
// Sync in-memory chain_state to persisted and flush
// Prepare chain_state before parallel write
vecs.chain_state.truncate_if_needed(Height::ZERO)?;
for block_state in chain_state {
vecs.chain_state.push(block_state.supply.clone());
}
vecs.chain_state
.stamped_write_maybe_with_changes(stamp, with_changes)?;
// Write all vecs in parallel using chained iterators
vecs.any_address_indexes
.par_iter_mut()
.chain(vecs.addresses_data.par_iter_mut())
.chain(vecs.addresstype_to_height_to_addr_count.par_iter_mut())
.chain(
vecs.addresstype_to_height_to_empty_addr_count
.par_iter_mut(),
)
.chain(rayon::iter::once(
&mut vecs.txoutindex_to_txinindex as &mut dyn AnyStoredVec,
))
.chain(rayon::iter::once(
&mut vecs.chain_state as &mut dyn AnyStoredVec,
))
.chain(rayon::iter::once(
&mut vecs.height_to_unspendable_supply as &mut dyn AnyStoredVec,
))
.chain(rayon::iter::once(
&mut vecs.height_to_opreturn_supply as &mut dyn AnyStoredVec,
))
.chain(vecs.utxo_cohorts.par_iter_vecs_mut())
.chain(vecs.address_cohorts.par_iter_vecs_mut())
.try_for_each(|v| v.any_stamped_write_maybe_with_changes(stamp, with_changes))?;
// Commit states after vec writes
let cleanup = with_changes;
vecs.utxo_cohorts.commit_all_states(height, cleanup)?;
vecs.address_cohorts.commit_all_states(height, cleanup)?;
info!("Wrote in {:?}", i.elapsed());

11
crates/brk_computer/src/stateful/metrics/activity.rs
View File

@@ -5,6 +5,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Height, Sats, StoredF64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use crate::{
@@ -121,6 +122,16 @@ impl ActivityMetrics {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.height_to_sent as &mut dyn AnyStoredVec,
&mut self.height_to_satblocks_destroyed as &mut dyn AnyStoredVec,
&mut self.height_to_satdays_destroyed as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs

25
crates/brk_computer/src/stateful/metrics/mod.rs
View File

@@ -28,7 +28,8 @@ use brk_error::Result;
use brk_grouper::Filter;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Exit, IterableVec};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Exit, IterableVec};
use crate::{Indexes, indexes, price, stateful::states::CohortState};
@@ -125,6 +126,28 @@ impl CohortMetrics {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = Vec::new();
vecs.extend(self.supply.par_iter_mut().collect::<Vec<_>>());
vecs.extend(self.activity.par_iter_mut().collect::<Vec<_>>());
if let Some(realized) = self.realized.as_mut() {
vecs.extend(realized.par_iter_mut().collect::<Vec<_>>());
}
if let Some(unrealized) = self.unrealized.as_mut() {
vecs.extend(unrealized.par_iter_mut().collect::<Vec<_>>());
}
if let Some(price_paid) = self.price_paid.as_mut() {
vecs.extend(price_paid.par_iter_mut().collect::<Vec<_>>());
}
vecs.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.supply.validate_computed_versions(base_version)?;

19
crates/brk_computer/src/stateful/metrics/price_paid.rs
View File

@@ -5,6 +5,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use crate::{
@@ -121,6 +122,24 @@ impl PricePaidMetrics {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = vec![
&mut self.height_to_min_price_paid,
&mut self.height_to_max_price_paid,
];
if let Some(pp) = self.price_percentiles.as_mut() {
vecs.extend(
pp.vecs
.iter_mut()
.flatten()
.filter_map(|v| v.dateindex.as_mut())
.map(|v| v as &mut dyn AnyStoredVec),
);
}
vecs.into_par_iter()
}
/// Validate computed versions or reset if mismatched.
pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
if let Some(price_percentiles) = self.price_percentiles.as_mut() {

19
crates/brk_computer/src/stateful/metrics/realized.rs
View File

@@ -5,6 +5,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, PcoVec};
use crate::{
@@ -448,6 +449,24 @@ impl RealizedMetrics {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
let mut vecs: Vec<&mut dyn AnyStoredVec> = vec![
&mut self.height_to_realized_cap,
&mut self.height_to_realized_profit,
&mut self.height_to_realized_loss,
&mut self.height_to_value_created,
&mut self.height_to_value_destroyed,
];
if let Some(v) = self.height_to_adjusted_value_created.as_mut() {
vecs.push(v);
}
if let Some(v) = self.height_to_adjusted_value_destroyed.as_mut() {
vecs.push(v);
}
vecs.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs

10
crates/brk_computer/src/stateful/metrics/supply.rs
View File

@@ -5,6 +5,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, PcoVec,
TypedVecIterator,
@@ -137,6 +138,15 @@ impl SupplyMetrics {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.height_to_supply as &mut dyn AnyStoredVec,
&mut self.height_to_utxo_count as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
/// Validate computed versions against base version.
pub fn validate_computed_versions(&mut self, _base_version: Version) -> Result<()> {
// Validation logic for computed vecs

16
crates/brk_computer/src/stateful/metrics/unrealized.rs
View File

@@ -5,6 +5,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Sats, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec,
};
@@ -231,6 +232,21 @@ impl UnrealizedMetrics {
Ok(())
}
/// Returns a parallel iterator over all vecs for parallel writing.
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
vec![
&mut self.height_to_supply_in_profit as &mut dyn AnyStoredVec,
&mut self.height_to_supply_in_loss as &mut dyn AnyStoredVec,
&mut self.height_to_unrealized_profit as &mut dyn AnyStoredVec,
&mut self.height_to_unrealized_loss as &mut dyn AnyStoredVec,
&mut self.dateindex_to_supply_in_profit as &mut dyn AnyStoredVec,
&mut self.dateindex_to_supply_in_loss as &mut dyn AnyStoredVec,
&mut self.dateindex_to_unrealized_profit as &mut dyn AnyStoredVec,
&mut self.dateindex_to_unrealized_loss as &mut dyn AnyStoredVec,
]
.into_par_iter()
}
/// Compute aggregate values from separate cohorts.
pub fn compute_from_stateful(
&mut self,

11
crates/brk_computer/src/stateful/states/address_cohort.rs
View File

@@ -173,14 +173,11 @@ impl AddressCohortState {
)
});
self.inner.decrement_(
&addr_supply,
addressdata.realized_cap,
realized_price,
);
self.inner
.decrement_(&addr_supply, addressdata.realized_cap, realized_price);
}
pub fn commit(&mut self, height: Height) -> Result<()> {
self.inner.commit(height)
pub fn write(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.inner.write(height, cleanup)
}
}

4
crates/brk_computer/src/stateful/states/cohort.rs
View File

@@ -369,9 +369,9 @@ impl CohortState {
}
/// Flush state to disk at checkpoint.
pub fn commit(&mut self, height: Height) -> Result<()> {
pub fn write(&mut self, height: Height, cleanup: bool) -> Result<()> {
if let Some(p) = self.price_to_amount.as_mut() {
p.flush(height)?;
p.write(height, cleanup)?;
}
Ok(())
}

21
crates/brk_computer/src/stateful/states/price_to_amount.rs
View File

@@ -144,9 +144,7 @@ impl PriceToAmount {
for (&price, &amount) in state.iter() {
cumsum += u64::from(amount);
while idx < PERCENTILES_LEN
&& cumsum >= total * u64::from(PERCENTILES[idx]) / 100
{
while idx < PERCENTILES_LEN && cumsum >= total * u64::from(PERCENTILES[idx]) / 100 {
result[idx] = price;
idx += 1;
}
@@ -181,16 +179,19 @@ impl PriceToAmount {
.collect::<BTreeMap<Height, PathBuf>>())
}
pub fn flush(&mut self, height: Height) -> Result<()> {
/// Flush state to disk, optionally cleaning up old state files.
pub fn write(&mut self, height: Height, cleanup: bool) -> Result<()> {
self.apply_pending();
let files = self.read_dir(Some(height))?;
if cleanup {
let files = self.read_dir(Some(height))?;
for (_, path) in files
.iter()
.take(files.len().saturating_sub(STATE_TO_KEEP - 1))
{
fs::remove_file(path)?;
for (_, path) in files
.iter()
.take(files.len().saturating_sub(STATE_TO_KEEP - 1))
{
fs::remove_file(path)?;
}
}
fs::write(self.path_state(height), self.state.u().serialize()?)?;

5
crates/brk_indexer/src/vecs/address.rs
View File

@@ -6,6 +6,7 @@ use brk_types::{
P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, P2WPKHBytes, P2WSHAddressIndex,
P2WSHBytes, TypeIndex, Version,
};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Reader, Stamp,
TypedVecIterator,
@@ -136,7 +137,7 @@ impl AddressVecs {
Ok(())
}
pub fn iter_mut_any(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_first_p2pk65addressindex as &mut dyn AnyStoredVec,
&mut self.height_to_first_p2pk33addressindex,
@@ -155,7 +156,7 @@ impl AddressVecs {
&mut self.p2traddressindex_to_p2trbytes,
&mut self.p2aaddressindex_to_p2abytes,
]
.into_iter()
.into_par_iter()
}
/// Get address bytes by output type, using the reader for the specific address type.

5
crates/brk_indexer/src/vecs/blocks.rs
View File

@@ -1,6 +1,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{BlockHash, Height, StoredF64, StoredU64, Timestamp, Version, Weight};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
#[derive(Clone, Traversable)]
@@ -38,7 +39,7 @@ impl BlockVecs {
Ok(())
}
pub fn iter_mut_any(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_blockhash as &mut dyn AnyStoredVec,
&mut self.height_to_difficulty,
@@ -46,6 +47,6 @@ impl BlockVecs {
&mut self.height_to_total_size,
&mut self.height_to_weight,
]
.into_iter()
.into_par_iter()
}
}

32
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -121,16 +121,14 @@ impl Vecs {
}
pub fn flush(&mut self, height: Height) -> Result<()> {
self.iter_mut_any_stored_vec()
// self.par_iter_mut_any_stored_vec()
.par_bridge()
self.par_iter_mut_any_stored_vec()
.try_for_each(|vec| vec.stamped_write(Stamp::from(height)))?;
self.db.flush()?;
Ok(())
}
pub fn starting_height(&mut self) -> Height {
self.iter_mut_any_stored_vec()
self.par_iter_mut_any_stored_vec()
.map(|vec| {
let h = Height::from(vec.stamp());
if h > Height::ZERO { h.incremented() } else { h }
@@ -152,26 +150,18 @@ impl Vecs {
self.address.iter_hashes_from(address_type, height)
}
fn iter_mut_any_stored_vec(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
fn par_iter_mut_any_stored_vec(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.block
.iter_mut_any()
.chain(self.tx.iter_mut_any())
.chain(self.txin.iter_mut_any())
.chain(self.txout.iter_mut_any())
.chain(self.address.iter_mut_any())
.chain(self.output.iter_mut_any())
.par_iter_mut_any()
.chain(self.tx.par_iter_mut_any())
.chain(self.txin.par_iter_mut_any())
.chain(self.txout.par_iter_mut_any())
.chain(self.address.par_iter_mut_any())
.chain(self.output.par_iter_mut_any())
}
// fn par_iter_mut_any_stored_vec(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
// self.block
// .iter_mut_any()
// .chain(self.tx.iter_mut_any())
// .chain(self.txin.iter_mut_any())
// .chain(self.txout.iter_mut_any())
// .chain(self.address.iter_mut_any())
// .chain(self.output.iter_mut_any())
// }
pub fn db(&self) -> &Database {
&self.db
}

5
crates/brk_indexer/src/vecs/output.rs
View File

@@ -3,6 +3,7 @@ use brk_traversable::Traversable;
use brk_types::{
EmptyOutputIndex, Height, OpReturnIndex, P2MSOutputIndex, TxIndex, UnknownOutputIndex, Version,
};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
#[derive(Clone, Traversable)]
@@ -77,7 +78,7 @@ impl OutputVecs {
Ok(())
}
pub fn iter_mut_any(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_first_emptyoutputindex as &mut dyn AnyStoredVec,
&mut self.height_to_first_opreturnindex,
@@ -88,6 +89,6 @@ impl OutputVecs {
&mut self.p2msoutputindex_to_txindex,
&mut self.unknownoutputindex_to_txindex,
]
.into_iter()
.into_par_iter()
}
}

5
crates/brk_indexer/src/vecs/tx.rs
View File

@@ -4,6 +4,7 @@ use brk_types::{
Height, RawLockTime, StoredBool, StoredU32, TxInIndex, TxIndex, TxOutIndex, TxVersion, Txid,
Version,
};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
#[derive(Clone, Traversable)]
@@ -60,7 +61,7 @@ impl TxVecs {
Ok(())
}
pub fn iter_mut_any(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_first_txindex as &mut dyn AnyStoredVec,
&mut self.txindex_to_height,
@@ -73,6 +74,6 @@ impl TxVecs {
&mut self.txindex_to_first_txinindex,
&mut self.txindex_to_first_txoutindex,
]
.into_iter()
.into_par_iter()
}
}

5
crates/brk_indexer/src/vecs/txin.rs
View File

@@ -1,6 +1,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, OutPoint, TxInIndex, TxIndex, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
#[derive(Clone, Traversable)]
@@ -29,12 +30,12 @@ impl TxinVecs {
Ok(())
}
pub fn iter_mut_any(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_first_txinindex as &mut dyn AnyStoredVec,
&mut self.txinindex_to_outpoint,
&mut self.txinindex_to_txindex,
]
.into_iter()
.into_par_iter()
}
}

5
crates/brk_indexer/src/vecs/txout.rs
View File

@@ -1,6 +1,7 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, OutputType, Sats, TxIndex, TxOutIndex, TypeIndex, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
#[derive(Clone, Traversable)]
@@ -37,7 +38,7 @@ impl TxoutVecs {
Ok(())
}
pub fn iter_mut_any(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_first_txoutindex as &mut dyn AnyStoredVec,
&mut self.txoutindex_to_value,
@@ -45,6 +46,6 @@ impl TxoutVecs {
&mut self.txoutindex_to_typeindex,
&mut self.txoutindex_to_txindex,
]
.into_iter()
.into_par_iter()
}
}

81
docs/CHANGELOG.md
View File

@@ -4,6 +4,87 @@ All notable changes to the Bitcoin Research Kit (BRK) project will be documented
> *This changelog was generated by Claude Code*
## [v0.1.0-alpha.1](https://github.com/bitcoinresearchkit/brk/releases/tag/v0.1.0-alpha.1) - 2025-12-21
### New Features
#### `brk_binder`
- Implemented complete multi-language client generation system that produces typed API clients from the metric catalog and OpenAPI specification ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/lib.rs))
- Added JavaScript client generator with full JSDoc type annotations for IDE autocomplete support across 20k+ metrics ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/javascript.rs))
- Added Python client generator with type hints and httpx for HTTP requests ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/python.rs))
- Added Rust client generator with strong typing using reqwest ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/rust.rs))
- Implemented OpenAPI 3.1 integration using the `oas3` crate for parsing endpoint definitions ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/openapi.rs))
- Created structural pattern detection system that identifies repeating tree structures for type reuse ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/types/patterns.rs))
- Added generic pattern detection that groups type-parameterized structures to reduce generated code size
- Created `ClientMetadata` for extracting catalog structure, patterns, and index sets from `brk_query::Vecs` ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_binder/src/types/mod.rs))
#### `brk_types`
- Added `MetricLeaf` struct containing metric name, value type, and available indexes ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_types/src/treenode.rs))
- Added `MetricLeafWithSchema` that wraps `MetricLeaf` with a JSON Schema for client type generation
- Changed `TreeNode::Leaf` from `String` to `MetricLeafWithSchema` to carry full metadata through the tree
- Added `JsonSchema` derive to byte array types (`U8x2`, `U8x20`, `U8x32`) with custom implementations for `U8x33` and `U8x65`
- Added `JsonSchema` derive to OHLC types (`OHLCCents`, `OHLCDollars`, `OHLCSats`, `Open`, `High`, `Low`, `Close`)
- Added `JsonSchema` derive to all index types (date, week, month, quarter, semester, year, decade, halving epoch, difficulty epoch)
- Added `JsonSchema` derive to address index types (P2PKH, P2SH, P2WPKH, P2WSH, P2TR, P2PK33, P2PK65, P2A, OP_RETURN, empty, unknown)
- Implemented index merging when collapsing tree nodes with the same metric name
#### `brk_traversable`
- Added `make_leaf` helper function that generates `MetricLeafWithSchema` nodes with schemars-based JSON Schema generation ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_traversable/src/lib.rs))
- Updated all `Traversable` implementations to require `JsonSchema` bound and generate full leaf metadata
#### `brk_computer`
- Added `neg_realized_loss` metric (realized_loss * -1) for charting convenience ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_computer/src/stateful/metrics/realized.rs))
- Added `net_realized_pnl` metric (realized_profit - realized_loss)
- Added `realized_value` metric (realized_profit + realized_loss)
- Added `total_realized_pnl` at both height and dateindex levels
- Added `realized_price` metric (realized_cap / supply)
- Added `realized_cap_30d_delta` for 30-day realized cap changes
- Added `sopr` (Spent Output Profit Ratio) with 7-day and 30-day EMA variants
- Added `adjusted_sopr` with EMA variants for coinbase-adjusted analysis
- Added `sell_side_risk_ratio` (realized_value / realized_cap) with 7-day and 30-day EMAs
- Added realized profit/loss ratios relative to realized cap
- Added `net_realized_pnl_cumulative_30d_delta` with ratios relative to realized cap and market cap
- Added `neg_unrealized_loss_rel_to_market_cap` and `net_unrealized_pnl_rel_to_market_cap` metrics ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_computer/src/stateful/metrics/relative.rs))
- Added `supply_in_profit/loss_rel_to_circulating_supply` metrics
- Added unrealized profit/loss relative to own market cap at both height and indexes levels
- Added full benchmark example with indexer, computer, and bencher integration ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_computer/examples/full_bench.rs))
#### `brk_grouper`
- Added public constants for age boundaries (`DAYS_1D`, `DAYS_1W`, `DAYS_1M`, etc.) for external use ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_grouper/src/by_age_range.rs))
- Added `get_mut_by_days_old()` method to `ByAgeRange` for O(1) direct bucket access by days
- Added `AmountBucket` type for O(1) amount range classification ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_grouper/src/by_amount_range.rs))
- Added `amounts_in_different_buckets()` helper for cheap bucket comparison
- Optimized `get()` and `get_mut()` in `ByAmountRange` using match on bucket index instead of if-else chain
#### `brk_server`
- Integrated automatic client generation on startup when `brk_binder/clients/` directory exists ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_server/src/lib.rs))
- Added panic catching for client generation to prevent server startup failures
#### `brk_cli`
- Enabled distribution via cargo-dist (`dist = true` in package metadata)
- Added performance benchmarks section to README with timing and resource usage data
#### `workspace`
- Added `rust-toolchain.toml` pinning toolchain to Rust 1.92.0
- Created comprehensive `scripts/publish.sh` for automated crate publishing in dependency order ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/scripts/publish.sh))
- Updated `scripts/update.sh` to also update rust-toolchain version
### Internal Changes
#### `brk_computer`
- Removed `FenwickTree` state module (O(log n) prefix sum data structure)
- Removed `PriceBuckets` state module (logarithmic price bucket distribution)
- Simplified cohort state management after removal of Fenwick-based percentile computation
#### `brk_indexer`
- Renamed `push_if_needed()` to `checked_push()` throughout for API consistency ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.1.0-alpha.1/crates/brk_indexer/src/processor.rs))
- Renamed `Indexes::push_if_needed()` to `Indexes::checked_push()`
#### `brk_cli`
- Updated `zip` dependency from 6.0.0 to 7.0.0
[View changes](https://github.com/bitcoinresearchkit/brk/compare/v0.1.0-alpha.0...v0.1.0-alpha.1)
## [v0.1.0-alpha.0](https://github.com/bitcoinresearchkit/brk/releases/tag/v0.1.0-alpha.0) - 2025-12-18
### Breaking Changes