Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-27 01:54:47 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-12-26 22:41:36 +01:00
parent d538280f4b
commit de93f08e93
120 changed files with 1125 additions and 1773 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
crates/brk_computer/src/chain/compute.rs
View File

@@ -6,7 +6,7 @@ use brk_types::{
};
use vecdb::{Exit, IterableVec, TypedVecIterator, VecIndex, unlikely};
use crate::{grouped::ComputedVecsFromHeight, indexes, price, utils::OptionExt, Indexes};
use crate::{grouped::ComputedVecsFromHeight, indexes, price, txins, utils::OptionExt, Indexes};
use super::{Vecs, ONE_TERA_HASH, TARGET_BLOCKS_PER_DAY_F32, TARGET_BLOCKS_PER_DAY_F64};
@@ -15,11 +15,12 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
txins: &txins::Vecs,
starting_indexes: &Indexes,
price: Option<&price::Vecs>,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, price, exit)?;
self.compute_(indexer, indexes, txins, starting_indexes, price, exit)?;
self.db.compact()?;
Ok(())
}
@@ -28,6 +29,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
txins: &txins::Vecs,
starting_indexes: &Indexes,
price: Option<&price::Vecs>,
exit: &Exit,
@@ -271,15 +273,11 @@ impl Vecs {
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
// ---
// TxInIndex
// ---
self.txindex_to_input_value.compute_sum_from_indexes(
starting_indexes.txindex,
&indexer.vecs.tx.txindex_to_first_txinindex,
&indexes.txindex_to_input_count,
&indexer.vecs.txin.txinindex_to_value,
&txins.txinindex_to_value,
exit,
)?;
@@ -365,8 +363,8 @@ impl Vecs {
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_txoutdata_iter =
indexer.vecs.txout.txoutindex_to_txoutdata.iter()?;
let mut txoutindex_to_value_iter =
indexer.vecs.txout.txoutindex_to_value.iter()?;
vec.compute_transform(
starting_indexes.height,
&indexer.vecs.tx.height_to_first_txindex,
@@ -378,9 +376,8 @@ impl Vecs {
let mut sats = Sats::ZERO;
(first_txoutindex..first_txoutindex + usize::from(output_count)).for_each(
|txoutindex| {
sats += txoutindex_to_txoutdata_iter
.get_unwrap(TxOutIndex::from(txoutindex))
.value;
sats += txoutindex_to_value_iter
.get_unwrap(TxOutIndex::from(txoutindex));
},
);
(height, sats)

8
crates/brk_computer/src/chain/import.rs
View File

@@ -18,9 +18,9 @@ use crate::{
};
use super::{
Vecs, TARGET_BLOCKS_PER_DAY, TARGET_BLOCKS_PER_DECADE, TARGET_BLOCKS_PER_MONTH,
TARGET_BLOCKS_PER_DAY, TARGET_BLOCKS_PER_DECADE, TARGET_BLOCKS_PER_MONTH,
TARGET_BLOCKS_PER_QUARTER, TARGET_BLOCKS_PER_SEMESTER, TARGET_BLOCKS_PER_WEEK,
TARGET_BLOCKS_PER_YEAR,
TARGET_BLOCKS_PER_YEAR, Vecs,
};
impl Vecs {
@@ -42,7 +42,6 @@ impl Vecs {
let v4 = Version::new(4);
let v5 = Version::new(5);
// Helper macros for common patterns
macro_rules! eager {
($name:expr) => {
EagerVec::forced_import(&db, $name, version + v0)?
@@ -125,8 +124,6 @@ impl Vecs {
.add_cumulative()
};
let txinindex_to_value = eager!("value");
let txindex_to_weight = LazyVecFrom2::init(
"weight",
version + Version::ZERO,
@@ -451,7 +448,6 @@ impl Vecs {
indexes_to_inputs_per_sec: computed_di!("inputs_per_sec", v2, last()),
txindex_to_is_coinbase,
txinindex_to_value,
txindex_to_input_value,
txindex_to_output_value,
txindex_to_fee,

3
crates/brk_computer/src/chain/mod.rs
View File

@@ -5,7 +5,7 @@ use brk_traversable::Traversable;
use brk_types::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, FeeRate, HalvingEpoch, Height,
MonthIndex, QuarterIndex, Sats, SemesterIndex, StoredBool, StoredF32, StoredF64, StoredU32,
StoredU64, Timestamp, TxInIndex, TxIndex, VSize, WeekIndex, Weight, YearIndex,
StoredU64, Timestamp, TxIndex, VSize, WeekIndex, Weight, YearIndex,
};
use vecdb::{Database, EagerVec, LazyVecFrom1, LazyVecFrom2, PcoVec};
@@ -86,7 +86,6 @@ pub struct Vecs {
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<VSize>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU64>,
pub txinindex_to_value: EagerVec<PcoVec<TxInIndex, Sats>>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase: LazyVecFrom2<TxIndex, StoredBool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredU64>,

43
crates/brk_computer/src/indexes.rs
View File

@@ -245,10 +245,6 @@ impl Vecs {
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> Result<Indexes> {
// ---
// TxIndex
// ---
self.txindex_to_input_count.compute_count_from_indexes(
starting_indexes.txindex,
&indexer.vecs.tx.txindex_to_first_txinindex,
@@ -270,10 +266,6 @@ impl Vecs {
exit,
)?;
// ---
// Height
// ---
self.height_to_height.compute_from_index(
starting_indexes.height,
&indexer.vecs.block.height_to_weight,
@@ -318,10 +310,6 @@ impl Vecs {
let decremented_starting_height = starting_indexes.height.decremented().unwrap_or_default();
// ---
// DateIndex
// ---
let starting_dateindex = self
.height_to_dateindex
.into_iter()
@@ -370,10 +358,6 @@ impl Vecs {
exit,
)?;
// ---
// WeekIndex
// ---
let starting_weekindex = self
.dateindex_to_weekindex
.into_iter()
@@ -407,10 +391,6 @@ impl Vecs {
exit,
)?;
// ---
// DifficultyEpoch
// ---
let starting_difficultyepoch = self
.height_to_difficultyepoch
.into_iter()
@@ -443,10 +423,6 @@ impl Vecs {
exit,
)?;
// ---
// MonthIndex
// ---
let starting_monthindex = self
.dateindex_to_monthindex
.into_iter()
@@ -480,10 +456,6 @@ impl Vecs {
exit,
)?;
// ---
// QuarterIndex
// ---
let starting_quarterindex = self
.monthindex_to_quarterindex
.into_iter()
@@ -518,10 +490,6 @@ impl Vecs {
exit,
)?;
// ---
// SemesterIndex
// ---
let starting_semesterindex = self
.monthindex_to_semesterindex
.into_iter()
@@ -556,10 +524,6 @@ impl Vecs {
exit,
)?;
// ---
// YearIndex
// ---
let starting_yearindex = self
.monthindex_to_yearindex
.into_iter()
@@ -591,9 +555,6 @@ impl Vecs {
&self.monthindex_to_monthindex,
exit,
)?;
// ---
// HalvingEpoch
// ---
let starting_halvingepoch = self
.height_to_halvingepoch
@@ -619,10 +580,6 @@ impl Vecs {
exit,
)?;
// ---
// DecadeIndex
// ---
let starting_decadeindex = self
.yearindex_to_decadeindex
.into_iter()

56
crates/brk_computer/src/lib.rs
View File

@@ -23,6 +23,8 @@ mod pools;
mod price;
mod stateful;
mod traits;
mod txins;
mod txouts;
mod utils;
use indexes::Indexes;
@@ -40,6 +42,8 @@ pub struct Computer {
pub pools: pools::Vecs,
pub price: Option<price::Vecs>,
pub stateful: stateful::Vecs,
pub txins: txins::Vecs,
pub txouts: txouts::Vecs,
}
const VERSION: Version = Version::new(4);
@@ -60,7 +64,7 @@ impl Computer {
let big_thread = || thread::Builder::new().stack_size(STACK_SIZE);
let i = Instant::now();
let (indexes, fetched, blks) = thread::scope(|s| -> Result<_> {
let (indexes, fetched, blks, txins, txouts) = thread::scope(|s| -> Result<_> {
let fetched_handle = fetcher
.map(|fetcher| {
big_thread().spawn_scoped(s, move || {
@@ -72,13 +76,21 @@ impl Computer {
let blks_handle = big_thread()
.spawn_scoped(s, || blks::Vecs::forced_import(&computed_path, VERSION))?;
let txins_handle = big_thread()
.spawn_scoped(s, || txins::Vecs::forced_import(&computed_path, VERSION))?;
let txouts_handle = big_thread()
.spawn_scoped(s, || txouts::Vecs::forced_import(&computed_path, VERSION))?;
let indexes = indexes::Vecs::forced_import(&computed_path, VERSION, indexer)?;
let fetched = fetched_handle.map(|h| h.join().unwrap()).transpose()?;
let blks = blks_handle.join().unwrap()?;
let txins = txins_handle.join().unwrap()?;
let txouts = txouts_handle.join().unwrap()?;
Ok((indexes, fetched, blks))
Ok((indexes, fetched, blks, txins, txouts))
})?;
info!("Imported indexes/fetched/blks in {:?}", i.elapsed());
info!("Imported indexes/fetched/blks/txins/txouts in {:?}", i.elapsed());
let i = Instant::now();
let (price, constants, market) = thread::scope(|s| -> Result<_> {
@@ -144,8 +156,10 @@ impl Computer {
pools,
cointime,
indexes,
txins,
fetched,
price,
txouts,
})
}
@@ -195,20 +209,33 @@ impl Computer {
Ok(())
});
let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
// Txins must complete before txouts (txouts needs txinindex_to_txoutindex)
// and before chain (chain needs txinindex_to_value)
info!("Computing txins...");
let i = Instant::now();
self.txins.compute(indexer, &starting_indexes, exit)?;
info!("Computed txins in {:?}", i.elapsed());
let txouts = scope.spawn(|| -> Result<()> {
info!("Computing txouts...");
let i = Instant::now();
self.chain.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
exit,
)?;
info!("Computed chain in {:?}", i.elapsed());
self.txouts.compute(indexer, &self.txins, &starting_indexes, exit)?;
info!("Computed txouts in {:?}", i.elapsed());
Ok(())
});
info!("Computing chain...");
let i = Instant::now();
self.chain.compute(
indexer,
&self.indexes,
&self.txins,
&starting_indexes,
self.price.as_ref(),
exit,
)?;
info!("Computed chain in {:?}", i.elapsed());
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
let i = Instant::now();
@@ -218,7 +245,7 @@ impl Computer {
blks.join().unwrap()?;
constants.join().unwrap()?;
chain.join().unwrap()?;
txouts.join().unwrap()?;
Ok(())
})?;
@@ -244,6 +271,7 @@ impl Computer {
self.stateful.compute(
indexer,
&self.indexes,
&self.txins,
&self.chain,
self.price.as_ref(),
&mut starting_indexes,

11
crates/brk_computer/src/pools/mod.rs
View File

@@ -126,8 +126,10 @@ impl Vecs {
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_txoutdata_iter =
indexer.vecs.txout.txoutindex_to_txoutdata.iter()?;
let mut txoutindex_to_outputtype_iter =
indexer.vecs.txout.txoutindex_to_outputtype.iter()?;
let mut txoutindex_to_typeindex_iter =
indexer.vecs.txout.txoutindex_to_typeindex.iter()?;
let mut p2pk65addressindex_to_p2pk65bytes_iter = indexer
.vecs
.address
@@ -180,9 +182,8 @@ impl Vecs {
let pool = (*txoutindex..(*txoutindex + *outputcount))
.map(TxOutIndex::from)
.find_map(|txoutindex| {
let txoutdata = txoutindex_to_txoutdata_iter.get_unwrap(txoutindex);
let outputtype = txoutdata.outputtype;
let typeindex = txoutdata.typeindex;
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex);
let typeindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex);
match outputtype {
OutputType::P2PK65 => Some(AddressBytes::from(

6
crates/brk_computer/src/price.rs
View File

@@ -469,9 +469,6 @@ impl Vecs {
exit,
)?;
// self.halvingepoch_to_price_ohlc
// .compute_transform(starting_indexes.halvingepoch, other, t, exit)?;
self.decadeindex_to_price_ohlc.compute_transform4(
starting_indexes.decadeindex,
self.timeindexes_to_price_open.decadeindex.unwrap_first(),
@@ -798,9 +795,6 @@ impl Vecs {
exit,
)?;
// self.halvingepoch_to_price_ohlc
// _in_sats.compute_transform(starting_indexes.halvingepoch, other, t, exit)?;
self.decadeindex_to_price_ohlc_in_sats.compute_transform4(
starting_indexes.decadeindex,
self.timeindexes_to_price_open_in_sats

2
crates/brk_computer/src/stateful/address/address_count.rs
View File

@@ -1,5 +1,3 @@
//! Address count types per address type.
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_traversable::Traversable;

2
crates/brk_computer/src/stateful/address/any_address_indexes.rs
View File

@@ -1,5 +1,3 @@
//! Storage for address indexes by type.
use brk_error::{Error, Result};
use brk_traversable::Traversable;
use brk_types::{

2
crates/brk_computer/src/stateful/address/data.rs
View File

@@ -1,5 +1,3 @@
//! Storage for address data (loaded and empty addresses).
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{

2
crates/brk_computer/src/stateful/address/height_type_vec.rs
View File

@@ -1,5 +1,3 @@
//! Height to AddressTypeToVec hashmap.
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;

12
crates/brk_computer/src/stateful/address/mod.rs
View File

@@ -1,15 +1,3 @@
//! Address handling with macro-generated code for 8 address types.
//!
//! This module provides:
//! - `AnyAddressIndexesVecs` for storing address indexes by type
//! - `AddressesDataVecs` for storing address data (loaded/empty)
//! - `AddressTypeToTypeIndexMap` for per-type hashmaps
//! - `AddressTypeToVec` for per-type vectors
//! - `HeightToAddressTypeToVec` for height-keyed per-type vectors
//! - `AddressTypeToAddressCount` for runtime address counts
//! - `AddressTypeToHeightToAddressCount` for height-indexed address counts
//! - `AddressTypeToIndexesToAddressCount` for computed address counts
mod address_count;
mod any_address_indexes;
mod data;

5
crates/brk_computer/src/stateful/address/type_index_map.rs
View File

@@ -1,13 +1,10 @@
//! Per-address-type hashmap keyed by TypeIndex.
use std::mem;
use std::{collections::hash_map::Entry, mem};
use brk_grouper::ByAddressType;
use brk_types::{OutputType, TypeIndex};
use derive_deref::{Deref, DerefMut};
use rustc_hash::FxHashMap;
use smallvec::{Array, SmallVec};
use std::collections::hash_map::Entry;
/// A hashmap for each address type, keyed by TypeIndex.
#[derive(Debug, Clone, Deref, DerefMut)]

2
crates/brk_computer/src/stateful/address/type_vec.rs
View File

@@ -1,5 +1,3 @@
//! Per-address-type vector.
use brk_grouper::ByAddressType;
use derive_deref::{Deref, DerefMut};

5
crates/brk_computer/src/stateful/cohorts/address.rs
View File

@@ -1,5 +1,3 @@
//! Address cohort vectors with metrics and state.
use std::path::Path;
use brk_error::Result;
@@ -19,8 +17,7 @@ use crate::{
stateful::states::AddressCohortState,
};
use super::super::metrics::{CohortMetrics, ImportConfig};
use super::traits::{CohortVecs, DynCohortVecs};
use super::{super::metrics::{CohortMetrics, ImportConfig}, traits::{CohortVecs, DynCohortVecs}};
const VERSION: Version = Version::ZERO;

2
crates/brk_computer/src/stateful/cohorts/address_cohorts.rs
View File

@@ -1,5 +1,3 @@
//! Container for all Address cohorts organized by filter type.
use std::path::Path;
use brk_error::Result;

8
crates/brk_computer/src/stateful/cohorts/mod.rs
View File

@@ -1,11 +1,3 @@
//! Cohort management for UTXO and address groupings.
//!
//! Cohorts are groups of UTXOs or addresses filtered by criteria like:
//! - Age (0-1d, 1-7d, etc.)
//! - Amount (< 1 BTC, 1-10 BTC, etc.)
//! - Type (P2PKH, P2SH, etc.)
//! - Term (short-term holder, long-term holder)
mod address;
mod address_cohorts;
mod traits;

2
crates/brk_computer/src/stateful/cohorts/traits.rs
View File

@@ -1,5 +1,3 @@
//! Traits for cohort vector operations.
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{Exit, IterableVec};

2
crates/brk_computer/src/stateful/cohorts/utxo.rs
View File

@@ -1,5 +1,3 @@
//! UTXO cohort vectors with metrics and state.
use std::path::Path;
use brk_error::Result;

2
crates/brk_computer/src/stateful/cohorts/utxo_cohorts/mod.rs
View File

@@ -1,5 +1,3 @@
//! Container for all UTXO cohorts organized by filter type.
mod receive;
mod send;
mod tick_tock;

2
crates/brk_computer/src/stateful/cohorts/utxo_cohorts/receive.rs
View File

@@ -1,5 +1,3 @@
//! Processing received outputs (new UTXOs).
use brk_types::{Dollars, Height, Timestamp};
use crate::stateful::states::Transacted;

2
crates/brk_computer/src/stateful/cohorts/utxo_cohorts/send.rs
View File

@@ -1,5 +1,3 @@
//! Processing spent inputs (UTXOs being spent).
use brk_types::{CheckedSub, Height};
use rustc_hash::FxHashMap;
use vecdb::VecIndex;

8
crates/brk_computer/src/stateful/cohorts/utxo_cohorts/tick_tock.rs
View File

@@ -1,11 +1,3 @@
//! Age-based state transitions for UTXO cohorts.
//!
//! When a new block arrives, UTXOs age. Some cross day boundaries
//! and need to move between age-based cohorts.
//!
//! Optimization: Instead of iterating all ~800k blocks O(n), we binary search
//! for blocks at each day boundary O(k * log n) where k = number of boundaries.
use brk_grouper::AGE_BOUNDARIES;
use brk_types::{ONE_DAY_IN_SEC, Timestamp};

7
crates/brk_computer/src/stateful/compute/aggregates.rs
View File

@@ -1,10 +1,3 @@
//! Aggregate cohort computation.
//!
//! After block processing, compute derived metrics:
//! 1. Overlapping cohorts (e.g., ">=1d" from sum of age_range cohorts)
//! 2. Index-based transforms (height -> dateindex, etc.)
//! 3. Relative metrics (supply ratios, market cap ratios)
use brk_error::Result;
use brk_types::{Bitcoin, DateIndex, Dollars, Height};
use log::info;

30
crates/brk_computer/src/stateful/compute/block_loop.rs
View File

@@ -1,25 +1,15 @@
//! Main block processing loop.
//!
//! Iterates through blocks and processes each one:
//! 1. Reset per-block state values
//! 2. Tick-tock age transitions
//! 3. Process outputs (receive) in parallel
//! 4. Process inputs (send) in parallel
//! 5. Push to height-indexed vectors
//! 6. Periodically flush checkpoints
use std::thread;
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_indexer::Indexer;
use brk_types::{DateIndex, Height, OutputType, Sats, TypeIndex};
use brk_types::{DateIndex, Height, OutputType, Sats, TxIndex, TypeIndex};
use log::info;
use rayon::prelude::*;
use vecdb::{Exit, GenericStoredVec, IterableVec, TypedVecIterator, VecIndex};
use crate::{
chain, indexes, price,
chain, indexes, price, txins,
stateful::{
address::AddressTypeToAddressCount,
compute::write::{process_address_updates, write},
@@ -36,6 +26,7 @@ use super::{
super::{
cohorts::{AddressCohorts, DynCohortVecs, UTXOCohorts},
vecs::Vecs,
RangeMap,
},
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, TxInIterators, TxOutIterators,
@@ -48,6 +39,7 @@ pub fn process_blocks(
vecs: &mut Vecs,
indexer: &Indexer,
indexes: &indexes::Vecs,
txins: &txins::Vecs,
chain: &chain::Vecs,
price: Option<&price::Vecs>,
starting_height: Height,
@@ -128,9 +120,19 @@ pub fn process_blocks(
let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
// Build txindex -> height lookup map for efficient prev_height computation
info!("Building txindex_to_height map...");
let mut txindex_to_height: RangeMap<TxIndex, Height> = {
let mut map = RangeMap::with_capacity(last_height.to_usize() + 1);
for first_txindex in indexer.vecs.tx.height_to_first_txindex.into_iter() {
map.push(first_txindex);
}
map
};
// Create reusable iterators for sequential txout/txin reads (16KB buffered)
let mut txout_iters = TxOutIterators::new(indexer);
let mut txin_iters = TxInIterators::new(indexer);
let mut txin_iters = TxInIterators::new(indexer, txins, &mut txindex_to_height);
info!("Creating address iterators...");
@@ -270,7 +272,7 @@ pub fn process_blocks(
let (input_values, input_prev_heights, input_outputtypes, input_typeindexes) =
if input_count > 1 {
txin_iters.collect_block_inputs(first_txinindex + 1, input_count - 1)
txin_iters.collect_block_inputs(first_txinindex + 1, input_count - 1, height)
} else {
(Vec::new(), Vec::new(), Vec::new(), Vec::new())
};

2
crates/brk_computer/src/stateful/compute/context.rs
View File

@@ -1,5 +1,3 @@
//! Computation context holding shared state during block processing.
use brk_types::{Dollars, Height, Timestamp};
use vecdb::VecIndex;

11
crates/brk_computer/src/stateful/compute/mod.rs
View File

@@ -1,12 +1,3 @@
//! Block processing pipeline.
//!
//! This module handles the main computation loop that processes blocks:
//! 1. Recover state from checkpoint or start fresh
//! 2. Process each block's outputs and inputs
//! 3. Update cohort states
//! 4. Periodically flush to disk
//! 5. Compute aggregate cohorts from separate cohorts
pub mod aggregates;
mod block_loop;
mod context;
@@ -17,7 +8,7 @@ mod write;
pub use block_loop::process_blocks;
pub use context::ComputeContext;
pub use readers::{
TxInIterators, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
TxInIterators, TxOutData, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
build_txoutindex_to_txindex,
};
pub use recover::{StartMode, determine_start_mode, recover_state, reset_state};

61
crates/brk_computer/src/stateful/compute/readers.rs
View File

@@ -1,31 +1,42 @@
//! Cached readers for efficient data access during computation.
//!
//! Readers provide mmap-based access to indexed data without repeated syscalls.
use brk_grouper::{ByAddressType, ByAnyAddress};
use brk_indexer::Indexer;
use brk_types::{
Height, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutData, TxOutIndex, TypeIndex,
Height, OutPoint, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutIndex, TypeIndex,
};
use vecdb::{
BoxedVecIterator, BytesVecIterator, GenericStoredVec, PcodecVecIterator, Reader, VecIndex,
VecIterator,
};
use crate::stateful::address::{AddressesDataVecs, AnyAddressIndexesVecs};
use crate::{
stateful::{address::{AddressesDataVecs, AnyAddressIndexesVecs}, RangeMap},
txins,
};
/// Output data collected from separate vecs.
#[derive(Debug, Clone, Copy)]
pub struct TxOutData {
pub value: Sats,
pub outputtype: OutputType,
pub typeindex: TypeIndex,
}
/// Reusable iterators for txout vectors (16KB buffered reads).
///
/// Iterators are created once and re-positioned each block to avoid
/// creating new file handles repeatedly.
pub struct TxOutIterators<'a> {
txoutdata_iter: BytesVecIterator<'a, TxOutIndex, TxOutData>,
value_iter: BytesVecIterator<'a, TxOutIndex, Sats>,
outputtype_iter: BytesVecIterator<'a, TxOutIndex, OutputType>,
typeindex_iter: BytesVecIterator<'a, TxOutIndex, TypeIndex>,
}
impl<'a> TxOutIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self {
Self {
txoutdata_iter: indexer.vecs.txout.txoutindex_to_txoutdata.into_iter(),
value_iter: indexer.vecs.txout.txoutindex_to_value.into_iter(),
outputtype_iter: indexer.vecs.txout.txoutindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txout.txoutindex_to_typeindex.into_iter(),
}
}
@@ -36,7 +47,11 @@ impl<'a> TxOutIterators<'a> {
output_count: usize,
) -> Vec<TxOutData> {
(first_txoutindex..first_txoutindex + output_count)
.map(|i| self.txoutdata_iter.get_at_unwrap(i))
.map(|i| TxOutData {
value: self.value_iter.get_at_unwrap(i),
outputtype: self.outputtype_iter.get_at_unwrap(i),
typeindex: self.typeindex_iter.get_at_unwrap(i),
})
.collect()
}
}
@@ -44,26 +59,34 @@ impl<'a> TxOutIterators<'a> {
/// Reusable iterators for txin vectors (PcoVec - avoids repeated page decompression).
pub struct TxInIterators<'a> {
value_iter: PcodecVecIterator<'a, TxInIndex, Sats>,
prev_height_iter: PcodecVecIterator<'a, TxInIndex, Height>,
outpoint_iter: PcodecVecIterator<'a, TxInIndex, OutPoint>,
outputtype_iter: PcodecVecIterator<'a, TxInIndex, OutputType>,
typeindex_iter: PcodecVecIterator<'a, TxInIndex, TypeIndex>,
txindex_to_height: &'a mut RangeMap<TxIndex, Height>,
}
impl<'a> TxInIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self {
pub fn new(
indexer: &'a Indexer,
txins: &'a txins::Vecs,
txindex_to_height: &'a mut RangeMap<TxIndex, Height>,
) -> Self {
Self {
value_iter: indexer.vecs.txin.txinindex_to_value.into_iter(),
prev_height_iter: indexer.vecs.txin.txinindex_to_prev_height.into_iter(),
value_iter: txins.txinindex_to_value.into_iter(),
outpoint_iter: indexer.vecs.txin.txinindex_to_outpoint.into_iter(),
outputtype_iter: indexer.vecs.txin.txinindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txin.txinindex_to_typeindex.into_iter(),
txindex_to_height,
}
}
/// Collect input data for a block range using buffered iteration.
/// Computes prev_height on-the-fly from outpoint using RangeMap lookup.
pub fn collect_block_inputs(
&mut self,
first_txinindex: usize,
input_count: usize,
current_height: Height,
) -> (Vec<Sats>, Vec<Height>, Vec<OutputType>, Vec<TypeIndex>) {
let mut values = Vec::with_capacity(input_count);
let mut prev_heights = Vec::with_capacity(input_count);
@@ -72,7 +95,17 @@ impl<'a> TxInIterators<'a> {
for i in first_txinindex..first_txinindex + input_count {
values.push(self.value_iter.get_at_unwrap(i));
prev_heights.push(self.prev_height_iter.get_at_unwrap(i));
let outpoint = self.outpoint_iter.get_at_unwrap(i);
let prev_height = if outpoint.is_coinbase() {
current_height
} else {
self.txindex_to_height
.get(outpoint.txindex())
.unwrap_or(current_height)
};
prev_heights.push(prev_height);
outputtypes.push(self.outputtype_iter.get_at_unwrap(i));
typeindexes.push(self.typeindex_iter.get_at_unwrap(i));
}

15
crates/brk_computer/src/stateful/compute/recover.rs
View File

@@ -1,17 +1,14 @@
//! State recovery logic for checkpoint/resume.
//!
//! Determines starting height and imports saved state from checkpoints.
use std::cmp::Ordering;
use std::collections::BTreeSet;
use std::{cmp::Ordering, collections::BTreeSet};
use brk_error::Result;
use brk_types::Height;
use vecdb::Stamp;
use super::super::AddressesDataVecs;
use super::super::address::AnyAddressIndexesVecs;
use super::super::cohorts::{AddressCohorts, UTXOCohorts};
use super::super::{
AddressesDataVecs,
address::AnyAddressIndexesVecs,
cohorts::{AddressCohorts, UTXOCohorts},
};
/// Result of state recovery.
pub struct RecoveredState {

6
crates/brk_computer/src/stateful/compute/write.rs
View File

@@ -1,9 +1,3 @@
//! State flushing logic for checkpoints.
//!
//! Separates processing (mutations) from flushing (I/O):
//! - `process_address_updates`: applies cached address changes to storage
//! - `flush`: writes all data to disk
use std::time::Instant;
use brk_error::Result;

4
crates/brk_computer/src/stateful/metrics/activity.rs
View File

@@ -1,7 +1,3 @@
//! Transaction activity metrics.
//!
//! These metrics track amounts sent and destruction of satoshi-days/blocks.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Height, Sats, StoredF64, Version};

2
crates/brk_computer/src/stateful/metrics/config.rs
View File

@@ -1,5 +1,3 @@
//! Configuration for metric imports.
use brk_grouper::{CohortContext, Filter};
use brk_types::Version;
use vecdb::Database;

10
crates/brk_computer/src/stateful/metrics/mod.rs
View File

@@ -1,13 +1,3 @@
//! Metric vectors organized by category.
//!
//! Instead of a single 80+ field struct, metrics are grouped into logical categories:
//! - `supply`: Supply and UTXO count metrics (always computed)
//! - `activity`: Transaction activity metrics (always computed)
//! - `realized`: Realized cap, profit/loss, SOPR (requires price)
//! - `unrealized`: Unrealized profit/loss (requires price)
//! - `price`: Price paid metrics and percentiles (requires price)
//! - `relative`: Ratios relative to market cap, etc. (requires price)
mod activity;
mod config;
mod price_paid;

4
crates/brk_computer/src/stateful/metrics/price_paid.rs
View File

@@ -1,7 +1,3 @@
//! Price paid metrics and percentiles.
//!
//! Tracks min/max price paid for UTXOs and price distribution percentiles.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Version};

4
crates/brk_computer/src/stateful/metrics/realized.rs
View File

@@ -1,7 +1,3 @@
//! Realized cap and profit/loss metrics.
//!
//! These metrics require price data and track realized value based on acquisition price.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version};

4
crates/brk_computer/src/stateful/metrics/relative.rs
View File

@@ -1,7 +1,3 @@
//! Relative metrics (ratios to market cap, realized cap, supply, etc.)
//!
//! These are computed ratios comparing cohort metrics to global metrics.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredF32, StoredF64, Version};

4
crates/brk_computer/src/stateful/metrics/supply.rs
View File

@@ -1,7 +1,3 @@
//! Supply and UTXO count metrics.
//!
//! These metrics are always computed regardless of price data availability.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Sats, StoredU64, Version};

4
crates/brk_computer/src/stateful/metrics/unrealized.rs
View File

@@ -1,7 +1,3 @@
//! Unrealized profit/loss metrics.
//!
//! These metrics track paper gains/losses based on current vs acquisition price.
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Sats, Version};

30
crates/brk_computer/src/stateful/mod.rs
View File

@@ -1,41 +1,15 @@
//! Stateful computation for Bitcoin UTXO and address cohort metrics.
//!
//! This module processes blockchain data to compute metrics for various cohorts
//! (groups of UTXOs or addresses filtered by age, amount, type, etc.).
//!
//! ## Module Structure
//!
//! ```text
//! stateful/
//! ├── address/ # Address type collections (type_vec, type_index_map, etc.)
//! ├── cohorts/ # Cohort traits and state management
//! ├── compute/ # Block processing loop and I/O
//! ├── metrics/ # Metric vectors organized by category
//! ├── process/ # Transaction processing (inputs, outputs, cache)
//! └── vecs.rs # Main vectors container
//! ```
//!
//! ## Data Flow
//!
//! 1. **Import**: Load from checkpoint or start fresh
//! 2. **Process blocks**: For each block, process outputs/inputs in parallel
//! 3. **Update cohorts**: Track supply, realized/unrealized P&L per cohort
//! 4. **Flush**: Periodically checkpoint state to disk
//! 5. **Compute aggregates**: Derive aggregate cohorts from separate cohorts
pub mod address;
pub mod cohorts;
pub mod compute;
pub mod metrics;
mod process;
mod range_map;
mod states;
mod vecs;
use states::*;
pub use range_map::RangeMap;
pub use vecs::Vecs;
// Address re-exports
pub use address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs};
// Cohort re-exports
pub use cohorts::{AddressCohorts, CohortVecs, DynCohortVecs, UTXOCohorts};

7
crates/brk_computer/src/stateful/process/address_updates.rs
View File

@@ -1,10 +1,3 @@
//! Address data update processing for flush operations.
//!
//! Handles transitions between loaded (non-zero balance) and empty (zero balance) states:
//! - New addresses: push to storage
//! - Updated addresses: update in place
//! - State transitions: delete from source, push to destination
use brk_error::Result;
use brk_types::{
AnyAddressIndex, EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex,

11
crates/brk_computer/src/stateful/process/cache.rs
View File

@@ -1,15 +1,12 @@
//! Address data cache for flush intervals.
//!
//! Accumulates address data across blocks within a flush interval.
//! Data is flushed to disk at checkpoints.
use brk_grouper::ByAddressType;
use brk_types::{AnyAddressDataIndexEnum, LoadedAddressData, OutputType, TypeIndex};
use vecdb::GenericStoredVec;
use super::super::address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs};
use super::super::compute::VecsReaders;
use super::{
super::{
address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs},
compute::VecsReaders,
},
AddressLookup, EmptyAddressDataWithSource, LoadedAddressDataWithSource, TxIndexVec,
WithAddressDataSource,
};

2
crates/brk_computer/src/stateful/process/inputs.rs
View File

@@ -1,5 +1,3 @@
//! Parallel input processing.
use brk_grouper::ByAddressType;
use brk_types::{Height, OutputType, Sats, TxIndex, TypeIndex};
use rayon::prelude::*;

8
crates/brk_computer/src/stateful/process/lookup.rs
View File

@@ -1,9 +1,9 @@
//! Address data lookup during block processing.
use brk_types::{LoadedAddressData, OutputType, TypeIndex};
use super::super::address::AddressTypeToTypeIndexMap;
use super::{EmptyAddressDataWithSource, LoadedAddressDataWithSource, WithAddressDataSource};
use super::{
super::address::AddressTypeToTypeIndexMap,
EmptyAddressDataWithSource, LoadedAddressDataWithSource, WithAddressDataSource,
};
/// Tracking status of an address - determines cohort update strategy.
#[derive(Clone, Copy)]

22
crates/brk_computer/src/stateful/process/outputs.rs
View File

@@ -1,20 +1,16 @@
//! Output processing.
//!
//! Processes a block's outputs (new UTXOs), building:
//! - Transacted: aggregated supply by output type and amount range
//! - Address data for address cohort tracking (optional)
use brk_grouper::ByAddressType;
use brk_types::{Sats, TxIndex, TxOutData, TypeIndex};
use brk_types::{Sats, TxIndex, TypeIndex};
use crate::stateful::address::{
AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs,
use crate::stateful::{
address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs},
compute::{TxOutData, VecsReaders},
states::Transacted,
};
use crate::stateful::compute::VecsReaders;
use crate::stateful::states::Transacted;
use super::super::address::AddressTypeToVec;
use super::{load_uncached_address_data, AddressCache, LoadedAddressDataWithSource, TxIndexVec};
use super::{
super::address::AddressTypeToVec,
load_uncached_address_data, AddressCache, LoadedAddressDataWithSource, TxIndexVec,
};
/// Result of processing outputs for a block.
pub struct OutputsResult {

34
crates/brk_computer/src/stateful/process/received.rs
View File

@@ -1,12 +1,11 @@
//! Process received outputs for address cohorts.
use brk_grouper::{amounts_in_different_buckets, ByAddressType};
use brk_grouper::{AmountBucket, ByAddressType};
use brk_types::{Dollars, Sats, TypeIndex};
use rustc_hash::FxHashMap;
use super::super::address::AddressTypeToVec;
use super::super::cohorts::AddressCohorts;
use super::lookup::{AddressLookup, TrackingStatus};
use super::{
super::{address::AddressTypeToVec, cohorts::AddressCohorts},
lookup::{AddressLookup, TrackingStatus},
};
pub fn process_received(
received_data: AddressTypeToVec<(TypeIndex, Sats)>,
@@ -21,6 +20,10 @@ pub fn process_received(
continue;
}
// Cache mutable refs for this address type
let type_addr_count = addr_count.get_mut(output_type).unwrap();
let type_empty_count = empty_addr_count.get_mut(output_type).unwrap();
// Aggregate receives by address - each address processed exactly once
// Track (total_value, output_count) for correct UTXO counting
let mut aggregated: FxHashMap<TypeIndex, (Sats, u32)> = FxHashMap::default();
@@ -35,11 +38,11 @@ pub fn process_received(
match status {
TrackingStatus::New => {
*addr_count.get_mut(output_type).unwrap() += 1;
*type_addr_count += 1;
}
TrackingStatus::WasEmpty => {
*addr_count.get_mut(output_type).unwrap() += 1;
*empty_addr_count.get_mut(output_type).unwrap() -= 1;
*type_addr_count += 1;
*type_empty_count -= 1;
}
TrackingStatus::Tracked => {}
}
@@ -49,9 +52,10 @@ pub fn process_received(
if is_new_entry {
// New/was-empty address - just add to cohort
addr_data.receive_outputs(total_value, price, output_count);
let new_bucket = AmountBucket::from(total_value);
cohorts
.amount_range
.get_mut(total_value) // new_balance = 0 + total_value
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
@@ -59,12 +63,14 @@ pub fn process_received(
} else {
let prev_balance = addr_data.balance();
let new_balance = prev_balance + total_value;
let prev_bucket = AmountBucket::from(prev_balance);
let new_bucket = AmountBucket::from(new_balance);
if amounts_in_different_buckets(prev_balance, new_balance) {
if let Some((old_bucket, new_bucket)) = prev_bucket.transition_to(new_bucket) {
// Crossing cohort boundary - subtract from old, add to new
let cohort_state = cohorts
.amount_range
.get_mut(prev_balance)
.get_mut_by_bucket(old_bucket)
.state
.as_mut()
.unwrap();
@@ -89,7 +95,7 @@ pub fn process_received(
addr_data.receive_outputs(total_value, price, output_count);
cohorts
.amount_range
.get_mut(new_balance)
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
@@ -98,7 +104,7 @@ pub fn process_received(
// Staying in same cohort - just receive
cohorts
.amount_range
.get_mut(new_balance)
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()

36
crates/brk_computer/src/stateful/process/sent.rs
View File

@@ -1,18 +1,12 @@
//! Process sent outputs for address cohorts.
//!
//! Updates address cohort states when addresses send funds:
//! - Addresses may cross cohort boundaries
//! - Addresses may become empty (0 balance)
//! - Age metrics (blocks_old, days_old) are tracked for sent UTXOs
use brk_error::Result;
use brk_grouper::{amounts_in_different_buckets, ByAddressType};
use brk_grouper::{AmountBucket, ByAddressType};
use brk_types::{CheckedSub, Dollars, Height, Sats, Timestamp, TypeIndex};
use vecdb::{VecIndex, unlikely};
use super::super::address::HeightToAddressTypeToVec;
use super::super::cohorts::AddressCohorts;
use super::lookup::AddressLookup;
use super::{
super::{address::HeightToAddressTypeToVec, cohorts::AddressCohorts},
lookup::AddressLookup,
};
/// Process sent outputs for address cohorts.
///
@@ -49,6 +43,10 @@ pub fn process_sent(
.is_more_than_hour();
for (output_type, vec) in by_type.unwrap().into_iter() {
// Cache mutable refs for this address type
let type_addr_count = addr_count.get_mut(output_type).unwrap();
let type_empty_count = empty_addr_count.get_mut(output_type).unwrap();
for (type_index, value) in vec {
let addr_data = lookup.get_for_send(output_type, type_index);
@@ -56,14 +54,16 @@ pub fn process_sent(
let new_balance = prev_balance.checked_sub(value).unwrap();
let will_be_empty = addr_data.has_1_utxos();
// Check if crossing cohort boundary
let crossing_boundary = amounts_in_different_buckets(prev_balance, new_balance);
// Compute buckets once
let prev_bucket = AmountBucket::from(prev_balance);
let new_bucket = AmountBucket::from(new_balance);
let crossing_boundary = prev_bucket != new_bucket;
if will_be_empty || crossing_boundary {
// Subtract from old cohort
let cohort_state = cohorts
.amount_range
.get_mut(prev_balance)
.get_mut_by_bucket(prev_bucket)
.state
.as_mut()
.unwrap();
@@ -101,8 +101,8 @@ pub fn process_sent(
unreachable!()
}
*addr_count.get_mut(output_type).unwrap() -= 1;
*empty_addr_count.get_mut(output_type).unwrap() += 1;
*type_addr_count -= 1;
*type_empty_count += 1;
// Move from loaded to empty
lookup.move_to_empty(output_type, type_index);
@@ -110,7 +110,7 @@ pub fn process_sent(
// Add to new cohort
cohorts
.amount_range
.get_mut(new_balance)
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()
@@ -120,7 +120,7 @@ pub fn process_sent(
// Address staying in same cohort - update in place
cohorts
.amount_range
.get_mut(new_balance)
.get_mut_by_bucket(new_bucket)
.state
.as_mut()
.unwrap()

4
crates/brk_computer/src/stateful/process/tx_counts.rs
View File

@@ -1,7 +1,3 @@
//! Transaction count tracking per address.
//!
//! Updates tx_count on address data after deduplicating transaction indexes.
use crate::stateful::address::AddressTypeToTypeIndexMap;
use super::{EmptyAddressDataWithSource, LoadedAddressDataWithSource, TxIndexVec};

2
crates/brk_computer/src/stateful/process/with_source.rs
View File

@@ -1,5 +1,3 @@
//! Address data types with source tracking for flush operations.
use brk_types::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex, TxIndex};
use smallvec::SmallVec;

133
crates/brk_computer/src/stateful/range_map.rs Normal file
View File

@@ -0,0 +1,133 @@
use std::marker::PhantomData;
/// Number of ranges to cache. Small enough for O(1) linear scan,
/// large enough to cover the "hot" source blocks in a typical block.
const CACHE_SIZE: usize = 8;
/// Maps ranges of indices to values for efficient reverse lookups.
///
/// Instead of storing a value for every index, stores first_index values
/// in a sorted Vec and uses binary search to find the value for any index.
/// The value is derived from the position in the Vec.
///
/// Includes an LRU cache of recently accessed ranges to avoid binary search
/// when there's locality in access patterns.
#[derive(Debug)]
pub struct RangeMap<I, V> {
/// Sorted vec of first_index values. Position in vec = value.
first_indexes: Vec<I>,
/// LRU cache: (range_low, range_high, value, age). Lower age = more recent.
cache: [(I, I, V, u8); CACHE_SIZE],
cache_len: u8,
_phantom: PhantomData<V>,
}
impl<I: Default + Copy, V: Default + Copy> Default for RangeMap<I, V> {
fn default() -> Self {
Self {
first_indexes: Vec::new(),
cache: [(I::default(), I::default(), V::default(), 0); CACHE_SIZE],
cache_len: 0,
_phantom: PhantomData,
}
}
}
impl<I: Ord + Copy + Default, V: From<usize> + Copy + Default> RangeMap<I, V> {
/// Create with pre-allocated capacity.
pub fn with_capacity(capacity: usize) -> Self {
Self {
first_indexes: Vec::with_capacity(capacity),
cache: [(I::default(), I::default(), V::default(), 0); CACHE_SIZE],
cache_len: 0,
_phantom: PhantomData,
}
}
/// Push a new first_index. Value is implicitly the current length.
/// Must be called in order (first_index must be >= all previous).
#[inline]
pub fn push(&mut self, first_index: I) {
debug_assert!(
self.first_indexes
.last()
.is_none_or(|&last| first_index >= last),
"RangeMap: first_index must be monotonically increasing"
);
self.first_indexes.push(first_index);
}
/// Look up value for an index, checking cache first.
/// Returns the value (position) of the largest first_index <= given index.
#[inline]
pub fn get(&mut self, index: I) -> Option<V> {
if self.first_indexes.is_empty() {
return None;
}
let cache_len = self.cache_len as usize;
// Check cache first (linear scan of small array)
for i in 0..cache_len {
let (low, high, value, _) = self.cache[i];
if index >= low && index < high {
// Cache hit - mark as most recently used
if self.cache[i].3 != 0 {
for j in 0..cache_len {
self.cache[j].3 = self.cache[j].3.saturating_add(1);
}
self.cache[i].3 = 0;
}
return Some(value);
}
}
// Cache miss - binary search
let pos = self.first_indexes.partition_point(|&first| first <= index);
if pos > 0 {
let value = V::from(pos - 1);
let low = self.first_indexes[pos - 1];
// For last range, use low as high (special marker)
// The check `index < high` will fail, but `index >= low` handles it
let high = self.first_indexes.get(pos).copied().unwrap_or(low);
let is_last = pos == self.first_indexes.len();
// Add to cache (skip if last range - unbounded high is tricky)
if !is_last {
self.add_to_cache(low, high, value);
}
Some(value)
} else {
None
}
}
#[inline]
fn add_to_cache(&mut self, low: I, high: I, value: V) {
let cache_len = self.cache_len as usize;
// Age all entries
for i in 0..cache_len {
self.cache[i].3 = self.cache[i].3.saturating_add(1);
}
if cache_len < CACHE_SIZE {
// Not full - append
self.cache[cache_len] = (low, high, value, 0);
self.cache_len += 1;
} else {
// Full - evict oldest (highest age)
let mut oldest_idx = 0;
let mut oldest_age = 0u8;
for i in 0..CACHE_SIZE {
if self.cache[i].3 > oldest_age {
oldest_age = self.cache[i].3;
oldest_idx = i;
}
}
self.cache[oldest_idx] = (low, high, value, 0);
}
}
}

4
crates/brk_computer/src/stateful/states/cohort.rs
View File

@@ -1,7 +1,3 @@
//! Cohort state tracking during computation.
//!
//! This state is maintained in memory during block processing and periodically flushed.
use std::path::Path;
use brk_error::Result;

12
crates/brk_computer/src/stateful/vecs.rs
View File

@@ -1,5 +1,3 @@
//! Main Vecs struct for stateful computation.
use std::path::Path;
use brk_error::Result;
@@ -21,7 +19,7 @@ use crate::{
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source,
VecBuilderOptions,
},
indexes, price,
indexes, price, txins,
stateful::{
compute::{StartMode, determine_start_mode, process_blocks, recover_state, reset_state},
states::BlockState,
@@ -43,9 +41,6 @@ pub struct Vecs {
#[traversable(skip)]
db: Database,
// ---
// States
// ---
pub chain_state: BytesVec<Height, SupplyState>,
pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs,
@@ -57,9 +52,6 @@ pub struct Vecs {
pub addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount,
pub addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount,
// ---
// Computed
// ---
pub addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount,
pub addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount,
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
@@ -245,6 +237,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
txins: &txins::Vecs,
chain: &chain::Vecs,
price: Option<&price::Vecs>,
starting_indexes: &mut Indexes,
@@ -365,6 +358,7 @@ impl Vecs {
self,
indexer,
indexes,
txins,
chain,
price,
starting_height,

139
crates/brk_computer/src/txins.rs Normal file
View File

@@ -0,0 +1,139 @@
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{Sats, TxInIndex, TxIndex, TxOutIndex, Version, Vout};
use log::info;
use vecdb::{
AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, ImportableVec, PAGE_SIZE, PcoVec,
TypedVecIterator, VecIndex,
};
use super::Indexes;
const ONE_GB: usize = 1024 * 1024 * 1024;
#[derive(Clone, Traversable)]
pub struct Vecs {
db: Database,
pub txinindex_to_txoutindex: PcoVec<TxInIndex, TxOutIndex>,
pub txinindex_to_value: PcoVec<TxInIndex, Sats>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("txins"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let version = parent_version + Version::ZERO;
let this = Self {
txinindex_to_txoutindex: PcoVec::forced_import(&db, "txoutindex", version)?,
txinindex_to_value: PcoVec::forced_import(&db, "value", version)?,
db,
};
this.db.retain_regions(
this.iter_any_exportable()
.flat_map(|v| v.region_names())
.collect(),
)?;
this.db.compact()?;
Ok(this)
}
pub fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let target = indexer.vecs.txin.txinindex_to_outpoint.len();
if target == 0 {
return Ok(());
}
let min = self
.txinindex_to_txoutindex
.len()
.min(self.txinindex_to_value.len())
.min(starting_indexes.txinindex.to_usize());
if min >= target {
return Ok(());
}
info!("TxIns: computing {} entries ({} to {})", target - min, min, target);
const BATCH_SIZE: usize = ONE_GB / size_of::<Entry>();
let mut outpoint_iter = indexer.vecs.txin.txinindex_to_outpoint.iter()?;
let mut first_txoutindex_iter = indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut value_iter = indexer.vecs.txout.txoutindex_to_value.iter()?;
let mut entries: Vec<Entry> = Vec::with_capacity(BATCH_SIZE);
let mut batch_start = min;
while batch_start < target {
let batch_end = (batch_start + BATCH_SIZE).min(target);
entries.clear();
for i in batch_start..batch_end {
let txinindex = TxInIndex::from(i);
let outpoint = outpoint_iter.get_unwrap(txinindex);
entries.push(Entry {
txinindex,
txindex: outpoint.txindex(),
vout: outpoint.vout(),
txoutindex: TxOutIndex::COINBASE,
value: Sats::MAX,
});
}
// Coinbase entries (txindex MAX) sorted to end
entries.sort_unstable_by_key(|e| e.txindex);
for entry in &mut entries {
if entry.txindex.is_coinbase() {
break;
}
entry.txoutindex = first_txoutindex_iter.get_unwrap(entry.txindex) + entry.vout;
}
entries.sort_unstable_by_key(|e| e.txoutindex);
for entry in &mut entries {
if entry.txoutindex.is_coinbase() {
break;
}
entry.value = value_iter.get_unwrap(entry.txoutindex);
}
entries.sort_unstable_by_key(|e| e.txinindex);
for entry in &entries {
self.txinindex_to_txoutindex
.truncate_push(entry.txinindex, entry.txoutindex)?;
self.txinindex_to_value
.truncate_push(entry.txinindex, entry.value)?;
}
batch_start = batch_end;
}
{
let _lock = exit.lock();
self.txinindex_to_txoutindex.flush()?;
self.txinindex_to_value.flush()?;
}
self.db.compact()?;
Ok(())
}
}
struct Entry {
txinindex: TxInIndex,
txindex: TxIndex,
vout: Vout,
txoutindex: TxOutIndex,
value: Sats,
}

128
crates/brk_computer/src/txouts.rs Normal file
View File

@@ -0,0 +1,128 @@
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{Height, TxInIndex, TxOutIndex, Version};
use log::info;
use vecdb::{
AnyVec, BytesVec, Database, Exit, GenericStoredVec, ImportableVec, PAGE_SIZE, Stamp,
TypedVecIterator,
};
use super::{txins, Indexes};
const ONE_GB: usize = 1024 * 1024 * 1024;
const BATCH_SIZE: usize = ONE_GB / size_of::<(TxOutIndex, TxInIndex)>();
#[derive(Clone, Traversable)]
pub struct Vecs {
db: Database,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("txouts"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let version = parent_version + Version::ZERO;
let this = Self {
txoutindex_to_txinindex: BytesVec::forced_import(&db, "txinindex", version)?,
db,
};
this.db.retain_regions(
this.iter_any_exportable()
.flat_map(|v| v.region_names())
.collect(),
)?;
this.db.compact()?;
Ok(this)
}
pub fn compute(
&mut self,
indexer: &Indexer,
txins: &txins::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, txins, starting_indexes, exit)?;
self.db.compact()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
txins: &txins::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let target_txoutindex = indexer.vecs.txout.txoutindex_to_value.len();
let target_txinindex = txins.txinindex_to_txoutindex.len();
if target_txoutindex == 0 {
return Ok(());
}
let target_height = Height::from(indexer.vecs.block.height_to_blockhash.len() - 1);
let min_txoutindex =
TxOutIndex::from(self.txoutindex_to_txinindex.len()).min(starting_indexes.txoutindex);
let min_txinindex = usize::from(starting_indexes.txinindex);
let starting_stamp = Stamp::from(starting_indexes.height);
let _ = self.txoutindex_to_txinindex.rollback_before(starting_stamp);
self.txoutindex_to_txinindex
.truncate_if_needed(min_txoutindex)?;
self.txoutindex_to_txinindex
.fill_to(target_txoutindex, TxInIndex::UNSPENT)?;
if min_txinindex < target_txinindex {
info!(
"TxOuts: computing spend mappings ({} to {})",
min_txinindex, target_txinindex
);
let mut txoutindex_iter = txins.txinindex_to_txoutindex.iter()?;
let mut pairs: Vec<(TxOutIndex, TxInIndex)> = Vec::with_capacity(BATCH_SIZE);
let mut batch_start = min_txinindex;
while batch_start < target_txinindex {
let batch_end = (batch_start + BATCH_SIZE).min(target_txinindex);
pairs.clear();
for i in batch_start..batch_end {
let txinindex = TxInIndex::from(i);
let txoutindex = txoutindex_iter.get_unwrap(txinindex);
if txoutindex.is_coinbase() {
continue;
}
pairs.push((txoutindex, txinindex));
}
pairs.sort_unstable_by_key(|(txoutindex, _)| *txoutindex);
for &(txoutindex, txinindex) in &pairs {
self.txoutindex_to_txinindex.update(txoutindex, txinindex)?;
}
batch_start = batch_end;
}
}
let _lock = exit.lock();
self.txoutindex_to_txinindex
.stamped_write_with_changes(Stamp::from(target_height))?;
Ok(())
}
}