Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2026-03-31 22:53:25 +02:00
parent d038141a8a
commit ae26db6df2
83 changed files with 3398 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/brk_computer/src/distribution/compute/block_loop.rs
View File

@@ -7,7 +7,7 @@ use brk_types::{
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use tracing::{debug, info};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely};
use crate::{
distribution::{
@@ -243,7 +243,11 @@ pub(crate) fn process_blocks(
for height in starting_height.to_usize()..=last_height.to_usize() {
let height = Height::from(height);
info!("Processing chain at {}...", height);
if unlikely(height.is_multiple_of(100)) {
info!("Processing chain at {}...", height);
} else {
debug!("Processing chain at {}...", height);
}
// Get block metadata from pre-collected vecs
let offset = height.to_usize() - start_usize;

39
crates/brk_computer/src/lib.rs
View File

@@ -4,7 +4,6 @@ use std::{fs, path::Path, thread, time::Instant};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_reader::Reader;
use brk_traversable::Traversable;
use brk_types::Version;
use tracing::info;
@@ -23,7 +22,6 @@ mod market;
mod mining;
mod outputs;
mod pools;
mod positions;
pub mod prices;
mod scripts;
mod supply;
@@ -35,7 +33,6 @@ pub struct Computer<M: StorageMode = Rw> {
pub mining: Box<mining::Vecs<M>>,
pub transactions: Box<transactions::Vecs<M>>,
pub scripts: Box<scripts::Vecs<M>>,
pub positions: Box<positions::Vecs<M>>,
pub cointime: Box<cointime::Vecs<M>>,
pub constants: Box<constants::Vecs>,
pub indexes: Box<indexes::Vecs<M>>,
@@ -63,24 +60,12 @@ impl Computer {
const STACK_SIZE: usize = 8 * 1024 * 1024;
let big_thread = || thread::Builder::new().stack_size(STACK_SIZE);
let (indexes, positions) = timed("Imported indexes/positions", || {
thread::scope(|s| -> Result<_> {
let positions_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(positions::Vecs::forced_import(
&computed_path,
VERSION,
)?))
})?;
let indexes = Box::new(indexes::Vecs::forced_import(
&computed_path,
VERSION,
indexer,
)?);
let positions = positions_handle.join().unwrap()?;
Ok((indexes, positions))
})
let indexes = timed("Imported indexes", || -> Result<_> {
Ok(Box::new(indexes::Vecs::forced_import(
&computed_path,
VERSION,
indexer,
)?))
})?;
let (constants, prices) = timed("Imported prices/constants", || -> Result<_> {
@@ -257,7 +242,6 @@ impl Computer {
market,
distribution,
supply,
positions,
pools,
cointime,
indexes,
@@ -278,7 +262,6 @@ impl Computer {
mining::DB_NAME,
transactions::DB_NAME,
scripts::DB_NAME,
positions::DB_NAME,
cointime::DB_NAME,
indicators::DB_NAME,
indexes::DB_NAME,
@@ -319,7 +302,6 @@ impl Computer {
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
reader: &Reader,
exit: &Exit,
) -> Result<()> {
internal::cache_clear_all();
@@ -387,13 +369,6 @@ impl Computer {
)
})?;
let positions = scope.spawn(|| {
timed("Computed positions", || {
self.positions
.compute(indexer, &starting_indexes, reader, exit)
})
});
timed("Computed transactions", || {
self.transactions.compute(
indexer,
@@ -419,7 +394,6 @@ impl Computer {
)
})?;
positions.join().unwrap()?;
market.join().unwrap()?;
Ok(())
})?;
@@ -561,7 +535,6 @@ impl_iter_named!(
mining,
transactions,
scripts,
positions,
cointime,
constants,
indicators,

2
crates/brk_computer/src/mining/compute.rs
View File

@@ -25,7 +25,7 @@ impl Vecs {
indexer,
indexes,
&blocks.lookback,
&transactions.fees,
transactions,
prices,
starting_indexes,
exit,

11
crates/brk_computer/src/mining/rewards/compute.rs
View File

@@ -17,7 +17,7 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
lookback: &blocks::LookbackVecs,
transactions_fees: &transactions::FeesVecs,
transactions: &transactions::Vecs,
prices: &prices::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
@@ -67,7 +67,7 @@ impl Vecs {
starting_indexes.height,
&indexer.vecs.transactions.first_tx_index,
&indexes.height.tx_index_count,
&transactions_fees.fee.tx_index,
&transactions.fees.fee.tx_index,
exit,
)?;
Ok(())
@@ -95,6 +95,13 @@ impl Vecs {
self.subsidy
.compute_rest(starting_indexes.height, prices, exit)?;
self.output_volume.compute_subtract(
starting_indexes.height,
&transactions.volume.transfer_volume.block.sats,
&self.fees.block.sats,
exit,
)?;
self.unclaimed.block.sats.compute_transform(
starting_indexes.height,
&self.subsidy.block.sats,

3
crates/brk_computer/src/mining/rewards/import.rs
View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::Database;
use vecdb::{Database, EagerVec, ImportableVec};
use super::Vecs;
use crate::{
@@ -44,6 +44,7 @@ impl Vecs {
cached_starts,
)?,
fees: AmountPerBlockFull::forced_import(db, "fees", version, indexes, cached_starts)?,
output_volume: EagerVec::forced_import(db, "output_volume", version)?,
unclaimed: AmountPerBlockCumulative::forced_import(
db,
"unclaimed_rewards",

5
crates/brk_computer/src/mining/rewards/vecs.rs
View File

@@ -1,6 +1,6 @@
use brk_traversable::Traversable;
use brk_types::{BasisPoints16, BasisPoints32};
use vecdb::{Rw, StorageMode};
use brk_types::{BasisPoints16, BasisPoints32, Height, Sats};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
use crate::internal::{
AmountPerBlockCumulative, AmountPerBlockCumulativeRolling, AmountPerBlockFull,
@@ -12,6 +12,7 @@ pub struct Vecs<M: StorageMode = Rw> {
pub coinbase: AmountPerBlockCumulativeRolling<M>,
pub subsidy: AmountPerBlockCumulativeRolling<M>,
pub fees: AmountPerBlockFull<M>,
pub output_volume: M::Stored<EagerVec<PcoVec<Height, Sats>>>,
pub unclaimed: AmountPerBlockCumulative<M>,
#[traversable(wrap = "fees", rename = "dominance")]
pub fee_dominance: PercentPerBlock<BasisPoints16, M>,

30
crates/brk_computer/src/pools/mod.rs
View File

@@ -2,7 +2,6 @@ use std::{collections::BTreeMap, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_store::AnyStore;
use brk_traversable::Traversable;
use brk_types::{Addr, AddrBytes, Height, Indexes, OutputType, PoolSlug, Pools, TxOutIndex, pools};
use rayon::prelude::*;
@@ -114,8 +113,18 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let dep_version = indexer.vecs.blocks.coinbase_tag.version();
let pool_vec_version = self.pool.header().vec_version();
let pool_computed = self.pool.header().computed_version();
let expected = pool_vec_version + dep_version;
if expected != pool_computed {
tracing::warn!(
"Pool version mismatch: vec_version={pool_vec_version:?} + dep={dep_version:?} = {expected:?}, stored computed={pool_computed:?}, len={}",
self.pool.len()
);
}
self.pool
.validate_computed_version_or_reset(indexer.stores.height_to_coinbase_tag.version())?;
.validate_computed_version_or_reset(dep_version)?;
let first_txout_index = indexer.vecs.transactions.first_txout_index.reader();
let output_type = indexer.vecs.outputs.output_type.reader();
@@ -142,12 +151,12 @@ impl Vecs {
self.pool.truncate_if_needed_at(min)?;
indexer
.stores
.height_to_coinbase_tag
.iter()
.skip(min)
.try_for_each(|(_, coinbase_tag)| -> Result<()> {
let len = indexer.vecs.blocks.coinbase_tag.len();
indexer.vecs.blocks.coinbase_tag.try_for_each_range_at(
min,
len,
|coinbase_tag| -> Result<()> {
let tx_index = first_tx_index_cursor.next().unwrap();
let out_start = first_txout_index.get(tx_index.to_usize());
@@ -174,12 +183,13 @@ impl Vecs {
.map(|bytes| Addr::try_from(&bytes).unwrap())
.and_then(|addr| self.pools.find_from_addr(&addr))
})
.or_else(|| self.pools.find_from_coinbase_tag(&coinbase_tag))
.or_else(|| self.pools.find_from_coinbase_tag(&coinbase_tag.as_str()))
.unwrap_or(unknown);
self.pool.push(pool.slug);
Ok(())
})?;
},
)?;
let _lock = exit.lock();
self.pool.write()?;

147
crates/brk_computer/src/positions.rs
View File

@@ -1,147 +0,0 @@
use std::{fs, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_reader::{Reader, XOR_LEN, XORBytes};
use brk_traversable::Traversable;
use brk_types::{BlkPosition, Height, Indexes, TxIndex, Version};
use tracing::info;
use vecdb::{
AnyStoredVec, AnyVec, Database, Exit, ImportableVec, PcoVec, ReadableVec, Rw, StorageMode,
WritableVec,
};
use crate::internal::db_utils::{finalize_db, open_db};
pub const DB_NAME: &str = "positions";
#[derive(Traversable)]
#[traversable(hidden)]
pub struct Vecs<M: StorageMode = Rw> {
db: Database,
pub block: M::Stored<PcoVec<Height, BlkPosition>>,
pub tx: M::Stored<PcoVec<TxIndex, BlkPosition>>,
}
impl Vecs {
pub(crate) fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = open_db(parent_path, DB_NAME, 1_000_000)?;
let version = parent_version;
let this = Self {
block: PcoVec::forced_import(&db, "position", version + Version::TWO)?,
tx: PcoVec::forced_import(&db, "position", version + Version::TWO)?,
db,
};
finalize_db(&this.db, &this)?;
Ok(this)
}
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
reader: &Reader,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
self.compute_(indexer, starting_indexes, reader, exit)?;
let exit = exit.clone();
self.db.run_bg(move |db| {
let _lock = exit.lock();
db.compact_deferred_default()
});
Ok(())
}
fn check_xor_bytes(&mut self, reader: &Reader) -> Result<()> {
let xor_path = self.db.path().join("xor.dat");
let current = reader.xor_bytes();
let cached = fs::read(&xor_path)
.ok()
.and_then(|b| <[u8; XOR_LEN]>::try_from(b).ok())
.map(XORBytes::from);
match cached {
Some(c) if c == current => return Ok(()),
Some(_) => {
info!("XOR bytes changed, resetting positions...");
self.block.reset()?;
self.tx.reset()?;
}
None => {}
}
fs::write(&xor_path, *current)?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
parser: &Reader,
exit: &Exit,
) -> Result<()> {
self.check_xor_bytes(parser)?;
// Validate computed versions against dependencies
let dep_version = indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.height.version();
self.block.validate_computed_version_or_reset(dep_version)?;
self.tx.validate_computed_version_or_reset(dep_version)?;
let min_tx_index = TxIndex::from(self.tx.len()).min(starting_indexes.tx_index);
let Some(min_height) = indexer
.vecs
.transactions
.height
.collect_one(min_tx_index)
.map(|h: Height| h.min(starting_indexes.height))
else {
return Ok(());
};
let first_tx_at_min_height = indexer
.vecs
.transactions
.first_tx_index
.collect_one(min_height)
.unwrap();
self.block.truncate_if_needed(min_height)?;
self.tx.truncate_if_needed(first_tx_at_min_height)?;
parser
.read(
Some(min_height),
Some((indexer.vecs.transactions.first_tx_index.len() - 1).into()),
)
.iter()
.try_for_each(|block| -> Result<()> {
self.block.push(block.metadata().position());
block.tx_metadata().iter().for_each(|metadata| {
self.tx.push(metadata.position());
});
if *block.height() % 1_000 == 0 {
let _lock = exit.lock();
self.block.write()?;
self.tx.write()?;
}
Ok(())
})?;
let _lock = exit.lock();
self.block.write()?;
self.tx.write()?;
Ok(())
}
}

209
crates/brk_computer/src/transactions/fees/compute.rs
View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{FeeRate, Indexes, Sats};
use brk_types::{FeeRate, Indexes, OutPoint, Sats, TxInIndex, VSize};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely};
use super::super::size;
@@ -33,26 +33,47 @@ impl Vecs {
exit,
)?;
self.compute_fee_and_fee_rate(size_vecs, starting_indexes, exit)?;
self.compute_fees(indexer, indexes, size_vecs, starting_indexes, exit)?;
let (r3, r4) = rayon::join(
let (r1, (r2, r3)) = rayon::join(
|| {
self.fee
.derive_from_with_skip(indexer, indexes, starting_indexes, exit, 1)
},
|| {
self.fee_rate
.derive_from_with_skip(indexer, indexes, starting_indexes, exit, 1)
rayon::join(
|| {
self.fee_rate.derive_from_with_skip(
indexer,
indexes,
starting_indexes,
exit,
1,
)
},
|| {
self.effective_fee_rate.derive_from_with_skip(
indexer,
indexes,
starting_indexes,
exit,
1,
)
},
)
},
);
r1?;
r2?;
r3?;
r4?;
Ok(())
}
fn compute_fee_and_fee_rate(
fn compute_fees(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
size_vecs: &size::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
@@ -67,6 +88,9 @@ impl Vecs {
self.fee_rate
.tx_index
.validate_computed_version_or_reset(dep_version)?;
self.effective_fee_rate
.tx_index
.validate_computed_version_or_reset(dep_version)?;
let target = self
.input_value
@@ -78,6 +102,7 @@ impl Vecs {
.tx_index
.len()
.min(self.fee_rate.tx_index.len())
.min(self.effective_fee_rate.tx_index.len())
.min(starting_indexes.tx_index.to_usize());
if min >= target {
@@ -90,39 +115,171 @@ impl Vecs {
self.fee_rate
.tx_index
.truncate_if_needed(starting_indexes.tx_index)?;
self.effective_fee_rate
.tx_index
.truncate_if_needed(starting_indexes.tx_index)?;
loop {
let skip = self.fee.tx_index.len();
let end = self.fee.tx_index.batch_end(target);
if skip >= end {
let start_tx = self.fee.tx_index.len();
let max_height = indexer.vecs.transactions.first_tx_index.len();
let start_height = if start_tx == 0 {
0
} else {
indexer
.vecs
.transactions
.height
.collect_one_at(start_tx)
.unwrap()
.to_usize()
};
for h in start_height..max_height {
let first_tx: usize = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(h)
.unwrap()
.to_usize();
let n = *indexes.height.tx_index_count.collect_one_at(h).unwrap() as usize;
if first_tx + n > target {
break;
}
let input_batch = self.input_value.collect_range_at(skip, end);
let output_batch = self.output_value.collect_range_at(skip, end);
let vsize_batch = size_vecs.vsize.tx_index.collect_range_at(skip, end);
// Batch read all per-tx data for this block
let input_values = self.input_value.collect_range_at(first_tx, first_tx + n);
let output_values = self.output_value.collect_range_at(first_tx, first_tx + n);
let vsizes: Vec<VSize> = size_vecs
.vsize
.tx_index
.collect_range_at(first_tx, first_tx + n);
let txin_starts: Vec<TxInIndex> = indexer
.vecs
.transactions
.first_txin_index
.collect_range_at(first_tx, first_tx + n);
let input_begin = txin_starts[0].to_usize();
let input_end = if h + 1 < max_height {
indexer
.vecs
.inputs
.first_txin_index
.collect_one_at(h + 1)
.unwrap()
.to_usize()
} else {
indexer.vecs.inputs.outpoint.len()
};
let outpoints: Vec<OutPoint> = indexer
.vecs
.inputs
.outpoint
.collect_range_at(input_begin, input_end);
for j in 0..input_batch.len() {
let fee = if unlikely(input_batch[j].is_max()) {
// Compute fee + fee_rate per tx
let mut fees = Vec::with_capacity(n);
for j in 0..n {
let fee = if unlikely(input_values[j].is_max()) {
Sats::ZERO
} else {
input_batch[j] - output_batch[j]
input_values[j] - output_values[j]
};
self.fee.tx_index.push(fee);
self.fee_rate
.tx_index
.push(FeeRate::from((fee, vsize_batch[j])));
self.fee_rate.tx_index.push(FeeRate::from((fee, vsizes[j])));
fees.push(fee);
}
let _lock = exit.lock();
let (r1, r2) = rayon::join(
|| self.fee.tx_index.write(),
|| self.fee_rate.tx_index.write(),
// Effective fee rate via same-block CPFP clustering
let effective = cluster_fee_rates(
&txin_starts,
&outpoints,
input_begin,
first_tx,
&fees,
&vsizes,
);
r1?;
r2?;
for rate in effective {
self.effective_fee_rate.tx_index.push(rate);
}
if h % 1_000 == 0 {
let _lock = exit.lock();
self.fee.tx_index.write()?;
self.fee_rate.tx_index.write()?;
self.effective_fee_rate.tx_index.write()?;
}
}
let _lock = exit.lock();
self.fee.tx_index.write()?;
self.fee_rate.tx_index.write()?;
self.effective_fee_rate.tx_index.write()?;
Ok(())
}
}
/// Clusters same-block parent-child txs and computes effective fee rate per cluster.
fn cluster_fee_rates(
txin_starts: &[TxInIndex],
outpoints: &[OutPoint],
outpoint_base: usize,
first_tx: usize,
fees: &[Sats],
vsizes: &[VSize],
) -> Vec<FeeRate> {
let n = fees.len();
let mut parent: Vec<usize> = (0..n).collect();
for j in 1..n {
let start = txin_starts[j].to_usize() - outpoint_base;
let end = if j + 1 < txin_starts.len() {
txin_starts[j + 1].to_usize() - outpoint_base
} else {
outpoints.len()
};
for op in &outpoints[start..end] {
if op.is_coinbase() {
continue;
}
let parent_tx = op.tx_index().to_usize();
if parent_tx >= first_tx && parent_tx < first_tx + n {
union(&mut parent, j, parent_tx - first_tx);
}
}
}
let mut cluster_fee = vec![Sats::ZERO; n];
let mut cluster_vsize = vec![VSize::from(0u64); n];
for j in 0..n {
let root = find(&mut parent, j);
cluster_fee[root] += fees[j];
cluster_vsize[root] += vsizes[j];
}
(0..n)
.map(|j| {
let root = find(&mut parent, j);
FeeRate::from((cluster_fee[root], cluster_vsize[root]))
})
.collect()
}
fn find(parent: &mut [usize], mut i: usize) -> usize {
while parent[i] != i {
parent[i] = parent[parent[i]];
i = parent[i];
}
i
}
fn union(parent: &mut [usize], a: usize, b: usize) {
let ra = find(parent, a);
let rb = find(parent, b);
if ra != rb {
parent[ra] = rb;
}
}

6
crates/brk_computer/src/transactions/fees/import.rs
View File

@@ -20,6 +20,12 @@ impl Vecs {
output_value: EagerVec::forced_import(db, "output_value", version)?,
fee: PerTxDistribution::forced_import(db, "fee", v, indexes)?,
fee_rate: PerTxDistribution::forced_import(db, "fee_rate", v, indexes)?,
effective_fee_rate: PerTxDistribution::forced_import(
db,
"effective_fee_rate",
v,
indexes,
)?,
})
}
}

1
crates/brk_computer/src/transactions/fees/vecs.rs
View File

@@ -10,4 +10,5 @@ pub struct Vecs<M: StorageMode = Rw> {
pub output_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub fee: PerTxDistribution<Sats, M>,
pub fee_rate: PerTxDistribution<FeeRate, M>,
pub effective_fee_rate: PerTxDistribution<FeeRate, M>,
}