Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-01 09:59:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: big snapshot part 2

Browse Source
This commit is contained in:
nym21
2026-04-13 22:47:08 +02:00
parent 765261648d
commit 283baca848
93 changed files with 3242 additions and 3067 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
crates/brk_computer/src/inputs/by_type/compute.rs
View File

@@ -3,15 +3,11 @@ use brk_indexer::Indexer;
use brk_types::{Indexes, StoredU64};
use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::Vecs;
use crate::internal::{
PerBlockFull, compute_by_addr_type_block_counts, compute_by_addr_type_tx_percents,
};
use super::{Vecs, WithInputTypes};
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
impl Vecs {
/// Phase 1: walk inputs and populate `input_count` + `tx_count`.
/// Independent of transactions, can run alongside other inputs work.
pub(crate) fn compute_counts(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
@@ -22,83 +18,93 @@ impl Vecs {
+ indexer.vecs.transactions.first_txin_index.version()
+ indexer.vecs.transactions.txid.version();
for (_, v) in self.input_count.iter_mut() {
v.block
.validate_and_truncate(dep_version, starting_indexes.height)?;
}
for (_, v) in self.tx_count.iter_mut() {
v.block
.validate_and_truncate(dep_version, starting_indexes.height)?;
}
self.input_count
.validate_and_truncate(dep_version, starting_indexes.height)?;
self.tx_count
.validate_and_truncate(dep_version, starting_indexes.height)?;
let skip = self
.input_count
.values()
.map(|v| v.block.len())
.min()
.unwrap()
.min(self.tx_count.values().map(|v| v.block.len()).min().unwrap());
.min_stateful_len()
.min(self.tx_count.min_stateful_len());
let first_tx_index = &indexer.vecs.transactions.first_tx_index;
let end = first_tx_index.len();
if skip >= end {
return Ok(());
if skip < end {
self.input_count.truncate_if_needed_at(skip)?;
self.tx_count.truncate_if_needed_at(skip)?;
let fi_batch = first_tx_index.collect_range_at(skip, end);
let txid_len = indexer.vecs.transactions.txid.len();
let total_txin_len = indexer.vecs.inputs.output_type.len();
let mut itype_cursor = indexer.vecs.inputs.output_type.cursor();
let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor();
walk_blocks(
&fi_batch,
txid_len,
CoinbasePolicy::Skip,
|tx_pos, per_tx| {
let fi_in = fi_in_cursor.get(tx_pos).data()?.to_usize();
let next_fi_in = if tx_pos + 1 < txid_len {
fi_in_cursor.get(tx_pos + 1).data()?.to_usize()
} else {
total_txin_len
};
itype_cursor.advance(fi_in - itype_cursor.position());
for _ in fi_in..next_fi_in {
let otype = itype_cursor.next().unwrap();
per_tx[otype as usize] += 1;
}
Ok(())
},
|agg| {
push_block(&mut self.input_count, agg.entries_all, &agg.entries_per_type);
push_block(&mut self.tx_count, agg.txs_all, &agg.txs_per_type);
if self.input_count.all.block.batch_limit_reached() {
let _lock = exit.lock();
self.input_count.write()?;
self.tx_count.write()?;
}
Ok(())
},
)?;
{
let _lock = exit.lock();
self.input_count.write()?;
self.tx_count.write()?;
}
self.input_count
.compute_rest(starting_indexes.height, exit)?;
self.tx_count
.compute_rest(starting_indexes.height, exit)?;
}
for (_, v) in self.input_count.iter_mut() {
v.block.truncate_if_needed_at(skip)?;
for (otype, source) in self.tx_count.by_type.iter_typed() {
self.tx_percent.get_mut(otype).compute_count_ratio(
source,
&self.tx_count.all,
starting_indexes.height,
exit,
)?;
}
for (_, v) in self.tx_count.iter_mut() {
v.block.truncate_if_needed_at(skip)?;
}
let fi_batch = first_tx_index.collect_range_at(skip, end);
let txid_len = indexer.vecs.transactions.txid.len();
let total_txin_len = indexer.vecs.inputs.output_type.len();
let mut itype_cursor = indexer.vecs.inputs.output_type.cursor();
let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor();
compute_by_addr_type_block_counts(
&mut self.input_count,
&mut self.tx_count,
&fi_batch,
txid_len,
true, // skip coinbase (1 fake input)
starting_indexes.height,
exit,
|tx_pos, per_tx| {
let fi_in = fi_in_cursor.get(tx_pos).data()?.to_usize();
let next_fi_in = if tx_pos + 1 < txid_len {
fi_in_cursor.get(tx_pos + 1).data()?.to_usize()
} else {
total_txin_len
};
itype_cursor.advance(fi_in - itype_cursor.position());
for _ in fi_in..next_fi_in {
let otype = itype_cursor.next().unwrap();
per_tx[otype as usize] += 1;
}
Ok(())
},
)
}
/// Phase 2: derive `tx_percent` from `tx_count` and the total tx count.
/// Must run after `transactions::Vecs::compute`.
pub(crate) fn compute_percents(
&mut self,
transactions_count_total: &PerBlockFull<StoredU64>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
compute_by_addr_type_tx_percents(
&self.tx_count,
&mut self.tx_percent,
transactions_count_total,
starting_indexes,
exit,
)
Ok(())
}
}
#[inline]
fn push_block(
metric: &mut WithInputTypes<PerBlockCumulativeRolling<StoredU64, StoredU64>>,
total: u64,
per_type: &[u64; 12],
) {
metric.all.block.push(StoredU64::from(total));
for (otype, vec) in metric.by_type.iter_typed_mut() {
vec.block.push(StoredU64::from(per_type[otype as usize]));
}
}

68
crates/brk_computer/src/inputs/by_type/import.rs
View File

@@ -1,12 +1,14 @@
use brk_cohort::ByAddrType;
use brk_cohort::SpendableType;
use brk_error::Result;
use brk_types::Version;
use brk_types::{StoredU64, Version};
use vecdb::Database;
use super::Vecs;
use super::{Vecs, WithInputTypes};
use crate::{
indexes,
internal::{PerBlockCumulativeRolling, PercentCumulativeRolling, WindowStartVec, Windows},
internal::{
PerBlockCumulativeRolling, PercentCumulativeRolling, WindowStartVec, Windows,
},
};
impl Vecs {
@@ -16,33 +18,39 @@ impl Vecs {
indexes: &indexes::Vecs,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let input_count = WithInputTypes::<
PerBlockCumulativeRolling<StoredU64, StoredU64>,
>::forced_import_with(
db,
"input_count_bis",
|t| format!("{t}_prevout_count"),
version,
indexes,
cached_starts,
)?;
let tx_count = WithInputTypes::<
PerBlockCumulativeRolling<StoredU64, StoredU64>,
>::forced_import_with(
db,
"non_coinbase_tx_count",
|t| format!("tx_count_with_{t}_prevout"),
version,
indexes,
cached_starts,
)?;
let tx_percent = SpendableType::try_new(|_, name| {
PercentCumulativeRolling::forced_import(
db,
&format!("tx_percent_with_{name}_prevout"),
version,
indexes,
)
})?;
Ok(Self {
input_count: ByAddrType::new_with_name(|name| {
PerBlockCumulativeRolling::forced_import(
db,
&format!("{name}_input_count"),
version,
indexes,
cached_starts,
)
})?,
tx_count: ByAddrType::new_with_name(|name| {
PerBlockCumulativeRolling::forced_import(
db,
&format!("tx_count_with_{name}_in"),
version,
indexes,
cached_starts,
)
})?,
tx_percent: ByAddrType::new_with_name(|name| {
PercentCumulativeRolling::forced_import(
db,
&format!("tx_count_with_{name}_in_rel_to_all"),
version,
indexes,
)
})?,
input_count,
tx_count,
tx_percent,
})
}
}

2
crates/brk_computer/src/inputs/by_type/mod.rs
View File

@@ -1,5 +1,7 @@
mod compute;
mod import;
mod vecs;
mod with_input_types;
pub use vecs::Vecs;
pub(crate) use with_input_types::WithInputTypes;

14
crates/brk_computer/src/inputs/by_type/vecs.rs
View File

@@ -1,18 +1,14 @@
use brk_cohort::ByAddrType;
use brk_cohort::SpendableType;
use brk_traversable::Traversable;
use brk_types::{BasisPoints16, StoredU64};
use vecdb::{Rw, StorageMode};
use super::WithInputTypes;
use crate::internal::{PerBlockCumulativeRolling, PercentCumulativeRolling};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
/// Per-block, per-type total input count (granular). The "type" is the
/// type of the spent output that the input consumes.
pub input_count: ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
/// Per-block, per-type count of TXs containing at least one input that
/// spends an output of this type.
pub tx_count: ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
/// Per-type tx_count as a percent of total tx count.
pub tx_percent: ByAddrType<PercentCumulativeRolling<BasisPoints16, M>>,
pub input_count: WithInputTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
pub tx_count: WithInputTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
pub tx_percent: SpendableType<PercentCumulativeRolling<BasisPoints16, M>>,
}

93
crates/brk_computer/src/inputs/by_type/with_input_types.rs Normal file
View File

@@ -0,0 +1,93 @@
//! Generic `all` + per-input-type container (11 spendable types — no
//! op_return since op_return outputs are non-spendable). Used by
//! `inputs/by_type/`. Mirrors `WithAddrTypes` and `WithOutputTypes`.
use brk_cohort::SpendableType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, Version};
use schemars::JsonSchema;
use vecdb::{AnyStoredVec, AnyVec, Database, Exit, WritableVec};
use crate::{
indexes,
internal::{NumericValue, PerBlockCumulativeRolling, WindowStartVec, Windows},
};
/// `all` aggregate plus per-input-type breakdown across the 11 spendable
/// output types (everything except op_return). The "type" of an input is
/// the type of the previous output it spends.
#[derive(Clone, Traversable)]
pub struct WithInputTypes<T> {
pub all: T,
#[traversable(flatten)]
pub by_type: SpendableType<T>,
}
impl<T, C> WithInputTypes<PerBlockCumulativeRolling<T, C>>
where
T: NumericValue + JsonSchema + Into<C>,
C: NumericValue + JsonSchema,
{
pub(crate) fn forced_import_with(
db: &Database,
all_name: &str,
per_type_name: impl Fn(&str) -> String,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let make = |name: &str| {
PerBlockCumulativeRolling::forced_import(db, name, version, indexes, cached_starts)
};
Ok(Self {
all: make(all_name)?,
by_type: SpendableType::try_new(|_, name| make(&per_type_name(name)))?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.by_type
.iter()
.map(|v| v.block.len())
.min()
.unwrap()
.min(self.all.block.len())
}
pub(crate) fn write(&mut self) -> Result<()> {
self.all.block.write()?;
for v in self.by_type.iter_mut() {
v.block.write()?;
}
Ok(())
}
pub(crate) fn validate_and_truncate(
&mut self,
dep_version: Version,
at_height: Height,
) -> Result<()> {
self.all.block.validate_and_truncate(dep_version, at_height)?;
for v in self.by_type.iter_mut() {
v.block.validate_and_truncate(dep_version, at_height)?;
}
Ok(())
}
pub(crate) fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.all.block.truncate_if_needed_at(len)?;
for v in self.by_type.iter_mut() {
v.block.truncate_if_needed_at(len)?;
}
Ok(())
}
pub(crate) fn compute_rest(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.all.compute_rest(max_from, exit)?;
for v in self.by_type.iter_mut() {
v.compute_rest(max_from, exit)?;
}
Ok(())
}
}

3
crates/brk_computer/src/inputs/compute.rs
View File

@@ -20,6 +20,9 @@ impl Vecs {
self.spent.compute(indexer, starting_indexes, exit)?;
self.count
.compute(indexer, indexes, blocks, starting_indexes, exit)?;
self.per_sec
.compute(&self.count, starting_indexes, exit)?;
self.by_type.compute(indexer, starting_indexes, exit)?;
let exit = exit.clone();
self.db.run_bg(move |db| {

12
crates/brk_computer/src/inputs/import.rs
View File

@@ -11,7 +11,7 @@ use crate::{
},
};
use super::{CountVecs, SpentVecs, Vecs};
use super::{ByTypeVecs, CountVecs, PerSecVecs, SpentVecs, Vecs};
impl Vecs {
pub(crate) fn forced_import(
@@ -25,8 +25,16 @@ impl Vecs {
let spent = SpentVecs::forced_import(&db, version)?;
let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?;
let per_sec = PerSecVecs::forced_import(&db, version, indexes)?;
let by_type = ByTypeVecs::forced_import(&db, version, indexes, cached_starts)?;
let this = Self { db, spent, count };
let this = Self {
db,
spent,
count,
per_sec,
by_type,
};
finalize_db(&this.db, &this)?;
Ok(this)
}

6
crates/brk_computer/src/inputs/mod.rs
View File

@@ -1,4 +1,6 @@
pub mod by_type;
pub mod count;
pub mod per_sec;
pub mod spent;
mod compute;
@@ -7,7 +9,9 @@ mod import;
use brk_traversable::Traversable;
use vecdb::{Database, Rw, StorageMode};
pub use by_type::Vecs as ByTypeVecs;
pub use count::Vecs as CountVecs;
pub use per_sec::Vecs as PerSecVecs;
pub use spent::Vecs as SpentVecs;
pub const DB_NAME: &str = "inputs";
@@ -19,4 +23,6 @@ pub struct Vecs<M: StorageMode = Rw> {
pub spent: SpentVecs<M>,
pub count: CountVecs<M>,
pub per_sec: PerSecVecs<M>,
pub by_type: ByTypeVecs<M>,
}

28
crates/brk_computer/src/inputs/per_sec/compute.rs Normal file
View File

@@ -0,0 +1,28 @@
use brk_error::Result;
use brk_types::{Indexes, StoredF32};
use vecdb::Exit;
use super::Vecs;
use crate::{inputs::CountVecs, internal::Windows};
impl Vecs {
pub(crate) fn compute(
&mut self,
count: &CountVecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let h = starting_indexes.height;
let sums = count.rolling.sum.0.as_array();
let per_sec = self.0.as_mut_array();
for (i, &secs) in Windows::<()>::SECS.iter().enumerate() {
per_sec[i].height.compute_transform(
h,
&sums[i].height,
|(h, sum, ..)| (h, StoredF32::from(*sum as f64 / secs)),
exit,
)?;
}
Ok(())
}
}

21
crates/brk_computer/src/inputs/per_sec/import.rs Normal file
View File

@@ -0,0 +1,21 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::Database;
use super::Vecs;
use crate::{
indexes,
internal::{PerBlock, Windows},
};
impl Vecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(Windows::try_from_fn(|suffix| {
PerBlock::forced_import(db, &format!("inputs_per_sec_{suffix}"), version, indexes)
})?))
}
}

5
crates/brk_computer/src/inputs/per_sec/mod.rs Normal file
View File

@@ -0,0 +1,5 @@
mod compute;
mod import;
mod vecs;
pub use vecs::Vecs;

8
crates/brk_computer/src/inputs/per_sec/vecs.rs Normal file
View File

@@ -0,0 +1,8 @@
use brk_traversable::Traversable;
use brk_types::StoredF32;
use vecdb::{Rw, StorageMode};
use crate::internal::{PerBlock, Windows};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw>(#[traversable(flatten)] pub Windows<PerBlock<StoredF32, M>>);