Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-23 17:08:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: opreturn

Browse Source
This commit is contained in:
nym21
2026-07-16 17:39:05 +02:00
parent 8d43f340a0
commit 29ad37803e
100 changed files with 6084 additions and 1924 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/distribution/cohorts/utxo/groups.rs
+1 -1
View File
@@ -16,6 +16,7 @@ use vecdb::{
use crate::{
blocks,
distribution::{
DynCohortVecs,
metrics::{
AllCohortMetrics, BasicCohortMetrics, CohortMetricsBase, CoreCohortMetrics,
ExtendedAdjustedCohortMetrics, ExtendedCohortMetrics, ImportConfig,
@@ -23,7 +24,6 @@ use crate::{
TypeCohortMetrics,
},
state::UTXOCohortState,
DynCohortVecs,
},
indexes,
internal::{ValuePerBlockCumulativeRolling, WindowStartVec, Windows},
crates/brk_computer/src/indexes/addr.rs
+1 -6
View File
@@ -250,12 +250,7 @@ impl Vecs {
identity: LazyVecFrom1::init(
"op_return_index",
version,
indexer
.vecs
.scripts
.op_return
.to_tx_index
.read_only_boxed_clone(),
indexer.vecs.op_return.to_tx_index.read_only_boxed_clone(),
|index, _| index,
),
},
crates/brk_computer/src/inputs/by_type/compute.rs
+9 -5
View File
@@ -6,6 +6,8 @@ use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, WithInputTypes};
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -37,6 +39,7 @@ impl Vecs {
let mut itype_cursor = indexer.vecs.inputs.output_type.cursor();
let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor();
let mut height = skip;
walk_blocks(
&fi_batch,
@@ -65,7 +68,8 @@ impl Vecs {
);
push_block(&mut self.tx_count, agg.txs_all, &agg.txs_per_type);
if self.input_count.all.block.batch_limit_reached() {
height += 1;
if height.is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.input_count.write()?;
self.tx_count.write()?;
@@ -79,12 +83,12 @@ impl Vecs {
self.input_count.write()?;
self.tx_count.write()?;
}
self.input_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
}
self.input_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
for (otype, source) in self.input_count.by_type.iter_typed() {
self.input_share.get_mut(otype).compute_count_ratio(
source,
crates/brk_computer/src/inputs/compute.rs
+2 -3
View File
@@ -3,13 +3,12 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -18,7 +17,7 @@ impl Vecs {
let starting_lengths = indexer.safe_lengths();
self.spent.compute(indexer, exit)?;
self.count.compute(indexer, indexes, blocks, exit)?;
self.count.compute(indexer, blocks, exit)?;
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
self.by_type.compute(indexer, exit)?;
crates/brk_computer/src/inputs/count/compute.rs
+6 -8
View File
@@ -3,27 +3,25 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let window_starts = blocks.lookback.window_starts();
self.0.compute(
self.sum.compute_count_from_indexes(
starting_height,
&indexes.tx_index.input_count,
&indexer.vecs.transactions.first_tx_index,
&indexes.height.tx_index_count,
&window_starts,
&indexer.vecs.inputs.first_txin_index,
&indexer.vecs.inputs.outpoint,
exit,
0,
)?;
self.compute_rest(starting_height, &window_starts, exit)?;
Ok(())
}
crates/brk_computer/src/internal/per_block/computed/aggregated.rs
+5 -71
View File
@@ -2,10 +2,7 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Height;
use schemars::JsonSchema;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw,
StorageMode, VecIndex, VecValue, Version, WritableVec,
};
use vecdb::{Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode, Version};
use crate::{
indexes,
@@ -52,82 +49,19 @@ where
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute<A>(
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
source: &impl ReadableVec<A, T>,
first_indexes: &impl ReadableVec<Height, A>,
count_indexes: &impl ReadableVec<Height, brk_types::StoredU64>,
windows: &WindowStarts<'_>,
exit: &Exit,
skip_count: usize,
) -> Result<()>
where
T: From<f64> + Default + Copy + Ord,
f64: From<T>,
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
{
let combined_version = source.version() + first_indexes.version() + count_indexes.version();
let mut index = max_from;
index = {
self.sum
.validate_computed_version_or_reset(combined_version)?;
index.min(Height::from(self.sum.len()))
};
index = {
self.cumulative
.height
.validate_computed_version_or_reset(combined_version)?;
index.min(Height::from(self.cumulative.height.len()))
};
let start = index.to_usize();
self.sum.truncate_if_needed_at(start)?;
self.cumulative.height.truncate_if_needed_at(start)?;
let mut cumulative_val = index.decremented().map_or(T::from(0_usize), |idx| {
self.cumulative
.height
.collect_one_at(idx.to_usize())
.unwrap_or(T::from(0_usize))
});
let fi_len = first_indexes.len();
let first_indexes_batch: Vec<A> = first_indexes.collect_range_at(start, fi_len);
let count_indexes_batch: Vec<brk_types::StoredU64> =
count_indexes.collect_range_at(start, fi_len);
first_indexes_batch
.into_iter()
.zip(count_indexes_batch)
.try_for_each(|(first_index, count_index)| -> Result<()> {
let count = u64::from(count_index) as usize;
let effective_count = count.saturating_sub(skip_count);
let effective_first_index = first_index + skip_count.min(count);
let efi = effective_first_index.to_usize();
let sum_val = source.fold_range_at(
efi,
efi + effective_count,
T::from(0_usize),
|acc, val| acc + val,
);
self.sum.push(sum_val);
cumulative_val += sum_val;
self.cumulative.height.push(cumulative_val);
Ok(())
})?;
let _lock = exit.lock();
self.sum.write()?;
self.cumulative.height.write()?;
drop(_lock);
self.cumulative
.height
.compute_cumulative(max_from, &self.sum, exit)?;
self.rolling.compute(max_from, windows, &self.sum, exit)?;
Ok(())
}
crates/brk_computer/src/internal/per_tx/lazy_distribution.rs
+8 -10
View File
@@ -3,7 +3,7 @@ use brk_indexer::{Indexer, Lengths};
use brk_traversable::Traversable;
use brk_types::TxIndex;
use schemars::JsonSchema;
use vecdb::{Database, Exit, LazyVecFrom2, ReadableVec, Rw, StorageMode, Version};
use vecdb::{Database, Exit, LazyVecFrom1, ReadableVec, Rw, StorageMode, Version};
use crate::{
indexes,
@@ -11,29 +11,27 @@ use crate::{
};
#[derive(Traversable)]
pub struct LazyPerTxDistribution<T, S1, S2, M: StorageMode = Rw>
pub struct LazyPerTxDistribution<T, S, M: StorageMode = Rw>
where
T: ComputedVecValue + PartialOrd + JsonSchema,
S1: ComputedVecValue,
S2: ComputedVecValue,
S: ComputedVecValue,
{
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
pub tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
#[traversable(flatten)]
pub distribution: TxDerivedDistribution<T, M>,
}
impl<T, S1, S2> LazyPerTxDistribution<T, S1, S2>
impl<T, S> LazyPerTxDistribution<T, S>
where
T: NumericValue + JsonSchema,
S1: ComputedVecValue + JsonSchema,
S2: ComputedVecValue + JsonSchema,
S: ComputedVecValue + JsonSchema,
{
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
) -> Result<Self> {
let distribution = TxDerivedDistribution::forced_import(db, name, version, indexes)?;
Ok(Self {
@@ -52,7 +50,7 @@ where
where
T: Copy + Ord + From<f64> + Default,
f64: From<T>,
LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>: ReadableVec<TxIndex, T>,
LazyVecFrom1<TxIndex, T, TxIndex, S>: ReadableVec<TxIndex, T>,
{
self.distribution
.derive_from(indexer, indexes, starting_lengths, &self.tx_index, exit)
crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs
+7 -9
View File
@@ -1,36 +1,34 @@
use brk_traversable::Traversable;
use brk_types::{TxIndex, Version};
use schemars::JsonSchema;
use vecdb::{LazyVecFrom2, UnaryTransform};
use vecdb::{LazyVecFrom1, UnaryTransform};
use crate::internal::{ComputedVecValue, LazyTxDerivedDistribution, TxDerivedDistribution};
/// Like `LazyPerTxDistribution` but with a lazy-derived distribution
/// (transformed from another type's distribution rather than eagerly computed).
#[derive(Clone, Traversable)]
pub struct LazyPerTxDistributionTransformed<T, S1, S2, DSource>
pub struct LazyPerTxDistributionTransformed<T, S, DSource>
where
T: ComputedVecValue + JsonSchema,
S1: ComputedVecValue,
S2: ComputedVecValue,
S: ComputedVecValue,
DSource: ComputedVecValue,
{
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
pub tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
#[traversable(flatten)]
pub distribution: LazyTxDerivedDistribution<T, DSource>,
}
impl<T, S1, S2, DSource> LazyPerTxDistributionTransformed<T, S1, S2, DSource>
impl<T, S, DSource> LazyPerTxDistributionTransformed<T, S, DSource>
where
T: ComputedVecValue + JsonSchema + 'static,
S1: ComputedVecValue + JsonSchema,
S2: ComputedVecValue + JsonSchema,
S: ComputedVecValue + JsonSchema,
DSource: ComputedVecValue + JsonSchema,
{
pub(crate) fn new<F: UnaryTransform<DSource, T>>(
name: &str,
version: Version,
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
source_distribution: &TxDerivedDistribution<DSource>,
) -> Self {
let distribution =
crates/brk_computer/src/internal/transform/arithmetic.rs
+4 -4
View File
@@ -146,11 +146,11 @@ impl UnaryTransform<StoredU64, Weight> for VBytesToWeight {
}
}
pub struct VSizeToWeight;
pub struct WeightToVSize;
impl UnaryTransform<VSize, Weight> for VSizeToWeight {
impl UnaryTransform<Weight, VSize> for WeightToVSize {
#[inline(always)]
fn apply(vsize: VSize) -> Weight {
Weight::from(vsize)
fn apply(weight: Weight) -> VSize {
VSize::from(weight)
}
}
crates/brk_computer/src/internal/transform/mod.rs
+1 -1
View File
@@ -8,7 +8,7 @@ mod specialized;
pub use arithmetic::{
BlocksToDaysF32, DifficultyToHashF64, HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin,
Identity, MaskSats, OneMinusBp16, OneMinusF64, ReturnF32Tenths, ReturnI8, ReturnU16,
ThsToPhsF32, VBytesToWeight, VSizeToWeight,
ThsToPhsF32, VBytesToWeight, WeightToVSize,
};
pub use bps::{
Bp16ToFloat, Bp16ToPercent, Bp32ToFloat, Bp32ToPercent, Bps16ToFloat, Bps16ToPercent,
crates/brk_computer/src/lib.rs
+40 -16
View File
@@ -20,6 +20,7 @@ mod internal;
mod investing;
mod market;
mod mining;
mod op_return;
mod outputs;
mod pools;
pub mod price;
@@ -44,9 +45,10 @@ pub struct Computer<M: StorageMode = Rw> {
pub supply: Box<supply::Vecs<M>>,
pub inputs: Box<inputs::Vecs<M>>,
pub outputs: Box<outputs::Vecs<M>>,
pub op_return: Box<op_return::Vecs<M>>,
}
const VERSION: Version = Version::new(6);
const VERSION: Version = Version::new(7);
impl Computer {
pub fn forced_import(outputs_path: &Path, indexer: &Indexer) -> Result<Self> {
@@ -87,8 +89,9 @@ impl Computer {
let cached_starts = blocks.lookback.cached_window_starts();
let (inputs, outputs, mining, transactions, pools, cointime) =
timed("Imported inputs/outputs/mining/tx/pools/cointime", || {
let (inputs, outputs, mining, transactions, pools, cointime, op_return) = timed(
"Imported inputs/outputs/mining/tx/pools/cointime/op_return",
|| {
thread::scope(|s| -> Result<_> {
let inputs_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(inputs::Vecs::forced_import(
@@ -143,15 +146,33 @@ impl Computer {
&cached_starts,
)?);
let op_return_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(op_return::Vecs::forced_import(
&computed_path,
VERSION,
&indexes,
)?))
})?;
let inputs = inputs_handle.join().unwrap()?;
let outputs = outputs_handle.join().unwrap()?;
let mining = mining_handle.join().unwrap()?;
let transactions = transactions_handle.join().unwrap()?;
let pools = pools_handle.join().unwrap()?;
let op_return = op_return_handle.join().unwrap()?;
Ok((inputs, outputs, mining, transactions, pools, cointime))
Ok((
inputs,
outputs,
mining,
transactions,
pools,
cointime,
op_return,
))
})
})?;
},
)?;
// Market, indicators, and distribution are independent; import in parallel.
// Supply depends on distribution so it runs after.
@@ -225,6 +246,7 @@ impl Computer {
inputs,
price,
outputs,
op_return,
};
Self::retain_databases(&computed_path)?;
@@ -249,6 +271,7 @@ impl Computer {
supply::DB_NAME,
inputs::DB_NAME,
outputs::DB_NAME,
op_return::DB_NAME,
];
if !computed_path.exists() {
@@ -292,8 +315,7 @@ impl Computer {
let (inputs_result, prices_result) = rayon::join(
|| {
timed("Computed inputs", || {
self.inputs
.compute(indexer, &self.indexes, &self.blocks, exit)
self.inputs.compute(indexer, &self.blocks, exit)
})
},
|| {
@@ -337,18 +359,19 @@ impl Computer {
})
});
let op_return = scope.spawn(|| {
timed("Computed OP_RETURN", || {
self.op_return.compute(indexer, exit)
})
});
timed("Computed outputs", || {
self.outputs.compute(
indexer,
&self.indexes,
&self.inputs,
&self.blocks,
&self.price,
exit,
)
self.outputs
.compute(indexer, &self.inputs, &self.blocks, &self.price, exit)
})?;
tx_mining.join().unwrap()?;
op_return.join().unwrap()?;
market.join().unwrap()?;
Ok(())
})?;
@@ -502,7 +525,8 @@ impl_iter_named!(
distribution,
supply,
inputs,
outputs
outputs,
op_return
);
fn timed<T>(label: &str, f: impl FnOnce() -> T) -> T {
crates/brk_computer/src/op_return/by_kind.rs
+89
View File
@@ -0,0 +1,89 @@
use brk_traversable::Traversable;
use brk_types::OpReturnKind;
macro_rules! define_by_kind {
($($field:ident => $kind:ident),+ $(,)?) => {
#[derive(Traversable)]
pub struct ByKind<T> {
$(pub $field: T),+
}
impl<T> ByKind<T> {
pub fn try_new<E>(
mut create: impl FnMut(OpReturnKind, &'static str) -> Result<T, E>,
) -> Result<Self, E> {
Ok(Self {
$($field: create(OpReturnKind::$kind, stringify!($field))?),+
})
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
[$( &self.$field ),+].into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
let Self { $($field),+ } = self;
[$($field),+].into_iter()
}
pub fn iter_typed(&self) -> impl Iterator<Item = (OpReturnKind, &T)> {
[$( (OpReturnKind::$kind, &self.$field) ),+].into_iter()
}
pub fn iter_typed_mut(
&mut self,
) -> impl Iterator<Item = (OpReturnKind, &mut T)> {
let Self { $($field),+ } = self;
[$( (OpReturnKind::$kind, $field) ),+].into_iter()
}
}
};
}
define_by_kind! {
runes => Runes,
veri_block => VeriBlock,
omni => Omni,
stacks => Stacks,
blockstack => Blockstack,
colu => Colu,
open_assets => OpenAssets,
komodo => Komodo,
coin_spark => CoinSpark,
poet => Poet,
docproof => Docproof,
open_timestamps => OpenTimestamps,
factom => Factom,
eternity_wall => EternityWall,
memo => Memo,
bitproof => Bitproof,
ascribe => Ascribe,
stampery => Stampery,
epobc => Epobc,
bare_hash => BareHash,
text => Text,
empty => Empty,
unknown => Unknown,
}
#[cfg(test)]
mod tests {
use std::convert::Infallible;
use brk_types::OpReturnKind;
use super::ByKind;
#[test]
fn covers_every_kind_in_discriminant_order() {
let by_kind =
ByKind::try_new(|kind, _| Ok::<_, Infallible>(kind)).expect("infallible constructor");
let kinds: Vec<_> = by_kind.iter_typed().collect();
assert_eq!(kinds.len(), OpReturnKind::Unknown as usize + 1);
for (index, (kind, value)) in kinds.into_iter().enumerate() {
assert_eq!(kind as usize, index);
assert_eq!(kind, *value);
}
}
}
crates/brk_computer/src/op_return/compute.rs
+242
View File
@@ -0,0 +1,242 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{OpReturnKind, VSize};
use vecdb::{AnyVec, Exit, ReadableVec, VecIndex};
use super::{Vecs, vecs::Totals};
const KIND_COUNT: usize = OpReturnKind::Unknown as usize + 1;
const OLD_STANDARD_MAX_POST_OP_RETURN_BYTES: u64 = 82;
const WRITE_INTERVAL: usize = 10_000;
#[derive(Clone, Copy, Default)]
struct PolicyTotals {
oversized: Totals,
multiple: Totals,
pre_v30_nonstandard: Totals,
}
#[derive(Clone, Copy, Default)]
struct Carrier {
kinds: u32,
output_count: u64,
post_op_return_bytes: u64,
oversized_output_count: u64,
oversized_post_op_return_bytes: u64,
vsize: VSize,
}
impl Carrier {
fn add_output(&mut self, kind: OpReturnKind, post_op_return_bytes: u64) {
self.kinds |= kind_bit(kind);
self.output_count += 1;
self.post_op_return_bytes += post_op_return_bytes;
if post_op_return_bytes > OLD_STANDARD_MAX_POST_OP_RETURN_BYTES {
self.oversized_output_count += 1;
self.oversized_post_op_return_bytes += post_op_return_bytes;
}
}
}
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
self.db.sync_bg_tasks()?;
let starting_lengths = indexer.safe_lengths();
let raw = &indexer.vecs.op_return;
let txs = &indexer.vecs.transactions;
let version = raw.first_index.version()
+ raw.to_tx_index.version()
+ raw.kind.version()
+ raw.post_op_return_bytes.version()
+ txs.weight.version();
self.validate_and_truncate(version, starting_lengths.height)?;
let skip = self.min_len();
let end = raw
.first_index
.len()
.min(starting_lengths.height.to_usize());
if skip < end {
self.truncate_if_needed_at(skip)?;
let op_return_len = raw
.to_tx_index
.len()
.min(raw.kind.len())
.min(raw.post_op_return_bytes.len())
.min(starting_lengths.op_return_index.to_usize());
let mut tx_cursor = raw.to_tx_index.cursor();
let mut kind_cursor = raw.kind.cursor();
let mut post_op_return_bytes = raw.post_op_return_bytes.cursor();
let mut first_index_cursor = raw.first_index.cursor();
let mut weight_cursor = txs.weight.cursor();
first_index_cursor.advance(skip);
let mut start = first_index_cursor.next().unwrap().to_usize();
for height in skip..end {
let block_end = if height + 1 < end {
first_index_cursor.next().unwrap().to_usize()
} else {
op_return_len
};
tx_cursor.advance(start - tx_cursor.position());
kind_cursor.advance(start - kind_cursor.position());
post_op_return_bytes.advance(start - post_op_return_bytes.position());
let mut total = Totals::default();
let mut by_kind = [Totals::default(); KIND_COUNT];
let mut policy = PolicyTotals::default();
let mut current_tx = None;
let mut carrier = Carrier::default();
for _ in start..block_end {
let tx_index = tx_cursor.next().unwrap();
let kind = kind_cursor.next().unwrap();
let bytes = u32::from(post_op_return_bytes.next().unwrap()) as u64;
let kind_index = kind as usize;
if current_tx != Some(tx_index) {
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
current_tx = Some(tx_index);
carrier = Carrier::default();
let tx_position = tx_index.to_usize();
weight_cursor.advance(tx_position - weight_cursor.position());
carrier.vsize = VSize::from(weight_cursor.next().unwrap());
}
total.output_count += 1;
total.post_op_return_bytes += bytes;
by_kind[kind_index].output_count += 1;
by_kind[kind_index].post_op_return_bytes += bytes;
carrier.add_output(kind, bytes);
}
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
self.total.push(total);
for (kind, metrics) in self.by_kind.iter_typed_mut() {
metrics.push(by_kind[kind as usize]);
}
self.policy.oversized.push(policy.oversized);
self.policy.multiple.push(policy.multiple);
self.policy
.pre_v30_nonstandard
.push(policy.pre_v30_nonstandard);
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.write()?;
}
start = block_end;
}
let _lock = exit.lock();
self.write()?;
}
self.compute_cumulative(starting_lengths.height, exit)?;
let exit = exit.clone();
self.db.run_bg(move |db| {
let _lock = exit.lock();
db.compact_deferred_default()
});
Ok(())
}
}
fn finalize_transaction(
total: &mut Totals,
by_kind: &mut [Totals; KIND_COUNT],
policy: &mut PolicyTotals,
carrier: Carrier,
) {
if carrier.output_count == 0 {
return;
}
add_carrier(total, carrier.vsize);
let mut kinds = carrier.kinds;
while kinds != 0 {
let kind_index = kinds.trailing_zeros() as usize;
add_carrier(&mut by_kind[kind_index], carrier.vsize);
kinds &= kinds - 1;
}
if carrier.oversized_output_count > 0 {
policy.oversized.output_count += carrier.oversized_output_count;
policy.oversized.post_op_return_bytes += carrier.oversized_post_op_return_bytes;
add_carrier(&mut policy.oversized, carrier.vsize);
}
if carrier.output_count > 1 {
policy.multiple.output_count += carrier.output_count;
policy.multiple.post_op_return_bytes += carrier.post_op_return_bytes;
add_carrier(&mut policy.multiple, carrier.vsize);
}
if carrier.oversized_output_count > 0 || carrier.output_count > 1 {
policy.pre_v30_nonstandard.output_count += carrier.output_count;
policy.pre_v30_nonstandard.post_op_return_bytes += carrier.post_op_return_bytes;
add_carrier(&mut policy.pre_v30_nonstandard, carrier.vsize);
}
}
fn add_carrier(metrics: &mut Totals, vsize: VSize) {
metrics.carrier_tx_count += 1;
metrics.carrier_vsize += vsize;
}
const fn kind_bit(kind: OpReturnKind) -> u32 {
1_u32 << kind as u8
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn multiple_kinds_count_one_total_carrier() {
let mut total = Totals::default();
let mut by_kind = [Totals::default(); KIND_COUNT];
let mut policy = PolicyTotals::default();
let mut carrier = Carrier {
vsize: VSize::new(100),
..Carrier::default()
};
carrier.add_output(OpReturnKind::Runes, 15);
carrier.add_output(OpReturnKind::Omni, 15);
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
assert_eq!(total.carrier_tx_count, 1);
assert_eq!(by_kind[OpReturnKind::Runes as usize].carrier_tx_count, 1);
assert_eq!(by_kind[OpReturnKind::Omni as usize].carrier_tx_count, 1);
assert_eq!(policy.multiple.output_count, 2);
assert_eq!(policy.pre_v30_nonstandard.carrier_tx_count, 1);
assert_eq!(policy.oversized.carrier_tx_count, 0);
}
#[test]
fn oversized_output_marks_pre_v30_nonstandard_once() {
let mut total = Totals::default();
let mut by_kind = [Totals::default(); KIND_COUNT];
let mut policy = PolicyTotals::default();
let mut carrier = Carrier {
vsize: VSize::new(120),
..Carrier::default()
};
carrier.add_output(OpReturnKind::Unknown, 83);
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
assert_eq!(policy.oversized.output_count, 1);
assert_eq!(policy.oversized.carrier_vsize, VSize::new(120));
assert_eq!(policy.pre_v30_nonstandard.carrier_tx_count, 1);
assert_eq!(policy.multiple.carrier_tx_count, 0);
}
}
crates/brk_computer/src/op_return/import.rs
+34
View File
@@ -0,0 +1,34 @@
use std::path::Path;
use brk_error::Result;
use brk_types::Version;
use super::{ByKind, Metrics, Policy, Vecs};
use crate::{
indexes,
internal::db_utils::{finalize_db, open_db},
};
impl Vecs {
pub(crate) fn forced_import(
parent_path: &Path,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let db = open_db(parent_path, super::DB_NAME, 1_000_000)?;
let total = Metrics::forced_import(&db, "op_return", version, indexes)?;
let by_kind = ByKind::try_new(|_, name| {
Metrics::forced_import(&db, &format!("op_return_{name}"), version, indexes)
})?;
let policy = Policy::forced_import(&db, version, indexes)?;
let this = Self {
db,
total,
by_kind,
policy,
};
finalize_db(&this.db, &this)?;
Ok(this)
}
}
crates/brk_computer/src/op_return/mod.rs
+9
View File
@@ -0,0 +1,9 @@
mod by_kind;
mod compute;
mod import;
mod vecs;
pub use by_kind::ByKind;
pub use vecs::{Metrics, Policy, Vecs};
pub const DB_NAME: &str = "op_return";
crates/brk_computer/src/op_return/vecs.rs
+274
View File
@@ -0,0 +1,274 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, VSize, Version};
use schemars::JsonSchema;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode,
WritableVec,
};
use super::ByKind;
use crate::{
indexes,
internal::{NumericValue, PerBlock},
};
#[derive(Traversable)]
pub struct Series<T, M: StorageMode = Rw>
where
T: NumericValue + JsonSchema,
{
pub block: M::Stored<EagerVec<PcoVec<Height, T>>>,
pub cumulative: PerBlock<T, M>,
}
impl<T> Series<T>
where
T: NumericValue + JsonSchema,
{
fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let block = EagerVec::forced_import(db, name, version)?;
let cumulative =
PerBlock::forced_import(db, &format!("{name}_cumulative"), version, indexes)?;
Ok(Self { block, cumulative })
}
fn len(&self) -> usize {
self.block.len()
}
fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
self.block.validate_and_truncate(version, height)?;
Ok(())
}
fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.block.truncate_if_needed_at(len)?;
Ok(())
}
fn push(&mut self, block: T) {
self.block.push(block);
}
fn write(&mut self) -> Result<()> {
self.block.write()?;
Ok(())
}
fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.cumulative
.height
.compute_cumulative(max_from, &self.block, exit)?;
Ok(())
}
}
#[derive(Clone, Copy, Default)]
pub(super) struct Totals {
pub output_count: u64,
pub post_op_return_bytes: u64,
pub carrier_tx_count: u64,
pub carrier_vsize: VSize,
}
#[derive(Traversable)]
pub struct Metrics<M: StorageMode = Rw> {
pub output_count: Series<StoredU64, M>,
pub post_op_return_bytes: Series<StoredU64, M>,
pub carrier_tx_count: Series<StoredU64, M>,
pub carrier_vsize: Series<VSize, M>,
}
impl Metrics {
pub(crate) fn forced_import(
db: &Database,
prefix: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
output_count: Series::forced_import(
db,
&format!("{prefix}_output_count"),
version,
indexes,
)?,
post_op_return_bytes: Series::forced_import(
db,
&format!("{prefix}_post_op_return_bytes"),
version,
indexes,
)?,
carrier_tx_count: Series::forced_import(
db,
&format!("{prefix}_carrier_tx_count"),
version,
indexes,
)?,
carrier_vsize: Series::forced_import(
db,
&format!("{prefix}_carrier_vsize"),
version,
indexes,
)?,
})
}
fn len(&self) -> usize {
self.output_count
.len()
.min(self.post_op_return_bytes.len())
.min(self.carrier_tx_count.len())
.min(self.carrier_vsize.len())
}
pub(super) fn push(&mut self, block: Totals) {
self.output_count.push(block.output_count.into());
self.post_op_return_bytes
.push(block.post_op_return_bytes.into());
self.carrier_tx_count.push(block.carrier_tx_count.into());
self.carrier_vsize.push(block.carrier_vsize);
}
fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
self.output_count.validate_and_truncate(version, height)?;
self.post_op_return_bytes
.validate_and_truncate(version, height)?;
self.carrier_tx_count
.validate_and_truncate(version, height)?;
self.carrier_vsize.validate_and_truncate(version, height)?;
Ok(())
}
fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.output_count.truncate_if_needed_at(len)?;
self.post_op_return_bytes.truncate_if_needed_at(len)?;
self.carrier_tx_count.truncate_if_needed_at(len)?;
self.carrier_vsize.truncate_if_needed_at(len)?;
Ok(())
}
fn write(&mut self) -> Result<()> {
self.output_count.write()?;
self.post_op_return_bytes.write()?;
self.carrier_tx_count.write()?;
self.carrier_vsize.write()?;
Ok(())
}
fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.output_count.compute_cumulative(max_from, exit)?;
self.post_op_return_bytes
.compute_cumulative(max_from, exit)?;
self.carrier_tx_count.compute_cumulative(max_from, exit)?;
self.carrier_vsize.compute_cumulative(max_from, exit)?;
Ok(())
}
}
#[derive(Traversable)]
pub struct Policy<M: StorageMode = Rw> {
pub oversized: Metrics<M>,
pub multiple: Metrics<M>,
pub pre_v30_nonstandard: Metrics<M>,
}
impl Policy {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let import = |name| {
Metrics::forced_import(db, &format!("op_return_policy_{name}"), version, indexes)
};
Ok(Self {
oversized: import("oversized")?,
multiple: import("multiple")?,
pre_v30_nonstandard: import("pre_v30_nonstandard")?,
})
}
fn iter(&self) -> impl Iterator<Item = &Metrics> {
[&self.oversized, &self.multiple, &self.pre_v30_nonstandard].into_iter()
}
fn iter_mut(&mut self) -> impl Iterator<Item = &mut Metrics> {
[
&mut self.oversized,
&mut self.multiple,
&mut self.pre_v30_nonstandard,
]
.into_iter()
}
}
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
#[traversable(skip)]
pub(crate) db: Database,
pub total: Metrics<M>,
pub by_kind: ByKind<Metrics<M>>,
pub policy: Policy<M>,
}
impl Vecs {
pub(crate) fn min_len(&self) -> usize {
self.by_kind
.iter()
.chain(self.policy.iter())
.map(Metrics::len)
.fold(self.total.len(), usize::min)
}
pub(crate) fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
self.total.validate_and_truncate(version, height)?;
for metrics in self.by_kind.iter_mut() {
metrics.validate_and_truncate(version, height)?;
}
for metrics in self.policy.iter_mut() {
metrics.validate_and_truncate(version, height)?;
}
Ok(())
}
pub(crate) fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.total.truncate_if_needed_at(len)?;
for metrics in self.by_kind.iter_mut() {
metrics.truncate_if_needed_at(len)?;
}
for metrics in self.policy.iter_mut() {
metrics.truncate_if_needed_at(len)?;
}
Ok(())
}
pub(crate) fn write(&mut self) -> Result<()> {
self.total.write()?;
for metrics in self.by_kind.iter_mut() {
metrics.write()?;
}
for metrics in self.policy.iter_mut() {
metrics.write()?;
}
Ok(())
}
pub(crate) fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.total.compute_cumulative(max_from, exit)?;
for metrics in self.by_kind.iter_mut() {
metrics.compute_cumulative(max_from, exit)?;
}
for metrics in self.policy.iter_mut() {
metrics.compute_cumulative(max_from, exit)?;
}
Ok(())
}
}
crates/brk_computer/src/outputs/by_type/compute.rs
+11 -7
View File
@@ -6,6 +6,8 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, WithOutputTypes};
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -44,6 +46,7 @@ impl Vecs {
let mut otype_cursor = indexer.vecs.outputs.output_type.cursor();
let mut fo_cursor = indexer.vecs.transactions.first_txout_index.cursor();
let mut height = skip;
walk_blocks(
&fi_batch,
@@ -77,7 +80,8 @@ impl Vecs {
.block
.push(StoredU64::from(spendable_total));
if self.output_count.all.block.batch_limit_reached() {
height += 1;
if height.is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.output_count.write()?;
self.spendable_output_count.block.write()?;
@@ -93,14 +97,14 @@ impl Vecs {
self.spendable_output_count.block.write()?;
self.tx_count.write()?;
}
self.output_count
.compute_rest(starting_lengths.height, exit)?;
self.spendable_output_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
}
self.output_count
.compute_rest(starting_lengths.height, exit)?;
self.spendable_output_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
for (otype, source) in self.output_count.by_type.iter_typed() {
self.output_share.get_mut(otype).compute_count_ratio(
source,
crates/brk_computer/src/outputs/compute.rs
+2 -3
View File
@@ -3,14 +3,13 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes, inputs, price};
use crate::{blocks, inputs, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
inputs: &inputs::Vecs,
blocks: &blocks::Vecs,
prices: &price::Vecs,
@@ -20,7 +19,7 @@ impl Vecs {
let starting_lengths = indexer.safe_lengths();
self.count.compute(indexer, indexes, blocks, exit)?;
self.count.compute(indexer, blocks, exit)?;
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
self.value.compute(indexer, prices, exit)?;
self.by_type.compute(indexer, exit)?;
crates/brk_computer/src/outputs/count/compute.rs
+7 -8
View File
@@ -3,27 +3,26 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let window_starts = blocks.lookback.window_starts();
self.total.compute(
self.total.sum.compute_count_from_indexes(
starting_height,
&indexes.tx_index.output_count,
&indexer.vecs.transactions.first_tx_index,
&indexes.height.tx_index_count,
&window_starts,
&indexer.vecs.outputs.first_txout_index,
&indexer.vecs.outputs.value,
exit,
0,
)?;
self.total
.compute_rest(starting_height, &window_starts, exit)?;
Ok(())
}
}
crates/brk_computer/src/price/compute.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use std::ops::Range;
use brk_error::Result;
use brk_indexer::{Indexer, Lengths};
use brk_oracle::{
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW, bin_to_cents, cents_to_bin,
};
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
use tracing::info;
crates/brk_computer/src/transactions/compute.rs
+6 -1
View File
@@ -19,7 +19,7 @@ impl Vecs {
self.db.sync_bg_tasks()?;
let (r1, (r2, r3)) = rayon::join(
|| self.count.compute(indexer, &blocks.lookback, exit),
|| self.count.compute(indexer, indexes, &blocks.lookback, exit),
|| {
rayon::join(
|| self.versions.compute(indexer, exit),
@@ -34,6 +34,11 @@ impl Vecs {
self.fees
.compute(indexer, indexes, &inputs.spent, &self.size, exit)?;
self.patterns
.compute(indexer, indexes, &inputs.spent, exit)?;
self.policy.compute(indexer, indexes, &self.fees, exit)?;
self.volume
.compute(indexer, indexes, prices, &self.count, &self.fees, exit)?;
crates/brk_computer/src/transactions/count/compute.rs
+5 -4
View File
@@ -3,12 +3,13 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::blocks;
use crate::{blocks, indexes};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
lookback: &blocks::LookbackVecs,
exit: &Exit,
) -> Result<()> {
@@ -17,10 +18,10 @@ impl Vecs {
let window_starts = lookback.window_starts();
self.total
.compute(starting_height, &window_starts, exit, |height| {
Ok(height.compute_count_from_indexes(
Ok(height.compute_transform(
starting_height,
&indexer.vecs.transactions.first_tx_index,
&indexer.vecs.transactions.txid,
&indexes.height.tx_index_count,
|(height, count, ..)| (height, count),
exit,
)?)
})?;
crates/brk_computer/src/transactions/fees/compute.rs
+361 -96
View File
@@ -1,6 +1,10 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize};
use brk_types::{
ChunkInput, CpfpClusterTxIndex, FeeRate, OutPoint, Sats, StoredBool, StoredU64, TxInIndex,
TxIndex, VSize, linearize,
};
use smallvec::SmallVec;
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely};
use super::super::size;
@@ -70,7 +74,11 @@ impl Vecs {
let dep_version = self.input_value.version()
+ self.output_value.version()
+ size_vecs.vsize.tx_index.version();
+ size_vecs.vsize.tx_index.version()
+ indexer.vecs.inputs.outpoint.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.first_txin_index.version()
+ indexes.height.tx_index_count.version();
self.fee
.tx_index
@@ -80,92 +88,135 @@ impl Vecs {
self.effective_fee_rate
.tx_index
.validate_computed_version_or_reset(dep_version)?;
self.is_cpfp_parent
.validate_computed_version_or_reset(dep_version)?;
self.is_cpfp_child
.validate_computed_version_or_reset(dep_version)?;
self.count
.cpfp_parent
.validate_computed_version_or_reset(dep_version)?;
self.count
.cpfp_child
.validate_computed_version_or_reset(dep_version)?;
let target = self
.input_value
.len()
.min(self.output_value.len())
.min(size_vecs.vsize.tx_index.len());
let min = self
.min(size_vecs.vsize.tx_index.len())
.min(starting_lengths.tx_index.to_usize());
let tx_len = self
.fee
.tx_index
.len()
.min(self.fee_rate.len())
.min(self.effective_fee_rate.tx_index.len())
.min(self.is_cpfp_parent.len())
.min(self.is_cpfp_child.len())
.min(starting_lengths.tx_index.to_usize());
if min >= target {
return Ok(());
}
self.fee
.tx_index
.truncate_if_needed(starting_lengths.tx_index)?;
self.fee_rate
.truncate_if_needed(starting_lengths.tx_index)?;
self.effective_fee_rate
.tx_index
.truncate_if_needed(starting_lengths.tx_index)?;
let start_tx = self.fee.tx_index.len();
let max_height = indexer.vecs.transactions.first_tx_index.len();
let start_height = if start_tx == 0 {
0
let max_height = indexer
.vecs
.transactions
.first_tx_index
.len()
.min(starting_lengths.height.to_usize())
.min(indexes.height.tx_index_count.len());
let next_height = if tx_len >= target {
max_height
} else {
indexes
.tx_heights
.get_shared(TxIndex::from(start_tx))
.get_shared(TxIndex::from(tx_len))
.unwrap()
.to_usize()
};
let count_len = self
.count
.cpfp_parent
.len()
.min(self.count.cpfp_child.len())
.min(max_height);
let start_height = count_len.min(next_height);
if start_height >= max_height {
return Ok(());
}
let start_tx = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(start_height)
.unwrap()
.to_usize();
self.fee
.tx_index
.truncate_if_needed(TxIndex::from(start_tx))?;
self.fee_rate.truncate_if_needed(TxIndex::from(start_tx))?;
self.effective_fee_rate
.tx_index
.truncate_if_needed(TxIndex::from(start_tx))?;
self.is_cpfp_parent
.truncate_if_needed(TxIndex::from(start_tx))?;
self.is_cpfp_child
.truncate_if_needed(TxIndex::from(start_tx))?;
self.count.cpfp_parent.truncate_if_needed_at(start_height)?;
self.count.cpfp_child.truncate_if_needed_at(start_height)?;
let mut tx_count = indexes.height.tx_index_count.cursor();
let mut next_block_input = indexer.vecs.inputs.first_txin_index.cursor();
tx_count.advance(start_height);
next_block_input.advance(start_height + 1);
let mut input_values = Vec::new();
let mut output_values = Vec::new();
let mut vsizes = Vec::new();
let mut txin_starts = Vec::new();
let mut outpoints = Vec::new();
let mut fees = Vec::new();
let mut cluster = Cluster::default();
let mut first_tx = start_tx;
for h in start_height..max_height {
let first_tx: usize = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(h)
.unwrap()
.to_usize();
let n = *indexes.height.tx_index_count.collect_one_at(h).unwrap() as usize;
let n = u64::from(tx_count.next().unwrap()) as usize;
if first_tx + n > target {
break;
}
// Batch read all per-tx data for this block
let input_values = self.input_value.collect_range_at(first_tx, first_tx + n);
let output_values = self.output_value.collect_range_at(first_tx, first_tx + n);
let vsizes: Vec<VSize> = size_vecs
self.input_value
.collect_range_into_at(first_tx, first_tx + n, &mut input_values);
self.output_value
.collect_range_into_at(first_tx, first_tx + n, &mut output_values);
size_vecs
.vsize
.tx_index
.collect_range_at(first_tx, first_tx + n);
let txin_starts: Vec<TxInIndex> = indexer
.collect_range_into_at(first_tx, first_tx + n, &mut vsizes);
indexer
.vecs
.transactions
.first_txin_index
.collect_range_at(first_tx, first_tx + n);
.collect_range_into_at(first_tx, first_tx + n, &mut txin_starts);
let input_begin = txin_starts[0].to_usize();
let input_end = if h + 1 < max_height {
next_block_input.next().unwrap().to_usize()
} else {
indexer
.vecs
.inputs
.first_txin_index
.collect_one_at(h + 1)
.unwrap()
.to_usize()
} else {
indexer.vecs.inputs.outpoint.len()
.outpoint
.len()
.min(starting_lengths.txin_index.to_usize())
};
let outpoints: Vec<OutPoint> = indexer
.vecs
.inputs
.outpoint
.collect_range_at(input_begin, input_end);
indexer.vecs.inputs.outpoint.collect_range_into_at(
input_begin,
input_end,
&mut outpoints,
);
// Compute fee + fee_rate per tx
let mut fees = Vec::with_capacity(n);
fees.clear();
fees.reserve(n);
for j in 0..n {
let fee = if unlikely(input_values[j].is_max()) {
Sats::ZERO
@@ -178,36 +229,56 @@ impl Vecs {
}
// Effective fee rate via same-block CPFP clustering
let effective = cluster_fee_rates(
cluster_fee_rates(
&txin_starts,
&outpoints,
input_begin,
first_tx,
&fees,
&vsizes,
&mut cluster,
);
for rate in effective {
self.effective_fee_rate.tx_index.push(rate);
let mut parent_count = 0;
let mut child_count = 0;
for ((&effective, &fee), &vsize) in cluster.rates.iter().zip(&fees).zip(&vsizes) {
let (is_parent, is_child) = cpfp_roles(effective, FeeRate::from((fee, vsize)));
parent_count += is_parent as u64;
child_count += is_child as u64;
self.effective_fee_rate.tx_index.push(effective);
self.is_cpfp_parent.push(StoredBool::from(is_parent));
self.is_cpfp_child.push(StoredBool::from(is_child));
}
self.count.cpfp_parent.push(StoredU64::from(parent_count));
self.count.cpfp_child.push(StoredU64::from(child_count));
if h % 1_000 == 0 {
let _lock = exit.lock();
self.fee.tx_index.write()?;
self.fee_rate.write()?;
self.effective_fee_rate.tx_index.write()?;
self.is_cpfp_parent.write()?;
self.is_cpfp_child.write()?;
self.count.cpfp_parent.write()?;
self.count.cpfp_child.write()?;
}
first_tx += n;
}
let _lock = exit.lock();
self.fee.tx_index.write()?;
self.fee_rate.write()?;
self.effective_fee_rate.tx_index.write()?;
self.is_cpfp_parent.write()?;
self.is_cpfp_child.write()?;
self.count.cpfp_parent.write()?;
self.count.cpfp_child.write()?;
Ok(())
}
}
/// Clusters same-block parent-child txs and computes effective fee rate per cluster.
/// Computes SFL chunk rates for each same-block dependency component.
fn cluster_fee_rates(
txin_starts: &[TxInIndex],
outpoints: &[OutPoint],
@@ -215,57 +286,251 @@ fn cluster_fee_rates(
first_tx: usize,
fees: &[Sats],
vsizes: &[VSize],
) -> Vec<FeeRate> {
cluster: &mut Cluster,
) {
let n = fees.len();
let mut parent: Vec<usize> = (0..n).collect();
cluster.rates.clear();
cluster.rates.extend(
fees.iter()
.zip(vsizes)
.map(|(&fee, &vsize)| FeeRate::from((fee, vsize))),
);
cluster.parents.clear();
cluster.parents.resize_with(n, SmallVec::new);
cluster.roots.clear();
cluster.roots.extend(0..n);
cluster.members.clear();
cluster.local_index.clear();
cluster.local_index.resize(n, usize::MAX);
for j in 1..n {
let start = txin_starts[j].to_usize() - outpoint_base;
let end = if j + 1 < txin_starts.len() {
txin_starts[j + 1].to_usize() - outpoint_base
} else {
outpoints.len()
};
for child in 0..n {
let mut parents: SmallVec<[usize; 2]> =
same_block_parents(child, txin_starts, outpoints, outpoint_base, first_tx, n).collect();
parents.sort_unstable();
parents.dedup();
for &parent in &parents {
union(&mut cluster.roots, child, parent);
}
cluster.parents[child] = parents;
}
for op in &outpoints[start..end] {
if op.is_coinbase() {
continue;
}
let parent_tx = op.tx_index().to_usize();
if parent_tx >= first_tx && parent_tx < first_tx + n {
union(&mut parent, j, parent_tx - first_tx);
}
for tx in 0..n {
cluster.members.push((root(&mut cluster.roots, tx), tx));
}
cluster.members.sort_unstable();
let mut start = 0;
while start < n {
let component_root = cluster.members[start].0;
let end = cluster.members[start..]
.partition_point(|&(candidate, _)| candidate == component_root)
+ start;
if end - start > 1 {
linearize_component(
&cluster.members[start..end],
&cluster.parents,
fees,
vsizes,
&mut cluster.rates,
&mut cluster.local_index,
&mut cluster.local_parents,
);
}
start = end;
}
}
#[allow(clippy::too_many_arguments)]
fn linearize_component(
members: &[(usize, usize)],
parents: &[SmallVec<[usize; 2]>],
fees: &[Sats],
vsizes: &[VSize],
rates: &mut [FeeRate],
local_index: &mut [usize],
local_parents: &mut Vec<SmallVec<[CpfpClusterTxIndex; 2]>>,
) {
for (local, &(_, tx)) in members.iter().enumerate() {
local_index[tx] = local;
}
local_parents.clear();
local_parents.extend(members.iter().map(|&(_, tx)| {
parents[tx]
.iter()
.map(|&parent| CpfpClusterTxIndex::from(local_index[parent] as u32))
.collect()
}));
let inputs: Vec<ChunkInput<'_>> = members
.iter()
.enumerate()
.map(|(local, &(_, tx))| ChunkInput {
fee: fees[tx],
vsize: vsizes[tx],
parents: local_parents[local].as_slice(),
})
.collect();
for chunk in linearize(&inputs) {
for local in chunk.txs {
rates[members[u32::from(local) as usize].1] = chunk.feerate;
}
}
let mut cluster_fee = vec![Sats::ZERO; n];
let mut cluster_vsize = vec![VSize::from(0u64); n];
for j in 0..n {
let root = find(&mut parent, j);
cluster_fee[root] += fees[j];
cluster_vsize[root] += vsizes[j];
}
(0..n)
.map(|j| {
let root = find(&mut parent, j);
FeeRate::from((cluster_fee[root], cluster_vsize[root]))
})
.collect()
}
fn find(parent: &mut [usize], mut i: usize) -> usize {
while parent[i] != i {
parent[i] = parent[parent[i]];
i = parent[i];
fn union(roots: &mut [usize], left: usize, right: usize) {
let left = root(roots, left);
let right = root(roots, right);
if left != right {
roots[right] = left;
}
i
}
fn union(parent: &mut [usize], a: usize, b: usize) {
let ra = find(parent, a);
let rb = find(parent, b);
if ra != rb {
parent[ra] = rb;
fn root(roots: &mut [usize], node: usize) -> usize {
let mut root = node;
while roots[root] != root {
root = roots[root];
}
let mut current = node;
while roots[current] != current {
let next = roots[current];
roots[current] = root;
current = next;
}
root
}
fn same_block_parents<'a>(
tx: usize,
txin_starts: &'a [TxInIndex],
outpoints: &'a [OutPoint],
outpoint_base: usize,
first_tx: usize,
tx_count: usize,
) -> impl Iterator<Item = usize> + 'a {
let start = txin_starts[tx].to_usize() - outpoint_base;
let end = txin_starts
.get(tx + 1)
.map_or(outpoints.len(), |index| index.to_usize() - outpoint_base);
outpoints[start..end].iter().filter_map(move |outpoint| {
let parent = outpoint.tx_index().to_usize();
(parent >= first_tx && parent < first_tx + tx_count).then_some(parent - first_tx)
})
}
fn cpfp_roles(effective: FeeRate, raw: FeeRate) -> (bool, bool) {
(effective > raw, effective < raw)
}
#[derive(Default)]
struct Cluster {
rates: Vec<FeeRate>,
parents: Vec<SmallVec<[usize; 2]>>,
roots: Vec<usize>,
members: Vec<(usize, usize)>,
local_index: Vec<usize>,
local_parents: Vec<SmallVec<[CpfpClusterTxIndex; 2]>>,
}
#[cfg(test)]
mod tests {
use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize, Vout};
use super::{Cluster, cluster_fee_rates, cpfp_roles};
#[test]
fn marks_actual_cpfp_roles() {
let mut cluster = Cluster::default();
cluster_fee_rates(
&[TxInIndex::from(0usize), TxInIndex::from(1usize)],
&[
OutPoint::COINBASE,
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
],
0,
10,
&[Sats::new(100), Sats::new(200)],
&[VSize::new(100), VSize::new(100)],
&mut cluster,
);
assert_eq!(cluster.rates, [FeeRate::new(1.5), FeeRate::new(1.5)]);
assert_eq!(
[
cpfp_roles(cluster.rates[0], FeeRate::new(1.0)),
cpfp_roles(cluster.rates[1], FeeRate::new(2.0)),
],
[(true, false), (false, true)]
);
}
#[test]
fn keeps_independent_transaction_rates_separate() {
let mut cluster = Cluster::default();
cluster_fee_rates(
&[TxInIndex::from(0usize), TxInIndex::from(1usize)],
&[OutPoint::COINBASE, OutPoint::COINBASE],
0,
10,
&[Sats::new(100), Sats::new(300)],
&[VSize::new(100), VSize::new(100)],
&mut cluster,
);
assert_eq!(cluster.rates, [FeeRate::new(1.0), FeeRate::new(3.0)]);
assert_eq!(
[
cpfp_roles(cluster.rates[0], FeeRate::new(1.0)),
cpfp_roles(cluster.rates[1], FeeRate::new(3.0)),
],
[(false, false), (false, false)]
);
}
#[test]
fn linearizes_shared_parent_branches_independently_of_sibling_order() {
let txin_starts = [
TxInIndex::from(0usize),
TxInIndex::from(1usize),
TxInIndex::from(2usize),
];
let outpoints = [
OutPoint::COINBASE,
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
];
let vsizes = [VSize::new(100); 3];
let mut cluster = Cluster::default();
cluster_fee_rates(
&txin_starts,
&outpoints,
0,
10,
&[Sats::ZERO, Sats::ZERO, Sats::new(3_000)],
&vsizes,
&mut cluster,
);
assert_eq!(
cluster.rates,
[FeeRate::new(15.0), FeeRate::new(0.0), FeeRate::new(15.0)]
);
cluster_fee_rates(
&txin_starts,
&outpoints,
0,
10,
&[Sats::ZERO, Sats::new(3_000), Sats::ZERO],
&vsizes,
&mut cluster,
);
assert_eq!(
cluster.rates,
[FeeRate::new(15.0), FeeRate::new(15.0), FeeRate::new(0.0)]
);
}
}
crates/brk_computer/src/transactions/fees/import.rs
+7 -1
View File
@@ -2,7 +2,7 @@ use brk_error::Result;
use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::Vecs;
use super::{CountVecs, Vecs};
use crate::{indexes, internal::PerTxDistribution};
/// Bump this when fee/feerate aggregation logic changes (e.g., skip coinbase, skip zero-fee).
@@ -16,6 +16,10 @@ impl Vecs {
) -> Result<Self> {
let v = version + VERSION;
Ok(Self {
count: CountVecs {
cpfp_parent: EagerVec::forced_import(db, "cpfp_parent_count", version)?,
cpfp_child: EagerVec::forced_import(db, "cpfp_child_count", version)?,
},
input_value: EagerVec::forced_import(db, "input_value", version)?,
output_value: EagerVec::forced_import(db, "output_value", version)?,
fee: PerTxDistribution::forced_import(db, "fee", v, indexes)?,
@@ -26,6 +30,8 @@ impl Vecs {
v,
indexes,
)?,
is_cpfp_parent: EagerVec::forced_import(db, "is_cpfp_parent", version)?,
is_cpfp_child: EagerVec::forced_import(db, "is_cpfp_child", version)?,
})
}
}
crates/brk_computer/src/transactions/fees/mod.rs
+1 -1
View File
@@ -2,4 +2,4 @@ mod compute;
mod import;
mod vecs;
pub use vecs::Vecs;
pub use vecs::{CountVecs, Vecs};
crates/brk_computer/src/transactions/fees/vecs.rs
+10 -1
View File
@@ -1,14 +1,23 @@
use brk_traversable::Traversable;
use brk_types::{FeeRate, Sats, TxIndex};
use brk_types::{FeeRate, Height, Sats, StoredBool, StoredU64, TxIndex};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
use crate::internal::PerTxDistribution;
#[derive(Traversable)]
pub struct CountVecs<M: StorageMode = Rw> {
pub cpfp_parent: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub cpfp_child: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
}
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub count: CountVecs<M>,
pub input_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub output_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub fee: PerTxDistribution<Sats, M>,
pub fee_rate: M::Stored<EagerVec<PcoVec<TxIndex, FeeRate>>>,
pub effective_fee_rate: PerTxDistribution<FeeRate, M>,
pub is_cpfp_parent: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
pub is_cpfp_child: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
}
crates/brk_computer/src/transactions/import.rs
+7 -1
View File
@@ -12,7 +12,9 @@ use crate::{
},
};
use super::{CountVecs, FeesVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs};
use super::{
CountVecs, FeesVecs, PatternsVecs, PolicyVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs,
};
impl Vecs {
pub(crate) fn forced_import(
@@ -28,6 +30,8 @@ impl Vecs {
let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?;
let size = SizeVecs::forced_import(&db, version, indexer, indexes)?;
let fees = FeesVecs::forced_import(&db, version, indexes)?;
let patterns = PatternsVecs::forced_import(&db, version)?;
let policy = PolicyVecs::forced_import(&db, version)?;
let versions = VersionsVecs::forced_import(&db, version, indexes, cached_starts)?;
let volume = VolumeVecs::forced_import(&db, version, indexes, cached_starts)?;
@@ -36,6 +40,8 @@ impl Vecs {
count,
size,
fees,
patterns,
policy,
versions,
volume,
};
crates/brk_computer/src/transactions/mod.rs
+6
View File
@@ -1,5 +1,7 @@
pub mod count;
pub mod fees;
pub mod patterns;
pub mod policy;
pub mod size;
pub mod versions;
pub mod volume;
@@ -12,6 +14,8 @@ use vecdb::{Database, Rw, StorageMode};
pub use count::Vecs as CountVecs;
pub use fees::Vecs as FeesVecs;
pub use patterns::Vecs as PatternsVecs;
pub use policy::Vecs as PolicyVecs;
pub use size::Vecs as SizeVecs;
pub use versions::Vecs as VersionsVecs;
pub use volume::Vecs as VolumeVecs;
@@ -26,6 +30,8 @@ pub struct Vecs<M: StorageMode = Rw> {
pub count: CountVecs<M>,
pub size: SizeVecs<M>,
pub fees: FeesVecs<M>,
pub patterns: PatternsVecs<M>,
pub policy: PolicyVecs<M>,
pub versions: VersionsVecs<M>,
pub volume: VolumeVecs<M>,
}
crates/brk_computer/src/transactions/patterns/coinjoin.rs
+127
View File
@@ -0,0 +1,127 @@
use brk_types::{OutputType, Sats, TypeIndex};
use rustc_hash::FxHashSet;
type Address = (OutputType, TypeIndex);
#[derive(Default)]
pub(super) struct Candidate {
values: FxHashSet<Sats>,
input_addresses: FxHashSet<Address>,
output_addresses: FxHashSet<Address>,
zero_values: usize,
address_reuse: bool,
}
impl Candidate {
pub(super) fn clear(&mut self) {
self.values.clear();
self.input_addresses.clear();
self.output_addresses.clear();
self.zero_values = 0;
self.address_reuse = false;
}
pub(super) fn add_input(
&mut self,
value: Sats,
output_type: OutputType,
type_index: TypeIndex,
) {
self.add_value(value);
if has_script_address(output_type) {
self.address_reuse |= !self.input_addresses.insert((output_type, type_index));
}
}
pub(super) fn add_output(
&mut self,
value: Sats,
output_type: OutputType,
type_index: TypeIndex,
) {
self.add_value(value);
if has_script_address(output_type) {
let address = (output_type, type_index);
self.address_reuse |=
self.input_addresses.contains(&address) || !self.output_addresses.insert(address);
}
}
pub(super) fn is_match(&self, input_count: usize, output_count: usize) -> bool {
!self.address_reuse
&& self.values.len() + self.zero_values <= (input_count + output_count) / 2
}
fn add_value(&mut self, value: Sats) {
if value.is_zero() {
self.zero_values += 1;
} else {
self.values.insert(value);
}
}
}
fn has_script_address(output_type: OutputType) -> bool {
matches!(
output_type,
OutputType::P2PK65
| OutputType::P2PK33
| OutputType::P2PKH
| OutputType::P2SH
| OutputType::P2WPKH
| OutputType::P2WSH
| OutputType::P2TR
| OutputType::P2A
)
}
#[cfg(test)]
mod tests {
use brk_types::{OutputType, Sats, TypeIndex};
use super::Candidate;
#[test]
fn repeated_values_match_without_address_reuse() {
let mut candidate = Candidate::default();
for index in 0usize..5 {
candidate.add_input(
Sats::new(10_000),
OutputType::P2WPKH,
TypeIndex::from(index),
);
candidate.add_output(
Sats::new(9_000),
OutputType::P2WPKH,
TypeIndex::from(index + 10),
);
}
assert!(candidate.is_match(5, 5));
}
#[test]
fn reused_output_address_does_not_match() {
let mut candidate = Candidate::default();
for _ in 0..5 {
candidate.add_output(
Sats::new(9_000),
OutputType::P2WPKH,
TypeIndex::from(1usize),
);
}
assert!(!candidate.is_match(5, 5));
}
#[test]
fn reused_input_address_does_not_match() {
let mut candidate = Candidate::default();
for _ in 0..5 {
candidate.add_input(
Sats::new(10_000),
OutputType::P2PK33,
TypeIndex::from(1usize),
);
}
assert!(!candidate.is_match(5, 5));
}
}
crates/brk_computer/src/transactions/patterns/compute.rs
+251
View File
@@ -0,0 +1,251 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{StoredBool, StoredU64, TxIndex};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, coinjoin::Candidate};
use crate::{indexes, inputs};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
spent: &inputs::SpentVecs,
exit: &Exit,
) -> Result<()> {
let features = &indexer.vecs.transaction_features;
let version = indexes.tx_index.input_count.version()
+ indexes.tx_index.output_count.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.first_txin_index.version()
+ indexer.vecs.transactions.first_txout_index.version()
+ spent.value.version()
+ indexer.vecs.inputs.output_type.version()
+ indexer.vecs.inputs.type_index.version()
+ indexer.vecs.outputs.value.version()
+ indexer.vecs.outputs.output_type.version()
+ indexer.vecs.outputs.type_index.version()
+ features.has_op_return.version()
+ features.has_inscription.version()
+ indexes.height.tx_index_count.version();
for vec in [
&mut self.is_coinjoin,
&mut self.is_consolidation,
&mut self.is_batch_payout,
] {
vec.validate_computed_version_or_reset(version)?;
}
for vec in [
&mut self.count.coinjoin,
&mut self.count.consolidation,
&mut self.count.batch_payout,
] {
vec.validate_computed_version_or_reset(version)?;
}
let lengths = indexer.safe_lengths();
let target_tx = lengths.tx_index.to_usize();
let target_height = lengths.height.to_usize();
let tx_len = self
.is_coinjoin
.len()
.min(self.is_consolidation.len())
.min(self.is_batch_payout.len())
.min(target_tx);
let count_len = self
.count
.coinjoin
.len()
.min(self.count.consolidation.len())
.min(self.count.batch_payout.len())
.min(target_height);
let start_height = count_len.min(next_height(indexes, tx_len, target_tx, target_height));
if start_height >= target_height {
return Ok(());
}
let first_tx = &indexer.vecs.transactions.first_tx_index;
let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize();
self.is_coinjoin.truncate_if_needed_at(start_tx)?;
self.is_consolidation.truncate_if_needed_at(start_tx)?;
self.is_batch_payout.truncate_if_needed_at(start_tx)?;
self.count.coinjoin.truncate_if_needed_at(start_height)?;
self.count
.consolidation
.truncate_if_needed_at(start_height)?;
self.count
.batch_payout
.truncate_if_needed_at(start_height)?;
let first_txin = indexer
.vecs
.transactions
.first_txin_index
.collect_one_at(start_tx)
.unwrap()
.to_usize();
let first_txout = indexer
.vecs
.transactions
.first_txout_index
.collect_one_at(start_tx)
.unwrap()
.to_usize();
let mut input_count = indexes.tx_index.input_count.cursor();
let mut output_count = indexes.tx_index.output_count.cursor();
let mut input_value = spent.value.cursor();
let mut input_type = indexer.vecs.inputs.output_type.cursor();
let mut input_type_index = indexer.vecs.inputs.type_index.cursor();
let mut output_value = indexer.vecs.outputs.value.cursor();
let mut output_type = indexer.vecs.outputs.output_type.cursor();
let mut output_type_index = indexer.vecs.outputs.type_index.cursor();
let mut has_op_return = features.has_op_return.cursor();
let mut has_inscription = features.has_inscription.cursor();
let mut tx_count = indexes.height.tx_index_count.cursor();
input_count.advance(start_tx);
output_count.advance(start_tx);
input_value.advance(first_txin);
input_type.advance(first_txin);
input_type_index.advance(first_txin);
output_value.advance(first_txout);
output_type.advance(first_txout);
output_type_index.advance(first_txout);
has_op_return.advance(start_tx);
has_inscription.advance(start_tx);
tx_count.advance(start_height);
let mut candidate = Candidate::default();
let mut block_start = start_tx;
for height in start_height..target_height {
let block_end = block_start + u64::from(tx_count.next().unwrap()) as usize;
let mut coinjoin_count = 0;
let mut consolidation_count = 0;
let mut batch_payout_count = 0;
for tx_index in block_start..block_end {
let inputs = usize::from(input_count.next().unwrap());
let outputs = usize::from(output_count.next().unwrap());
let op_return = has_op_return.next().unwrap().is_true();
let inscription = has_inscription.next().unwrap().is_true();
let token_related = op_return || inscription;
let consolidation = is_consolidation(inputs, outputs);
let batch_payout = is_batch_payout(inputs, outputs, tx_index == block_start);
let coinjoin_candidate = is_coinjoin_candidate(inputs, outputs, token_related);
let coinjoin = if coinjoin_candidate {
candidate.clear();
for _ in 0..inputs {
candidate.add_input(
input_value.next().unwrap(),
input_type.next().unwrap(),
input_type_index.next().unwrap(),
);
}
for _ in 0..outputs {
candidate.add_output(
output_value.next().unwrap(),
output_type.next().unwrap(),
output_type_index.next().unwrap(),
);
}
candidate.is_match(inputs, outputs)
} else {
input_value.advance(inputs);
input_type.advance(inputs);
input_type_index.advance(inputs);
output_value.advance(outputs);
output_type.advance(outputs);
output_type_index.advance(outputs);
false
};
coinjoin_count += coinjoin as u64;
consolidation_count += consolidation as u64;
batch_payout_count += batch_payout as u64;
self.is_coinjoin.push(StoredBool::from(coinjoin));
self.is_consolidation.push(StoredBool::from(consolidation));
self.is_batch_payout.push(StoredBool::from(batch_payout));
}
self.count.coinjoin.push(StoredU64::from(coinjoin_count));
self.count
.consolidation
.push(StoredU64::from(consolidation_count));
self.count
.batch_payout
.push(StoredU64::from(batch_payout_count));
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.write()?;
}
block_start = block_end;
}
let _lock = exit.lock();
self.write()
}
fn write(&mut self) -> Result<()> {
self.is_coinjoin.write()?;
self.is_consolidation.write()?;
self.is_batch_payout.write()?;
self.count.coinjoin.write()?;
self.count.consolidation.write()?;
self.count.batch_payout.write()?;
Ok(())
}
}
fn next_height(
indexes: &indexes::Vecs,
tx_len: usize,
target_tx: usize,
target_height: usize,
) -> usize {
if tx_len >= target_tx {
target_height
} else {
indexes
.tx_heights
.get_shared(TxIndex::from(tx_len))
.unwrap()
.to_usize()
}
}
fn is_consolidation(inputs: usize, outputs: usize) -> bool {
inputs >= outputs * 5
}
fn is_batch_payout(inputs: usize, outputs: usize, is_coinbase: bool) -> bool {
!is_coinbase && outputs >= inputs * 5
}
fn is_coinjoin_candidate(inputs: usize, outputs: usize, token_related: bool) -> bool {
inputs >= 5 && outputs >= 5 && inputs < outputs * 5 && outputs < inputs * 5 && !token_related
}
#[cfg(test)]
mod tests {
use super::{is_batch_payout, is_coinjoin_candidate, is_consolidation};
#[test]
fn ratio_boundaries_match_filter_semantics() {
assert!(is_consolidation(25, 5));
assert!(is_batch_payout(5, 25, false));
assert!(!is_batch_payout(1, 5, true));
assert!(!is_coinjoin_candidate(25, 5, false));
assert!(!is_coinjoin_candidate(5, 25, false));
assert!(is_coinjoin_candidate(5, 5, false));
assert!(!is_coinjoin_candidate(5, 5, true));
}
}
crates/brk_computer/src/transactions/patterns/import.rs
+20
View File
@@ -0,0 +1,20 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::{CountVecs, Vecs};
impl Vecs {
pub(crate) fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
count: CountVecs {
coinjoin: EagerVec::forced_import(db, "coinjoin_count", version)?,
consolidation: EagerVec::forced_import(db, "consolidation_count", version)?,
batch_payout: EagerVec::forced_import(db, "batch_payout_count", version)?,
},
is_coinjoin: EagerVec::forced_import(db, "is_coinjoin", version)?,
is_consolidation: EagerVec::forced_import(db, "is_consolidation", version)?,
is_batch_payout: EagerVec::forced_import(db, "is_batch_payout", version)?,
})
}
}
crates/brk_computer/src/transactions/patterns/mod.rs
+6
View File
@@ -0,0 +1,6 @@
mod coinjoin;
mod compute;
mod import;
mod vecs;
pub use vecs::{CountVecs, Vecs};
crates/brk_computer/src/transactions/patterns/vecs.rs
+18
View File
@@ -0,0 +1,18 @@
use brk_traversable::Traversable;
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
#[derive(Traversable)]
pub struct CountVecs<M: StorageMode = Rw> {
pub coinjoin: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub consolidation: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub batch_payout: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
}
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub count: CountVecs<M>,
pub is_coinjoin: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
pub is_consolidation: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
pub is_batch_payout: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
}
crates/brk_computer/src/transactions/policy/compute.rs
+127
View File
@@ -0,0 +1,127 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{Sats, StoredBool, StoredU64, TxIndex};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::Vecs;
use crate::{indexes, transactions::fees};
const FIRST_EPHEMERAL_DUST_HEIGHT: usize = 905_000;
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fees: &fees::Vecs,
exit: &Exit,
) -> Result<()> {
let features = &indexer.vecs.transaction_features;
let version = features.is_unconditionally_nonstandard.version()
+ features.has_dust_output.version()
+ fees.fee.tx_index.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexes.height.tx_index_count.version();
self.is_nonstandard
.validate_computed_version_or_reset(version)?;
self.count.validate_computed_version_or_reset(version)?;
let lengths = indexer.safe_lengths();
let target_tx = lengths.tx_index.to_usize().min(fees.fee.tx_index.len());
let target_height = lengths.height.to_usize();
let tx_len = self.is_nonstandard.len().min(target_tx);
let count_len = self.count.len().min(target_height);
let next_height = if tx_len >= target_tx {
target_height
} else {
indexes
.tx_heights
.get_shared(TxIndex::from(tx_len))
.unwrap()
.to_usize()
};
let start_height = count_len.min(next_height);
if start_height >= target_height {
return Ok(());
}
let first_tx = &indexer.vecs.transactions.first_tx_index;
let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize();
self.is_nonstandard.truncate_if_needed_at(start_tx)?;
self.count.truncate_if_needed_at(start_height)?;
let mut unconditional = features.is_unconditionally_nonstandard.cursor();
let mut has_dust = features.has_dust_output.cursor();
let mut fee = fees.fee.tx_index.cursor();
let mut tx_count = indexes.height.tx_index_count.cursor();
unconditional.advance(start_tx);
has_dust.advance(start_tx);
fee.advance(start_tx);
tx_count.advance(start_height);
let mut block_start = start_tx;
for height in start_height..target_height {
let block_end =
(block_start + u64::from(tx_count.next().unwrap()) as usize).min(target_tx);
let mut count = 0;
for _ in block_start..block_end {
let raw = unconditional.next().unwrap().is_true();
let dust = has_dust.next().unwrap().is_true();
let nonstandard = if raw {
fee.advance(1);
true
} else if dust {
dust_is_nonstandard(height, fee.next().unwrap())
} else {
fee.advance(1);
false
};
count += nonstandard as u64;
self.is_nonstandard.push(StoredBool::from(nonstandard));
}
self.count.push(StoredU64::from(count));
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.is_nonstandard.write()?;
self.count.write()?;
}
block_start = block_end;
}
let _lock = exit.lock();
self.is_nonstandard.write()?;
self.count.write()?;
Ok(())
}
}
fn dust_is_nonstandard(height: usize, fee: Sats) -> bool {
height < FIRST_EPHEMERAL_DUST_HEIGHT || fee != Sats::ZERO
}
#[cfg(test)]
mod tests {
use brk_types::Sats;
use super::{FIRST_EPHEMERAL_DUST_HEIGHT, dust_is_nonstandard};
#[test]
fn zero_fee_ephemeral_dust_starts_at_activation() {
assert!(dust_is_nonstandard(
FIRST_EPHEMERAL_DUST_HEIGHT - 1,
Sats::ZERO
));
assert!(!dust_is_nonstandard(
FIRST_EPHEMERAL_DUST_HEIGHT,
Sats::ZERO
));
assert!(dust_is_nonstandard(
FIRST_EPHEMERAL_DUST_HEIGHT,
Sats::new(1)
));
}
}
crates/brk_computer/src/transactions/policy/import.rs
+14
View File
@@ -0,0 +1,14 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::Vecs;
impl Vecs {
pub(crate) fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
count: EagerVec::forced_import(db, "nonstandard_count", version)?,
is_nonstandard: EagerVec::forced_import(db, "is_nonstandard", version)?,
})
}
}
crates/brk_computer/src/transactions/policy/mod.rs
+5
View File
@@ -0,0 +1,5 @@
mod compute;
mod import;
mod vecs;
pub use vecs::Vecs;
crates/brk_computer/src/transactions/policy/vecs.rs
+9
View File
@@ -0,0 +1,9 @@
use brk_traversable::Traversable;
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub count: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub is_nonstandard: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
}
crates/brk_computer/src/transactions/size/compute.rs
+1 -1
View File
@@ -14,7 +14,7 @@ impl Vecs {
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
self.vsize
self.weight
.derive_from(indexer, indexes, &starting_lengths, exit)?;
Ok(())
crates/brk_computer/src/transactions/size/import.rs
+16 -22
View File
@@ -1,12 +1,12 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{TxIndex, VSize, Version, Weight};
use vecdb::{Database, LazyVecFrom2, ReadableCloneableVec};
use brk_types::{Version, Weight};
use vecdb::{Database, LazyVecFrom1, ReadableCloneableVec};
use super::Vecs;
use crate::{
indexes,
internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed, VSizeToWeight},
internal::{Identity, LazyPerTxDistribution, LazyPerTxDistributionTransformed, WeightToVSize},
};
impl Vecs {
@@ -16,37 +16,31 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
) -> Result<Self> {
let tx_index_to_vsize = LazyVecFrom2::init(
"tx_vsize",
let tx_index_to_weight = LazyVecFrom1::transformed::<Identity<Weight>>(
"tx_weight",
version,
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|_index: TxIndex, base_size, total_size| {
VSize::from(Weight::from_sizes(*base_size, *total_size))
},
indexer.vecs.transactions.weight.read_only_boxed_clone(),
);
let vsize = LazyPerTxDistribution::forced_import(
let weight = LazyPerTxDistribution::forced_import(
db,
"tx_vsize",
"tx_weight",
version,
indexes,
tx_index_to_vsize,
tx_index_to_weight,
)?;
let tx_index_to_weight = LazyVecFrom2::init(
"tx_weight",
let tx_index_to_vsize = LazyVecFrom1::transformed::<WeightToVSize>(
"tx_vsize",
version,
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|_index: TxIndex, base_size, total_size| Weight::from_sizes(*base_size, *total_size),
indexer.vecs.transactions.weight.read_only_boxed_clone(),
);
let weight = LazyPerTxDistributionTransformed::new::<VSizeToWeight>(
"tx_weight",
let vsize = LazyPerTxDistributionTransformed::new::<WeightToVSize>(
"tx_vsize",
version,
tx_index_to_weight,
&vsize.distribution,
tx_index_to_vsize,
&weight.distribution,
);
Ok(Self { vsize, weight })
crates/brk_computer/src/transactions/size/vecs.rs
+3 -3
View File
@@ -1,11 +1,11 @@
use brk_traversable::Traversable;
use brk_types::{StoredU32, VSize, Weight};
use brk_types::{VSize, Weight};
use vecdb::{Rw, StorageMode};
use crate::internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub vsize: LazyPerTxDistribution<VSize, StoredU32, StoredU32, M>,
pub weight: LazyPerTxDistributionTransformed<Weight, StoredU32, StoredU32, VSize>,
pub vsize: LazyPerTxDistributionTransformed<VSize, Weight, Weight>,
pub weight: LazyPerTxDistribution<Weight, Weight, M>,
}
crates/brk_computer/src/transactions/versions/compute.rs
+18 -77
View File
@@ -1,90 +1,31 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{StoredU64, TxVersion};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use vecdb::Exit;
use super::Vecs;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let lengths = indexer.safe_lengths();
let starting_height = lengths.height;
let counts = &indexer.vecs.transaction_features.count;
let dep_version = indexer.vecs.transactions.tx_version.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.txid.version();
for vec in [&mut self.v1.block, &mut self.v2.block, &mut self.v3.block] {
vec.validate_and_truncate(dep_version, starting_height)?;
for (metrics, source) in [
(&mut self.v1, &counts.v1),
(&mut self.v2, &counts.v2),
(&mut self.v3, &counts.v3),
(&mut self.other, &counts.other_version),
] {
metrics.compute(starting_height, exit, |block| {
Ok(block.compute_transform(
starting_height,
source,
|(height, count, ..)| (height, count),
exit,
)?)
})?;
}
let skip = self
.v1
.block
.len()
.min(self.v2.block.len())
.min(self.v3.block.len());
let first_tx_index = &indexer.vecs.transactions.first_tx_index;
let end = first_tx_index.len();
if skip >= end {
return Ok(());
}
// Truncate all 3 to skip, then push (no per-element bounds checks).
self.v1.block.truncate_if_needed_at(skip)?;
self.v2.block.truncate_if_needed_at(skip)?;
self.v3.block.truncate_if_needed_at(skip)?;
// Single cursor over tx_version — scanned once for all 3 version counts.
let mut cursor = indexer.vecs.transactions.tx_version.cursor();
let fi_batch = first_tx_index.collect_range_at(skip, end);
let txid_len = indexer.vecs.transactions.txid.len();
for (j, first_index) in fi_batch.iter().enumerate() {
let next_first = fi_batch
.get(j + 1)
.map(|fi| fi.to_usize())
.unwrap_or(txid_len);
let mut c1: usize = 0;
let mut c2: usize = 0;
let mut c3: usize = 0;
let fi = first_index.to_usize();
cursor.advance(fi - cursor.position());
for _ in fi..next_first {
match cursor.next().unwrap() {
TxVersion::ONE => c1 += 1,
TxVersion::TWO => c2 += 1,
TxVersion::THREE => c3 += 1,
_ => {}
}
}
self.v1.block.push(StoredU64::from(c1 as u64));
self.v2.block.push(StoredU64::from(c2 as u64));
self.v3.block.push(StoredU64::from(c3 as u64));
if self.v1.block.batch_limit_reached() {
let _lock = exit.lock();
self.v1.block.write()?;
self.v2.block.write()?;
self.v3.block.write()?;
}
}
{
let _lock = exit.lock();
self.v1.block.write()?;
self.v2.block.write()?;
self.v3.block.write()?;
}
// Derive cumulative + sums from base
self.v1.compute_rest(starting_height, exit)?;
self.v2.compute_rest(starting_height, exit)?;
self.v3.compute_rest(starting_height, exit)?;
Ok(())
}
}
crates/brk_computer/src/transactions/versions/import.rs
+7
View File
@@ -37,6 +37,13 @@ impl Vecs {
indexes,
cached_starts,
)?,
other: PerBlockCumulativeRolling::forced_import(
db,
"tx_other_version",
version,
indexes,
cached_starts,
)?,
})
}
}
crates/brk_computer/src/transactions/versions/vecs.rs
+1
View File
@@ -9,4 +9,5 @@ pub struct Vecs<M: StorageMode = Rw> {
pub v1: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
pub v2: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
pub v3: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
pub other: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
}