Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: utxos part 3

Browse Source
This commit is contained in:
nym21
2025-05-18 11:52:14 +02:00
parent c8a25934a6
commit c2a77072d2
7 changed files with 249 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/brk_computer/src/states/block.rs
View File

@@ -5,4 +5,5 @@ use brk_core::{Sats, StoredU32};
pub struct BlockState {
utxos: StoredU32,
value: Sats,
unspendable: Sats,
}

6
crates/brk_computer/src/states/mod.rs
View File

@@ -1,7 +1,13 @@
mod block;
mod cohort;
mod outputs;
mod realized;
mod received;
mod sent;
pub use block::*;
pub use cohort::*;
pub use outputs::*;
pub use realized::*;
pub use received::*;
pub use sent::*;

11
crates/brk_computer/src/states/realized.rs Normal file
View File

@@ -0,0 +1,11 @@
use brk_core::Dollars;
#[derive(Debug, Default)]
pub struct RealizedState {
realized_profit: Dollars,
realized_loss: Dollars,
value_created: Dollars,
adjusted_value_created: Dollars,
value_destroyed: Dollars,
adjusted_value_destroyed: Dollars,
}

8
crates/brk_computer/src/states/received.rs Normal file
View File

@@ -0,0 +1,8 @@
use brk_core::{Sats, StoredUsize};
#[derive(Debug, Default)]
pub struct ReceivedState {
utxos: StoredUsize,
sats: Sats,
unspendable: Sats,
}

7
crates/brk_computer/src/states/sent.rs Normal file
View File

@@ -0,0 +1,7 @@
use brk_core::{Sats, StoredUsize};
#[derive(Debug, Default)]
pub struct SentState {
utxos: StoredUsize,
sats: Sats,
}

45
crates/brk_computer/src/vecs/transactions.rs
View File

@@ -1,8 +1,8 @@
use std::{fs, path::Path};
use brk_core::{
CheckedSub, Feerate, Height, InputIndex, OutputIndex, Sats, StoredU32, StoredUsize, TxIndex,
TxVersion, Weight,
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
StoredUsize, TxIndex, TxVersion, Weight,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
@@ -65,6 +65,7 @@ pub struct Vecs {
pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight,
pub indexes_to_tx_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredUsize>,
@@ -476,6 +477,15 @@ impl Vecs {
.add_average(),
compute_dollars,
)?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
path,
"unclaimed_rewards",
true,
Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
compute_dollars,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
path,
"p2a_count",
@@ -850,9 +860,35 @@ impl Vecs {
.as_ref()
.unwrap()
.as_ref(),
|(height, subsidy, ..)| {
|(height, coinbase, ..)| {
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
(height, subsidy.checked_sub(fees).unwrap())
(height, coinbase.checked_sub(fees).unwrap())
},
exit,
)
},
)?;
self.indexes_to_unclaimed_rewards.compute_all(
indexer,
indexes,
fetched,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_transform(
starting_indexes.height,
self.indexes_to_subsidy
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
|(height, subsidy, ..)| {
let halving = HalvingEpoch::from(height);
let expected =
Sats::FIFTY_BTC / 2_usize.pow(halving.unwrap_to_usize() as u32);
(height, expected.checked_sub(subsidy).unwrap())
},
exit,
)
@@ -1131,6 +1167,7 @@ impl Vecs {
self.indexes_to_tx_weight.vecs(),
self.indexes_to_unknownoutput_count.vecs(),
self.indexes_to_exact_utxo_count.vecs(),
self.indexes_to_unclaimed_rewards.vecs(),
]
.concat()
}

301
crates/brk_computer/src/vecs/utxos.rs
View File

@@ -1,4 +1,4 @@
use std::{fs, path::Path, thread};
use std::{collections::BTreeMap, fs, path::Path, thread};
use brk_core::{
Bitcoin, CheckedSub, Dollars, Height, InputIndex, OutputIndex, OutputType, Sats, StoredUsize,
@@ -9,11 +9,10 @@ use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, Compressed, Computation, EagerVec, StoredIndex,
VecIterator, Version,
};
use color_eyre::eyre::ContextCompat;
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::states::{CohortState, Outputs};
use crate::states::{CohortState, Outputs, RealizedState, ReceivedState, SentState};
use super::{
Indexes, fetched,
@@ -21,23 +20,11 @@ use super::{
indexes, transactions,
};
const VERSION: Version = Version::new(3);
#[derive(Clone, Deref, DerefMut)]
pub struct Vecs(Outputs<Vecs_>);
#[derive(Clone)]
pub struct Vecs_ {
pub height_to_realized_cap: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_supply: EagerVec<Height, Sats>,
pub indexes_to_supply: ComputedValueVecsFromHeight,
pub height_to_unspendable_supply: EagerVec<Height, Sats>,
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
const VERSION: Version = Version::new(3);
impl Vecs {
pub fn forced_import(
path: &Path,
@@ -45,73 +32,8 @@ impl Vecs {
compressed: Compressed,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
fs::create_dir_all(path)?;
Ok(Self(Outputs {
all: Vecs_ {
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
&path.join("height_to_realized_cap"),
VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
"realized_cap",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply: EagerVec::forced_import(
&path.join("height_to_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_supply: ComputedValueVecsFromHeight::forced_import(
path,
"supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
&path.join("height_to_unspendable_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
path,
"unspendable_supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_utxo_count: EagerVec::forced_import(
&path.join("height_to_utxo_count"),
VERSION + Version::new(111),
compressed,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
"utxo_count",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
},
all: Vecs_::forced_import(path, _computation, compressed, fetched)?,
}))
}
@@ -133,8 +55,16 @@ impl Vecs {
let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex;
let outputindex_to_value = &indexer_vecs.outputindex_to_value;
let txindex_to_height = &indexes.txindex_to_height;
let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed;
let outputindex_to_txindex = &indexes.outputindex_to_txindex;
let outputindex_to_outputtype = &indexer_vecs.outputindex_to_outputtype;
let height_to_unclaimed_rewards = transactions
.indexes_to_unclaimed_rewards
.sats
.height
.as_ref()
.unwrap()
.as_ref();
let height_to_close = &fetched
.as_ref()
.map(|fetched| &fetched.chainindexes_to_close.height);
@@ -157,10 +87,13 @@ impl Vecs {
let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter());
let mut height_to_opreturn_count_iter = height_to_opreturn_count.into_iter();
let mut outputindex_to_outputtype_iter = outputindex_to_outputtype.into_iter();
let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter();
let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter();
let base_version = Version::ZERO
+ height_to_first_outputindex.version()
+ height_to_first_inputindex.version()
+ height_to_timestamp_fixed.version()
+ height_to_output_count.version()
+ height_to_input_count.version()
+ inputindex_to_outputindex.version()
@@ -169,6 +102,7 @@ impl Vecs {
+ outputindex_to_txindex.version()
+ height_to_opreturn_count.version()
+ outputindex_to_outputtype.version()
+ height_to_unclaimed_rewards.version()
+ height_to_close
.as_ref()
.map_or(Version::ZERO, |v| v.version());
@@ -227,6 +161,10 @@ impl Vecs {
(starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
.map(Height::from)
.try_for_each(|height| -> color_eyre::Result<()> {
let sent_state = SentState::default();
let received_state = ReceivedState::default();
let realized_state = RealizedState::default();
let first_outputindex = height_to_first_outputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
@@ -237,35 +175,52 @@ impl Vecs {
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let opreturn_count = height_to_opreturn_count_iter.unwrap_get_inner(height);
let (sent_sats_dollars, (mut received_spendable, mut received_unspendable)) =
let (sent_sats_price_tuple, (mut received_spendable, mut received_unspendable)) =
thread::scope(|s| {
// Skip coinbase
let sent_sats_dollars = s.spawn(|| {
let sent_sats_price_tuple = s.spawn(|| {
let mut txindex_to_height = BTreeMap::new();
let mut height_to_timestamp_price_sats = BTreeMap::new();
(first_inputindex + 1..first_inputindex + *input_count)
.map(InputIndex::from)
.map(|inputindex| {
inputindex_to_outputindex_iter.unwrap_get_inner(inputindex)
})
.map(|outputindex| {
.for_each(|outputindex| {
let value =
outputindex_to_value_iter.unwrap_get_inner(outputindex);
if let Some(height_to_close_iter) =
height_to_close_iter.as_mut()
{
let txindex = outputindex_to_txindex_iter
.unwrap_get_inner(outputindex);
let height =
txindex_to_height_iter.unwrap_get_inner(txindex);
let dollars =
*height_to_close_iter.unwrap_get_inner(height);
let txindex =
outputindex_to_txindex_iter.unwrap_get_inner(outputindex);
(value, dollars)
} else {
(value, Dollars::ZERO)
}
})
.collect::<Vec<_>>()
let height =
*txindex_to_height.entry(txindex).or_insert_with(|| {
txindex_to_height_iter.unwrap_get_inner(txindex)
});
let entry = height_to_timestamp_price_sats
.entry(height)
.or_insert_with(|| {
let timestamp = height_to_timestamp_fixed_iter
.unwrap_get_inner(height);
if let Some(height_to_close_iter) =
height_to_close_iter.as_mut()
{
let dollars =
*height_to_close_iter.unwrap_get_inner(height);
(timestamp, dollars, Sats::ZERO)
} else {
(timestamp, Dollars::ZERO, Sats::ZERO)
}
});
entry.2 += value;
});
height_to_timestamp_price_sats
});
let received = s.spawn(|| {
@@ -277,8 +232,12 @@ impl Vecs {
let value =
outputindex_to_value_iter_2.unwrap_get_inner(outputindex);
if outputindex_to_outputtype_iter.unwrap_get_inner(outputindex)
== OutputType::OpReturn
let outputtype = outputindex_to_outputtype_iter
.unwrap_get_inner(outputindex);
if outputtype == OutputType::OpReturn
|| outputtype == OutputType::Empty
|| outputtype == OutputType::Unknown
{
unspendable += value
} else {
@@ -288,12 +247,19 @@ impl Vecs {
(spendable, unspendable)
});
(sent_sats_dollars.join().unwrap(), received.join().unwrap())
(
sent_sats_price_tuple.join().unwrap(),
received.join().unwrap(),
)
});
let (sent, realized_cap_destroyed) = sent_sats_dollars
.into_par_iter()
.map(|(sats, dollars)| (sats, dollars * Bitcoin::from(sats)))
let (sent, realized_cap_destroyed) = sent_sats_price_tuple
.par_iter()
.map(|(_, (_, dollars, sats))| {
let dollars = *dollars;
let sats = *sats;
(sats, dollars * Bitcoin::from(sats))
})
.reduce(
|| (Sats::ZERO, Dollars::ZERO),
|acc, (sats, dollars)| (acc.0 + sats, acc.1 + dollars),
@@ -313,6 +279,8 @@ impl Vecs {
*input_count - 1
};
received_unspendable += height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
state.supply -= sent;
state.supply += received_spendable;
@@ -327,15 +295,6 @@ impl Vecs {
let realized_cap_created = price * Bitcoin::from(received);
state.realized_cap = (state.realized_cap + realized_cap_created)
.checked_sub(realized_cap_destroyed)
.context("to work")
.inspect_err(|_| {
dbg!((
height,
state.realized_cap,
realized_cap_created,
realized_cap_destroyed
));
})
.unwrap();
}
@@ -394,22 +353,112 @@ impl Vecs {
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[self.all.vecs()].concat()
}
}
#[derive(Clone)]
pub struct Vecs_ {
pub height_to_realized_cap: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_supply: EagerVec<Height, Sats>,
pub indexes_to_supply: ComputedValueVecsFromHeight,
pub height_to_unspendable_supply: EagerVec<Height, Sats>,
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
impl Vecs_ {
pub fn forced_import(
path: &Path,
_computation: Computation,
compressed: Compressed,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
fs::create_dir_all(path)?;
Ok(Self {
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
&path.join("height_to_realized_cap"),
VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
"realized_cap",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply: EagerVec::forced_import(
&path.join("height_to_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_supply: ComputedValueVecsFromHeight::forced_import(
path,
"supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
&path.join("height_to_unspendable_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
path,
"unspendable_supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_utxo_count: EagerVec::forced_import(
&path.join("height_to_utxo_count"),
VERSION + Version::new(111),
compressed,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
"utxo_count",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
})
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![
&self.all.height_to_supply as &dyn AnyCollectableVec,
&self.all.height_to_unspendable_supply,
&self.all.height_to_utxo_count,
&self.height_to_supply as &dyn AnyCollectableVec,
&self.height_to_unspendable_supply,
&self.height_to_utxo_count,
],
self.all
.height_to_realized_cap
self.height_to_realized_cap
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.all.indexes_to_supply.vecs(),
self.all.indexes_to_unspendable_supply.vecs(),
self.all.indexes_to_utxo_count.vecs(),
self.all
.indexes_to_realized_cap
self.indexes_to_supply.vecs(),
self.indexes_to_unspendable_supply.vecs(),
self.indexes_to_utxo_count.vecs(),
self.indexes_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
]