Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 14:49:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: utxos part 2

Browse Source
This commit is contained in:
nym21
2025-05-17 19:51:52 +02:00
parent 7b38355cd4
commit c8a25934a6
15 changed files with 435 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/brk_computer/src/states/cohort.rs
View File

@@ -4,9 +4,10 @@ use brk_core::{Dollars, Sats, StoredUsize};
// Vecs ? probably
#[derive(Default)]
pub struct CohortStates {
pub struct CohortState {
pub realized_cap: Dollars,
pub supply: Sats,
pub unspendable_supply: Sats,
pub utxo_count: StoredUsize,
// pub price_to_amount: PriceToValue<Amount>, save it not rounded in fjall
}

4
crates/brk_computer/src/states/outputs.rs
View File

@@ -10,6 +10,7 @@ pub struct Outputs<T> {
// pub by_epoch: OutputsByEpoch<T>,
// pub by_size: OutputsBySize<T>,
// pub by_value: OutputsByValue<T>,
// pub by_type: OutputsByType<T>, // all but op-return
}
#[derive(Default)]
@@ -105,3 +106,6 @@ pub struct OutputsByValue<T> {
pub from_1_000_000_000usd: T,
// ...
}
// #[derive(Default)]
// pub struct OutputsByType<T> {}

11
crates/brk_computer/src/vecs/grouped/value_from_height.rs
View File

@@ -136,8 +136,8 @@ impl ComputedValueVecsFromHeight {
)?;
}
let txindex = self.bitcoin.height.as_ref().unwrap().as_ref();
let price = &fetched.as_ref().unwrap().chainindexes_to_close.height;
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap().as_ref();
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
dollars.compute_all(
@@ -146,7 +146,12 @@ impl ComputedValueVecsFromHeight {
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_from_bitcoin(starting_indexes.height, txindex, price, exit)
v.compute_from_bitcoin(
starting_indexes.height,
height_to_bitcoin,
height_to_close,
exit,
)
},
)?;
}

14
crates/brk_computer/src/vecs/mod.rs
View File

@@ -25,7 +25,7 @@ pub struct Vecs {
pub mining: mining::Vecs,
pub market: market::Vecs,
pub transactions: transactions::Vecs,
// pub utxos: utxos::Vecs,
pub utxos: utxos::Vecs,
pub fetched: Option<fetched::Vecs>,
}
@@ -49,7 +49,7 @@ impl Vecs {
mining: mining::Vecs::forced_import(path, computation, compressed)?,
constants: constants::Vecs::forced_import(path, computation, compressed)?,
market: market::Vecs::forced_import(path, computation, compressed)?,
// utxos: utxos::Vecs::forced_import(path, computation, compressed)?,
utxos: utxos::Vecs::forced_import(path, computation, compressed, fetched.as_ref())?,
transactions: transactions::Vecs::forced_import(
path,
indexer,
@@ -110,6 +110,15 @@ impl Vecs {
)?;
}
self.utxos.compute(
indexer,
&self.indexes,
&self.transactions,
self.fetched.as_ref(),
&starting_indexes,
exit,
)?;
Ok(())
}
@@ -121,6 +130,7 @@ impl Vecs {
self.mining.vecs(),
self.market.vecs(),
self.transactions.vecs(),
self.utxos.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
]
.concat()

10
crates/brk_computer/src/vecs/transactions.rs
View File

@@ -82,7 +82,7 @@ pub struct Vecs {
ComputedVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
pub txindex_to_fee: ComputedVecFrom2<TxIndex, Sats, TxIndex, Sats, TxIndex, Sats>,
pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
impl Vecs {
@@ -632,9 +632,9 @@ impl Vecs {
.add_sum()
.add_total(),
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
path,
"utxo_count_bis",
"exact_utxo_count",
true,
Version::TWO,
compressed,
@@ -1039,7 +1039,7 @@ impl Vecs {
},
)?;
self.indexes_to_utxo_count.compute_all(
self.indexes_to_exact_utxo_count.compute_all(
indexer,
indexes,
starting_indexes,
@@ -1130,7 +1130,7 @@ impl Vecs {
self.indexes_to_tx_vsize.vecs(),
self.indexes_to_tx_weight.vecs(),
self.indexes_to_unknownoutput_count.vecs(),
self.indexes_to_utxo_count.vecs(),
self.indexes_to_exact_utxo_count.vecs(),
]
.concat()
}

315
crates/brk_computer/src/vecs/utxos.rs
View File

@@ -1,19 +1,23 @@
use std::{fs, path::Path};
use std::{fs, path::Path, thread};
use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize};
use brk_core::{
Bitcoin, CheckedSub, Dollars, Height, InputIndex, OutputIndex, OutputType, Sats, StoredUsize,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, Compressed, Computation, EagerVec, StoredIndex,
VecIterator, Version,
};
use color_eyre::eyre::ContextCompat;
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::states::{CohortStates, Outputs};
use crate::states::{CohortState, Outputs};
use super::{
Indexes,
grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
Indexes, fetched,
grouped::{ComputedValueVecsFromHeight, ComputedVecsFromHeight, StorableVecGeneatorOptions},
indexes, transactions,
};
@@ -22,51 +26,77 @@ pub struct Vecs(Outputs<Vecs_>);
#[derive(Clone)]
pub struct Vecs_ {
pub height_to_realized_cap: EagerVec<Height, Dollars>,
pub indexes_to_realized_cap: ComputedVecsFromHeight<Dollars>,
pub height_to_realized_cap: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_cap: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_supply: EagerVec<Height, Sats>,
pub indexes_to_supply: ComputedVecsFromHeight<Sats>,
pub indexes_to_supply: ComputedValueVecsFromHeight,
pub height_to_unspendable_supply: EagerVec<Height, Sats>,
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
}
const VERSION: Version = Version::ZERO;
const VERSION: Version = Version::new(3);
impl Vecs {
pub fn forced_import(
path: &Path,
_computation: Computation,
compressed: Compressed,
fetched: Option<&fetched::Vecs>,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
fs::create_dir_all(path)?;
Ok(Self(Outputs {
all: Vecs_ {
height_to_realized_cap: EagerVec::forced_import(
&path.join("height_to_realized_cap"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_realized_cap: ComputedVecsFromHeight::forced_import(
path,
"realized_cap",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)?,
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
&path.join("height_to_realized_cap"),
VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
"realized_cap",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply: EagerVec::forced_import(
&path.join("height_to_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_supply: ComputedVecsFromHeight::forced_import(
indexes_to_supply: ComputedValueVecsFromHeight::forced_import(
path,
"supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
&path.join("height_to_unspendable_supply"),
VERSION + Version::ZERO,
compressed,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
path,
"unspendable_supply",
false,
VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_utxo_count: EagerVec::forced_import(
&path.join("height_to_utxo_count"),
@@ -90,6 +120,7 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
transactions: &transactions::Vecs,
fetched: Option<&fetched::Vecs>,
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
@@ -97,12 +128,22 @@ impl Vecs {
let height_to_first_outputindex = &indexer_vecs.height_to_first_outputindex;
let height_to_first_inputindex = &indexer_vecs.height_to_first_inputindex;
let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_last();
let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_last();
let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_sum();
let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_sum();
let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex;
let outputindex_to_value = &indexer_vecs.outputindex_to_value;
let txindex_to_height = &indexes.txindex_to_height;
let outputindex_to_txindex = &indexes.outputindex_to_txindex;
let outputindex_to_outputtype = &indexer_vecs.outputindex_to_outputtype;
let height_to_close = &fetched
.as_ref()
.map(|fetched| &fetched.chainindexes_to_close.height);
let height_to_opreturn_count = &transactions
.indexes_to_opreturn_count
.height
.as_ref()
.unwrap()
.as_ref();
let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter();
let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter();
@@ -110,8 +151,12 @@ impl Vecs {
let mut height_to_input_count_iter = height_to_input_count.into_iter();
let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter();
let mut outputindex_to_value_iter = outputindex_to_value.into_iter();
let mut outputindex_to_value_iter_2 = outputindex_to_value.into_iter();
let mut txindex_to_height_iter = txindex_to_height.into_iter();
let mut outputindex_to_txindex_iter = outputindex_to_txindex.into_iter();
let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter());
let mut height_to_opreturn_count_iter = height_to_opreturn_count.into_iter();
let mut outputindex_to_outputtype_iter = outputindex_to_outputtype.into_iter();
let base_version = Version::ZERO
+ height_to_first_outputindex.version()
@@ -121,26 +166,40 @@ impl Vecs {
+ inputindex_to_outputindex.version()
+ outputindex_to_value.version()
+ txindex_to_height.version()
+ outputindex_to_txindex.version();
+ outputindex_to_txindex.version()
+ height_to_opreturn_count.version()
+ outputindex_to_outputtype.version()
+ height_to_close
.as_ref()
.map_or(Version::ZERO, |v| v.version());
let height_to_realized_cap = &mut self.0.all.height_to_realized_cap;
let mut height_to_realized_cap = self.0.all.height_to_realized_cap.as_mut();
let height_to_supply = &mut self.0.all.height_to_supply;
let height_to_unspendable_supply = &mut self.0.all.height_to_unspendable_supply;
let height_to_utxo_count = &mut self.0.all.height_to_utxo_count;
height_to_realized_cap.validate_computed_version_or_reset_file(
base_version + height_to_realized_cap.inner_version(),
)?;
height_to_supply.validate_computed_version_or_reset_file(
base_version + height_to_supply.inner_version(),
)?;
height_to_unspendable_supply.validate_computed_version_or_reset_file(
base_version + height_to_unspendable_supply.inner_version(),
)?;
height_to_utxo_count.validate_computed_version_or_reset_file(
base_version + height_to_utxo_count.inner_version(),
)?;
if let Some(height_to_realized_cap) = height_to_realized_cap.as_mut() {
height_to_realized_cap.validate_computed_version_or_reset_file(
base_version + height_to_realized_cap.inner_version(),
)?;
}
let starting_height = [
height_to_realized_cap.len(),
height_to_supply.len(),
height_to_unspendable_supply.len(),
height_to_utxo_count.len(),
height_to_realized_cap
.as_ref()
.map_or(usize::MAX, |v| v.len()),
]
.into_iter()
.map(Height::from)
@@ -148,34 +207,212 @@ impl Vecs {
.unwrap()
.min(starting_indexes.height);
let mut states = CohortStates::default();
let mut state = CohortState::default();
if let Some(prev_height) = starting_height.checked_sub(Height::new(1)) {
states.realized_cap = height_to_realized_cap
state.supply = height_to_supply.into_iter().unwrap_get_inner(prev_height);
state.unspendable_supply = height_to_unspendable_supply
.into_iter()
.unwrap_get_inner(prev_height);
states.supply = height_to_supply.into_iter().unwrap_get_inner(prev_height);
states.utxo_count = height_to_utxo_count
state.utxo_count = height_to_utxo_count
.into_iter()
.unwrap_get_inner(prev_height);
if let Some(height_to_realized_cap) = height_to_realized_cap.as_mut() {
state.realized_cap = height_to_realized_cap
.into_iter()
.unwrap_get_inner(prev_height);
}
}
(starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
.map(Height::from)
.try_for_each(|height| -> color_eyre::Result<()> {
let first_outputindex = height_to_first_outputindex_iter.unwrap_get_inner(height);
let first_inputindex = height_to_first_inputindex_iter.unwrap_get_inner(height);
let first_outputindex = height_to_first_outputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
let first_inputindex = height_to_first_inputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let opreturn_count = height_to_opreturn_count_iter.unwrap_get_inner(height);
let (sent_sats_dollars, (mut received_spendable, mut received_unspendable)) =
thread::scope(|s| {
// Skip coinbase
let sent_sats_dollars = s.spawn(|| {
(first_inputindex + 1..first_inputindex + *input_count)
.map(InputIndex::from)
.map(|inputindex| {
inputindex_to_outputindex_iter.unwrap_get_inner(inputindex)
})
.map(|outputindex| {
let value =
outputindex_to_value_iter.unwrap_get_inner(outputindex);
if let Some(height_to_close_iter) =
height_to_close_iter.as_mut()
{
let txindex = outputindex_to_txindex_iter
.unwrap_get_inner(outputindex);
let height =
txindex_to_height_iter.unwrap_get_inner(txindex);
let dollars =
*height_to_close_iter.unwrap_get_inner(height);
(value, dollars)
} else {
(value, Dollars::ZERO)
}
})
.collect::<Vec<_>>()
});
let received = s.spawn(|| {
let mut spendable = Sats::ZERO;
let mut unspendable = Sats::ZERO;
(first_outputindex..first_outputindex + *output_count)
.map(OutputIndex::from)
.for_each(|outputindex| {
let value =
outputindex_to_value_iter_2.unwrap_get_inner(outputindex);
if outputindex_to_outputtype_iter.unwrap_get_inner(outputindex)
== OutputType::OpReturn
{
unspendable += value
} else {
spendable += value
}
});
(spendable, unspendable)
});
(sent_sats_dollars.join().unwrap(), received.join().unwrap())
});
let (sent, realized_cap_destroyed) = sent_sats_dollars
.into_par_iter()
.map(|(sats, dollars)| (sats, dollars * Bitcoin::from(sats)))
.reduce(
|| (Sats::ZERO, Dollars::ZERO),
|acc, (sats, dollars)| (acc.0 + sats, acc.1 + dollars),
);
let utxos_created = *output_count - *opreturn_count;
// Three invalid coinbases which all have 1 output
let utxos_destroyed = if height == Height::new(0)
|| height == Height::new(91_842)
|| height == Height::new(91_880)
{
received_spendable -= Sats::FIFTY_BTC;
received_unspendable += Sats::FIFTY_BTC;
*input_count
} else {
*input_count - 1
};
state.supply -= sent;
state.supply += received_spendable;
state.unspendable_supply += received_unspendable;
*state.utxo_count += utxos_created;
*state.utxo_count -= utxos_destroyed;
if let Some(height_to_close_iter) = height_to_close_iter.as_mut() {
let received = received_spendable + received_unspendable;
let price = *height_to_close_iter.unwrap_get_inner(height);
let realized_cap_created = price * Bitcoin::from(received);
state.realized_cap = (state.realized_cap + realized_cap_created)
.checked_sub(realized_cap_destroyed)
.context("to work")
.inspect_err(|_| {
dbg!((
height,
state.realized_cap,
realized_cap_created,
realized_cap_destroyed
));
})
.unwrap();
}
height_to_supply.forced_push_at(height, state.supply, exit)?;
height_to_unspendable_supply.forced_push_at(
height,
state.unspendable_supply,
exit,
)?;
height_to_utxo_count.forced_push_at(height, state.utxo_count, exit)?;
if let Some(height_to_realized_cap) = height_to_realized_cap.as_mut() {
height_to_realized_cap.forced_push_at(height, state.realized_cap, exit)?;
}
Ok(())
})?;
height_to_supply.safe_flush(exit)?;
height_to_unspendable_supply.safe_flush(exit)?;
height_to_utxo_count.safe_flush(exit)?;
if let Some(height_to_realized_cap) = height_to_realized_cap.as_mut() {
height_to_realized_cap.safe_flush(exit)?;
}
self.0.all.indexes_to_supply.compute_rest(
indexer,
indexes,
fetched,
starting_indexes,
exit,
Some(&self.0.all.height_to_supply),
)?;
self.0.all.indexes_to_unspendable_supply.compute_rest(
indexer,
indexes,
fetched,
starting_indexes,
exit,
Some(&self.0.all.height_to_unspendable_supply),
)?;
self.0.all.indexes_to_utxo_count.compute_rest(
indexes,
starting_indexes,
exit,
Some(&self.0.all.height_to_utxo_count),
)?;
if let Some(indexes_to_realized_cap) = self.0.all.indexes_to_realized_cap.as_mut() {
indexes_to_realized_cap.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.0.all.height_to_realized_cap.as_ref().unwrap()),
)?;
}
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
// [].concat()
vec![]
[
vec![
&self.all.height_to_supply as &dyn AnyCollectableVec,
&self.all.height_to_unspendable_supply,
&self.all.height_to_utxo_count,
],
self.all
.height_to_realized_cap
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.all.indexes_to_supply.vecs(),
self.all.indexes_to_unspendable_supply.vecs(),
self.all.indexes_to_utxo_count.vecs(),
self.all
.indexes_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.concat()
}
}