global: MASSIVE snapshot

This commit is contained in:
nym21
2026-02-23 17:22:12 +01:00
parent be0d749f9c
commit 3b7aa8242a
703 changed files with 29130 additions and 30779 deletions

View File

@@ -2,16 +2,17 @@ use brk_error::Result;
use brk_indexer::Indexer;
use vecdb::Exit;
use crate::{indexes, inputs, outputs, ComputeIndexes};
use crate::{blocks, indexes, inputs, outputs, ComputeIndexes};
use super::Vecs;
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn compute(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
inputs: &inputs::Vecs,
outputs: &outputs::Vecs,
starting_indexes: &ComputeIndexes,
@@ -19,11 +20,11 @@ impl Vecs {
) -> Result<()> {
// Count computes first
self.count
.compute(indexer, indexes, starting_indexes, exit)?;
.compute(indexer, starting_indexes, exit)?;
// Versions depends on count
self.versions
.compute(indexer, indexes, starting_indexes, exit)?;
.compute(indexer, starting_indexes, exit)?;
// Size computes next
self.size
@@ -39,10 +40,11 @@ impl Vecs {
exit,
)?;
// Volume depends on fees and input/output counts
// Volume depends on fees, counts, and blocks (lookback vecs, interval)
self.volume.compute(
indexer,
indexes,
blocks,
&self.count,
&self.fees,
&inputs.count,

View File

@@ -3,18 +3,17 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{ComputeIndexes, indexes};
use crate::ComputeIndexes;
impl Vecs {
pub fn compute(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.tx_count
.compute_all(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_count_from_indexes(
starting_indexes.height,
&indexer.vecs.transactions.first_txindex,

View File

@@ -1,13 +1,13 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{StoredBool, TxIndex, Version};
use vecdb::{Database, IterableCloneableVec, LazyVecFrom2};
use vecdb::{Database, ReadableCloneableVec, LazyVecFrom2};
use super::Vecs;
use crate::{indexes, internal::ComputedFromHeightFull};
impl Vecs {
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexer: &Indexer,
@@ -16,13 +16,10 @@ impl Vecs {
let txindex_to_is_coinbase = LazyVecFrom2::init(
"is_coinbase",
version,
indexer.vecs.transactions.height.boxed_clone(),
indexer.vecs.transactions.first_txindex.boxed_clone(),
|index: TxIndex, txindex_to_height_iter, height_to_first_txindex_iter| {
txindex_to_height_iter.get(index).map(|height| {
let txindex = height_to_first_txindex_iter.get_unwrap(height);
StoredBool::from(index == txindex)
})
indexer.vecs.transactions.height.read_only_boxed_clone(),
indexer.vecs.transactions.first_txindex.read_only_boxed_clone(),
|index: TxIndex, _height, first_txindex| {
StoredBool::from(index == first_txindex)
},
);

View File

@@ -1,11 +1,11 @@
use brk_traversable::Traversable;
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
use vecdb::LazyVecFrom2;
use vecdb::{LazyVecFrom2, Rw, StorageMode};
use crate::internal::ComputedFromHeightFull;
#[derive(Clone, Traversable)]
pub struct Vecs {
pub tx_count: ComputedFromHeightFull<StoredU64>,
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub tx_count: ComputedFromHeightFull<StoredU64, M>,
pub is_coinbase: LazyVecFrom2<TxIndex, StoredBool, TxIndex, Height, Height, TxIndex>,
}

View File

@@ -9,7 +9,7 @@ use crate::{ComputeIndexes, indexes, inputs};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn compute(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,

View File

@@ -4,24 +4,28 @@ use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::Vecs;
use crate::{indexes, internal::{ComputedFromTxDistribution, ValueFromTxFull}, price};
use crate::{
indexes,
internal::{ComputedFromTxDistribution, ValueFromTxFull},
prices,
};
/// Bump this when fee/feerate aggregation logic changes (e.g., skip coinbase).
const VERSION: Version = Version::ONE;
impl Vecs {
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
prices: &prices::Vecs,
) -> Result<Self> {
let v = version + VERSION;
Ok(Self {
input_value: EagerVec::forced_import(db, "input_value", version)?,
output_value: EagerVec::forced_import(db, "output_value", version)?,
fee: ValueFromTxFull::forced_import(db, "fee", v, indexes, indexer, price)?,
fee: ValueFromTxFull::forced_import(db, "fee", v, indexes, indexer, prices)?,
fee_rate: ComputedFromTxDistribution::forced_import(db, "fee_rate", v, indexes)?,
})
}

View File

@@ -1,13 +1,13 @@
use brk_traversable::Traversable;
use brk_types::{FeeRate, Sats, TxIndex};
use vecdb::{EagerVec, PcoVec};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
use crate::internal::{ComputedFromTxDistribution, ValueFromTxFull};
#[derive(Clone, Traversable)]
pub struct Vecs {
pub input_value: EagerVec<PcoVec<TxIndex, Sats>>,
pub output_value: EagerVec<PcoVec<TxIndex, Sats>>,
pub fee: ValueFromTxFull,
pub fee_rate: ComputedFromTxDistribution<FeeRate>,
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub input_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub output_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub fee: ValueFromTxFull<M>,
pub fee_rate: ComputedFromTxDistribution<FeeRate, M>,
}

View File

@@ -6,17 +6,17 @@ use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{Database, PAGE_SIZE};
use crate::{indexes, price};
use crate::{indexes, prices};
use super::{CountVecs, FeesVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs};
impl Vecs {
pub fn forced_import(
pub(crate) fn forced_import(
parent_path: &Path,
parent_version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
prices: &prices::Vecs,
) -> Result<Self> {
let db = Database::open(&parent_path.join(super::DB_NAME))?;
db.set_min_len(PAGE_SIZE * 50_000_000)?;
@@ -25,9 +25,9 @@ impl Vecs {
let count = CountVecs::forced_import(&db, version, indexer, indexes)?;
let size = SizeVecs::forced_import(&db, version, indexer, indexes)?;
let fees = FeesVecs::forced_import(&db, version, indexer, indexes, price)?;
let fees = FeesVecs::forced_import(&db, version, indexer, indexes, prices)?;
let versions = VersionsVecs::forced_import(&db, version, indexes)?;
let volume = VolumeVecs::forced_import(&db, version, indexes, price)?;
let volume = VolumeVecs::forced_import(&db, version, indexes, prices)?;
let this = Self {
db,

View File

@@ -8,7 +8,7 @@ mod compute;
mod import;
use brk_traversable::Traversable;
use vecdb::Database;
use vecdb::{Database, Rw, StorageMode};
pub use count::Vecs as CountVecs;
pub use fees::Vecs as FeesVecs;
@@ -18,14 +18,14 @@ pub use volume::Vecs as VolumeVecs;
pub const DB_NAME: &str = "transactions";
#[derive(Clone, Traversable)]
pub struct Vecs {
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
#[traversable(skip)]
pub(crate) db: Database,
pub count: CountVecs,
pub size: SizeVecs,
pub fees: FeesVecs,
pub versions: VersionsVecs,
pub volume: VolumeVecs,
pub count: CountVecs<M>,
pub size: SizeVecs<M>,
pub fees: FeesVecs<M>,
pub versions: VersionsVecs<M>,
pub volume: VolumeVecs<M>,
}

View File

@@ -6,7 +6,7 @@ use super::Vecs;
use crate::{indexes, ComputeIndexes};
impl Vecs {
pub fn compute(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,

View File

@@ -1,13 +1,13 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{TxIndex, VSize, Version, Weight};
use vecdb::{Database, IterableCloneableVec, LazyVecFrom2, VecIndex};
use vecdb::{Database, ReadableCloneableVec, LazyVecFrom2};
use super::Vecs;
use crate::{indexes, internal::LazyFromTxDistribution};
impl Vecs {
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexer: &Indexer,
@@ -16,28 +16,20 @@ impl Vecs {
let txindex_to_weight = LazyVecFrom2::init(
"tx_weight",
version,
indexer.vecs.transactions.base_size.boxed_clone(),
indexer.vecs.transactions.total_size.boxed_clone(),
|index: TxIndex, base_size_iter, total_size_iter| {
let index = index.to_usize();
base_size_iter.get_at(index).map(|base_size| {
let total_size = total_size_iter.get_at_unwrap(index);
Weight::from_sizes(*base_size, *total_size)
})
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|_index: TxIndex, base_size, total_size| {
Weight::from_sizes(*base_size, *total_size)
},
);
let txindex_to_vsize = LazyVecFrom2::init(
"tx_vsize",
version,
indexer.vecs.transactions.base_size.boxed_clone(),
indexer.vecs.transactions.total_size.boxed_clone(),
|index: TxIndex, base_size_iter, total_size_iter| {
let index = index.to_usize();
base_size_iter.get_at(index).map(|base_size| {
let total_size = total_size_iter.get_at_unwrap(index);
VSize::from(Weight::from_sizes(*base_size, *total_size))
})
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|_index: TxIndex, base_size, total_size| {
VSize::from(Weight::from_sizes(*base_size, *total_size))
},
);

View File

@@ -1,10 +1,11 @@
use brk_traversable::Traversable;
use brk_types::{StoredU32, VSize, Weight};
use vecdb::{Rw, StorageMode};
use crate::internal::LazyFromTxDistribution;
#[derive(Clone, Traversable)]
pub struct Vecs {
pub vsize: LazyFromTxDistribution<VSize, StoredU32, StoredU32>,
pub weight: LazyFromTxDistribution<Weight, StoredU32, StoredU32>,
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub vsize: LazyFromTxDistribution<VSize, StoredU32, StoredU32, M>,
pub weight: LazyFromTxDistribution<Weight, StoredU32, StoredU32, M>,
}

View File

@@ -1,29 +1,32 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{StoredU64, TxVersion};
use vecdb::{Exit, TypedVecIterator};
use vecdb::{Exit, ReadableVec, VecIndex};
use super::Vecs;
use crate::{ComputeIndexes, indexes, internal::ComputedFromHeightSumCum};
use crate::{ComputeIndexes, internal::ComputedFromHeightSumCum};
impl Vecs {
pub fn compute(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
let tx_vany = |tx_vany: &mut ComputedFromHeightSumCum<StoredU64>, txversion: TxVersion| {
let mut txversion_iter = indexer.vecs.transactions.txversion.iter()?;
tx_vany.compute_all(indexes, starting_indexes, exit, |vec| {
let txversion_vec = &indexer.vecs.transactions.txversion;
// Cursor avoids per-transaction PcoVec page decompression.
// Txindex values are sequential, so the cursor only advances forward.
let mut cursor = txversion_vec.cursor();
tx_vany.compute(starting_indexes, exit, |vec| {
vec.compute_filtered_count_from_indexes(
starting_indexes.height,
&indexer.vecs.transactions.first_txindex,
&indexer.vecs.transactions.txid,
|txindex| {
let v = txversion_iter.get_unwrap(txindex);
v == txversion
let ti = txindex.to_usize();
cursor.advance(ti - cursor.position());
cursor.next().unwrap() == txversion
},
exit,
)?;

View File

@@ -6,7 +6,7 @@ use super::Vecs;
use crate::{indexes, internal::ComputedFromHeightSumCum};
impl Vecs {
pub fn forced_import(db: &Database, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
pub(crate) fn forced_import(db: &Database, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
Ok(Self {
v1: ComputedFromHeightSumCum::forced_import(db, "tx_v1", version, indexes)?,
v2: ComputedFromHeightSumCum::forced_import(db, "tx_v2", version, indexes)?,

View File

@@ -1,11 +1,12 @@
use brk_traversable::Traversable;
use brk_types::StoredU64;
use vecdb::{Rw, StorageMode};
use crate::internal::ComputedFromHeightSumCum;
#[derive(Clone, Traversable)]
pub struct Vecs {
pub v1: ComputedFromHeightSumCum<StoredU64>,
pub v2: ComputedFromHeightSumCum<StoredU64>,
pub v3: ComputedFromHeightSumCum<StoredU64>,
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub v1: ComputedFromHeightSumCum<StoredU64, M>,
pub v2: ComputedFromHeightSumCum<StoredU64, M>,
pub v3: ComputedFromHeightSumCum<StoredU64, M>,
}

View File

@@ -1,18 +1,19 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{ONE_DAY_IN_SEC_F64, StoredF32};
use brk_types::StoredF32;
use vecdb::Exit;
use super::super::{count, fees};
use super::Vecs;
use crate::{ComputeIndexes, indexes, inputs, outputs};
use crate::{blocks, ComputeIndexes, indexes, inputs, outputs};
use crate::transactions::{count, fees};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn compute(
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
count_vecs: &count::Vecs,
fees_vecs: &fees::Vecs,
inputs_count: &inputs::CountVecs,
@@ -20,115 +21,81 @@ impl Vecs {
starting_indexes: &ComputeIndexes,
exit: &Exit,
) -> Result<()> {
self.sent_sum
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_filtered_sum_from_indexes(
starting_indexes.height,
&indexer.vecs.transactions.first_txindex,
&indexes.height.txindex_count,
&fees_vecs.input_value,
|sats| !sats.is_max(),
exit,
)?;
Ok(())
})?;
self.sent_sum.sats.height.compute_filtered_sum_from_indexes(
starting_indexes.height,
&indexer.vecs.transactions.first_txindex,
&indexes.height.txindex_count,
&fees_vecs.input_value,
|sats| !sats.is_max(),
exit,
)?;
self.received_sum
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_sum_from_indexes(
starting_indexes.height,
&indexer.vecs.transactions.first_txindex,
&indexes.height.txindex_count,
&fees_vecs.output_value,
exit,
)?;
Ok(())
})?;
self.received_sum.sats.height.compute_sum_from_indexes(
starting_indexes.height,
&indexer.vecs.transactions.first_txindex,
&indexes.height.txindex_count,
&fees_vecs.output_value,
exit,
)?;
self.annualized_volume.compute_sats(|v| {
v.compute_sum(
starting_indexes.dateindex,
&self.sent_sum.sats.dateindex.0,
365,
exit,
)?;
Ok(())
})?;
// Annualized volume: rolling 1y sum of per-block sent volume
self.annualized_volume.sats.height.compute_rolling_sum(
starting_indexes.height,
&blocks.count.height_1y_ago,
&self.sent_sum.sats.height,
exit,
)?;
if let Some(sent_sum_dollars) = self.sent_sum.dollars.as_ref() {
self.annualized_volume.compute_dollars(|dollars| {
dollars.compute_all(starting_indexes, exit, |v| {
v.compute_sum(
starting_indexes.dateindex,
&sent_sum_dollars.dateindex.0,
365,
exit,
)?;
Ok(())
})
})?;
}
// tx_per_sec: per-block tx count / block interval
self.tx_per_sec.height.compute_transform2(
starting_indexes.height,
&count_vecs.tx_count.height,
&blocks.interval.interval.height,
|(h, tx_count, interval, ..)| {
let interval_f64 = f64::from(*interval);
let per_sec = if interval_f64 > 0.0 {
StoredF32::from(*tx_count as f64 / interval_f64)
} else {
StoredF32::NAN
};
(h, per_sec)
},
exit,
)?;
self.tx_per_sec.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
&count_vecs.tx_count.dateindex.sum_cum.sum.0,
&indexes.dateindex.date,
|(i, tx_count, date, ..)| {
let completion = date.completion();
let per_sec = if completion == 0.0 {
StoredF32::NAN
} else {
StoredF32::from(*tx_count as f64 / (completion * ONE_DAY_IN_SEC_F64))
};
(i, per_sec)
},
exit,
)?;
Ok(())
})?;
// inputs_per_sec: per-block input count / block interval
self.inputs_per_sec.height.compute_transform2(
starting_indexes.height,
&inputs_count.height.sum_cum.sum.0,
&blocks.interval.interval.height,
|(h, input_count, interval, ..)| {
let interval_f64 = f64::from(*interval);
let per_sec = if interval_f64 > 0.0 {
StoredF32::from(*input_count as f64 / interval_f64)
} else {
StoredF32::NAN
};
(h, per_sec)
},
exit,
)?;
self.inputs_per_sec
.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
&inputs_count.dateindex.sum_cum.sum.0,
&indexes.dateindex.date,
|(i, input_count, date, ..)| {
let completion = date.completion();
let per_sec = if completion == 0.0 {
StoredF32::NAN
} else {
StoredF32::from(*input_count as f64 / (completion * ONE_DAY_IN_SEC_F64))
};
(i, per_sec)
},
exit,
)?;
Ok(())
})?;
self.outputs_per_sec
.compute_all(starting_indexes, exit, |v| {
v.compute_transform2(
starting_indexes.dateindex,
&outputs_count.total_count.dateindex.sum_cum.sum.0,
&indexes.dateindex.date,
|(i, output_count, date, ..)| {
let completion = date.completion();
let per_sec = if completion == 0.0 {
StoredF32::NAN
} else {
StoredF32::from(
*output_count as f64 / (completion * ONE_DAY_IN_SEC_F64),
)
};
(i, per_sec)
},
exit,
)?;
Ok(())
})?;
// outputs_per_sec: per-block output count / block interval
self.outputs_per_sec.height.compute_transform2(
starting_indexes.height,
&outputs_count.total_count.height.sum_cum.sum.0,
&blocks.interval.interval.height,
|(h, output_count, interval, ..)| {
let interval_f64 = f64::from(*interval);
let per_sec = if interval_f64 > 0.0 {
StoredF32::from(*output_count as f64 / interval_f64)
} else {
StoredF32::NAN
};
(h, per_sec)
},
exit,
)?;
Ok(())
}

View File

@@ -3,48 +3,49 @@ use brk_types::Version;
use vecdb::Database;
use super::Vecs;
use crate::{indexes, internal::{ComputedFromDateLast, ValueFromHeightSum, ValueFromDateLast}, price};
use crate::{
indexes,
internal::{ComputedFromHeightLast, ValueFromHeightLast, ValueFromHeightSum},
prices,
};
impl Vecs {
pub fn forced_import(
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
prices: &prices::Vecs,
) -> Result<Self> {
let v2 = Version::TWO;
let compute_dollars = price.is_some();
Ok(Self {
sent_sum: ValueFromHeightSum::forced_import(
db,
"sent_sum",
version,
indexes,
price,
)?,
sent_sum: ValueFromHeightSum::forced_import(db, "sent_sum", version, indexes, prices)?,
received_sum: ValueFromHeightSum::forced_import(
db,
"received_sum",
version,
indexes,
price,
prices,
)?,
annualized_volume: ValueFromDateLast::forced_import(
annualized_volume: ValueFromHeightLast::forced_import(
db,
"annualized_volume",
version,
compute_dollars,
indexes,
prices,
)?,
tx_per_sec: ComputedFromHeightLast::forced_import(
db,
"tx_per_sec",
version + v2,
indexes,
)?,
tx_per_sec: ComputedFromDateLast::forced_import(db, "tx_per_sec", version + v2, indexes)?,
outputs_per_sec: ComputedFromDateLast::forced_import(
outputs_per_sec: ComputedFromHeightLast::forced_import(
db,
"outputs_per_sec",
version + v2,
indexes,
)?,
inputs_per_sec: ComputedFromDateLast::forced_import(
inputs_per_sec: ComputedFromHeightLast::forced_import(
db,
"inputs_per_sec",
version + v2,

View File

@@ -1,15 +1,16 @@
use brk_traversable::Traversable;
use brk_types::StoredF32;
use vecdb::{Rw, StorageMode};
use crate::internal::{ComputedFromDateLast, ValueFromHeightSum, ValueFromDateLast};
use crate::internal::{ComputedFromHeightLast, ValueFromHeightLast, ValueFromHeightSum};
/// Volume metrics
#[derive(Clone, Traversable)]
pub struct Vecs {
pub sent_sum: ValueFromHeightSum,
pub received_sum: ValueFromHeightSum,
pub annualized_volume: ValueFromDateLast,
pub tx_per_sec: ComputedFromDateLast<StoredF32>,
pub outputs_per_sec: ComputedFromDateLast<StoredF32>,
pub inputs_per_sec: ComputedFromDateLast<StoredF32>,
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub sent_sum: ValueFromHeightSum<M>,
pub received_sum: ValueFromHeightSum<M>,
pub annualized_volume: ValueFromHeightLast<M>,
pub tx_per_sec: ComputedFromHeightLast<StoredF32, M>,
pub outputs_per_sec: ComputedFromHeightLast<StoredF32, M>,
pub inputs_per_sec: ComputedFromHeightLast<StoredF32, M>,
}