computer: add more up to and from datasets

This commit is contained in:
nym21
2025-07-07 23:53:59 +02:00
parent bff22b5182
commit 8cc1f8d691
15 changed files with 1251 additions and 247 deletions

View File

@@ -186,9 +186,9 @@ impl Vecs {
)?,
},
by_up_to_size: GroupedByUpToSize {
_1_000sats: address_cohort::Vecs::forced_import(
_10sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_1_000sats"),
Some("addresses_up_to_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
@@ -196,9 +196,59 @@ impl Vecs {
states_path,
true,
)?,
_10_000sats: address_cohort::Vecs::forced_import(
_100sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_10_000sats"),
Some("addresses_up_to_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
@@ -236,11 +286,111 @@ impl Vecs {
states_path,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100k_btc: address_cohort::Vecs::forced_import(
path,
Some("addresses_up_to_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
},
by_from_size: GroupedByFromSize {
_1_000sats: address_cohort::Vecs::forced_import(
_1sat: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_1_000sats"),
Some("addresses_from_1sat"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
@@ -278,6 +428,26 @@ impl Vecs {
states_path,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
path,
Some("addresses_from_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
},
}
.into(),

View File

@@ -2,13 +2,13 @@ use brk_core::{GroupedByAddressType, Height};
use brk_vec::VecIterator;
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_addresscount_vec::AddressTypeToAddressCountVec;
use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(GroupedByAddressType<usize>);
impl From<(&AddressTypeToAddressCountVec, Height)> for AddressTypeToAddressCount {
fn from((groups, starting_height): (&AddressTypeToAddressCountVec, Height)) -> Self {
impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddressCount {
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(GroupedByAddressType {
p2pk65: groups

View File

@@ -6,15 +6,17 @@ use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct AddressTypeToAddressCountVec(GroupedByAddressType<EagerVec<Height, StoredUsize>>);
pub struct AddressTypeToHeightToAddressCount(GroupedByAddressType<EagerVec<Height, StoredUsize>>);
impl From<GroupedByAddressType<EagerVec<Height, StoredUsize>>> for AddressTypeToAddressCountVec {
impl From<GroupedByAddressType<EagerVec<Height, StoredUsize>>>
for AddressTypeToHeightToAddressCount
{
fn from(value: GroupedByAddressType<EagerVec<Height, StoredUsize>>) -> Self {
Self(value)
}
}
impl AddressTypeToAddressCountVec {
impl AddressTypeToHeightToAddressCount {
pub fn forced_push_at(
&mut self,
height: Height,

View File

@@ -0,0 +1,91 @@
use brk_core::{GroupedByAddressType, StoredUsize};
use brk_exit::Exit;
use brk_vec::AnyCollectableVec;
use derive_deref::{Deref, DerefMut};
use crate::vecs::{
Indexes, grouped::ComputedVecsFromHeight, indexes,
stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
};
#[derive(Clone, Deref, DerefMut)]
pub struct AddressTypeToIndexesToAddressCount(
GroupedByAddressType<ComputedVecsFromHeight<StoredUsize>>,
);
impl From<GroupedByAddressType<ComputedVecsFromHeight<StoredUsize>>>
for AddressTypeToIndexesToAddressCount
{
fn from(value: GroupedByAddressType<ComputedVecsFromHeight<StoredUsize>>) -> Self {
Self(value)
}
}
impl AddressTypeToIndexesToAddressCount {
pub fn compute(
&mut self,
// height: Height,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
addresstype_to_height_to_addresscount: &AddressTypeToHeightToAddressCount,
) -> color_eyre::Result<()> {
self.p2pk65.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk65),
)?;
self.p2pk33.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pk33),
)?;
self.p2pkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2pkh),
)?;
self.p2sh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2sh),
)?;
self.p2wpkh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wpkh),
)?;
self.p2wsh.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2wsh),
)?;
self.p2tr.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2tr),
)?;
self.p2a.compute_rest(
indexes,
starting_indexes,
exit,
Some(&addresstype_to_height_to_addresscount.p2a),
)?;
Ok(())
}
//
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
self.0
.as_typed_vec()
.into_iter()
.flat_map(|(_, v)| v.vecs())
.collect::<Vec<_>>()
}
}

View File

@@ -21,7 +21,9 @@ use crate::{
market,
stateful::{
addresstype_to_addresscount::AddressTypeToAddressCount,
addresstype_to_addresscount_vec::AddressTypeToAddressCountVec, r#trait::DynCohortVecs,
addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount,
r#trait::DynCohortVecs,
},
},
};
@@ -35,7 +37,8 @@ use super::{
mod address_cohort;
mod address_cohorts;
mod addresstype_to_addresscount;
mod addresstype_to_addresscount_vec;
mod addresstype_to_height_to_addresscount;
mod addresstype_to_indexes_to_addresscount;
mod addresstype_to_typeindex_tree;
mod addresstype_to_typeindex_vec;
mod common;
@@ -59,8 +62,10 @@ pub struct Vecs {
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
pub height_to_opreturn_supply: EagerVec<Height, Sats>,
pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight,
pub addresstype_to_height_to_address_count: AddressTypeToAddressCountVec,
pub addresstype_to_height_to_empty_address_count: AddressTypeToAddressCountVec,
pub addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount,
pub addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount,
pub addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount,
pub addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount,
pub utxo_vecs: utxo_cohorts::Vecs,
pub address_vecs: address_cohorts::Vecs,
@@ -137,7 +142,7 @@ impl Vecs {
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
addresstype_to_height_to_address_count: AddressTypeToAddressCountVec::from(
addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from(
GroupedByAddressType {
p2pk65: EagerVec::forced_import(
path,
@@ -189,7 +194,7 @@ impl Vecs {
)?,
},
),
addresstype_to_height_to_empty_address_count: AddressTypeToAddressCountVec::from(
addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from(
GroupedByAddressType {
p2pk65: EagerVec::forced_import(
path,
@@ -241,6 +246,142 @@ impl Vecs {
)?,
},
),
addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from(
GroupedByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
path,
"p2pk65_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
path,
"p2pk33_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
path,
"p2pkh_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
path,
"p2sh_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
path,
"p2wpkh_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
path,
"p2wsh_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
path,
"p2tr_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
path,
"p2a_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
},
),
addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from(
GroupedByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
path,
"p2pk65_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
path,
"p2pk33_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
path,
"p2pkh_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
path,
"p2sh_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
path,
"p2wpkh_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
path,
"p2wsh_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
path,
"p2tr_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
path,
"p2a_empty_address_count",
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
},
),
utxo_vecs: utxo_cohorts::Vecs::forced_import(
path,
version,
@@ -1084,6 +1225,19 @@ impl Vecs {
},
)?;
self.addresstype_to_indexes_to_address_count.compute(
indexes,
starting_indexes,
exit,
&self.addresstype_to_height_to_address_count,
)?;
self.addresstype_to_indexes_to_empty_address_count.compute(
indexes,
starting_indexes,
exit,
&self.addresstype_to_height_to_empty_address_count,
)?;
thread::scope(|scope| {
scope.spawn(|| {
self.utxo_vecs
@@ -1241,6 +1395,8 @@ impl Vecs {
self.indexes_to_opreturn_supply.vecs(),
self.indexes_to_address_count.vecs(),
self.indexes_to_empty_address_count.vecs(),
self.addresstype_to_indexes_to_address_count.vecs(),
self.addresstype_to_indexes_to_empty_address_count.vecs(),
self.addresstype_to_height_to_address_count
.as_typed_vec()
.into_iter()

View File

@@ -827,9 +827,9 @@ impl Vecs {
)?,
},
by_up_to_size: GroupedByUpToSize {
_1_000sats: utxo_cohort::Vecs::forced_import(
_10sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_1_000sats"),
Some("up_to_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
@@ -837,9 +837,59 @@ impl Vecs {
states_path,
true,
)?,
_10_000sats: utxo_cohort::Vecs::forced_import(
_100sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_10_000sats"),
Some("up_to_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1k_sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100k_sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1m_sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10m_sats: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
@@ -877,11 +927,111 @@ impl Vecs {
states_path,
true,
)?,
_1k_btc: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_btc: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100k_btc: utxo_cohort::Vecs::forced_import(
path,
Some("up_to_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
},
by_from_size: GroupedByFromSize {
_1_000sats: utxo_cohort::Vecs::forced_import(
_1sat: utxo_cohort::Vecs::forced_import(
path,
Some("from_1_000sats"),
Some("from_1sat"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1k_sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_100k_sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_1m_sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10m_sats: utxo_cohort::Vecs::forced_import(
path,
Some("from_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
@@ -919,6 +1069,26 @@ impl Vecs {
states_path,
true,
)?,
_1k_btc: utxo_cohort::Vecs::forced_import(
path,
Some("from_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
_10k_btc: utxo_cohort::Vecs::forced_import(
path,
Some("from_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
fetched,
states_path,
true,
)?,
},
by_type: GroupedBySpendableType {
p2pk65: utxo_cohort::Vecs::forced_import(