global: MASSIVE snapshot

crates/brk_computer/src/scripts/compute.rs

@@ -0,0 +1,28 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use vecdb::Exit;
+
+use crate::{indexes, price, ComputeIndexes};
+
+use super::Vecs;
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        price: Option<&price::Vecs>,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.count
+            .compute(indexer, indexes, starting_indexes, exit)?;
+
+        self.value
+            .compute(indexer, indexes, price, starting_indexes, exit)?;
+
+        let _lock = exit.lock();
+        self.db.compact()?;
+        Ok(())
+    }
+}

crates/brk_computer/src/scripts/count/compute.rs

@@ -0,0 +1,169 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_types::StoredU64;
+use vecdb::{Exit, TypedVecIterator};
+
+use super::Vecs;
+use crate::{indexes, utils::OptionExt, ComputeIndexes};
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.indexes_to_p2a_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2aaddressindex,
+                    &indexer.vecs.address.p2aaddressindex_to_p2abytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2ms_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.output.height_to_first_p2msoutputindex,
+                    &indexer.vecs.output.p2msoutputindex_to_txindex,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2pk33_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2pk33addressindex,
+                    &indexer.vecs.address.p2pk33addressindex_to_p2pk33bytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2pk65_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2pk65addressindex,
+                    &indexer.vecs.address.p2pk65addressindex_to_p2pk65bytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2pkh_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2pkhaddressindex,
+                    &indexer.vecs.address.p2pkhaddressindex_to_p2pkhbytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2sh_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2shaddressindex,
+                    &indexer.vecs.address.p2shaddressindex_to_p2shbytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2tr_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2traddressindex,
+                    &indexer.vecs.address.p2traddressindex_to_p2trbytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2wpkh_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2wpkhaddressindex,
+                    &indexer.vecs.address.p2wpkhaddressindex_to_p2wpkhbytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_p2wsh_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.address.height_to_first_p2wshaddressindex,
+                    &indexer.vecs.address.p2wshaddressindex_to_p2wshbytes,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_opreturn_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.output.height_to_first_opreturnindex,
+                    &indexer.vecs.output.opreturnindex_to_txindex,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_unknownoutput_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.output.height_to_first_unknownoutputindex,
+                    &indexer.vecs.output.unknownoutputindex_to_txindex,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        self.indexes_to_emptyoutput_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    &indexer.vecs.output.height_to_first_emptyoutputindex,
+                    &indexer.vecs.output.emptyoutputindex_to_txindex,
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        // Compute segwit_count = p2wpkh + p2wsh + p2tr
+        let mut p2wsh_iter = self.indexes_to_p2wsh_count.height.u().into_iter();
+        let mut p2tr_iter = self.indexes_to_p2tr_count.height.u().into_iter();
+
+        self.indexes_to_segwit_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.indexes_to_p2wpkh_count.height.u(),
+                    |(h, p2wpkh, ..)| {
+                        let sum = *p2wpkh + *p2wsh_iter.get_unwrap(h) + *p2tr_iter.get_unwrap(h);
+                        (h, StoredU64::from(sum))
+                    },
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        Ok(())
+    }
+}

crates/brk_computer/src/scripts/count/import.rs

@@ -0,0 +1,177 @@
+use brk_error::Result;
+use brk_types::Version;
+use vecdb::{Database, IterableCloneableVec};
+
+use super::Vecs;
+use crate::{
+    indexes,
+    internal::{
+        ComputedVecsFromHeight, LazyVecsFrom2FromHeight, PercentageU64F32, Source,
+        VecBuilderOptions,
+    },
+    outputs,
+    utils::OptionExt,
+};
+
+impl Vecs {
+    pub fn forced_import(
+        db: &Database,
+        version: Version,
+        indexes: &indexes::Vecs,
+        outputs: &outputs::Vecs,
+    ) -> Result<Self> {
+        let v0 = Version::ZERO;
+        let full_stats = || {
+            VecBuilderOptions::default()
+                .add_average()
+                .add_minmax()
+                .add_percentiles()
+                .add_sum()
+                .add_cumulative()
+        };
+
+        let indexes_to_p2a_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2a_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2ms_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2ms_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2pk33_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2pk33_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2pk65_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2pk65_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2pkh_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2pkh_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2sh_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2sh_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2tr_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2tr_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2wpkh_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2wpkh_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+        let indexes_to_p2wsh_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "p2wsh_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+
+        // Aggregate counts (computed from per-type counts)
+        let indexes_to_segwit_count = ComputedVecsFromHeight::forced_import(
+            db,
+            "segwit_count",
+            Source::Compute,
+            version + v0,
+            indexes,
+            full_stats(),
+        )?;
+
+        // Adoption ratios (lazy)
+        // Uses outputs.count.indexes_to_count as denominator (total output count)
+        // At height level: per-block ratio; at dateindex level: sum-based ratio (% of new outputs)
+        let indexes_to_taproot_adoption =
+            LazyVecsFrom2FromHeight::from_height_and_txindex::<PercentageU64F32>(
+                "taproot_adoption",
+                version + v0,
+                indexes_to_p2tr_count.height.u().boxed_clone(),
+                outputs.count.indexes_to_count.height.sum.u().boxed_clone(),
+                &indexes_to_p2tr_count,
+                &outputs.count.indexes_to_count,
+            );
+        let indexes_to_segwit_adoption =
+            LazyVecsFrom2FromHeight::from_height_and_txindex::<PercentageU64F32>(
+                "segwit_adoption",
+                version + v0,
+                indexes_to_segwit_count.height.u().boxed_clone(),
+                outputs.count.indexes_to_count.height.sum.u().boxed_clone(),
+                &indexes_to_segwit_count,
+                &outputs.count.indexes_to_count,
+            );
+
+        Ok(Self {
+            indexes_to_p2a_count,
+            indexes_to_p2ms_count,
+            indexes_to_p2pk33_count,
+            indexes_to_p2pk65_count,
+            indexes_to_p2pkh_count,
+            indexes_to_p2sh_count,
+            indexes_to_p2tr_count,
+            indexes_to_p2wpkh_count,
+            indexes_to_p2wsh_count,
+            indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
+                db,
+                "opreturn_count",
+                Source::Compute,
+                version + v0,
+                indexes,
+                full_stats(),
+            )?,
+            indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
+                db,
+                "emptyoutput_count",
+                Source::Compute,
+                version + v0,
+                indexes,
+                full_stats(),
+            )?,
+            indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
+                db,
+                "unknownoutput_count",
+                Source::Compute,
+                version + v0,
+                indexes,
+                full_stats(),
+            )?,
+            indexes_to_segwit_count,
+            indexes_to_taproot_adoption,
+            indexes_to_segwit_adoption,
+        })
+    }
+}

crates/brk_computer/src/scripts/count/mod.rs

@@ -0,0 +1,5 @@
+mod compute;
+mod import;
+mod vecs;
+
+pub use vecs::Vecs;

crates/brk_computer/src/scripts/count/vecs.rs

@@ -0,0 +1,32 @@
+use brk_traversable::Traversable;
+use brk_types::{StoredF32, StoredU64};
+
+use crate::internal::{ComputedVecsFromHeight, LazyVecsFrom2FromHeight};
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    // Per-type output counts
+    pub indexes_to_p2a_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2ms_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2pk33_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2pk65_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2pkh_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2sh_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2tr_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_opreturn_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_emptyoutput_count: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU64>,
+
+    // Aggregate counts
+    /// SegWit output count (p2wpkh + p2wsh + p2tr)
+    pub indexes_to_segwit_count: ComputedVecsFromHeight<StoredU64>,
+
+    // Adoption ratios (lazy)
+    // Denominator is outputs.count.indexes_to_count (total output count)
+    /// Taproot adoption: p2tr / total_outputs * 100
+    pub indexes_to_taproot_adoption: LazyVecsFrom2FromHeight<StoredF32, StoredU64, StoredU64>,
+    /// SegWit adoption: segwit / total_outputs * 100
+    pub indexes_to_segwit_adoption: LazyVecsFrom2FromHeight<StoredF32, StoredU64, StoredU64>,
+}

crates/brk_computer/src/scripts/import.rs

@@ -0,0 +1,40 @@
+use std::path::Path;
+
+use brk_error::Result;
+use brk_traversable::Traversable;
+use brk_types::Version;
+use vecdb::{Database, PAGE_SIZE};
+
+use crate::{indexes, outputs, price};
+
+use super::{CountVecs, ValueVecs, Vecs};
+
+impl Vecs {
+    pub fn forced_import(
+        parent_path: &Path,
+        parent_version: Version,
+        indexes: &indexes::Vecs,
+        price: Option<&price::Vecs>,
+        outputs: &outputs::Vecs,
+    ) -> Result<Self> {
+        let db = Database::open(&parent_path.join(super::DB_NAME))?;
+        db.set_min_len(PAGE_SIZE * 50_000_000)?;
+
+        let version = parent_version + Version::ZERO;
+        let compute_dollars = price.is_some();
+
+        let count = CountVecs::forced_import(&db, version, indexes, outputs)?;
+        let value = ValueVecs::forced_import(&db, version, indexes, compute_dollars)?;
+
+        let this = Self { db, count, value };
+
+        this.db.retain_regions(
+            this.iter_any_exportable()
+                .flat_map(|v| v.region_names())
+                .collect(),
+        )?;
+        this.db.compact()?;
+
+        Ok(this)
+    }
+}

crates/brk_computer/src/scripts/mod.rs

@@ -0,0 +1,22 @@
+pub mod count;
+pub mod value;
+
+mod compute;
+mod import;
+
+use brk_traversable::Traversable;
+use vecdb::Database;
+
+pub use count::Vecs as CountVecs;
+pub use value::Vecs as ValueVecs;
+
+pub const DB_NAME: &str = "scripts";
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    #[traversable(skip)]
+    pub(crate) db: Database,
+
+    pub count: CountVecs,
+    pub value: ValueVecs,
+}

crates/brk_computer/src/scripts/value/compute.rs

@@ -0,0 +1,87 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_types::{Height, OutputType, Sats, TxOutIndex};
+use vecdb::{AnyStoredVec, AnyVec, Exit, GenericStoredVec, TypedVecIterator, VecIndex};
+
+use super::Vecs;
+use crate::{indexes, price, ComputeIndexes};
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        price: Option<&price::Vecs>,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        // Validate computed versions against dependencies
+        let dep_version = indexer.vecs.txout.height_to_first_txoutindex.version()
+            + indexer.vecs.txout.txoutindex_to_outputtype.version()
+            + indexer.vecs.txout.txoutindex_to_value.version();
+        self.height_to_opreturn_value
+            .validate_computed_version_or_reset(dep_version)?;
+
+        // Get target height
+        let target_len = indexer.vecs.txout.height_to_first_txoutindex.len();
+        if target_len == 0 {
+            return Ok(());
+        }
+        let target_height = Height::from(target_len - 1);
+
+        // Find starting height for this vec
+        let current_len = self.height_to_opreturn_value.len();
+        let starting_height = Height::from(current_len.min(starting_indexes.height.to_usize()));
+
+        if starting_height > target_height {
+            return Ok(());
+        }
+
+        // Prepare iterators
+        let mut height_to_first_txoutindex =
+            indexer.vecs.txout.height_to_first_txoutindex.iter()?;
+        let mut txoutindex_to_outputtype = indexer.vecs.txout.txoutindex_to_outputtype.iter()?;
+        let mut txoutindex_to_value = indexer.vecs.txout.txoutindex_to_value.iter()?;
+
+        // Iterate blocks
+        for h in starting_height.to_usize()..=target_height.to_usize() {
+            let height = Height::from(h);
+
+            // Get output range for this block
+            let first_txoutindex = height_to_first_txoutindex.get_unwrap(height);
+            let next_first_txoutindex = if height < target_height {
+                height_to_first_txoutindex.get_unwrap(height.incremented())
+            } else {
+                TxOutIndex::from(indexer.vecs.txout.txoutindex_to_value.len())
+            };
+
+            // Sum opreturn values
+            let mut opreturn_value = Sats::ZERO;
+            for i in first_txoutindex.to_usize()..next_first_txoutindex.to_usize() {
+                let txoutindex = TxOutIndex::from(i);
+                let outputtype = txoutindex_to_outputtype.get_unwrap(txoutindex);
+
+                if outputtype == OutputType::OpReturn {
+                    let value = txoutindex_to_value.get_unwrap(txoutindex);
+                    opreturn_value += value;
+                }
+            }
+
+            self.height_to_opreturn_value
+                .truncate_push(height, opreturn_value)?;
+        }
+
+        self.height_to_opreturn_value.write()?;
+
+        // Compute derived vecs (dateindex aggregations, etc.)
+        self.indexes_to_opreturn_value.compute_rest(
+            indexes,
+            price,
+            starting_indexes,
+            exit,
+            Some(&self.height_to_opreturn_value),
+        )?;
+
+        Ok(())
+    }
+}

crates/brk_computer/src/scripts/value/import.rs

@@ -0,0 +1,42 @@
+use brk_error::Result;
+use brk_types::Version;
+use vecdb::{Database, EagerVec, ImportableVec, IterableCloneableVec};
+
+use super::Vecs;
+use crate::{
+    indexes,
+    internal::{ComputedValueVecsFromHeight, Source, VecBuilderOptions},
+};
+
+impl Vecs {
+    pub fn forced_import(
+        db: &Database,
+        version: Version,
+        indexes: &indexes::Vecs,
+        compute_dollars: bool,
+    ) -> Result<Self> {
+        let v0 = Version::ZERO;
+
+        let height_to_opreturn_value =
+            EagerVec::forced_import(db, "opreturn_value", version + v0)?;
+
+        let indexes_to_opreturn_value = ComputedValueVecsFromHeight::forced_import(
+            db,
+            "opreturn_value",
+            Source::Vec(height_to_opreturn_value.boxed_clone()),
+            version + v0,
+            VecBuilderOptions::default()
+                .add_sum()
+                .add_cumulative()
+                .add_average()
+                .add_minmax(),
+            compute_dollars,
+            indexes,
+        )?;
+
+        Ok(Self {
+            height_to_opreturn_value,
+            indexes_to_opreturn_value,
+        })
+    }
+}

crates/brk_computer/src/scripts/value/mod.rs

@@ -0,0 +1,5 @@
+mod compute;
+mod import;
+mod vecs;
+
+pub use vecs::Vecs;

crates/brk_computer/src/scripts/value/vecs.rs

@@ -0,0 +1,11 @@
+use brk_traversable::Traversable;
+use brk_types::{Height, Sats};
+use vecdb::{EagerVec, PcoVec};
+
+use crate::internal::ComputedValueVecsFromHeight;
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    pub height_to_opreturn_value: EagerVec<PcoVec<Height, Sats>>,
+    pub indexes_to_opreturn_value: ComputedValueVecsFromHeight,
+}