global: MASSIVE snapshot

crates/brk_computer/src/outputs/compute.rs

@@ -0,0 +1,33 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use vecdb::Exit;
+
+use super::Vecs;
+use crate::{indexes, inputs, scripts, ComputeIndexes};
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        inputs: &inputs::Vecs,
+        scripts: &scripts::Vecs,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.spent
+            .compute(&self.db, indexer, inputs, starting_indexes, exit)?;
+        self.count.compute(
+            indexer,
+            indexes,
+            &inputs.count,
+            &scripts.count,
+            starting_indexes,
+            exit,
+        )?;
+
+        let _lock = exit.lock();
+        self.db.compact()?;
+        Ok(())
+    }
+}

crates/brk_computer/src/outputs/count/compute.rs

@@ -0,0 +1,77 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_types::{Height, StoredU64};
+use vecdb::{Exit, TypedVecIterator};
+
+use super::Vecs;
+use crate::{indexes, inputs, scripts, ComputeIndexes};
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        inputs_count: &inputs::CountVecs,
+        scripts_count: &scripts::CountVecs,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.indexes_to_count.compute_rest(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            Some(&indexes.transaction.txindex_to_output_count),
+        )?;
+
+        self.indexes_to_utxo_count
+            .compute_all(indexes, starting_indexes, exit, |v| {
+                let mut input_count_iter = inputs_count
+                    .indexes_to_count
+                    .height
+                    .unwrap_cumulative()
+                    .into_iter();
+                let mut opreturn_count_iter = scripts_count
+                    .indexes_to_opreturn_count
+                    .height_extra
+                    .unwrap_cumulative()
+                    .into_iter();
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.indexes_to_count.height.unwrap_cumulative(),
+                    |(h, output_count, ..)| {
+                        let input_count = input_count_iter.get_unwrap(h);
+                        let opreturn_count = opreturn_count_iter.get_unwrap(h);
+                        let block_count = u64::from(h + 1_usize);
+                        // -1 > genesis output is unspendable
+                        let mut utxo_count =
+                            *output_count - (*input_count - block_count) - *opreturn_count - 1;
+
+                        // txid dup: e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468
+                        // Block 91_722 https://mempool.space/block/00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e
+                        // Block 91_880 https://mempool.space/block/00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721
+                        //
+                        // txid dup: d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599
+                        // Block 91_812 https://mempool.space/block/00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f
+                        // Block 91_842 https://mempool.space/block/00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec
+                        //
+                        // Warning: Dups invalidate the previous coinbase according to
+                        // https://chainquery.com/bitcoin-cli/gettxoutsetinfo
+
+                        if h >= Height::new(91_842) {
+                            utxo_count -= 1;
+                        }
+                        if h >= Height::new(91_880) {
+                            utxo_count -= 1;
+                        }
+
+                        (h, StoredU64::from(utxo_count))
+                    },
+                    exit,
+                )?;
+                Ok(())
+            })?;
+
+        Ok(())
+    }
+}

crates/brk_computer/src/outputs/count/import.rs

@@ -0,0 +1,41 @@
+use brk_error::Result;
+use brk_types::Version;
+use vecdb::{Database, IterableCloneableVec};
+
+use super::Vecs;
+use crate::{
+    indexes,
+    internal::{ComputedVecsFromHeight, ComputedVecsFromTxindex, Source, VecBuilderOptions},
+};
+
+impl Vecs {
+    pub fn forced_import(db: &Database, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
+        let full_stats = || {
+            VecBuilderOptions::default()
+                .add_average()
+                .add_minmax()
+                .add_percentiles()
+                .add_sum()
+                .add_cumulative()
+        };
+
+        Ok(Self {
+            indexes_to_count: ComputedVecsFromTxindex::forced_import(
+                db,
+                "output_count",
+                Source::Vec(indexes.transaction.txindex_to_output_count.boxed_clone()),
+                version + Version::ZERO,
+                indexes,
+                full_stats(),
+            )?,
+            indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
+                db,
+                "exact_utxo_count",
+                Source::Compute,
+                version + Version::ZERO,
+                indexes,
+                full_stats(),
+            )?,
+        })
+    }
+}

crates/brk_computer/src/outputs/count/mod.rs

@@ -0,0 +1,5 @@
+mod compute;
+mod import;
+mod vecs;
+
+pub use vecs::Vecs;

crates/brk_computer/src/outputs/count/vecs.rs

@@ -0,0 +1,10 @@
+use brk_traversable::Traversable;
+use brk_types::StoredU64;
+
+use crate::internal::{ComputedVecsFromHeight, ComputedVecsFromTxindex};
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    pub indexes_to_count: ComputedVecsFromTxindex<StoredU64>,
+    pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredU64>,
+}

crates/brk_computer/src/outputs/import.rs

@@ -0,0 +1,36 @@
+use std::path::Path;
+
+use brk_error::Result;
+use brk_traversable::Traversable;
+use brk_types::Version;
+use vecdb::{Database, PAGE_SIZE};
+
+use super::{CountVecs, SpentVecs, Vecs};
+use crate::indexes;
+
+impl Vecs {
+    pub fn forced_import(
+        parent_path: &Path,
+        parent_version: Version,
+        indexes: &indexes::Vecs,
+    ) -> Result<Self> {
+        let db = Database::open(&parent_path.join(super::DB_NAME))?;
+        db.set_min_len(PAGE_SIZE * 10_000_000)?;
+
+        let version = parent_version + Version::ZERO;
+
+        let spent = SpentVecs::forced_import(&db, version)?;
+        let count = CountVecs::forced_import(&db, version, indexes)?;
+
+        let this = Self { db, spent, count };
+
+        this.db.retain_regions(
+            this.iter_any_exportable()
+                .flat_map(|v| v.region_names())
+                .collect(),
+        )?;
+        this.db.compact()?;
+
+        Ok(this)
+    }
+}

crates/brk_computer/src/outputs/mod.rs

@@ -0,0 +1,22 @@
+pub mod count;
+pub mod spent;
+
+mod compute;
+mod import;
+
+use brk_traversable::Traversable;
+use vecdb::Database;
+
+pub use count::Vecs as CountVecs;
+pub use spent::Vecs as SpentVecs;
+
+pub const DB_NAME: &str = "outputs";
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    #[traversable(skip)]
+    pub(crate) db: Database,
+
+    pub spent: SpentVecs,
+    pub count: CountVecs,
+}

crates/brk_computer/src/outputs/spent/compute.rs

@@ -0,0 +1,128 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_types::{Height, TxInIndex, TxOutIndex};
+use log::info;
+use vecdb::{AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, Stamp, TypedVecIterator, VecIndex};
+
+use super::Vecs;
+use crate::{inputs, ComputeIndexes};
+
+const HEIGHT_BATCH: u32 = 10_000;
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        db: &Database,
+        indexer: &Indexer,
+        inputs: &inputs::Vecs,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        let target_height = indexer.vecs.block.height_to_blockhash.len();
+        if target_height == 0 {
+            return Ok(());
+        }
+        let target_height = Height::from(target_height - 1);
+
+        // Find min_height from current vec length
+        let current_txoutindex = self.txoutindex_to_txinindex.len();
+        let min_txoutindex = current_txoutindex.min(starting_indexes.txoutindex.to_usize());
+
+        let starting_stamp = Stamp::from(starting_indexes.height);
+        let _ = self.txoutindex_to_txinindex.rollback_before(starting_stamp);
+
+        self.txoutindex_to_txinindex
+            .truncate_if_needed(TxOutIndex::from(min_txoutindex))?;
+
+        let mut height_to_first_txoutindex =
+            indexer.vecs.txout.height_to_first_txoutindex.iter()?;
+        let mut height_to_first_txinindex = indexer.vecs.txin.height_to_first_txinindex.iter()?;
+        let mut txinindex_to_txoutindex = inputs.spent.txinindex_to_txoutindex.iter()?;
+
+        // Find starting height from min_txoutindex
+        let mut min_height = Height::ZERO;
+        for h in 0..=target_height.to_usize() {
+            let txoutindex = height_to_first_txoutindex.get_unwrap(Height::from(h));
+            if txoutindex.to_usize() > min_txoutindex {
+                break;
+            }
+            min_height = Height::from(h);
+        }
+
+        // Validate: computed height must not exceed starting height
+        assert!(
+            min_height <= starting_indexes.height,
+            "txouts min_height ({}) exceeds starting_indexes.height ({})",
+            min_height,
+            starting_indexes.height
+        );
+
+        let mut pairs: Vec<(TxOutIndex, TxInIndex)> = Vec::new();
+
+        let mut batch_start_height = min_height;
+        while batch_start_height <= target_height {
+            let batch_end_height = (batch_start_height + HEIGHT_BATCH).min(target_height);
+
+            // Fill txoutindex up to batch_end_height + 1
+            let batch_txoutindex = if batch_end_height >= target_height {
+                indexer.vecs.txout.txoutindex_to_value.len()
+            } else {
+                height_to_first_txoutindex
+                    .get_unwrap(batch_end_height + 1_u32)
+                    .to_usize()
+            };
+            self.txoutindex_to_txinindex
+                .fill_to(batch_txoutindex, TxInIndex::UNSPENT)?;
+
+            // Get txin range for this height batch
+            let txin_start = height_to_first_txinindex
+                .get_unwrap(batch_start_height)
+                .to_usize();
+            let txin_end = if batch_end_height >= target_height {
+                inputs.spent.txinindex_to_txoutindex.len()
+            } else {
+                height_to_first_txinindex
+                    .get_unwrap(batch_end_height + 1_u32)
+                    .to_usize()
+            };
+
+            // Collect and process txins
+            pairs.clear();
+            for i in txin_start..txin_end {
+                let txinindex = TxInIndex::from(i);
+                let txoutindex = txinindex_to_txoutindex.get_unwrap(txinindex);
+
+                if txoutindex.is_coinbase() {
+                    continue;
+                }
+
+                pairs.push((txoutindex, txinindex));
+            }
+
+            pairs.sort_unstable_by_key(|(txoutindex, _)| *txoutindex);
+
+            for &(txoutindex, txinindex) in &pairs {
+                self.txoutindex_to_txinindex.update(txoutindex, txinindex)?;
+            }
+
+            if batch_end_height < target_height {
+                let _lock = exit.lock();
+                self.txoutindex_to_txinindex.write()?;
+                info!(
+                    "TxOuts: {:.2}%",
+                    batch_end_height.to_usize() as f64 / target_height.to_usize() as f64 * 100.0
+                );
+                db.flush()?;
+            }
+
+            batch_start_height = batch_end_height + 1_u32;
+        }
+
+        let _lock = exit.lock();
+        self.txoutindex_to_txinindex
+            .stamped_write_with_changes(Stamp::from(target_height))?;
+        db.flush()?;
+
+        Ok(())
+    }
+}

crates/brk_computer/src/outputs/spent/import.rs

@@ -0,0 +1,13 @@
+use brk_error::Result;
+use brk_types::Version;
+use vecdb::{BytesVec, Database, ImportableVec};
+
+use super::Vecs;
+
+impl Vecs {
+    pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
+        Ok(Self {
+            txoutindex_to_txinindex: BytesVec::forced_import(db, "txinindex", version)?,
+        })
+    }
+}

crates/brk_computer/src/outputs/spent/mod.rs

@@ -0,0 +1,5 @@
+mod compute;
+mod import;
+mod vecs;
+
+pub use vecs::Vecs;

crates/brk_computer/src/outputs/spent/vecs.rs

@@ -0,0 +1,8 @@
+use brk_traversable::Traversable;
+use brk_types::{TxInIndex, TxOutIndex};
+use vecdb::BytesVec;
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
+}