global: MASSIVE snapshot

crates/brk_computer/src/inputs/compute.rs

@@ -0,0 +1,25 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use vecdb::Exit;
+
+use super::Vecs;
+use crate::{indexes, ComputeIndexes};
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.spent
+            .compute(&self.db, indexer, starting_indexes, exit)?;
+        self.count
+            .compute(indexer, indexes, starting_indexes, exit)?;
+
+        let _lock = exit.lock();
+        self.db.compact()?;
+        Ok(())
+    }
+}

crates/brk_computer/src/inputs/count/compute.rs

@@ -0,0 +1,26 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use vecdb::Exit;
+
+use super::Vecs;
+use crate::{indexes, ComputeIndexes};
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.indexes_to_count.compute_rest(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            Some(&indexes.transaction.txindex_to_input_count),
+        )?;
+
+        Ok(())
+    }
+}

crates/brk_computer/src/inputs/count/import.rs

@@ -0,0 +1,33 @@
+use brk_error::Result;
+use brk_types::Version;
+use vecdb::{Database, IterableCloneableVec};
+
+use super::Vecs;
+use crate::{
+    indexes,
+    internal::{ComputedVecsFromTxindex, Source, VecBuilderOptions},
+};
+
+impl Vecs {
+    pub fn forced_import(db: &Database, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
+        let full_stats = || {
+            VecBuilderOptions::default()
+                .add_average()
+                .add_minmax()
+                .add_percentiles()
+                .add_sum()
+                .add_cumulative()
+        };
+
+        Ok(Self {
+            indexes_to_count: ComputedVecsFromTxindex::forced_import(
+                db,
+                "input_count",
+                Source::Vec(indexes.transaction.txindex_to_input_count.boxed_clone()),
+                version + Version::ZERO,
+                indexes,
+                full_stats(),
+            )?,
+        })
+    }
+}

crates/brk_computer/src/inputs/count/mod.rs

@@ -0,0 +1,5 @@
+mod compute;
+mod import;
+mod vecs;
+
+pub use vecs::Vecs;

crates/brk_computer/src/inputs/count/vecs.rs

@@ -0,0 +1,9 @@
+use brk_traversable::Traversable;
+use brk_types::StoredU64;
+
+use crate::internal::ComputedVecsFromTxindex;
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    pub indexes_to_count: ComputedVecsFromTxindex<StoredU64>,
+}

crates/brk_computer/src/inputs/import.rs

@@ -0,0 +1,36 @@
+use std::path::Path;
+
+use brk_error::Result;
+use brk_traversable::Traversable;
+use brk_types::Version;
+use vecdb::{Database, PAGE_SIZE};
+
+use super::{CountVecs, SpentVecs, Vecs};
+use crate::indexes;
+
+impl Vecs {
+    pub fn forced_import(
+        parent_path: &Path,
+        parent_version: Version,
+        indexes: &indexes::Vecs,
+    ) -> Result<Self> {
+        let db = Database::open(&parent_path.join(super::DB_NAME))?;
+        db.set_min_len(PAGE_SIZE * 50_000_000)?;
+
+        let version = parent_version + Version::ZERO;
+
+        let spent = SpentVecs::forced_import(&db, version)?;
+        let count = CountVecs::forced_import(&db, version, indexes)?;
+
+        let this = Self { db, spent, count };
+
+        this.db.retain_regions(
+            this.iter_any_exportable()
+                .flat_map(|v| v.region_names())
+                .collect(),
+        )?;
+        this.db.compact()?;
+
+        Ok(this)
+    }
+}

crates/brk_computer/src/inputs/mod.rs

@@ -0,0 +1,22 @@
+pub mod count;
+pub mod spent;
+
+mod compute;
+mod import;
+
+use brk_traversable::Traversable;
+use vecdb::Database;
+
+pub use count::Vecs as CountVecs;
+pub use spent::Vecs as SpentVecs;
+
+pub const DB_NAME: &str = "inputs";
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    #[traversable(skip)]
+    pub(crate) db: Database,
+
+    pub spent: SpentVecs,
+    pub count: CountVecs,
+}

crates/brk_computer/src/inputs/spent/compute.rs

@@ -0,0 +1,112 @@
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_types::{Sats, TxInIndex, TxIndex, TxOutIndex, Vout};
+use log::info;
+use vecdb::{AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, TypedVecIterator, VecIndex};
+
+use super::Vecs;
+use crate::ComputeIndexes;
+
+const BATCH_SIZE: usize = 2 * 1024 * 1024 * 1024 / size_of::<Entry>();
+
+impl Vecs {
+    pub fn compute(
+        &mut self,
+        db: &Database,
+        indexer: &Indexer,
+        starting_indexes: &ComputeIndexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        // Validate computed versions against dependencies
+        let dep_version = indexer.vecs.txin.txinindex_to_outpoint.version()
+            + indexer.vecs.tx.txindex_to_first_txoutindex.version()
+            + indexer.vecs.txout.txoutindex_to_value.version();
+        self.txinindex_to_txoutindex
+            .validate_computed_version_or_reset(dep_version)?;
+        self.txinindex_to_value
+            .validate_computed_version_or_reset(dep_version)?;
+
+        let target = indexer.vecs.txin.txinindex_to_outpoint.len();
+        if target == 0 {
+            return Ok(());
+        }
+
+        let len1 = self.txinindex_to_txoutindex.len();
+        let len2 = self.txinindex_to_value.len();
+        let starting = starting_indexes.txinindex.to_usize();
+        let min = len1.min(len2).min(starting);
+
+        if min >= target {
+            return Ok(());
+        }
+
+        let mut outpoint_iter = indexer.vecs.txin.txinindex_to_outpoint.iter()?;
+        let mut first_txoutindex_iter = indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
+        let mut value_iter = indexer.vecs.txout.txoutindex_to_value.iter()?;
+        let mut entries: Vec<Entry> = Vec::with_capacity(BATCH_SIZE);
+
+        let mut batch_start = min;
+        while batch_start < target {
+            let batch_end = (batch_start + BATCH_SIZE).min(target);
+
+            entries.clear();
+            for i in batch_start..batch_end {
+                let txinindex = TxInIndex::from(i);
+                let outpoint = outpoint_iter.get_unwrap(txinindex);
+                entries.push(Entry {
+                    txinindex,
+                    txindex: outpoint.txindex(),
+                    vout: outpoint.vout(),
+                    txoutindex: TxOutIndex::COINBASE,
+                    value: Sats::MAX,
+                });
+            }
+
+            // Coinbase entries (txindex MAX) sorted to end
+            entries.sort_unstable_by_key(|e| e.txindex);
+            for entry in &mut entries {
+                if entry.txindex.is_coinbase() {
+                    break;
+                }
+                entry.txoutindex = first_txoutindex_iter.get_unwrap(entry.txindex) + entry.vout;
+            }
+
+            entries.sort_unstable_by_key(|e| e.txoutindex);
+            for entry in &mut entries {
+                if entry.txoutindex.is_coinbase() {
+                    break;
+                }
+                entry.value = value_iter.get_unwrap(entry.txoutindex);
+            }
+
+            entries.sort_unstable_by_key(|e| e.txinindex);
+            for entry in &entries {
+                self.txinindex_to_txoutindex
+                    .truncate_push(entry.txinindex, entry.txoutindex)?;
+                self.txinindex_to_value
+                    .truncate_push(entry.txinindex, entry.value)?;
+            }
+
+            if batch_end < target {
+                info!("TxIns: {:.2}%", batch_end as f64 / target as f64 * 100.0);
+            }
+
+            let _lock = exit.lock();
+            self.txinindex_to_txoutindex.write()?;
+            self.txinindex_to_value.write()?;
+            db.flush()?;
+
+            batch_start = batch_end;
+        }
+
+        Ok(())
+    }
+}
+
+struct Entry {
+    txinindex: TxInIndex,
+    txindex: TxIndex,
+    vout: Vout,
+    txoutindex: TxOutIndex,
+    value: Sats,
+}

crates/brk_computer/src/inputs/spent/import.rs

@@ -0,0 +1,14 @@
+use brk_error::Result;
+use brk_types::Version;
+use vecdb::{Database, ImportableVec, PcoVec};
+
+use super::Vecs;
+
+impl Vecs {
+    pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
+        Ok(Self {
+            txinindex_to_txoutindex: PcoVec::forced_import(db, "txoutindex", version)?,
+            txinindex_to_value: PcoVec::forced_import(db, "value", version)?,
+        })
+    }
+}

crates/brk_computer/src/inputs/spent/mod.rs

@@ -0,0 +1,5 @@
+mod compute;
+mod import;
+mod vecs;
+
+pub use vecs::Vecs;

crates/brk_computer/src/inputs/spent/vecs.rs

@@ -0,0 +1,9 @@
+use brk_traversable::Traversable;
+use brk_types::{Sats, TxInIndex, TxOutIndex};
+use vecdb::PcoVec;
+
+#[derive(Clone, Traversable)]
+pub struct Vecs {
+    pub txinindex_to_txoutindex: PcoVec<TxInIndex, TxOutIndex>,
+    pub txinindex_to_value: PcoVec<TxInIndex, Sats>,
+}