computer: add positions

crates/brk_computer/src/constants.rs

@@ -12,8 +12,6 @@ use super::{
     indexes,
 };
 
-const VERSION: Version = Version::ZERO;
-
 #[derive(Clone)]
 pub struct Vecs {
     db: Database,
@@ -35,15 +33,21 @@ pub struct Vecs {
 }
 
 impl Vecs {
-    pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
-        let db = Database::open(&parent.join("constants"))?;
+    pub fn forced_import(
+        parent_path: &Path,
+        parent_version: Version,
+        indexes: &indexes::Vecs,
+    ) -> Result<Self> {
+        let db = Database::open(&parent_path.join("constants"))?;
+
+        let version = parent_version + Version::ZERO;
 
         let this = Self {
             constant_0: ComputedVecsFromHeight::forced_import(
                 &db,
                 "constant_0",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -51,7 +55,7 @@ impl Vecs {
                 &db,
                 "constant_1",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -59,7 +63,7 @@ impl Vecs {
                 &db,
                 "constant_2",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -67,7 +71,7 @@ impl Vecs {
                 &db,
                 "constant_3",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -75,7 +79,7 @@ impl Vecs {
                 &db,
                 "constant_4",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -83,7 +87,7 @@ impl Vecs {
                 &db,
                 "constant_38_2",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -91,7 +95,7 @@ impl Vecs {
                 &db,
                 "constant_50",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -99,7 +103,7 @@ impl Vecs {
                 &db,
                 "constant_61_8",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -107,7 +111,7 @@ impl Vecs {
                 &db,
                 "constant_100",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -115,7 +119,7 @@ impl Vecs {
                 &db,
                 "constant_600",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -123,7 +127,7 @@ impl Vecs {
                 &db,
                 "constant_minus_1",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -131,7 +135,7 @@ impl Vecs {
                 &db,
                 "constant_minus_2",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -139,7 +143,7 @@ impl Vecs {
                 &db,
                 "constant_minus_3",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,
@@ -147,7 +151,7 @@ impl Vecs {
                 &db,
                 "constant_minus_4",
                 Source::Compute,
-                version + VERSION + Version::ZERO,
+                version + Version::ZERO,
                 indexes,
                 VecBuilderOptions::default().add_last(),
             )?,

crates/brk_computer/src/lib.rs

@@ -5,6 +5,7 @@ use std::path::Path;
 use brk_error::Result;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
+use brk_parser::Parser;
 use brk_structs::Version;
 use log::info;
 use vecdb::{AnyCollectableVec, Exit, Format};
@@ -17,6 +18,7 @@ mod grouped;
 mod indexes;
 mod market;
 mod pools;
+mod positions;
 mod price;
 mod stateful;
 mod states;
@@ -31,15 +33,16 @@ use states::*;
 
 #[derive(Clone)]
 pub struct Computer {
-    pub indexes: indexes::Vecs,
+    pub chain: chain::Vecs,
+    pub cointime: cointime::Vecs,
     pub constants: constants::Vecs,
+    pub fetched: Option<fetched::Vecs>,
+    pub indexes: indexes::Vecs,
     pub market: market::Vecs,
     pub pools: pools::Vecs,
+    pub positions: positions::Vecs,
     pub price: Option<price::Vecs>,
-    pub chain: chain::Vecs,
     pub stateful: stateful::Vecs,
-    pub fetched: Option<fetched::Vecs>,
-    pub cointime: cointime::Vecs,
 }
 
 const VERSION: Version = Version::new(4);
@@ -87,6 +90,7 @@ impl Computer {
                 &indexes,
                 price.as_ref(),
             )?,
+            positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
             pools: pools::Vecs::forced_import(
                 &computed_path,
                 VERSION + Version::ZERO,
@@ -109,15 +113,12 @@ impl Computer {
         &mut self,
         indexer: &Indexer,
         starting_indexes: brk_indexer::Indexes,
+        parser: &Parser,
         exit: &Exit,
     ) -> Result<()> {
         info!("Computing indexes...");
         let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
 
-        info!("Computing constants...");
-        self.constants
-            .compute(&self.indexes, &starting_indexes, exit)?;
-
         if let Some(fetched) = self.fetched.as_mut() {
             info!("Computing fetched...");
             fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
@@ -131,7 +132,25 @@ impl Computer {
             )?;
         }
 
+        info!("Computing positions...");
+        self.positions
+            .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
+
         std::thread::scope(|scope| -> Result<()> {
+            let constants = scope.spawn(|| -> Result<()> {
+                info!("Computing constants...");
+                self.constants
+                    .compute(&self.indexes, &starting_indexes, exit)?;
+                Ok(())
+            });
+
+            // let positions = scope.spawn(|| -> Result<()> {
+            //     info!("Computing positions...");
+            //     self.positions
+            //         .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
+            //     Ok(())
+            // });
+
             let chain = scope.spawn(|| -> Result<()> {
                 info!("Computing chain...");
                 self.chain.compute(
@@ -149,6 +168,8 @@ impl Computer {
                 self.market.compute(price, &starting_indexes, exit)?;
             }
 
+            constants.join().unwrap()?;
+            // positions.join().unwrap()?;
             chain.join().unwrap()?;
             Ok(())
         })?;
@@ -162,8 +183,6 @@ impl Computer {
             exit,
         )?;
 
-        // return Ok(());
-
         info!("Computing stateful...");
         self.stateful.compute(
             indexer,
@@ -191,14 +210,15 @@ impl Computer {
         let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
             Box::new(self.fetched.iter().flat_map(|v| v.iter_any_collectable()));
 
-        iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
-        iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
+        iter = Box::new(iter.chain(self.chain.iter_any_collectable()));
+        iter = Box::new(iter.chain(self.cointime.iter_any_collectable()));
         iter = Box::new(iter.chain(self.constants.iter_any_collectable()));
         iter = Box::new(iter.chain(self.indexes.iter_any_collectable()));
         iter = Box::new(iter.chain(self.market.iter_any_collectable()));
-        iter = Box::new(iter.chain(self.chain.iter_any_collectable()));
+        iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
+        iter = Box::new(iter.chain(self.positions.iter_any_collectable()));
+        iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
         iter = Box::new(iter.chain(self.stateful.iter_any_collectable()));
-        iter = Box::new(iter.chain(self.cointime.iter_any_collectable()));
 
         iter
     }

crates/brk_computer/src/positions.rs

@@ -0,0 +1,133 @@
+use std::path::Path;
+
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_parser::Parser;
+use brk_structs::{BlkPosition, Height, TxIndex, Version};
+use log::info;
+use vecdb::{
+    AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
+    GenericStoredVec, PAGE_SIZE, VecIterator,
+};
+
+use super::{Indexes, indexes};
+
+#[derive(Clone)]
+pub struct Vecs {
+    db: Database,
+
+    pub height_to_position: CompressedVec<Height, BlkPosition>,
+    pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
+}
+
+impl Vecs {
+    pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
+        let db = Database::open(&parent_path.join("positions"))?;
+        db.set_min_len(PAGE_SIZE * 1_000_000)?;
+
+        let version = parent_version + Version::ZERO;
+
+        let this = Self {
+            height_to_position: CompressedVec::forced_import(
+                &db,
+                "position",
+                version + Version::ZERO,
+            )?,
+            txindex_to_position: CompressedVec::forced_import(
+                &db,
+                "position",
+                version + Version::ZERO,
+            )?,
+
+            db,
+        };
+
+        this.db.retain_regions(
+            this.iter_any_collectable()
+                .flat_map(|v| v.region_names())
+                .collect(),
+        )?;
+
+        Ok(this)
+    }
+
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        parser: &Parser,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.compute_(indexer, indexes, starting_indexes, parser, exit)?;
+        self.db.flush_then_punch()?;
+        Ok(())
+    }
+
+    fn compute_(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        parser: &Parser,
+        exit: &Exit,
+    ) -> Result<()> {
+        let min_txindex =
+            TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
+
+        let min_height = indexes
+            .txindex_to_height
+            .iter()
+            .unwrap_get_inner(min_txindex)
+            .min(starting_indexes.height);
+
+        let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
+
+        parser
+            .parse(Some(min_height), None)
+            .iter()
+            .try_for_each(|block| -> Result<()> {
+                let height = block.height();
+
+                info!("{height}");
+
+                self.height_to_position
+                    .forced_push_at(height, *block.position(), exit)?;
+
+                let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
+
+                block.tx_positions().iter().enumerate().try_for_each(
+                    |(index, position)| -> Result<()> {
+                        let txindex = txindex + index;
+                        self.txindex_to_position
+                            .forced_push_at(txindex, *position, exit)?;
+                        Ok(())
+                    },
+                )?;
+
+                // Stuck, why ??
+                if *height % 1_000 == 0 {
+                    let _lock = exit.lock();
+                    self.height_to_position.flush()?;
+                    self.txindex_to_position.flush()?;
+                }
+                Ok(())
+            })?;
+
+        let _lock = exit.lock();
+        self.height_to_position.flush()?;
+        self.txindex_to_position.flush()?;
+
+        Ok(())
+    }
+
+    pub fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
+        Box::new(
+            [
+                &self.height_to_position as &dyn AnyCollectableVec,
+                &self.txindex_to_position,
+            ]
+            .into_iter(),
+        )
+    }
+}