brk: first commit

crates/brk_computer/src/lib.rs

@@ -0,0 +1,114 @@
+use std::path::{Path, PathBuf};
+
+use brk_indexer::Indexer;
+pub use brk_parser::rpc;
+use hodor::Exit;
+
+mod storage;
+mod structs;
+
+use brk_fetcher::Date;
+use storable_vec::SINGLE_THREAD;
+use storage::{Fjalls, StorableVecs};
+pub use structs::*;
+
+pub struct Computer<const MODE: u8> {
+    path: PathBuf,
+    pub vecs: StorableVecs<MODE>,
+    pub trees: Fjalls,
+}
+
+impl<const MODE: u8> Computer<MODE> {
+    pub fn import(computed_dir: &Path) -> color_eyre::Result<Self> {
+        let vecs = StorableVecs::import(&computed_dir.join("vecs"))?;
+        let trees = Fjalls::import(&computed_dir.join("fjall"))?;
+        Ok(Self {
+            path: computed_dir.to_owned(),
+            vecs,
+            trees,
+        })
+    }
+}
+
+impl Computer<SINGLE_THREAD> {
+    pub fn compute(&mut self, mut indexer: Indexer<SINGLE_THREAD>, exit: &Exit) -> color_eyre::Result<()> {
+        let height_count = indexer.vecs.height_to_size.len();
+        let txindexes_count = indexer.vecs.txindex_to_txid.len();
+        let txinindexes_count = indexer.vecs.txinindex_to_txoutindex.len();
+        let txoutindexes_count = indexer.vecs.txoutindex_to_addressindex.len();
+
+        // TODO: Remove all outdated
+
+        self.vecs
+            .txindex_to_last_txinindex
+            .compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txinindex, txinindexes_count)?;
+
+        self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
+            &mut indexer.vecs.txindex_to_first_txinindex,
+            &mut self.vecs.txindex_to_last_txinindex,
+        )?;
+
+        self.vecs
+            .txindex_to_last_txoutindex
+            .compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txoutindex, txoutindexes_count)?;
+
+        self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
+            &mut indexer.vecs.txindex_to_first_txoutindex,
+            &mut self.vecs.txindex_to_last_txoutindex,
+        )?;
+
+        self.vecs
+            .height_to_date
+            .compute_transform(&mut indexer.vecs.height_to_timestamp, |timestamp| {
+                Date::from(*timestamp)
+            })?;
+
+        self.vecs
+            .height_to_last_txindex
+            .compute_last_index_from_first(&mut indexer.vecs.height_to_first_txindex, height_count)?;
+
+        self.vecs.txindex_to_height.compute_inverse_less_to_more(
+            &mut indexer.vecs.height_to_first_txindex,
+            &mut self.vecs.height_to_last_txindex,
+        )?;
+
+        self.vecs.txindex_to_is_coinbase.compute_is_first_ordered(
+            &mut self.vecs.txindex_to_height,
+            &mut indexer.vecs.height_to_first_txindex,
+        )?;
+
+        // self.vecs.txindex_to_fee.compute_transform(
+        //     &mut self.vecs.txindex_to_height,
+        //     &mut indexer.vecs.height_to_first_txindex,
+        // )?;
+
+        let date_count = self.vecs.height_to_date.len();
+
+        // self.vecs.height_to_dateindex.compute(...)
+
+        self.vecs
+            .dateindex_to_first_height
+            .compute_inverse_more_to_less(&mut self.vecs.height_to_dateindex)?;
+
+        // ---
+        // Date to X
+        // ---
+        // ...
+
+        // ---
+        // Month to X
+        // ---
+        // ...
+
+        // ---
+        // Year to X
+        // ---
+        // ...
+
+        Ok(())
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+}

crates/brk_computer/src/main.rs

@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use brk_computer::Computer;
+use brk_indexer::Indexer;
+use hodor::Exit;
+
+mod structs;
+
+pub fn main() -> color_eyre::Result<()> {
+    color_eyre::install()?;
+
+    let exit = Exit::new();
+
+    let i = std::time::Instant::now();
+
+    let outputs_dir = Path::new("../_outputs");
+
+    Computer::import(&outputs_dir.join("computed"))?.compute(Indexer::import(&outputs_dir.join("indexes"))?, &exit)?;
+
+    dbg!(i.elapsed());
+
+    Ok(())
+}

crates/brk_computer/src/storage/fjalls.rs

@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use brk_indexer::Store;
+use storable_vec::Version;
+
+use crate::structs::{AddressindexTxoutindex, Unit};
+
+pub struct Fjalls {
+    pub address_to_utxos_received: Store<AddressindexTxoutindex, Unit>,
+    pub address_to_utxos_spent: Store<AddressindexTxoutindex, Unit>,
+}
+
+impl Fjalls {
+    pub fn import(path: &Path) -> color_eyre::Result<Self> {
+        let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?;
+        let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?;
+
+        Ok(Self {
+            address_to_utxos_received,
+            address_to_utxos_spent,
+        })
+    }
+}

crates/brk_computer/src/storage/mod.rs

@@ -0,0 +1,5 @@
+mod fjalls;
+mod storable_vecs;
+
+pub use fjalls::*;
+pub use storable_vecs::*;

crates/brk_computer/src/storage/storable_vecs.rs

@@ -0,0 +1,102 @@
+use std::{fs, path::Path};
+
+use brk_fetcher::{Date, Dateindex};
+use brk_indexer::{Addressindex, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex};
+use storable_vec::{StorableVec, Version};
+
+use crate::structs::Feerate;
+
+// mod base;
+
+// use base::*;
+
+pub struct StorableVecs<const MODE: u8> {
+    pub dateindex_to_first_height: StorableVec<Dateindex, Height, MODE>,
+    // pub dateindex_to_last_height: StorableVec<Dateindex, Height, MODE>,
+    // pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
+    pub height_to_date: StorableVec<Height, Date, MODE>,
+    pub height_to_dateindex: StorableVec<Height, Dateindex, MODE>,
+    // pub height_to_fee: StorableVec<Txindex, Amount, MODE>,
+    // pub height_to_inputcount: StorableVec<Height, u32, MODE>,
+    // pub height_to_last_addressindex: StorableVec<Height, Addressindex, MODE>,
+    pub height_to_last_txindex: StorableVec<Height, Txindex, MODE>,
+    // pub height_to_last_txoutindex: StorableVec<Height, Txoutindex, MODE>,
+    // pub height_to_maxfeerate: StorableVec<Height, Feerate, MODE>,
+    // pub height_to_medianfeerate: StorableVec<Height, Feerate, MODE>,
+    // pub height_to_minfeerate: StorableVec<Height, Feerate, MODE>,
+    // pub height_to_outputcount: StorableVec<Height, u32, MODE>,
+    // pub height_to_subsidy: StorableVec<Height, u32, MODE>,
+    // pub height_to_totalfees: StorableVec<Height, Amount, MODE>,
+    // pub height_to_txcount: StorableVec<Height, u32, MODE>,
+    pub txindex_to_fee: StorableVec<Txindex, Sats, MODE>,
+    pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
+    pub txindex_to_is_coinbase: StorableVec<Txindex, bool, MODE>,
+    // pub txindex_to_feerate: StorableVec<Txindex, Feerate, MODE>,
+    pub txindex_to_inputs_count: StorableVec<Txindex, u32, MODE>,
+    pub txindex_to_inputs_sum: StorableVec<Txindex, Sats, MODE>,
+    pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex, MODE>,
+    pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
+    pub txindex_to_outputs_count: StorableVec<Txindex, u32, MODE>,
+    pub txindex_to_outputs_sum: StorableVec<Txindex, Sats, MODE>,
+}
+
+impl<const MODE: u8> StorableVecs<MODE> {
+    pub fn import(path: &Path) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        Ok(Self {
+            dateindex_to_first_height: StorableVec::forced_import(
+                &path.join("dateindex_to_first_height"),
+                Version::from(1),
+            )?,
+            // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
+            height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?,
+            height_to_dateindex: StorableVec::forced_import(&path.join("height_to_dateindex"), Version::from(1))?,
+            // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
+            // height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
+            // height_to_last_addressindex: StorableVec::forced_import(
+            //     &path.join("height_to_last_addressindex"),
+            //     Version::from(1),
+            // )?,
+            height_to_last_txindex: StorableVec::forced_import(&path.join("height_to_last_txindex"), Version::from(1))?,
+            // height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
+            // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?,
+            // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?,
+            // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?,
+            // height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?,
+            // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?,
+            // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?,
+            // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?,
+            txindex_to_fee: StorableVec::forced_import(&path.join("txindex_to_fee"), Version::from(1))?,
+            txindex_to_height: StorableVec::forced_import(&path.join("txindex_to_height"), Version::from(1))?,
+            txindex_to_is_coinbase: StorableVec::forced_import(&path.join("txindex_to_is_coinbase"), Version::from(1))?,
+            // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
+            txindex_to_inputs_count: StorableVec::forced_import(
+                &path.join("txindex_to_inputs_count"),
+                Version::from(1),
+            )?,
+            txindex_to_inputs_sum: StorableVec::forced_import(&path.join("txindex_to_inputs_sum"), Version::from(1))?,
+            txindex_to_last_txinindex: StorableVec::forced_import(
+                &path.join("txindex_to_last_txinindex"),
+                Version::from(1),
+            )?,
+            txindex_to_last_txoutindex: StorableVec::forced_import(
+                &path.join("txindex_to_last_txoutindex"),
+                Version::from(1),
+            )?,
+            txindex_to_outputs_count: StorableVec::forced_import(
+                &path.join("txindex_to_outputs_count"),
+                Version::from(1),
+            )?,
+            txindex_to_outputs_sum: StorableVec::forced_import(&path.join("txindex_to_outputs_sum"), Version::from(1))?,
+        })
+    }
+
+    // pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] {
+    //     [&self.dateindex_to_first_height]
+    // }
+
+    // pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] {
+    //     [&mut self.dateindex_to_first_height]
+    // }
+}

crates/brk_computer/src/structs/addressindextxoutindex.rs

@@ -0,0 +1,22 @@
+use brk_indexer::{Addressindex, Txoutindex};
+use fjall::Slice;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)]
+pub struct AddressindexTxoutindex {
+    addressindex: Addressindex,
+    _padding: u32,
+    txoutindex: Txoutindex,
+}
+
+impl TryFrom<Slice> for AddressindexTxoutindex {
+    type Error = storable_vec::Error;
+    fn try_from(value: Slice) -> Result<Self, Self::Error> {
+        Ok(Self::read_from_bytes(&value)?)
+    }
+}
+impl From<AddressindexTxoutindex> for Slice {
+    fn from(value: AddressindexTxoutindex) -> Self {
+        Self::new(value.as_bytes())
+    }
+}

crates/brk_computer/src/structs/bitcoin.rs

@@ -0,0 +1,14 @@
+use brk_indexer::Sats;
+
+#[derive(Debug, Default, Clone, Copy)]
+pub struct Bitcoin(f64);
+
+impl Bitcoin {
+    const ONE: Self = Self(100_000_000.0);
+}
+
+impl From<Sats> for Bitcoin {
+    fn from(value: Sats) -> Self {
+        Self((*value as f64) / Self::ONE.0)
+    }
+}

crates/brk_computer/src/structs/feerate.rs

@@ -0,0 +1,4 @@
+use derive_deref::Deref;
+
+#[derive(Debug, Deref, Clone, Copy)]
+pub struct Feerate(f32);

crates/brk_computer/src/structs/mod.rs

@@ -0,0 +1,11 @@
+mod addressindextxoutindex;
+mod bitcoin;
+mod feerate;
+// mod ohlc;
+mod unit;
+
+pub use addressindextxoutindex::*;
+pub use bitcoin::*;
+pub use feerate::*;
+// pub use ohlc::*;
+pub use unit::*;

crates/brk_computer/src/structs/ohlc.rs

@@ -0,0 +1,36 @@
+use serde::Serialize;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use super::{Cents, Close, High, Low, Open};
+
+// #[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
+// #[repr(C)]
+// pub struct OHLCCents(OHLC<Cents>);
+
+#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
+#[repr(C)]
+pub struct OHLCCents(Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>);
+
+impl OHLCCents {
+    pub fn open(&self) -> Open<Cents> {
+        self.0
+    }
+
+    pub fn high(&self) -> High<Cents> {
+        self.1
+    }
+
+    pub fn low(&self) -> Low<Cents> {
+        self.2
+    }
+
+    pub fn close(&self) -> Close<Cents> {
+        self.3
+    }
+}
+
+impl From<(Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>)> for OHLCCents {
+    fn from(value: (Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>)) -> Self {
+        Self(value.0, value.1, value.2, value.3)
+    }
+}

crates/brk_computer/src/structs/unit.rs

@@ -0,0 +1,13 @@
+use fjall::Slice;
+
+pub struct Unit();
+impl From<Slice> for Unit {
+    fn from(_: Slice) -> Self {
+        Self()
+    }
+}
+impl From<Unit> for Slice {
+    fn from(_: Unit) -> Self {
+        Self::new(&[])
+    }
+}