bindex: contained fjall code

CHANGELOG.md

@@ -25,7 +25,6 @@
 - Created separate crate for indexing called `bindex`
 - Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
 - Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibō has its own indexes now
-- Tried different storage engines such as `fjall`, `canopydb` and `heed`, the first ended up being 3 times slower than `sanakirja` and had very fragile files (just looking at them would corrupt them), and the rest wouldn't play nice with `rayon` which is a dealbreaker
 - `snkrj` added a robust auto defragmentation to improve disk usage without the need for user's intervention
 
 ## Git

Cargo.lock

@@ -214,6 +214,14 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 [[package]]
 name = "bomputer"
 version = "0.1.0"
+dependencies = [
+ "bindex",
+ "color-eyre",
+ "derive_deref",
+ "exit",
+ "jiff",
+ "rayon",
+]
 
 [[package]]
 name = "byteorder"
@@ -546,9 +554,9 @@ checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
 [[package]]
 name = "jiff"
-version = "0.1.25"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb73dbeee753ae9411475ddd8861765fa7f25fe1eebf180c24e1bbabef3fbdcd"
+checksum = "a85348106ab244d90fe2d70faad939b71c5dad1258e5da9116e176064fc6c078"
 dependencies = [
  "jiff-tzdb-platform",
  "log",

Cargo.toml

@@ -4,7 +4,16 @@
 # edition = "2021"
 
 [workspace]
-members = ["bindex", "biter", "bomputer", "exit", "iterable", "snkrj", "storable_vec"]
+members = [
+    "bindex",
+    "biter",
+    "bomputer",
+    "exit",
+    # "server",
+    "snkrj",
+    "struct_iterable",
+    "storable_vec",
+]
 resolver = "2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

bindex/Cargo.toml

@@ -10,7 +10,7 @@ color-eyre = "0.6.3"
 derive_deref = "1.1.1"
 exit = { path = "../exit" }
 fjall = "2.5.0"
-jiff = "0.1.24"
+jiff = "0.1.27"
 rapidhash = "1.3.0"
 rayon = "1.10.0"
 snkrj = { path = "../snkrj" }

bindex/src/lib.rs

@@ -7,10 +7,9 @@ use std::{
     time::Duration,
 };
 
-use biter::{
-    bitcoin::{Transaction, TxIn, TxOut, Txid},
-    rpc,
-};
+pub use biter::*;
+
+use biter::bitcoin::{Transaction, TxIn, TxOut, Txid};
 use color_eyre::eyre::{eyre, ContextCompat};
 use exit::Exit;
 use fjall::{PersistMode, ReadTransaction, TransactionalKeyspace};
@@ -19,8 +18,10 @@ use rayon::prelude::*;
 mod storage;
 mod structs;
 
-use storage::{Partitions, Vecs};
-use structs::{
+pub use biter;
+
+use storage::{Fjalls, StorableVecs};
+pub use structs::{
     Addressbytes, AddressbytesPrefix, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp,
     TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout,
 };
@@ -28,18 +29,24 @@ use structs::{
 const UNSAFE_BLOCKS: u32 = 100;
 const SNAPSHOT_BLOCK_RANGE: usize = 1000;
 
-#[derive(Debug)]
-pub struct Indexer;
+pub struct Indexer {
+    vecs: StorableVecs,
+    parts: Fjalls,
+}
 
 impl Indexer {
-    pub fn index(indexes_dir: &Path, bitcoin_dir: &Path, rpc: rpc::Client, exit: Exit) -> color_eyre::Result<()> {
+    pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
+        let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?;
+        let parts = Fjalls::import(&indexes_dir.join("fjall"))?;
+
+        Ok(Self { vecs, parts })
+    }
+
+    pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
         let check_collisions = true;
 
-        let mut vecs = Vecs::import(&indexes_dir.join("vecs"))?;
-
-        let keyspace = fjall::Config::new(indexes_dir.join("fjall")).open_transactional()?;
-        let mut parts = Partitions::import(&keyspace, &exit)?;
-        let rtx = keyspace.read_tx();
+        let vecs = &mut self.vecs;
+        let parts = &mut self.parts;
 
         let mut height = vecs
             .min_height()
@@ -66,24 +73,14 @@ impl Indexer {
         let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
         let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
 
-        let export = |keyspace: &TransactionalKeyspace,
-                      rtx: ReadTransaction,
-                      parts: &mut Partitions,
-                      vecs: &mut Vecs,
-                      height: Height|
-         -> color_eyre::Result<()> {
+        let export = |parts: &mut Fjalls, vecs: &mut StorableVecs, height: Height| -> color_eyre::Result<()> {
             println!("Exporting...");
 
-            drop(rtx);
-
             exit.block();
 
             thread::scope(|scope| -> color_eyre::Result<()> {
                 let vecs_handle = scope.spawn(|| vecs.flush(height));
-
-                parts.write(keyspace, height)?;
-                keyspace.persist(PersistMode::SyncAll)?;
-
+                parts.commit(height)?;
                 vecs_handle.join().unwrap()?;
                 Ok(())
             })?;
@@ -93,8 +90,8 @@ impl Indexer {
             Ok(())
         };
 
-        // let mut stores_opt = Some(stores);
-        let mut rtx_opt = Some(rtx);
+        // // let mut stores_opt = Some(stores);
+        // let mut rtx_opt = Some(rtx);
 
         biter::new(bitcoin_dir, Some(height.into()), None, rpc)
             .iter()
@@ -105,7 +102,7 @@ impl Indexer {
                 let timestamp = Timestamp::try_from(block.header.time)?;
 
                 // let mut stores = stores_opt.take().context("option should have store")?;
-                let rtx = rtx_opt.take().context("option should have rtx")?;
+                // let rtx = rtx_opt.take().context("option should have rtx")?;
 
                 if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
                     if &blockhash != saved_blockhash.as_ref() {
@@ -118,7 +115,7 @@ impl Indexer {
 
                 if parts
                     .blockhash_prefix_to_height
-                    .get(&rtx, &blockhash_prefix)?
+                    .get(&blockhash_prefix)?
                     .is_some_and(|prev_height| prev_height != height)
                 {
                     dbg!(blockhash);
@@ -205,7 +202,7 @@ impl Indexer {
                                     let prev_txindex_slice_opt =
                                         if check_collisions && parts.txid_prefix_to_txindex.needs(height) {
                                             // Should only find collisions for two txids (duplicates), see below
-                                            parts.txid_prefix_to_txindex.get(&rtx, &txid_prefix)?
+                                            parts.txid_prefix_to_txindex.get(&txid_prefix)?
                                         } else {
                                             None
                                         };
@@ -246,7 +243,7 @@ impl Indexer {
 
                                 let prev_txindex = if let Some(txindex) = parts
                                     .txid_prefix_to_txindex
-                                    .get(&rtx, &TxidPrefix::try_from(&outpoint.txid)?)?
+                                    .get(&TxidPrefix::try_from(&outpoint.txid)?)?
                                     .and_then(|txindex| {
                                         // Checking if not finding txindex from the future
                                         (txindex < txindex_global).then_some(txindex)
@@ -323,7 +320,7 @@ impl Indexer {
                                     let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
                                         parts
                                             .addressbytes_prefix_to_addressindex
-                                            .get(&rtx, &AddressbytesPrefix::from((addressbytes, addresstype)))
+                                            .get(&AddressbytesPrefix::from((addressbytes, addresstype)))
                                             .unwrap()
                                             // Checking if not in the future
                                             .and_then(|addressindex_local| {
@@ -546,7 +543,10 @@ impl Indexer {
 
                                     let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
                                         .get(&TxidPrefix::try_from(&txid)?)
-                                        .context("txid should be in same block")?
+                                        .context("txid should be in same block").inspect_err(|_| {
+                                            dbg!(&txid_prefix_to_txid_and_block_txindex_and_prev_txindex);
+                                            // panic!();
+                                        })?
                                         .2;
                                     let prev_txindex = txindex_global + block_txindex;
 
@@ -654,22 +654,13 @@ impl Indexer {
 
                 let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active();
                 if should_snapshot {
-                    export(&keyspace, rtx, &mut parts, &mut vecs, height)?;
-                    rtx_opt.replace(keyspace.read_tx());
-                } else {
-                    rtx_opt.replace(rtx);
+                    export(parts, vecs, height)?;
                 }
 
                 Ok(())
             })?;
 
-        export(
-            &keyspace,
-            rtx_opt.take().context("option should have wtx")?,
-            &mut parts,
-            &mut vecs,
-            height,
-        )?;
+        export(parts, vecs, height)?;
 
         sleep(Duration::from_millis(100));
 
@@ -683,7 +674,6 @@ enum InputSource<'a> {
     SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)),
 }
 
-#[allow(unused)]
 fn pause() {
     let mut stdin = std::io::stdin();
     let mut stdout = std::io::stdout();

bindex/src/main.rs

@@ -4,8 +4,6 @@ use bindex::Indexer;
 use biter::rpc;
 use exit::Exit;
 
-// https://github.com/romanz/electrs/blob/master/doc/schema.md
-
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
 
@@ -18,7 +16,9 @@ fn main() -> color_eyre::Result<()> {
 
     let i = std::time::Instant::now();
 
-    Indexer::index(Path::new("indexes"), data_dir, rpc, exit)?;
+    let mut indexer = Indexer::import(Path::new("indexes"))?;
+
+    indexer.index(data_dir, rpc, &exit)?;
 
     dbg!(i.elapsed());
 

bindex/src/storage/fjall/base.rs

@@ -1,145 +0,0 @@
-use std::{collections::BTreeMap, mem};
-
-use exit::Exit;
-use fjall::{
-    PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
-    TransactionalPartitionHandle, TxKeyspace, WriteTransaction,
-};
-
-use crate::structs::{Height, Version};
-
-pub struct Partition<Key, Value> {
-    version: Version,
-    data: TransactionalPartitionHandle,
-    meta: TransactionalPartitionHandle,
-    height: Option<Height>,
-    puts: BTreeMap<Key, Value>,
-}
-
-impl<Key, Value> Partition<Key, Value>
-where
-    Key: Into<Slice> + Ord,
-    Value: Into<Slice> + TryFrom<Slice> + Clone,
-{
-    pub const VERSION: &str = "version";
-    pub const HEIGHT: &str = "height";
-
-    pub fn import(
-        keyspace: &TransactionalKeyspace,
-        name: &str,
-        version: Version,
-        exit: &Exit,
-    ) -> color_eyre::Result<Self> {
-        let data = Self::open_data(keyspace, name)?;
-        let meta = Self::open_meta(keyspace, name)?;
-
-        let height = if let Some(slice) = meta.get(Self::HEIGHT)? {
-            Some(Height::try_from(slice)?)
-        } else {
-            None
-        };
-
-        let mut this = Self {
-            version,
-            height,
-            data,
-            meta,
-            puts: BTreeMap::new(),
-        };
-
-        if let Some(slice) = this.meta.get(Self::VERSION)? {
-            if version != Version::try_from(slice)? {
-                this = this.reset(keyspace, name, exit)?;
-            }
-        }
-
-        Ok(this)
-    }
-
-    fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
-        keyspace.open_partition(&format!("{name}-data"), Self::create_options())
-    }
-
-    fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
-        keyspace.open_partition(&format!("{name}-meta"), Self::create_options())
-    }
-
-    fn create_options() -> PartitionCreateOptions {
-        PartitionCreateOptions::default().manual_journal_persist(true)
-    }
-
-    pub fn has(&self, height: Height) -> bool {
-        self.height.is_some_and(|self_height| self_height >= height)
-    }
-    pub fn needs(&self, height: Height) -> bool {
-        !self.has(height)
-    }
-
-    pub fn get<'a>(&self, rtx: &ReadTransaction, key: &'a Key) -> color_eyre::Result<Option<Value>>
-    where
-        fjall::Slice: std::convert::From<&'a Key>,
-        <Value as std::convert::TryFrom<fjall::Slice>>::Error: std::error::Error + Send + Sync,
-        <Value as std::convert::TryFrom<fjall::Slice>>::Error: 'static,
-    {
-        if let Some(v) = self.puts.get(key) {
-            return Ok(Some(v.clone()));
-        }
-
-        if let Some(slice) = rtx.get(&self.data, Slice::from(key))? {
-            let v_res = Value::try_from(slice);
-            let v = v_res?;
-            Ok(Some(v))
-        } else {
-            Ok(None)
-        }
-    }
-
-    pub fn insert_if_needed(&mut self, key: Key, value: Value, height: Height) {
-        if self.needs(height) {
-            self.puts.insert(key, value);
-        }
-    }
-
-    fn update_meta(&self, wtx: &mut WriteTransaction, height: Height) {
-        wtx.insert(&self.meta, Self::VERSION, self.version());
-        wtx.insert(&self.meta, Self::HEIGHT, height);
-    }
-
-    pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> Result<()> {
-        if self.has(height) && self.puts.is_empty() {
-            return Ok(());
-        }
-
-        let mut wtx = keyspace.write_tx();
-        mem::take(&mut self.puts)
-            .into_iter()
-            .for_each(|(key, value)| wtx.insert(&self.data, key, value));
-        self.update_meta(&mut wtx, height);
-        wtx.commit()
-    }
-
-    pub fn version(&self) -> Version {
-        self.version
-    }
-
-    fn reset(mut self, keyspace: &TransactionalKeyspace, name: &str, exit: &Exit) -> Result<Self> {
-        exit.block();
-
-        keyspace.delete_partition(self.data)?;
-        keyspace.delete_partition(self.meta)?;
-
-        keyspace.persist(PersistMode::SyncAll)?;
-
-        self.data = Self::open_data(keyspace, name)?;
-        self.meta = Self::open_meta(keyspace, name)?;
-        self.height = None;
-
-        exit.unblock();
-
-        Ok(self)
-    }
-
-    pub fn height(&self) -> Option<&Height> {
-        self.height.as_ref()
-    }
-}

bindex/src/storage/fjalls/base.rs

@@ -0,0 +1,102 @@
+use std::{collections::BTreeMap, mem, path::Path};
+
+use fjall::{
+    PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
+    TransactionalPartitionHandle,
+};
+
+use crate::structs::{Height, Version};
+
+use super::Meta;
+
+pub struct Partition<Key, Value> {
+    meta: Meta,
+    keyspace: TransactionalKeyspace,
+    part: TransactionalPartitionHandle,
+    rtx: ReadTransaction,
+    puts: BTreeMap<Key, Value>,
+}
+
+impl<Key, Value> Partition<Key, Value>
+where
+    Key: Into<Slice> + Ord,
+    Value: Into<Slice> + TryFrom<Slice> + Clone,
+{
+    pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
+        let meta = Meta::checked_open(path, version)?;
+
+        let keyspace = fjall::Config::new(path.join("fjall")).open_transactional()?;
+        let handle = keyspace.open_partition(
+            "partition",
+            PartitionCreateOptions::default().manual_journal_persist(true),
+        )?;
+        let rtx = keyspace.read_tx();
+
+        Ok(Self {
+            meta,
+            keyspace,
+            part: handle,
+            rtx,
+            puts: BTreeMap::new(),
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        self.meta.len() + self.puts.len()
+    }
+
+    pub fn has(&self, height: Height) -> bool {
+        self.height().is_some_and(|self_height| self_height >= &height)
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        !self.has(height)
+    }
+
+    pub fn get<'a>(&self, key: &'a Key) -> color_eyre::Result<Option<Value>>
+    where
+        fjall::Slice: std::convert::From<&'a Key>,
+        <Value as std::convert::TryFrom<fjall::Slice>>::Error: std::error::Error + Send + Sync,
+        <Value as std::convert::TryFrom<fjall::Slice>>::Error: 'static,
+    {
+        if let Some(v) = self.puts.get(key) {
+            return Ok(Some(v.clone()));
+        }
+
+        if let Some(slice) = self.rtx.get(&self.part, Slice::from(key))? {
+            let v_res = Value::try_from(slice);
+            let v = v_res?;
+            Ok(Some(v))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn insert_if_needed(&mut self, key: Key, value: Value, height: Height) {
+        if self.needs(height) {
+            self.puts.insert(key, value);
+        }
+    }
+
+    pub fn commit(&mut self, height: Height) -> Result<()> {
+        if self.has(height) && self.puts.is_empty() {
+            return Ok(());
+        }
+
+        let mut wtx = self.keyspace.write_tx();
+        mem::take(&mut self.puts)
+            .into_iter()
+            .for_each(|(key, value)| wtx.insert(&self.part, key, value));
+        self.meta.export(self.len(), height)?;
+        wtx.commit()?;
+
+        self.keyspace.persist(PersistMode::SyncAll)?;
+
+        self.rtx = self.keyspace.read_tx();
+
+        Ok(())
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.meta.height()
+    }
+}

bindex/src/storage/fjalls/meta.rs

@@ -0,0 +1,92 @@
+use std::{
+    fs, io,
+    path::{Path, PathBuf},
+};
+
+use storable_vec::UnsafeSizedSerDe;
+
+use super::{Height, Version};
+
+pub struct Meta {
+    pathbuf: PathBuf,
+    version: Version,
+    height: Option<Height>,
+    len: usize,
+}
+
+impl Meta {
+    pub fn checked_open(path: &Path, version: Version) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let is_same_version =
+            Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
+
+        if !is_same_version {
+            fs::remove_dir_all(path)?;
+            fs::create_dir(path)?;
+        }
+
+        let this = Self {
+            pathbuf: path.to_owned(),
+            version,
+            height: Height::try_from(Self::path_height_(path).as_path()).ok(),
+            len: Self::read_length_(path)?,
+        };
+
+        this.version.write(&this.path_version())?;
+
+        Ok(this)
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
+        self.len = len;
+        self.write_length()?;
+        self.height = Some(height);
+        height.write(&self.path_height())
+    }
+
+    fn path_version(&self) -> PathBuf {
+        Self::path_version_(&self.pathbuf)
+    }
+    fn path_version_(path: &Path) -> PathBuf {
+        path.join("version")
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.height.as_ref()
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        self.height.is_none_or(|self_height| height > self_height)
+    }
+    pub fn has(&self, height: Height) -> bool {
+        !self.needs(height)
+    }
+    fn path_height(&self) -> PathBuf {
+        Self::path_height_(&self.pathbuf)
+    }
+    fn path_height_(path: &Path) -> PathBuf {
+        path.join("height")
+    }
+
+    fn read_length(&self) -> color_eyre::Result<usize> {
+        Self::read_length_(&self.pathbuf)
+    }
+    fn read_length_(path: &Path) -> color_eyre::Result<usize> {
+        Ok(fs::read(Self::path_length(path))
+            .map(|v| usize::unsafe_try_from_slice(v.as_slice()).cloned().unwrap_or_default())
+            .unwrap_or_default())
+    }
+    fn write_length(&self) -> io::Result<()> {
+        Self::write_length_(&self.pathbuf, self.len)
+    }
+    fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> {
+        fs::write(Self::path_length(path), len.to_le_bytes())
+    }
+    fn path_length(path: &Path) -> PathBuf {
+        path.join("length")
+    }
+}

bindex/src/storage/fjalls/mod.rs

@@ -1,34 +1,28 @@
-use std::thread;
+use std::{path::Path, thread};
 
 use crate::{structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
 
 mod base;
-use base::*;
-use exit::Exit;
-use fjall::{TransactionalKeyspace, TxKeyspace};
+mod meta;
 
-pub struct Partitions {
+use base::*;
+use meta::*;
+
+pub struct Fjalls {
     pub addressbytes_prefix_to_addressindex: Partition<AddressbytesPrefix, Addressindex>,
     pub blockhash_prefix_to_height: Partition<BlockHashPrefix, Height>,
     pub txid_prefix_to_txindex: Partition<TxidPrefix, Txindex>,
 }
 
-impl Partitions {
-    pub fn import(keyspace: &TransactionalKeyspace, exit: &Exit) -> color_eyre::Result<Self> {
+impl Fjalls {
+    pub fn import(path: &Path) -> color_eyre::Result<Self> {
         Ok(Self {
             addressbytes_prefix_to_addressindex: Partition::import(
-                keyspace,
-                "addressbytes_prefix_to_addressindex",
+                &path.join("addressbytes_prefix_to_addressindex"),
                 Version::from(1),
-                exit,
             )?,
-            blockhash_prefix_to_height: Partition::import(
-                keyspace,
-                "blockhash_prefix_to_height",
-                Version::from(1),
-                exit,
-            )?,
-            txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?,
+            blockhash_prefix_to_height: Partition::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
+            txid_prefix_to_txindex: Partition::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
         })
     }
 
@@ -163,18 +157,17 @@ impl Partitions {
         .cloned()
     }
 
-    pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> fjall::Result<()> {
+    pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
         thread::scope(|scope| {
-            let addressbytes_prefix_to_addressindex_write_handle =
-                scope.spawn(|| self.addressbytes_prefix_to_addressindex.write(keyspace, height));
-            let blockhash_prefix_to_height_write_handle =
-                scope.spawn(|| self.blockhash_prefix_to_height.write(keyspace, height));
-            let txid_prefix_to_txindex_write_handle =
-                scope.spawn(|| self.txid_prefix_to_txindex.write(keyspace, height));
+            let addressbytes_prefix_to_addressindex_commit_handle =
+                scope.spawn(|| self.addressbytes_prefix_to_addressindex.commit(height));
+            let blockhash_prefix_to_height_commit_handle =
+                scope.spawn(|| self.blockhash_prefix_to_height.commit(height));
+            let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
 
-            addressbytes_prefix_to_addressindex_write_handle.join().unwrap()?;
-            blockhash_prefix_to_height_write_handle.join().unwrap()?;
-            txid_prefix_to_txindex_write_handle.join().unwrap()?;
+            addressbytes_prefix_to_addressindex_commit_handle.join().unwrap()?;
+            blockhash_prefix_to_height_commit_handle.join().unwrap()?;
+            txid_prefix_to_txindex_commit_handle.join().unwrap()?;
 
             Ok(())
         })

bindex/src/storage/mod.rs

@@ -1,9 +1,9 @@
 // mod canopy;
-mod fjall;
+mod fjalls;
 // mod sanakirja;
-mod storable_vec;
+mod storable_vecs;
 
 // pub use canopy::*;
-pub use fjall::*;
+pub use fjalls::*;
 // pub use sanakirja::*;
-pub use storable_vec::*;
+pub use storable_vecs::*;

bindex/src/storage/storable_vecs/mod.rs

@@ -14,7 +14,7 @@ mod base;
 
 use base::*;
 
-pub struct Vecs {
+pub struct StorableVecs {
     pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
     pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
     pub addressindex_to_height: StorableVec<Addressindex, Height>,
@@ -75,7 +75,7 @@ pub struct Vecs {
 
 // const UNSAFE_BLOCKS: usize = 100;
 
-impl Vecs {
+impl StorableVecs {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 

bomputer/Cargo.toml

@@ -4,3 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+bindex = { path = "../bindex" }
+color-eyre = "0.6.3"
+derive_deref = "1.1.1"
+exit = { path = "../exit" }
+jiff = "0.1.27"
+rayon = "1.10.0"

bomputer/src/lib.rs

@@ -1,18 +1,35 @@
+use std::path::Path;
+
+use bindex::{biter::rpc, Indexer};
+use exit::Exit;
+
 mod structs;
 
 use structs::*;
 
-pub fn add(left: u64, right: u64) -> u64 {
-    left + right
+pub struct Bomputer;
+
+impl Bomputer {
+    pub fn compute() {}
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
+pub fn main() -> color_eyre::Result<()> {
+    color_eyre::install()?;
 
-    #[test]
-    fn it_works() {
-        let result = add(2, 2);
-        assert_eq!(result, 4);
-    }
+    let data_dir = Path::new("../../bitcoin");
+    let rpc = rpc::Client::new(
+        "http://localhost:8332",
+        rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
+    )?;
+    let exit = Exit::new();
+
+    let i = std::time::Instant::now();
+
+    let mut indexer = Indexer::import(Path::new("indexes"))?;
+
+    indexer.index(data_dir, rpc, &exit)?;
+
+    dbg!(i.elapsed());
+
+    Ok(())
 }

bomputer/src/main.rs

@@ -0,0 +1,13 @@
+use structs::Date;
+
+mod structs;
+
+pub fn main() -> color_eyre::Result<()> {
+    let date1 = Date::from(jiff::civil::Date::constant(2009, 1, 9));
+    let date2 = Date::from(jiff::civil::Date::constant(2009, 1, 31));
+    let date3 = Date::from(jiff::civil::Date::constant(2019, 1, 9));
+    dbg!(usize::try_from(date1))?;
+    dbg!(usize::try_from(date2))?;
+    dbg!(usize::try_from(date3))?;
+    Ok(())
+}

bomputer/src/structs/date.rs

@@ -1,19 +1,41 @@
-use jiff::tz::TimeZone;
+use bindex::Timestamp;
+use color_eyre::eyre::eyre;
+use derive_deref::Deref;
+use jiff::{civil::Date as _Date, tz::TimeZone};
 
-use super::Timestamp;
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref)]
+pub struct Date(_Date);
 
-#[derive(Debug, Clone)]
-pub struct Date(jiff::civil::Date);
+impl Date {
+    const INDEX_ZERO: Self = Self(_Date::constant(2009, 1, 3));
+    const INDEX_ONE: Self = Self(_Date::constant(2009, 1, 9));
+}
 
-impl From<&Timestamp> for Date {
-    fn from(value: &Timestamp) -> Self {
-        Self(jiff::civil::Date::from(value.to_zoned(TimeZone::UTC)))
+impl From<_Date> for Date {
+    fn from(value: _Date) -> Self {
+        Self(value)
     }
 }
 
-impl From<Date> for usize {
-    // 2009-01-03 => 0
-    // 2009-01-09 => 1
-    // 2009-01-10 => 2
-    // ...
+impl From<&Timestamp> for Date {
+    fn from(value: &Timestamp) -> Self {
+        Self(_Date::from(value.to_zoned(TimeZone::UTC)))
+    }
+}
+
+impl TryFrom<Date> for usize {
+    type Error = color_eyre::Report;
+    fn try_from(value: Date) -> Result<Self, Self::Error> {
+        if value < Date::INDEX_ZERO {
+            Err(eyre!("Date is too early"))
+        } else if value == Date::INDEX_ZERO {
+            Ok(0)
+        } else if value < Date::INDEX_ONE {
+            Err(eyre!("Date is between first and second"))
+        } else if value == Date::INDEX_ONE {
+            Ok(1)
+        } else {
+            Ok(Date::INDEX_ONE.until(*value)?.get_days() as usize + 1)
+        }
+    }
 }

server/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "kibo_server"
+version = "0.6.0"
+edition = "2021"
+
+[dependencies]
+# allocative = "0.3.4"
+axum = "0.7.9"
+# bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
+#     "serde",
+# ] }
+# bitcoin_hashes = { version = "0.15.0" }
+# biter = { path = "./src/crates/biter" }
+# chrono = { version = "0.4.39", features = ["serde"] }
+# clap = { version = "4.5.26", features = ["derive"] }
+# color-eyre = "0.6.3"
+# ctrlc = { version = "3.4.5", features = ["termination"] }
+# derive_deref = "1.1.1"
+# env_logger = "0.11.6"
+# inferno = "0.12.1"
+# itertools = "0.13.0"
+# log = { version = "0.4.25", features = ["std", "serde"] }
+# ordered-float = "4.6.0"
+# rayon = "1.10.0"
+# regex = "1.11.1"
+# reqwest = { version = "0.12.12", features = ["blocking", "json"] }
+# rlimit = "0.10.2"
+# snkrj = { path = "./src/crates/snkrj" }
+# serde = { version = "1.0.217", features = ["derive"] }
+# serde_json = "1.0.135"
+# struct_iterable = { path = "./src/crates/iterable" }
+# swc = "9.0.2"
+# swc_common = "5.0.0"
+# tokio = { version = "1.43.0", features = ["full"] }
+# toml = "0.8.19"
+# tower-http = { version = "0.6.2", features = ["compression-full"] }
+# zstd = "0.13.2"