bindex: retrying fjall

bindex/Cargo.toml

@@ -9,6 +9,7 @@ biter = { path = "../biter" }
 color-eyre = "0.6.3"
 derive_deref = "1.1.1"
 exit = { path = "../exit" }
+fjall = "2.5.0"
 jiff = "0.1.24"
 rapidhash = "1.3.0"
 rayon = "1.10.0"

bindex/src/lib.rs

@@ -3,7 +3,8 @@ use std::{
     io::{Read, Write},
     path::Path,
     str::FromStr,
-    thread::{self},
+    thread::{self, sleep},
+    time::Duration,
 };
 
 use biter::{
@@ -12,12 +13,13 @@ use biter::{
 };
 use color_eyre::eyre::{eyre, ContextCompat};
 use exit::Exit;
+use fjall::{PersistMode, ReadTransaction, TransactionalKeyspace};
 use rayon::prelude::*;
 
 mod storage;
 mod structs;
 
-use storage::{Stores, Vecs};
+use storage::{Partitions, Vecs};
 use structs::{
     Addressbytes, AddressbytesPrefix, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp,
     TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout,
@@ -35,17 +37,17 @@ impl Indexer {
 
         let mut vecs = Vecs::import(&indexes_dir.join("vecs"))?;
 
-        let open_stores = || Stores::open(&indexes_dir.join("stores"));
-        let stores = open_stores()?;
+        let keyspace = fjall::Config::new(indexes_dir.join("fjall")).open_transactional()?;
+        let mut parts = Partitions::import(&keyspace, &exit)?;
+        let rtx = keyspace.read_tx();
 
         let mut height = vecs
             .min_height()
             .unwrap_or_default()
-            .min(stores.min_height())
+            .min(parts.min_height())
             .and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
             .map(Height::from)
             .unwrap_or_default();
-        // let mut height = Height::default();
 
         let mut txindex_global = vecs.height_to_first_txindex.get_or_default(height)?;
         let mut txinindex_global = vecs.height_to_first_txinindex.get_or_default(height)?;
@@ -64,31 +66,37 @@ impl Indexer {
         let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
         let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
 
-        let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> {
+        let export = |keyspace: &TransactionalKeyspace,
+                      rtx: ReadTransaction,
+                      parts: &mut Partitions,
+                      vecs: &mut Vecs,
+                      height: Height|
+         -> color_eyre::Result<()> {
             println!("Exporting...");
 
-            if height >= Height::from(400_000_u32) {
-                pause();
-                //     println!("Flushing vecs...");
-            }
+            drop(rtx);
 
             exit.block();
 
             thread::scope(|scope| -> color_eyre::Result<()> {
                 let vecs_handle = scope.spawn(|| vecs.flush(height));
-                let stores_handle = scope.spawn(|| stores.export(height));
+
+                parts.write(keyspace, height)?;
+                keyspace.persist(PersistMode::SyncAll)?;
+
                 vecs_handle.join().unwrap()?;
-                stores_handle.join().unwrap()?;
                 Ok(())
             })?;
 
             exit.unblock();
+
             Ok(())
         };
 
-        let mut stores_opt = Some(stores);
+        // let mut stores_opt = Some(stores);
+        let mut rtx_opt = Some(rtx);
 
-        biter::new(bitcoin_dir, Some(height.into()), Some(500_000), rpc)
+        biter::new(bitcoin_dir, Some(height.into()), None, rpc)
             .iter()
             .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
                 println!("Processing block {_height}...");
@@ -96,27 +104,28 @@ impl Indexer {
                 height = Height::from(_height);
                 let timestamp = Timestamp::try_from(block.header.time)?;
 
-                let mut stores = stores_opt.take().context("option should have wtx")?;
+                // let mut stores = stores_opt.take().context("option should have store")?;
+                let rtx = rtx_opt.take().context("option should have rtx")?;
 
                 if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
                     if &blockhash != saved_blockhash.as_ref() {
                         todo!("Rollback not implemented");
-                        // parts.rollback_from(&mut wtx, height, &exit)?;
+                        // parts.rollback_from(&mut rtx, height, &exit)?;
                     }
                 }
 
                 let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
 
-                if stores
+                if parts
                     .blockhash_prefix_to_height
-                    .get(&blockhash_prefix)?
-                    .is_some_and(|prev_height| *prev_height != height)
+                    .get(&rtx, &blockhash_prefix)?
+                    .is_some_and(|prev_height| prev_height != height)
                 {
                     dbg!(blockhash);
                     return Err(eyre!("Collision, expect prefix to need be set yet"));
                 }
 
-                stores
+                parts
                     .blockhash_prefix_to_height
                     .insert_if_needed(blockhash_prefix, height, height);
 
@@ -194,9 +203,9 @@ impl Indexer {
                                     let txid_prefix = TxidPrefix::try_from(&txid)?;
 
                                     let prev_txindex_slice_opt =
-                                        if check_collisions && stores.txid_prefix_to_txindex.needs(height) {
+                                        if check_collisions && parts.txid_prefix_to_txindex.needs(height) {
                                             // Should only find collisions for two txids (duplicates), see below
-                                            stores.txid_prefix_to_txindex.get(&txid_prefix)?.cloned()
+                                            parts.txid_prefix_to_txindex.get(&rtx, &txid_prefix)?
                                         } else {
                                             None
                                         };
@@ -235,14 +244,14 @@ impl Indexer {
                                     return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
                                 }
 
-                                let prev_txindex = if let Some(txindex) = stores
+                                let prev_txindex = if let Some(txindex) = parts
                                     .txid_prefix_to_txindex
-                                    .get(&TxidPrefix::try_from(&outpoint.txid)?)?
+                                    .get(&rtx, &TxidPrefix::try_from(&outpoint.txid)?)?
                                     .and_then(|txindex| {
                                         // Checking if not finding txindex from the future
-                                        (txindex < &txindex_global).then_some(txindex)
+                                        (txindex < txindex_global).then_some(txindex)
                                     }) {
-                                    *txindex
+                                    txindex
                                 } else {
                                     // dbg!(txindex_global + block_txindex, txindex, txin, vin);
                                     return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
@@ -312,11 +321,10 @@ impl Indexer {
                                         });
 
                                     let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
-                                        stores
+                                        parts
                                             .addressbytes_prefix_to_addressindex
-                                            .get(&AddressbytesPrefix::from((addressbytes, addresstype)))
+                                            .get(&rtx, &AddressbytesPrefix::from((addressbytes, addresstype)))
                                             .unwrap()
-                                            .cloned()
                                             // Checking if not in the future
                                             .and_then(|addressindex_local| {
                                                 (addressindex_local < addressindex_global).then_some(addressindex_local)
@@ -346,7 +354,7 @@ impl Indexer {
 
                                         if (vecs.addressindex_to_addresstype.hasnt(addressindex)
                                             && addresstype != prev_addresstype)
-                                            || (stores.addressbytes_prefix_to_addressindex.needs(height)
+                                            || (parts.addressbytes_prefix_to_addressindex.needs(height)
                                                 && prev_addressbytes != addressbytes)
                                         {
                                             let txid = tx.compute_txid();
@@ -494,9 +502,9 @@ impl Indexer {
                                 let addressbytes_prefix = addressbytes_prefix.unwrap();
 
                                 already_added_addressbytes_prefix
-                                    .insert(addressbytes_prefix.clone(), addressindex);
+                                    .insert(addressbytes_prefix, addressindex);
 
-                                stores.addressbytes_prefix_to_addressindex.insert_if_needed(
+                                parts.addressbytes_prefix_to_addressindex.insert_if_needed(
                                     addressbytes_prefix,
                                     addressindex,
                                     height,
@@ -580,7 +588,7 @@ impl Indexer {
 
                             match prev_txindex_opt {
                                 None => {
-                                    stores
+                                    parts
                                         .txid_prefix_to_txindex
                                         .insert_if_needed(txid_prefix, txindex, height);
                                 }
@@ -646,16 +654,24 @@ impl Indexer {
 
                 let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active();
                 if should_snapshot {
-                    export(stores, &mut vecs, height)?;
-                    stores_opt.replace(open_stores()?);
+                    export(&keyspace, rtx, &mut parts, &mut vecs, height)?;
+                    rtx_opt.replace(keyspace.read_tx());
                 } else {
-                    stores_opt.replace(stores);
+                    rtx_opt.replace(rtx);
                 }
 
                 Ok(())
             })?;
 
-        export(stores_opt.take().context("option should have wtx")?, &mut vecs, height)?;
+        export(
+            &keyspace,
+            rtx_opt.take().context("option should have wtx")?,
+            &mut parts,
+            &mut vecs,
+            height,
+        )?;
+
+        sleep(Duration::from_millis(100));
 
         Ok(())
     }

bindex/src/storage/canopy/database.rs

@@ -0,0 +1,50 @@
+use std::{
+    ops::{Deref, DerefMut},
+    time::Duration,
+};
+
+use canopydb::{Database as CanopyDatabase, DbOptions, Error, WriteTransaction};
+
+use super::Environment;
+
+#[derive(Debug)]
+pub struct Database {
+    db: CanopyDatabase,
+    // pub wtx: WriteTransaction,
+}
+impl Deref for Database {
+    type Target = CanopyDatabase;
+    fn deref(&self) -> &Self::Target {
+        &self.db
+    }
+}
+impl DerefMut for Database {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.db
+    }
+}
+
+impl Database {
+    pub fn new(environment: &Environment, name: &str) -> color_eyre::Result<Self> {
+        let mut options = DbOptions::default();
+        options.use_wal = false;
+        options.checkpoint_interval = Duration::from_secs(u64::MAX);
+        options.checkpoint_target_size = usize::MAX;
+        options.throttle_memory_limit = usize::MAX;
+        options.stall_memory_limit = usize::MAX;
+        options.write_txn_memory_limit = usize::MAX;
+
+        let db = environment.get_or_create_database_with(name, options)?;
+
+        Ok(Self {
+            // wtx: db.begin_write()?,
+            db,
+        })
+    }
+
+    pub fn flush(&mut self) -> Result<(), Error> {
+        // drop(blockhash_prefix_to_height_tree);
+        // blockhash_prefix_to_height_tx_opt.take().map(|tx| tx.commit());
+        self.checkpoint()
+    }
+}

bindex/src/storage/canopy/environment.rs

@@ -0,0 +1,20 @@
+use std::path::Path;
+
+use canopydb::{EnvOptions, Environment as CanopyEnvironment};
+use derive_deref::{Deref, DerefMut};
+
+#[derive(Debug, Deref, DerefMut)]
+pub struct Environment(CanopyEnvironment);
+
+impl Environment {
+    pub fn new(path: &Path) -> color_eyre::Result<Self> {
+        let mut options = EnvOptions::new(path);
+        // options.use_mmap = true;
+        options.disable_fsync = true;
+        options.wal_new_file_on_checkpoint = false;
+        options.wal_background_sync_interval = None;
+        options.wal_write_batch_memory_limit = usize::MAX;
+
+        Ok(Self(CanopyEnvironment::with_options(options)?))
+    }
+}

bindex/src/storage/canopy/mod.rs

@@ -0,0 +1,9 @@
+mod database;
+mod environment;
+// mod transaction;
+mod tree;
+
+pub use database::*;
+pub use environment::*;
+// pub use transaction::*;
+pub use tree::*;

bindex/src/storage/canopy/transaction.rs

@@ -0,0 +1,19 @@
+use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
+
+use super::{Database, Tree};
+
+#[derive(Debug)]
+pub struct Transaction<'a, K, V> {
+    tx: WriteTransaction,
+    tree: Tree<'a, K, V>,
+}
+
+impl<'a, K, V> Transaction<'a, K, V> {
+    pub fn new(db: &Database) -> color_eyre::Result<Self> {
+        let tx = db.begin_write()?;
+
+        let tree = Tree::new(&tx)?;
+
+        Ok(Self { tx, tree })
+    }
+}

bindex/src/storage/canopy/tree.rs

@@ -0,0 +1,84 @@
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    ops::{Deref, DerefMut},
+};
+
+use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
+use color_eyre::eyre::eyre;
+
+#[derive(Debug)]
+pub struct Tree<'a, K, V> {
+    tree: CanopyTree<'a>,
+    k: PhantomData<K>,
+    v: PhantomData<V>,
+}
+impl<'a, K, V> Deref for Tree<'a, K, V> {
+    type Target = CanopyTree<'a>;
+    fn deref(&self) -> &Self::Target {
+        &self.tree
+    }
+}
+impl<'a, K, V> DerefMut for Tree<'a, K, V> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.tree
+    }
+}
+
+impl<'a, K, V> Tree<'a, K, V>
+where
+    K: Debug + Sized,
+    V: Debug + Sized + Clone + Copy,
+{
+    const SIZE_OF_K: usize = size_of::<K>();
+    const SIZE_OF_V: usize = size_of::<V>();
+
+    pub fn new(tx: &'a WriteTransaction) -> color_eyre::Result<Self> {
+        let mut options = TreeOptions::new();
+        options.compress_overflow_values = None;
+        options.fixed_key_len = size_of::<K>() as i8;
+        options.fixed_value_len = size_of::<V>() as i8;
+
+        Ok(Self {
+            tree: tx.get_or_create_tree_with(b"tree", options)?,
+            k: PhantomData,
+            v: PhantomData,
+        })
+    }
+
+    pub fn get(&self, key: &K) -> color_eyre::Result<Option<V>> {
+        let slice = self.tree.get(Self::key_as_slice(key))?;
+
+        if slice.is_none() {
+            return Ok(None);
+        }
+
+        let slice = slice.unwrap();
+
+        let (prefix, shorts, suffix) = unsafe { slice.align_to::<V>() };
+
+        if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
+            dbg!(&key, &prefix, &shorts, &suffix);
+            return Err(eyre!("align_to issue"));
+        }
+
+        Ok(Some(shorts[0]))
+    }
+
+    pub fn insert(&mut self, key: &K, value: &V) -> Result<(), canopydb::Error> {
+        self.tree
+            .insert(Self::key_as_slice(key), Self::value_as_slice(value))
+    }
+
+    fn key_as_slice(key: &K) -> &[u8] {
+        let data: *const K = key;
+        let data: *const u8 = data as *const u8;
+        unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_K) }
+    }
+
+    fn value_as_slice(value: &V) -> &[u8] {
+        let data: *const V = value;
+        let data: *const u8 = data as *const u8;
+        unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_V) }
+    }
+}

bindex/src/storage/fjall/base.rs

@@ -0,0 +1,145 @@
+use std::{collections::BTreeMap, mem};
+
+use exit::Exit;
+use fjall::{
+    PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
+    TransactionalPartitionHandle, TxKeyspace, WriteTransaction,
+};
+
+use crate::structs::{Height, Version};
+
+pub struct Partition<Key, Value> {
+    version: Version,
+    data: TransactionalPartitionHandle,
+    meta: TransactionalPartitionHandle,
+    height: Option<Height>,
+    puts: BTreeMap<Key, Value>,
+}
+
+impl<Key, Value> Partition<Key, Value>
+where
+    Key: Into<Slice> + Ord,
+    Value: Into<Slice> + TryFrom<Slice> + Clone,
+{
+    pub const VERSION: &str = "version";
+    pub const HEIGHT: &str = "height";
+
+    pub fn import(
+        keyspace: &TransactionalKeyspace,
+        name: &str,
+        version: Version,
+        exit: &Exit,
+    ) -> color_eyre::Result<Self> {
+        let data = Self::open_data(keyspace, name)?;
+        let meta = Self::open_meta(keyspace, name)?;
+
+        let height = if let Some(slice) = meta.get(Self::HEIGHT)? {
+            Some(Height::try_from(slice)?)
+        } else {
+            None
+        };
+
+        let mut this = Self {
+            version,
+            height,
+            data,
+            meta,
+            puts: BTreeMap::new(),
+        };
+
+        if let Some(slice) = this.meta.get(Self::VERSION)? {
+            if version != Version::try_from(slice)? {
+                this = this.reset(keyspace, name, exit)?;
+            }
+        }
+
+        Ok(this)
+    }
+
+    fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
+        keyspace.open_partition(&format!("{name}-data"), Self::create_options())
+    }
+
+    fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
+        keyspace.open_partition(&format!("{name}-meta"), Self::create_options())
+    }
+
+    fn create_options() -> PartitionCreateOptions {
+        PartitionCreateOptions::default().manual_journal_persist(true)
+    }
+
+    pub fn has(&self, height: Height) -> bool {
+        self.height.is_some_and(|self_height| self_height >= height)
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        !self.has(height)
+    }
+
+    pub fn get<'a>(&self, rtx: &ReadTransaction, key: &'a Key) -> color_eyre::Result<Option<Value>>
+    where
+        fjall::Slice: std::convert::From<&'a Key>,
+        <Value as std::convert::TryFrom<fjall::Slice>>::Error: std::error::Error + Send + Sync,
+        <Value as std::convert::TryFrom<fjall::Slice>>::Error: 'static,
+    {
+        if let Some(v) = self.puts.get(key) {
+            return Ok(Some(v.clone()));
+        }
+
+        if let Some(slice) = rtx.get(&self.data, Slice::from(key))? {
+            let v_res = Value::try_from(slice);
+            let v = v_res?;
+            Ok(Some(v))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn insert_if_needed(&mut self, key: Key, value: Value, height: Height) {
+        if self.needs(height) {
+            self.puts.insert(key, value);
+        }
+    }
+
+    fn update_meta(&self, wtx: &mut WriteTransaction, height: Height) {
+        wtx.insert(&self.meta, Self::VERSION, self.version());
+        wtx.insert(&self.meta, Self::HEIGHT, height);
+    }
+
+    pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> Result<()> {
+        if self.has(height) && self.puts.is_empty() {
+            return Ok(());
+        }
+
+        let mut wtx = keyspace.write_tx();
+        mem::take(&mut self.puts)
+            .into_iter()
+            .for_each(|(key, value)| wtx.insert(&self.data, key, value));
+        self.update_meta(&mut wtx, height);
+        wtx.commit()
+    }
+
+    pub fn version(&self) -> Version {
+        self.version
+    }
+
+    fn reset(mut self, keyspace: &TransactionalKeyspace, name: &str, exit: &Exit) -> Result<Self> {
+        exit.block();
+
+        keyspace.delete_partition(self.data)?;
+        keyspace.delete_partition(self.meta)?;
+
+        keyspace.persist(PersistMode::SyncAll)?;
+
+        self.data = Self::open_data(keyspace, name)?;
+        self.meta = Self::open_meta(keyspace, name)?;
+        self.height = None;
+
+        exit.unblock();
+
+        Ok(self)
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.height.as_ref()
+    }
+}

bindex/src/storage/fjall/mod.rs

@@ -0,0 +1,200 @@
+use std::thread;
+
+use crate::{structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
+
+mod base;
+use base::*;
+use exit::Exit;
+use fjall::{TransactionalKeyspace, TxKeyspace};
+
+pub struct Partitions {
+    pub addressbytes_prefix_to_addressindex: Partition<AddressbytesPrefix, Addressindex>,
+    pub blockhash_prefix_to_height: Partition<BlockHashPrefix, Height>,
+    pub txid_prefix_to_txindex: Partition<TxidPrefix, Txindex>,
+}
+
+impl Partitions {
+    pub fn import(keyspace: &TransactionalKeyspace, exit: &Exit) -> color_eyre::Result<Self> {
+        Ok(Self {
+            addressbytes_prefix_to_addressindex: Partition::import(
+                keyspace,
+                "addressbytes_prefix_to_addressindex",
+                Version::from(1),
+                exit,
+            )?,
+            blockhash_prefix_to_height: Partition::import(
+                keyspace,
+                "blockhash_prefix_to_height",
+                Version::from(1),
+                exit,
+            )?,
+            txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?,
+        })
+    }
+
+    // pub fn rollback_from(
+    //     &mut self,
+    //     _wtx: &mut WriteTransaction,
+    //     _height: Height,
+    //     _exit: &Exit,
+    // ) -> color_eyre::Result<()> {
+    //     panic!();
+    // let mut txindex = None;
+
+    // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (height_slice, slice_blockhash) = slice?;
+    //         let blockhash = BlockHash::from_slice(&slice_blockhash)?;
+
+    //         wtx.remove(self.height_to_blockhash.data(), height_slice);
+
+    //         wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
+
+    //         if txindex.is_none() {
+    //             txindex.replace(
+    //                 wtx.get(self.height_to_first_txindex.data(), height_slice)?
+    //                     .context("for height to have first txindex")?,
+    //             );
+    //         }
+    //         wtx.remove(self.height_to_first_txindex.data(), height_slice);
+    //         wtx.remove(self.height_to_last_txindex.data(), height_slice);
+
+    //         Ok(())
+    //     })?;
+
+    // let txindex = txindex.context("txindex to not be none by now")?;
+
+    // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (slice_txindex, slice_txid) = slice?;
+    //         let txindex = Txindex::from(slice_txindex);
+    //         let txid = Txid::from_slice(&slice_txid)?;
+
+    //         wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
+    //         wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
+    //         wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
+
+    //         Ok(())
+    //     })?;
+
+    // let txoutindex = Txoutindex::from(txindex);
+
+    // let mut addressindexes = BTreeSet::new();
+
+    // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (txoutindex_slice, _) = slice?;
+
+    //         wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
+
+    //         if let Some(addressindex_slice) =
+    //             wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
+    //         {
+    //             wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
+
+    //             let addressindex = Addressindex::from(addressindex_slice);
+    //             addressindexes.insert(addressindex);
+
+    //             let txoutindex = Txoutindex::from(txoutindex_slice);
+    //             let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
+
+    //             wtx.remove(
+    //                 self.addressindex_to_txoutindexes.data(),
+    //                 Slice::from(addresstxoutindex),
+    //             );
+    //         }
+
+    //         Ok(())
+    //     })?;
+
+    // addressindexes
+    // .into_iter()
+    // .filter(|addressindex| {
+    //     let is_empty = wtx
+    //         .prefix(
+    //             self.addressindex_to_txoutindexes.data(),
+    //             Slice::from(*addressindex),
+    //         )
+    //         .next()
+    //         .is_none();
+    //     is_empty
+    // })
+    // .try_for_each(|addressindex| -> color_eyre::Result<()> {
+    //     let addressindex_slice = Slice::from(addressindex);
+
+    //     let addressbytes = Addressbytes::from(
+    //         wtx.get(
+    //             self.addressindex_to_addressbytes.data(),
+    //             &addressindex_slice,
+    //         )?
+    //         .context("addressindex_to_address to have value")?,
+    //     );
+    //     wtx.remove(
+    //         self.addressbytes_prefix_to_addressindex.data(),
+    //         addressbytes.prefix(),
+    //     );
+    //     wtx.remove(
+    //         self.addressindex_to_addressbytes.data(),
+    //         &addressindex_slice,
+    //     );
+    //     wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
+
+    //     Ok(())
+    // })?;
+    //
+
+    // todo!("clear addresstxoutindexes_out")
+    // todo!("clear addresstxoutindexes_in")
+    // todo!("clear zero_txoutindexes")
+    // todo!("clear txindexvout_to_txoutindex")
+
+    // Ok(())
+    // }
+
+    pub fn min_height(&self) -> Option<Height> {
+        [
+            self.addressbytes_prefix_to_addressindex.height(),
+            self.blockhash_prefix_to_height.height(),
+            self.txid_prefix_to_txindex.height(),
+        ]
+        .into_iter()
+        .min()
+        .flatten()
+        .cloned()
+    }
+
+    pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> fjall::Result<()> {
+        thread::scope(|scope| {
+            let addressbytes_prefix_to_addressindex_write_handle =
+                scope.spawn(|| self.addressbytes_prefix_to_addressindex.write(keyspace, height));
+            let blockhash_prefix_to_height_write_handle =
+                scope.spawn(|| self.blockhash_prefix_to_height.write(keyspace, height));
+            let txid_prefix_to_txindex_write_handle =
+                scope.spawn(|| self.txid_prefix_to_txindex.write(keyspace, height));
+
+            addressbytes_prefix_to_addressindex_write_handle.join().unwrap()?;
+            blockhash_prefix_to_height_write_handle.join().unwrap()?;
+            txid_prefix_to_txindex_write_handle.join().unwrap()?;
+
+            Ok(())
+        })
+    }
+
+    // pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
+    //     self.addressbytes_prefix_to_addressindex.update_meta(wtx, height);
+    //     self.blockhash_prefix_to_height.update_meta(wtx, height);
+    //     self.txid_prefix_to_txindex.update_meta(wtx, height);
+    // }
+
+    // pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
+    //     thread::scope(|scope| {
+    //         vec![
+    //             scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
+    //             scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
+    //             scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
+    //         ]
+    //         .into_iter()
+    //         .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
+    //     })
+    // }
+}

bindex/src/storage/mod.rs

@@ -1,5 +1,9 @@
-mod stores;
-mod vecs;
+// mod canopy;
+mod fjall;
+// mod sanakirja;
+mod storable_vec;
 
-pub use stores::*;
-pub use vecs::*;
+// pub use canopy::*;
+pub use fjall::*;
+// pub use sanakirja::*;
+pub use storable_vec::*;

bindex/src/structs/addressindex.rs

@@ -1,5 +1,6 @@
 use derive_deref::{Deref, DerefMut};
 use snkrj::{direct_repr, Storable, UnsizedStorable};
+use storable_vec::UnsafeSizedSerDe;
 
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
 pub struct Addressindex(u32);
@@ -48,3 +49,15 @@ impl From<Addressindex> for usize {
         value.0 as usize
     }
 }
+
+impl TryFrom<fjall::Slice> for Addressindex {
+    type Error = storable_vec::Error;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<Addressindex> for fjall::Slice {
+    fn from(value: Addressindex) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}

bindex/src/structs/height.rs

@@ -119,3 +119,15 @@ impl TryFrom<&rpc::Client> for Height {
         Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
     }
 }
+
+impl TryFrom<fjall::Slice> for Height {
+    type Error = storable_vec::Error;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<Height> for fjall::Slice {
+    fn from(value: Height) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}

bindex/src/structs/prefix.rs

@@ -3,10 +3,11 @@ use std::hash::Hasher;
 use biter::bitcoin::{BlockHash, Txid};
 use derive_deref::Deref;
 use snkrj::{direct_repr, Storable, UnsizedStorable};
+use storable_vec::UnsafeSizedSerDe;
 
 use super::{Addressbytes, Addresstype, SliceExtended};
 
-#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct AddressbytesPrefix([u8; 8]);
 direct_repr!(AddressbytesPrefix);
 impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix {
@@ -23,8 +24,24 @@ impl From<[u8; 8]> for AddressbytesPrefix {
         Self(value)
     }
 }
+impl TryFrom<fjall::Slice> for AddressbytesPrefix {
+    type Error = color_eyre::Report;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<&AddressbytesPrefix> for fjall::Slice {
+    fn from(value: &AddressbytesPrefix) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}
+impl From<AddressbytesPrefix> for fjall::Slice {
+    fn from(value: AddressbytesPrefix) -> Self {
+        Self::from(&value)
+    }
+}
 
-#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct BlockHashPrefix([u8; 8]);
 direct_repr!(BlockHashPrefix);
 impl TryFrom<&BlockHash> for BlockHashPrefix {
@@ -33,8 +50,24 @@ impl TryFrom<&BlockHash> for BlockHashPrefix {
         Ok(Self((&value[..]).read_8x_u8()?))
     }
 }
+impl TryFrom<fjall::Slice> for BlockHashPrefix {
+    type Error = color_eyre::Report;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<&BlockHashPrefix> for fjall::Slice {
+    fn from(value: &BlockHashPrefix) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}
+impl From<BlockHashPrefix> for fjall::Slice {
+    fn from(value: BlockHashPrefix) -> Self {
+        Self::from(&value)
+    }
+}
 
-#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct TxidPrefix([u8; 8]);
 direct_repr!(TxidPrefix);
 impl TryFrom<&Txid> for TxidPrefix {
@@ -43,3 +76,19 @@ impl TryFrom<&Txid> for TxidPrefix {
         Ok(Self((&value[..]).read_8x_u8()?))
     }
 }
+impl TryFrom<fjall::Slice> for TxidPrefix {
+    type Error = color_eyre::Report;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<&TxidPrefix> for fjall::Slice {
+    fn from(value: &TxidPrefix) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}
+impl From<TxidPrefix> for fjall::Slice {
+    fn from(value: TxidPrefix) -> Self {
+        Self::from(&value)
+    }
+}

bindex/src/structs/txindex.rs

@@ -2,6 +2,7 @@ use std::ops::{Add, AddAssign};
 
 use derive_deref::{Deref, DerefMut};
 use snkrj::{direct_repr, Storable, UnsizedStorable};
+use storable_vec::UnsafeSizedSerDe;
 
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
 pub struct Txindex(u32);
@@ -57,3 +58,15 @@ impl From<Txindex> for usize {
         value.0 as usize
     }
 }
+
+impl TryFrom<fjall::Slice> for Txindex {
+    type Error = storable_vec::Error;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<Txindex> for fjall::Slice {
+    fn from(value: Txindex) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}

bindex/src/structs/version.rs

@@ -23,3 +23,15 @@ impl TryFrom<&Path> for Version {
         Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
     }
 }
+
+impl TryFrom<fjall::Slice> for Version {
+    type Error = color_eyre::Report;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(*Self::unsafe_try_from_slice(&value)?)
+    }
+}
+impl From<Version> for fjall::Slice {
+    fn from(value: Version) -> Self {
+        Self::new(value.unsafe_as_slice())
+    }
+}