workspace: reorg

indexer/src/storage/canopies/database.rs

@@ -0,0 +1,50 @@
+use std::{
+    ops::{Deref, DerefMut},
+    time::Duration,
+};
+
+use canopydb::{Database as CanopyDatabase, DbOptions, Error, WriteTransaction};
+
+use super::Environment;
+
+#[derive(Debug)]
+pub struct Database {
+    db: CanopyDatabase,
+    // pub wtx: WriteTransaction,
+}
+impl Deref for Database {
+    type Target = CanopyDatabase;
+    fn deref(&self) -> &Self::Target {
+        &self.db
+    }
+}
+impl DerefMut for Database {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.db
+    }
+}
+
+impl Database {
+    pub fn new(environment: &Environment, name: &str) -> color_eyre::Result<Self> {
+        let mut options = DbOptions::default();
+        options.use_wal = false;
+        options.checkpoint_interval = Duration::from_secs(u64::MAX);
+        options.checkpoint_target_size = usize::MAX;
+        options.throttle_memory_limit = usize::MAX;
+        options.stall_memory_limit = usize::MAX;
+        options.write_txn_memory_limit = usize::MAX;
+
+        let db = environment.get_or_create_database_with(name, options)?;
+
+        Ok(Self {
+            // wtx: db.begin_write()?,
+            db,
+        })
+    }
+
+    pub fn flush(&mut self) -> Result<(), Error> {
+        // drop(blockhash_prefix_to_height_tree);
+        // blockhash_prefix_to_height_tx_opt.take().map(|tx| tx.commit());
+        self.checkpoint()
+    }
+}

indexer/src/storage/canopies/environment.rs

@@ -0,0 +1,20 @@
+use std::path::Path;
+
+use canopydb::{EnvOptions, Environment as CanopyEnvironment};
+use derive_deref::{Deref, DerefMut};
+
+#[derive(Debug, Deref, DerefMut)]
+pub struct Environment(CanopyEnvironment);
+
+impl Environment {
+    pub fn new(path: &Path) -> color_eyre::Result<Self> {
+        let mut options = EnvOptions::new(path);
+        // options.use_mmap = true;
+        options.disable_fsync = true;
+        options.wal_new_file_on_checkpoint = false;
+        options.wal_background_sync_interval = None;
+        options.wal_write_batch_memory_limit = usize::MAX;
+
+        Ok(Self(CanopyEnvironment::with_options(options)?))
+    }
+}

indexer/src/storage/canopies/mod.rs

@@ -0,0 +1,9 @@
+mod database;
+mod environment;
+// mod transaction;
+mod tree;
+
+pub use database::*;
+pub use environment::*;
+// pub use transaction::*;
+pub use tree::*;

indexer/src/storage/canopies/transaction.rs

@@ -0,0 +1,19 @@
+use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
+
+use super::{Database, Tree};
+
+#[derive(Debug)]
+pub struct Transaction<'a, K, V> {
+    tx: WriteTransaction,
+    tree: Tree<'a, K, V>,
+}
+
+impl<'a, K, V> Transaction<'a, K, V> {
+    pub fn new(db: &Database) -> color_eyre::Result<Self> {
+        let tx = db.begin_write()?;
+
+        let tree = Tree::new(&tx)?;
+
+        Ok(Self { tx, tree })
+    }
+}

indexer/src/storage/canopies/tree.rs

@@ -0,0 +1,84 @@
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    ops::{Deref, DerefMut},
+};
+
+use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
+use color_eyre::eyre::eyre;
+
+#[derive(Debug)]
+pub struct Tree<'a, K, V> {
+    tree: CanopyTree<'a>,
+    k: PhantomData<K>,
+    v: PhantomData<V>,
+}
+impl<'a, K, V> Deref for Tree<'a, K, V> {
+    type Target = CanopyTree<'a>;
+    fn deref(&self) -> &Self::Target {
+        &self.tree
+    }
+}
+impl<'a, K, V> DerefMut for Tree<'a, K, V> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.tree
+    }
+}
+
+impl<'a, K, V> Tree<'a, K, V>
+where
+    K: Debug + Sized,
+    V: Debug + Sized + Clone + Copy,
+{
+    const SIZE_OF_K: usize = size_of::<K>();
+    const SIZE_OF_V: usize = size_of::<V>();
+
+    pub fn new(tx: &'a WriteTransaction) -> color_eyre::Result<Self> {
+        let mut options = TreeOptions::new();
+        options.compress_overflow_values = None;
+        options.fixed_key_len = size_of::<K>() as i8;
+        options.fixed_value_len = size_of::<V>() as i8;
+
+        Ok(Self {
+            tree: tx.get_or_create_tree_with(b"tree", options)?,
+            k: PhantomData,
+            v: PhantomData,
+        })
+    }
+
+    pub fn get(&self, key: &K) -> color_eyre::Result<Option<V>> {
+        let slice = self.tree.get(Self::key_as_slice(key))?;
+
+        if slice.is_none() {
+            return Ok(None);
+        }
+
+        let slice = slice.unwrap();
+
+        let (prefix, shorts, suffix) = unsafe { slice.align_to::<V>() };
+
+        if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
+            dbg!(&key, &prefix, &shorts, &suffix);
+            return Err(eyre!("align_to issue"));
+        }
+
+        Ok(Some(shorts[0]))
+    }
+
+    pub fn insert(&mut self, key: &K, value: &V) -> Result<(), canopydb::Error> {
+        self.tree
+            .insert(Self::key_as_slice(key), Self::value_as_slice(value))
+    }
+
+    fn key_as_slice(key: &K) -> &[u8] {
+        let data: *const K = key;
+        let data: *const u8 = data as *const u8;
+        unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_K) }
+    }
+
+    fn value_as_slice(value: &V) -> &[u8] {
+        let data: *const V = value;
+        let data: *const u8 = data as *const u8;
+        unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_V) }
+    }
+}

indexer/src/storage/fjalls/base.rs

@@ -0,0 +1,116 @@
+use std::{collections::BTreeMap, error, mem, path::Path};
+
+use fjall::{
+    PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
+    TransactionalPartitionHandle,
+};
+use storable_vec::Value;
+use unsafe_slice_serde::UnsafeSliceSerde;
+
+use crate::structs::{Height, Version};
+
+use super::Meta;
+
+pub struct Partition<Key, Value> {
+    meta: Meta,
+    keyspace: TransactionalKeyspace,
+    part: TransactionalPartitionHandle,
+    rtx: ReadTransaction,
+    puts: BTreeMap<Key, Value>,
+}
+
+impl<K, V> Partition<K, V>
+where
+    K: Into<Slice> + Ord,
+    V: Into<Slice> + TryFrom<Slice>,
+    <V as TryFrom<Slice>>::Error: error::Error + Send + Sync + 'static,
+{
+    pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
+        let meta = Meta::checked_open(path, version)?;
+        let keyspace = if let Ok(keyspace) = Self::open_keyspace(path) {
+            keyspace
+        } else {
+            meta.reset()?;
+            return Self::import(path, version);
+        };
+        let part = if let Ok(part) = Self::open_partition_handle(&keyspace) {
+            part
+        } else {
+            drop(keyspace);
+            meta.reset()?;
+            return Self::import(path, version);
+        };
+        let rtx = keyspace.read_tx();
+
+        Ok(Self {
+            meta,
+            keyspace,
+            part,
+            rtx,
+            puts: BTreeMap::new(),
+        })
+    }
+
+    pub fn get(&self, key: &K) -> color_eyre::Result<Option<Value<V>>> {
+        if let Some(v) = self.puts.get(key) {
+            Ok(Some(Value::Ref(v)))
+        } else if let Some(slice) = self.rtx.get(&self.part, key.unsafe_as_slice())? {
+            Ok(Some(Value::Owned(V::try_from(slice)?)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
+        if self.needs(height) {
+            self.puts.insert(key, value);
+        }
+    }
+
+    pub fn commit(&mut self, height: Height) -> Result<()> {
+        if self.has(height) && self.puts.is_empty() {
+            return Ok(());
+        }
+
+        self.meta.export(self.len(), height)?;
+
+        let mut wtx = self.keyspace.write_tx();
+        mem::take(&mut self.puts)
+            .into_iter()
+            .for_each(|(key, value)| wtx.insert(&self.part, key, value));
+
+        wtx.commit()?;
+
+        self.keyspace.persist(PersistMode::SyncAll)?;
+
+        self.rtx = self.keyspace.read_tx();
+
+        Ok(())
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.meta.height()
+    }
+
+    pub fn len(&self) -> usize {
+        self.meta.len() + self.puts.len()
+    }
+
+    pub fn has(&self, height: Height) -> bool {
+        self.meta.has(height)
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        self.meta.needs(height)
+    }
+
+    fn open_keyspace(path: &Path) -> Result<TransactionalKeyspace> {
+        fjall::Config::new(path.join("fjall")).open_transactional()
+    }
+
+    fn open_partition_handle(keyspace: &TransactionalKeyspace) -> Result<TransactionalPartitionHandle> {
+        keyspace.open_partition(
+            "partition",
+            PartitionCreateOptions::default().manual_journal_persist(true),
+        )
+    }
+}

indexer/src/storage/fjalls/meta.rs

@@ -0,0 +1,99 @@
+use std::{
+    fs, io,
+    path::{Path, PathBuf},
+};
+
+use unsafe_slice_serde::UnsafeSliceSerde;
+
+use super::{Height, Version};
+
+pub struct Meta {
+    pathbuf: PathBuf,
+    version: Version,
+    height: Option<Height>,
+    len: usize,
+}
+
+impl Meta {
+    pub fn checked_open(path: &Path, version: Version) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let is_same_version =
+            Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
+
+        if !is_same_version {
+            Self::reset_(path)?;
+        }
+
+        let this = Self {
+            pathbuf: path.to_owned(),
+            version,
+            height: Height::try_from(Self::path_height_(path).as_path()).ok(),
+            len: Self::read_length_(path)?,
+        };
+
+        this.version.write(&this.path_version())?;
+
+        Ok(this)
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
+        self.len = len;
+        self.write_length()?;
+        self.height = Some(height);
+        height.write(&self.path_height())
+    }
+
+    pub fn reset(&self) -> io::Result<()> {
+        Self::reset_(self.pathbuf.as_path())
+    }
+    fn reset_(path: &Path) -> io::Result<()> {
+        fs::remove_dir_all(path)?;
+        fs::create_dir(path)
+    }
+
+    fn path_version(&self) -> PathBuf {
+        Self::path_version_(&self.pathbuf)
+    }
+    fn path_version_(path: &Path) -> PathBuf {
+        path.join("version")
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.height.as_ref()
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        self.height.is_none_or(|self_height| height > self_height)
+    }
+    pub fn has(&self, height: Height) -> bool {
+        !self.needs(height)
+    }
+    fn path_height(&self) -> PathBuf {
+        Self::path_height_(&self.pathbuf)
+    }
+    fn path_height_(path: &Path) -> PathBuf {
+        path.join("height")
+    }
+
+    // fn read_length(&self) -> color_eyre::Result<usize> {
+    //     Self::read_length_(&self.pathbuf)
+    // }
+    fn read_length_(path: &Path) -> color_eyre::Result<usize> {
+        Ok(fs::read(Self::path_length(path))
+            .map(|v| usize::unsafe_try_from_slice(v.as_slice()).cloned().unwrap_or_default())
+            .unwrap_or_default())
+    }
+    fn write_length(&self) -> io::Result<()> {
+        Self::write_length_(&self.pathbuf, self.len)
+    }
+    fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> {
+        fs::write(Self::path_length(path), len.to_le_bytes())
+    }
+    fn path_length(path: &Path) -> PathBuf {
+        path.join("length")
+    }
+}

indexer/src/storage/fjalls/mod.rs

@@ -0,0 +1,193 @@
+use std::{path::Path, thread};
+
+use crate::{structs::Version, AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
+
+mod base;
+mod meta;
+
+use base::*;
+use meta::*;
+
+pub struct Fjalls {
+    pub addresshash_to_addressindex: Partition<AddressHash, Addressindex>,
+    pub blockhash_prefix_to_height: Partition<BlockHashPrefix, Height>,
+    pub txid_prefix_to_txindex: Partition<TxidPrefix, Txindex>,
+}
+
+impl Fjalls {
+    pub fn import(path: &Path) -> color_eyre::Result<Self> {
+        Ok(Self {
+            addresshash_to_addressindex: Partition::import(
+                &path.join("addresshash_to_addressindex"),
+                Version::from(1),
+            )?,
+            blockhash_prefix_to_height: Partition::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
+            txid_prefix_to_txindex: Partition::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
+        })
+    }
+
+    // pub fn rollback_from(
+    //     &mut self,
+    //     _wtx: &mut WriteTransaction,
+    //     _height: Height,
+    //     _exit: &Exit,
+    // ) -> color_eyre::Result<()> {
+    //     panic!();
+    // let mut txindex = None;
+
+    // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (height_slice, slice_blockhash) = slice?;
+    //         let blockhash = BlockHash::from_slice(&slice_blockhash)?;
+
+    //         wtx.remove(self.height_to_blockhash.data(), height_slice);
+
+    //         wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
+
+    //         if txindex.is_none() {
+    //             txindex.replace(
+    //                 wtx.get(self.height_to_first_txindex.data(), height_slice)?
+    //                     .context("for height to have first txindex")?,
+    //             );
+    //         }
+    //         wtx.remove(self.height_to_first_txindex.data(), height_slice);
+    //         wtx.remove(self.height_to_last_txindex.data(), height_slice);
+
+    //         Ok(())
+    //     })?;
+
+    // let txindex = txindex.context("txindex to not be none by now")?;
+
+    // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (slice_txindex, slice_txid) = slice?;
+    //         let txindex = Txindex::from(slice_txindex);
+    //         let txid = Txid::from_slice(&slice_txid)?;
+
+    //         wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
+    //         wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
+    //         wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
+
+    //         Ok(())
+    //     })?;
+
+    // let txoutindex = Txoutindex::from(txindex);
+
+    // let mut addressindexes = BTreeSet::new();
+
+    // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (txoutindex_slice, _) = slice?;
+
+    //         wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
+
+    //         if let Some(addressindex_slice) =
+    //             wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
+    //         {
+    //             wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
+
+    //             let addressindex = Addressindex::from(addressindex_slice);
+    //             addressindexes.insert(addressindex);
+
+    //             let txoutindex = Txoutindex::from(txoutindex_slice);
+    //             let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
+
+    //             wtx.remove(
+    //                 self.addressindex_to_txoutindexes.data(),
+    //                 Slice::from(addresstxoutindex),
+    //             );
+    //         }
+
+    //         Ok(())
+    //     })?;
+
+    // addressindexes
+    // .into_iter()
+    // .filter(|addressindex| {
+    //     let is_empty = wtx
+    //         .prefix(
+    //             self.addressindex_to_txoutindexes.data(),
+    //             Slice::from(*addressindex),
+    //         )
+    //         .next()
+    //         .is_none();
+    //     is_empty
+    // })
+    // .try_for_each(|addressindex| -> color_eyre::Result<()> {
+    //     let addressindex_slice = Slice::from(addressindex);
+
+    //     let addressbytes = Addressbytes::from(
+    //         wtx.get(
+    //             self.addressindex_to_addressbytes.data(),
+    //             &addressindex_slice,
+    //         )?
+    //         .context("addressindex_to_address to have value")?,
+    //     );
+    //     wtx.remove(
+    //         self.addressbytes_prefix_to_addressindex.data(),
+    //         addressbytes.prefix(),
+    //     );
+    //     wtx.remove(
+    //         self.addressindex_to_addressbytes.data(),
+    //         &addressindex_slice,
+    //     );
+    //     wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
+
+    //     Ok(())
+    // })?;
+    //
+
+    // todo!("clear addresstxoutindexes_out")
+    // todo!("clear addresstxoutindexes_in")
+    // todo!("clear zero_txoutindexes")
+    // todo!("clear txindexvout_to_txoutindex")
+
+    // Ok(())
+    // }
+
+    pub fn min_height(&self) -> Option<Height> {
+        [
+            self.addresshash_to_addressindex.height(),
+            self.blockhash_prefix_to_height.height(),
+            self.txid_prefix_to_txindex.height(),
+        ]
+        .into_iter()
+        .min()
+        .flatten()
+        .cloned()
+    }
+
+    pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
+        thread::scope(|scope| {
+            let addresshash_to_addressindex_commit_handle =
+                scope.spawn(|| self.addresshash_to_addressindex.commit(height));
+            let blockhash_prefix_to_height_commit_handle =
+                scope.spawn(|| self.blockhash_prefix_to_height.commit(height));
+            let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
+
+            addresshash_to_addressindex_commit_handle.join().unwrap()?;
+            blockhash_prefix_to_height_commit_handle.join().unwrap()?;
+            txid_prefix_to_txindex_commit_handle.join().unwrap()?;
+
+            Ok(())
+        })
+    }
+
+    // pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
+    //     self.addressbytes_prefix_to_addressindex.update_meta(wtx, height);
+    //     self.blockhash_prefix_to_height.update_meta(wtx, height);
+    //     self.txid_prefix_to_txindex.update_meta(wtx, height);
+    // }
+
+    // pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
+    //     thread::scope(|scope| {
+    //         vec![
+    //             scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
+    //             scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
+    //             scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
+    //         ]
+    //         .into_iter()
+    //         .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
+    //     })
+    // }
+}

indexer/src/storage/mod.rs

@@ -0,0 +1,9 @@
+// mod canopy;
+mod fjalls;
+// mod sanakirja;
+mod storable_vecs;
+
+// pub use canopy::*;
+pub use fjalls::*;
+// pub use sanakirja::*;
+pub use storable_vecs::*;

indexer/src/storage/sanakirjas/meta.rs

@@ -0,0 +1,81 @@
+use std::{
+    fs, io,
+    path::{Path, PathBuf},
+};
+
+use snkrj::UnitDatabase;
+
+use super::{Height, Version};
+
+pub struct StoreMeta {
+    pathbuf: PathBuf,
+    version: Version,
+    height: Option<Height>,
+    pub len: usize,
+}
+
+impl StoreMeta {
+    pub fn checked_open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
+        fs::create_dir_all(path)?;
+
+        let is_same_version =
+            Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
+
+        if !is_same_version {
+            fs::remove_dir_all(path)?;
+            fs::create_dir(path)?;
+        }
+
+        let this = Self {
+            pathbuf: path.to_owned(),
+            version,
+            height: Height::try_from(Self::path_height_(path).as_path()).ok(),
+            len: UnitDatabase::read_length_(path),
+        };
+
+        this.version.write(&this.path_version())?;
+
+        Ok(this)
+    }
+
+    #[allow(unused)]
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn export(mut self, height: Height) -> Result<(), io::Error> {
+        self.height = Some(height);
+        height.write(&self.path_height())?;
+        UnitDatabase::write_length_(&self.pathbuf, self.len)
+    }
+
+    pub fn path_parts(&self) -> PathBuf {
+        Self::path_parts_(&self.pathbuf)
+    }
+    fn path_parts_(path: &Path) -> PathBuf {
+        path.join("parts")
+    }
+
+    fn path_version(&self) -> PathBuf {
+        Self::path_version_(&self.pathbuf)
+    }
+    fn path_version_(path: &Path) -> PathBuf {
+        path.join("version")
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.height.as_ref()
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        self.height.is_none_or(|self_height| height > self_height)
+    }
+    pub fn has(&self, height: Height) -> bool {
+        !self.needs(height)
+    }
+    fn path_height(&self) -> PathBuf {
+        Self::path_height_(&self.pathbuf)
+    }
+    fn path_height_(path: &Path) -> PathBuf {
+        path.join("height")
+    }
+}

indexer/src/storage/sanakirjas/mod.rs

@@ -0,0 +1,174 @@
+use std::{path::Path, thread};
+
+use crate::{
+    structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex, Txoutindex,
+};
+
+mod meta;
+mod multi;
+mod unique;
+
+use meta::*;
+use unique::*;
+
+pub struct Stores {
+    pub addressbytes_prefix_to_addressindex: StoreUnique<AddressbytesPrefix, Addressindex>,
+    pub blockhash_prefix_to_height: StoreUnique<BlockHashPrefix, Height>,
+    pub txid_prefix_to_txindex: StoreUnique<TxidPrefix, Txindex>,
+}
+
+impl Stores {
+    pub fn open(path: &Path) -> color_eyre::Result<Self> {
+        Ok(Self {
+            addressbytes_prefix_to_addressindex: StoreUnique::open(
+                &path.join("addressbytes_prefix_to_addressindex"),
+                Version::from(1),
+            )?,
+            blockhash_prefix_to_height: StoreUnique::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
+            txid_prefix_to_txindex: StoreUnique::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
+        })
+    }
+
+    // pub fn rollback_from(
+    //     &mut self,
+    //     _wtx: &mut WriteTransaction,
+    //     _height: Height,
+    //     _exit: &Exit,
+    // ) -> color_eyre::Result<()> {
+    //     panic!();
+    // let mut txindex = None;
+
+    // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (height_slice, slice_blockhash) = slice?;
+    //         let blockhash = BlockHash::from_slice(&slice_blockhash)?;
+
+    //         wtx.remove(self.height_to_blockhash.data(), height_slice);
+
+    //         wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
+
+    //         if txindex.is_none() {
+    //             txindex.replace(
+    //                 wtx.get(self.height_to_first_txindex.data(), height_slice)?
+    //                     .context("for height to have first txindex")?,
+    //             );
+    //         }
+    //         wtx.remove(self.height_to_first_txindex.data(), height_slice);
+    //         wtx.remove(self.height_to_last_txindex.data(), height_slice);
+
+    //         Ok(())
+    //     })?;
+
+    // let txindex = txindex.context("txindex to not be none by now")?;
+
+    // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (slice_txindex, slice_txid) = slice?;
+    //         let txindex = Txindex::from(slice_txindex);
+    //         let txid = Txid::from_slice(&slice_txid)?;
+
+    //         wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
+    //         wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
+    //         wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
+
+    //         Ok(())
+    //     })?;
+
+    // let txoutindex = Txoutindex::from(txindex);
+
+    // let mut addressindexes = BTreeSet::new();
+
+    // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
+    //     .try_for_each(|slice| -> color_eyre::Result<()> {
+    //         let (txoutindex_slice, _) = slice?;
+
+    //         wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
+
+    //         if let Some(addressindex_slice) =
+    //             wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
+    //         {
+    //             wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
+
+    //             let addressindex = Addressindex::from(addressindex_slice);
+    //             addressindexes.insert(addressindex);
+
+    //             let txoutindex = Txoutindex::from(txoutindex_slice);
+    //             let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
+
+    //             wtx.remove(
+    //                 self.addressindex_to_txoutindexes.data(),
+    //                 Slice::from(addresstxoutindex),
+    //             );
+    //         }
+
+    //         Ok(())
+    //     })?;
+
+    // addressindexes
+    // .into_iter()
+    // .filter(|addressindex| {
+    //     let is_empty = wtx
+    //         .prefix(
+    //             self.addressindex_to_txoutindexes.data(),
+    //             Slice::from(*addressindex),
+    //         )
+    //         .next()
+    //         .is_none();
+    //     is_empty
+    // })
+    // .try_for_each(|addressindex| -> color_eyre::Result<()> {
+    //     let addressindex_slice = Slice::from(addressindex);
+
+    //     let addressbytes = Addressbytes::from(
+    //         wtx.get(
+    //             self.addressindex_to_addressbytes.data(),
+    //             &addressindex_slice,
+    //         )?
+    //         .context("addressindex_to_address to have value")?,
+    //     );
+    //     wtx.remove(
+    //         self.addressbytes_prefix_to_addressindex.data(),
+    //         addressbytes.prefix(),
+    //     );
+    //     wtx.remove(
+    //         self.addressindex_to_addressbytes.data(),
+    //         &addressindex_slice,
+    //     );
+    //     wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
+
+    //     Ok(())
+    // })?;
+    //
+
+    // todo!("clear addresstxoutindexes_out")
+    // todo!("clear addresstxoutindexes_in")
+    // todo!("clear zero_txoutindexes")
+    // todo!("clear txindexvout_to_txoutindex")
+
+    // Ok(())
+    // }
+
+    pub fn min_height(&self) -> Option<Height> {
+        [
+            self.addressbytes_prefix_to_addressindex.height(),
+            self.blockhash_prefix_to_height.height(),
+            self.txid_prefix_to_txindex.height(),
+        ]
+        .into_iter()
+        .min()
+        .flatten()
+        .cloned()
+    }
+
+    pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
+        thread::scope(|scope| {
+            vec![
+                scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
+                scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
+                scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
+            ]
+            .into_iter()
+            .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
+        })
+    }
+}

indexer/src/storage/sanakirjas/multi.rs

@@ -0,0 +1,103 @@
+use std::{array, path::Path, sync::OnceLock};
+
+use rayon::prelude::*;
+use snkrj::{DatabaseKey, DatabaseMulti, DatabaseValue};
+
+use super::{Height, StoreMeta, Version};
+
+pub struct StoreMulti<K, V>
+where
+    K: DatabaseKey,
+    V: DatabaseValue,
+{
+    meta: StoreMeta,
+    pub parts: [OnceLock<Box<DatabaseMulti<K, V>>>; 256],
+}
+
+impl<K, V> StoreMulti<K, V>
+where
+    K: DatabaseKey,
+    V: DatabaseValue,
+{
+    pub fn open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
+        let meta = StoreMeta::checked_open(path, version)?;
+
+        Ok(Self {
+            meta,
+            parts: array::from_fn(|_| OnceLock::new()),
+        })
+    }
+
+    // pub fn len(&self) -> usize {
+    //     self.meta.len()
+    // }
+
+    fn get_or_init_store(&self, key: &K) -> &DatabaseMulti<K, V> {
+        self.get_or_init_store_(key.as_ne_byte() as usize)
+    }
+
+    fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseMulti<K, V> {
+        self.parts[storeindex]
+            .get_or_init(|| Box::new(DatabaseMulti::open(self.meta.path_parts().join(storeindex.to_string())).unwrap()))
+    }
+
+    fn get_or_init_mut_store(&mut self, key: &K) -> &mut DatabaseMulti<K, V> {
+        self.get_or_init_store(key);
+
+        self.parts
+            .get_mut(key.as_ne_byte() as usize)
+            .unwrap()
+            .get_mut()
+            .unwrap()
+    }
+
+    #[allow(unused)]
+    pub fn open_all(&self) {
+        (0..=(u8::MAX) as usize).for_each(|storeindex| {
+            self.get_or_init_store_(storeindex);
+        });
+    }
+
+    #[allow(unused)]
+    pub fn get(&self, key: &K) -> Result<Option<&V>, snkrj::Error> {
+        self.get_or_init_store(key).get(key)
+    }
+
+    pub fn insert(&mut self, key: K, value: V) -> Option<V> {
+        self.meta.len += 1;
+        self.get_or_init_mut_store(&key).insert(key, value)
+    }
+
+    pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
+        if self.meta.needs(height) {
+            self.insert(key, value);
+        }
+    }
+
+    pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
+        if self.has(height) {
+            return Ok(());
+        }
+
+        self.meta.export(height)?;
+
+        self.parts.into_par_iter().try_for_each(|s| {
+            if let Some(db) = s.into_inner() {
+                db.export()
+            } else {
+                Ok(())
+            }
+        })
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.meta.height()
+    }
+    #[allow(unused)]
+    pub fn needs(&self, height: Height) -> bool {
+        self.meta.needs(height)
+    }
+    pub fn has(&self, height: Height) -> bool {
+        self.meta.has(height)
+    }
+}

indexer/src/storage/sanakirjas/unique.rs

@@ -0,0 +1,102 @@
+use std::{array, path::Path, sync::OnceLock};
+
+use rayon::prelude::*;
+use snkrj::{DatabaseKey, DatabaseUnique, DatabaseValue};
+
+use super::{Height, StoreMeta, Version};
+
+pub struct StoreUnique<K, V>
+where
+    K: DatabaseKey,
+    V: DatabaseValue,
+{
+    meta: StoreMeta,
+    pub parts: [OnceLock<Box<DatabaseUnique<K, V>>>; 256],
+}
+
+impl<K, V> StoreUnique<K, V>
+where
+    K: DatabaseKey,
+    V: DatabaseValue,
+{
+    pub fn open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
+        let meta = StoreMeta::checked_open(path, version)?;
+
+        Ok(Self {
+            meta,
+            parts: array::from_fn(|_| OnceLock::new()),
+        })
+    }
+
+    // pub fn len(&self) -> usize {
+    //     self.meta.len()
+    // }
+
+    fn get_or_init_store(&self, key: &K) -> &DatabaseUnique<K, V> {
+        self.get_or_init_store_(key.as_ne_byte() as usize)
+    }
+
+    fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseUnique<K, V> {
+        self.parts[storeindex].get_or_init(|| {
+            Box::new(DatabaseUnique::open(self.meta.path_parts().join(storeindex.to_string())).unwrap())
+        })
+    }
+
+    fn get_or_init_mut_store(&mut self, key: &K) -> &mut DatabaseUnique<K, V> {
+        self.get_or_init_store(key);
+
+        self.parts
+            .get_mut(key.as_ne_byte() as usize)
+            .unwrap()
+            .get_mut()
+            .unwrap()
+    }
+
+    #[allow(unused)]
+    pub fn open_all(&self) {
+        (0..=(u8::MAX) as usize).for_each(|storeindex| {
+            self.get_or_init_store_(storeindex);
+        });
+    }
+
+    pub fn get(&self, key: &K) -> Result<Option<&V>, snkrj::Error> {
+        self.get_or_init_store(key).get(key)
+    }
+
+    pub fn insert(&mut self, key: K, value: V) -> Option<V> {
+        self.meta.len += 1;
+        self.get_or_init_mut_store(&key).insert(key, value)
+    }
+
+    pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
+        if self.meta.needs(height) {
+            self.insert(key, value);
+        }
+    }
+
+    pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
+        if self.has(height) {
+            return Ok(());
+        }
+
+        self.meta.export(height)?;
+
+        self.parts.into_par_iter().try_for_each(|s| {
+            if let Some(db) = s.into_inner() {
+                db.export()
+            } else {
+                Ok(())
+            }
+        })
+    }
+
+    pub fn height(&self) -> Option<&Height> {
+        self.meta.height()
+    }
+    pub fn needs(&self, height: Height) -> bool {
+        self.meta.needs(height)
+    }
+    pub fn has(&self, height: Height) -> bool {
+        self.meta.has(height)
+    }
+}

indexer/src/storage/storable_vecs/base.rs

@@ -0,0 +1,119 @@
+use std::{
+    fmt::Debug,
+    fs, io,
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+};
+
+use super::{Height, Version};
+
+pub struct StorableVec<I, T> {
+    height: Option<Height>,
+    pathbuf: PathBuf,
+    version: Version,
+    vec: storable_vec::StorableVec<I, T>,
+}
+
+impl<I, T> StorableVec<I, T>
+where
+    I: Into<usize>,
+    T: Sized + Debug + Clone,
+{
+    pub fn import(path: &Path, version: Version) -> io::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let pathbuf = path.to_owned();
+        let path_vec = Self::path_vec_(path);
+        let path_version = Self::path_version_(path);
+
+        let is_same_version =
+            Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version);
+        if !is_same_version {
+            let _ = fs::remove_file(&path_vec);
+            let _ = fs::remove_file(&path_version);
+            let _ = fs::remove_file(Self::path_height_(path));
+        }
+
+        let this = Self {
+            height: Height::try_from(Self::path_height_(path).as_path()).ok(),
+            pathbuf,
+            version,
+            vec: storable_vec::StorableVec::import(&path_vec)?,
+        };
+
+        this.version.write(&this.path_version())?;
+
+        Ok(this)
+    }
+
+    pub fn flush(&mut self, height: Height) -> io::Result<()> {
+        if self.needs(height) {
+            height.write(&self.path_height())?;
+        }
+
+        self.vec.flush()
+    }
+
+    // fn path_vec(&self) -> PathBuf {
+    //     Self::_path_vec(&self.path)
+    // }
+    fn path_vec_(path: &Path) -> PathBuf {
+        path.join("vec")
+    }
+
+    fn path_version(&self) -> PathBuf {
+        Self::path_version_(&self.pathbuf)
+    }
+    fn path_version_(path: &Path) -> PathBuf {
+        path.join("version")
+    }
+
+    pub fn height(&self) -> color_eyre::Result<Height> {
+        Height::try_from(self.path_height().as_path())
+    }
+    fn path_height(&self) -> PathBuf {
+        Self::path_height_(&self.pathbuf)
+    }
+    fn path_height_(path: &Path) -> PathBuf {
+        path.join("height")
+    }
+
+    pub fn needs(&self, height: Height) -> bool {
+        self.height.is_none_or(|self_height| height > self_height)
+    }
+    #[allow(unused)]
+    pub fn has(&self, height: Height) -> bool {
+        !self.needs(height)
+    }
+}
+
+impl<I, T> Deref for StorableVec<I, T> {
+    type Target = storable_vec::StorableVec<I, T>;
+    fn deref(&self) -> &Self::Target {
+        &self.vec
+    }
+}
+impl<I, T> DerefMut for StorableVec<I, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.vec
+    }
+}
+
+pub trait AnyBindexVec {
+    fn height(&self) -> color_eyre::Result<Height>;
+    fn flush(&mut self, height: Height) -> io::Result<()>;
+}
+
+impl<I, T> AnyBindexVec for StorableVec<I, T>
+where
+    I: Into<usize>,
+    T: Sized + Debug + Clone,
+{
+    fn height(&self) -> color_eyre::Result<Height> {
+        self.height()
+    }
+
+    fn flush(&mut self, height: Height) -> io::Result<()> {
+        self.flush(height)
+    }
+}

indexer/src/storage/storable_vecs/mod.rs

@@ -0,0 +1,473 @@
+use std::{fs, io, path::Path};
+
+use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight};
+use exit::Exit;
+use rayon::prelude::*;
+
+use crate::structs::{
+    Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes,
+    P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, Timestamp, Txindex,
+    Txinindex, Txoutindex, Version,
+};
+
+mod base;
+
+use base::*;
+
+pub struct StorableVecs {
+    pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
+    pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
+    pub addressindex_to_height: StorableVec<Addressindex, Height>,
+    pub height_to_blockhash: StorableVec<Height, BlockHash>,
+    pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
+    pub height_to_first_emptyindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_multisigindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_opreturnindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_pushonlyindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_txindex: StorableVec<Height, Txindex>,
+    pub height_to_first_txinindex: StorableVec<Height, Txinindex>,
+    pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
+    pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex>,
+    pub height_to_size: StorableVec<Height, usize>,
+    pub height_to_timestamp: StorableVec<Height, Timestamp>,
+    pub height_to_weight: StorableVec<Height, Weight>,
+    pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
+    pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
+    pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
+    pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
+    pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
+    pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
+    pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
+    pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex>,
+    pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex>,
+    pub txindex_to_height: StorableVec<Txindex, Height>,
+    pub txindex_to_locktime: StorableVec<Txindex, bitcoin::absolute::LockTime>,
+    pub txindex_to_txid: StorableVec<Txindex, Txid>,
+    pub txindex_to_txversion: StorableVec<Txindex, transaction::Version>,
+    pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex>,
+    pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
+    pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
+    // Can be computed later:
+    // pub height_to_date: StorableVec<Height, Date>,
+    // pub height_to_totalfees: StorableVec<Height, Amount>,
+    // pub height_to_inputcount: StorableVec<Txindex, u32>,
+    // pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
+    // pub height_to_last_txindex: StorableVec<Height, Txindex>,
+    // pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
+    // pub height_to_outputcount: StorableVec<Txindex, u32>,
+    // pub height_to_txcount: StorableVec<Txindex, u32>,
+    // pub height_to_subsidy: StorableVec<Txindex, u32>,
+    // pub height_to_minfeerate: StorableVec<Txindex, Feerate>,
+    // pub height_to_maxfeerate: StorableVec<Txindex, Feerate>,
+    // pub height_to_medianfeerate: StorableVec<Txindex, Feerate>,
+    // pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
+    // pub txindex_to_inputcount: StorableVec<Txindex, u32>,
+    // pub txindex_to_outputcount: StorableVec<Txindex, u32>,
+    // pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
+}
+
+// const UNSAFE_BLOCKS: usize = 100;
+
+impl StorableVecs {
+    pub fn import(path: &Path) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        Ok(Self {
+            addressindex_to_addresstype: StorableVec::import(
+                &path.join("addressindex_to_addresstype"),
+                Version::from(1),
+            )?,
+            addressindex_to_addresstypeindex: StorableVec::import(
+                &path.join("addressindex_to_addresstypeindex"),
+                Version::from(1),
+            )?,
+            addressindex_to_height: StorableVec::import(&path.join("addressindex_to_height"), Version::from(1))?,
+            height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?,
+            height_to_first_addressindex: StorableVec::import(
+                &path.join("height_to_first_addressindex"),
+                Version::from(1),
+            )?,
+            height_to_first_emptyindex: StorableVec::import(
+                &path.join("height_to_first_emptyindex"),
+                Version::from(1),
+            )?,
+            height_to_first_multisigindex: StorableVec::import(
+                &path.join("height_to_first_multisigindex"),
+                Version::from(1),
+            )?,
+            height_to_first_opreturnindex: StorableVec::import(
+                &path.join("height_to_first_opreturnindex"),
+                Version::from(1),
+            )?,
+            height_to_first_pushonlyindex: StorableVec::import(
+                &path.join("height_to_first_pushonlyindex"),
+                Version::from(1),
+            )?,
+            height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?,
+            height_to_first_txinindex: StorableVec::import(&path.join("height_to_first_txinindex"), Version::from(1))?,
+            height_to_first_txoutindex: StorableVec::import(
+                &path.join("height_to_first_txoutindex"),
+                Version::from(1),
+            )?,
+            height_to_first_unknownindex: StorableVec::import(
+                &path.join("height_to_first_unkownindex"),
+                Version::from(1),
+            )?,
+            height_to_first_p2pk33index: StorableVec::import(
+                &path.join("height_to_first_p2pk33index"),
+                Version::from(1),
+            )?,
+            height_to_first_p2pk65index: StorableVec::import(
+                &path.join("height_to_first_p2pk65index"),
+                Version::from(1),
+            )?,
+            height_to_first_p2pkhindex: StorableVec::import(
+                &path.join("height_to_first_p2pkhindex"),
+                Version::from(1),
+            )?,
+            height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?,
+            height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?,
+            height_to_first_p2wpkhindex: StorableVec::import(
+                &path.join("height_to_first_p2wpkhindex"),
+                Version::from(1),
+            )?,
+            height_to_first_p2wshindex: StorableVec::import(
+                &path.join("height_to_first_p2wshindex"),
+                Version::from(1),
+            )?,
+            height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?,
+            height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
+            height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?,
+            p2pk33index_to_p2pk33addressbytes: StorableVec::import(
+                &path.join("p2pk33index_to_p2pk33addressbytes"),
+                Version::from(1),
+            )?,
+            p2pk65index_to_p2pk65addressbytes: StorableVec::import(
+                &path.join("p2pk65index_to_p2pk65addressbytes"),
+                Version::from(1),
+            )?,
+            p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(
+                &path.join("p2pkhindex_to_p2pkhaddressbytes"),
+                Version::from(1),
+            )?,
+            p2shindex_to_p2shaddressbytes: StorableVec::import(
+                &path.join("p2shindex_to_p2shaddressbytes"),
+                Version::from(1),
+            )?,
+            p2trindex_to_p2traddressbytes: StorableVec::import(
+                &path.join("p2trindex_to_p2traddressbytes"),
+                Version::from(1),
+            )?,
+            p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(
+                &path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
+                Version::from(1),
+            )?,
+            p2wshindex_to_p2wshaddressbytes: StorableVec::import(
+                &path.join("p2wshindex_to_p2wshaddressbytes"),
+                Version::from(1),
+            )?,
+            txindex_to_first_txinindex: StorableVec::import(
+                &path.join("txindex_to_first_txinindex"),
+                Version::from(1),
+            )?,
+            txindex_to_first_txoutindex: StorableVec::import(
+                &path.join("txindex_to_first_txoutindex"),
+                Version::from(1),
+            )?,
+            txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
+            txindex_to_locktime: StorableVec::import(&path.join("txindex_to_locktime"), Version::from(1))?,
+            txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?,
+            txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?,
+            txinindex_to_txoutindex: StorableVec::import(&path.join("txinindex_to_txoutindex"), Version::from(1))?,
+            txoutindex_to_addressindex: StorableVec::import(
+                &path.join("txoutindex_to_addressindex"),
+                Version::from(1),
+            )?,
+            txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"), Version::from(1))?,
+        })
+    }
+
+    pub fn push_addressbytes_if_needed(
+        &mut self,
+        index: Addresstypeindex,
+        addressbytes: Addressbytes,
+    ) -> storable_vec::Result<()> {
+        match addressbytes {
+            Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
+            Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
+            Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
+            Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
+            Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
+            Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
+            Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
+        }
+    }
+
+    pub fn get_addressbytes(
+        &self,
+        addresstype: Addresstype,
+        addresstypeindex: Addresstypeindex,
+    ) -> storable_vec::Result<Option<Addressbytes>> {
+        Ok(match addresstype {
+            Addresstype::P2PK65 => self
+                .p2pk65index_to_p2pk65addressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            Addresstype::P2PK33 => self
+                .p2pk33index_to_p2pk33addressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            Addresstype::P2PKH => self
+                .p2pkhindex_to_p2pkhaddressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            Addresstype::P2SH => self
+                .p2shindex_to_p2shaddressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            Addresstype::P2WPKH => self
+                .p2wpkhindex_to_p2wpkhaddressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            Addresstype::P2WSH => self
+                .p2wshindex_to_p2wshaddressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            Addresstype::P2TR => self
+                .p2trindex_to_p2traddressbytes
+                .get(addresstypeindex)?
+                // .map(|v| Addressbytes::from(v.clone())),
+                .map(|v| Addressbytes::from(v.into_inner())),
+            _ => unreachable!(),
+        })
+    }
+
+    #[allow(unused)]
+    pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
+        panic!();
+        // let mut txindex = None;
+
+        // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
+        //     .try_for_each(|slice| -> color_eyre::Result<()> {
+        //         let (height_slice, slice_blockhash) = slice?;
+        //         let blockhash = BlockHash::from_slice(&slice_blockhash)?;
+
+        //         wtx.remove(self.height_to_blockhash.data(), height_slice);
+
+        //         wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
+
+        //         if txindex.is_none() {
+        //             txindex.replace(
+        //                 wtx.get(self.height_to_first_txindex.data(), height_slice)?
+        //                     .context("for height to have first txindex")?,
+        //             );
+        //         }
+        //         wtx.remove(self.height_to_first_txindex.data(), height_slice);
+        //         wtx.remove(self.height_to_last_txindex.data(), height_slice);
+
+        //         Ok(())
+        //     })?;
+
+        // let txindex = txindex.context("txindex to not be none by now")?;
+
+        // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
+        //     .try_for_each(|slice| -> color_eyre::Result<()> {
+        //         let (slice_txindex, slice_txid) = slice?;
+        //         let txindex = Txindex::from(slice_txindex);
+        //         let txid = Txid::from_slice(&slice_txid)?;
+
+        //         wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
+        //         wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
+        //         wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
+
+        //         Ok(())
+        //     })?;
+
+        // let txoutindex = Txoutindex::from(txindex);
+
+        // let mut addressindexes = BTreeSet::new();
+
+        // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
+        //     .try_for_each(|slice| -> color_eyre::Result<()> {
+        //         let (txoutindex_slice, _) = slice?;
+
+        //         wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
+
+        //         if let Some(addressindex_slice) =
+        //             wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
+        //         {
+        //             wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
+
+        //             let addressindex = Addressindex::from(addressindex_slice);
+        //             addressindexes.insert(addressindex);
+
+        //             let txoutindex = Txoutindex::from(txoutindex_slice);
+        //             let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
+
+        //             wtx.remove(
+        //                 self.addressindex_to_txoutindexes.data(),
+        //                 Slice::from(addresstxoutindex),
+        //             );
+        //         }
+
+        //         Ok(())
+        //     })?;
+
+        // addressindexes
+        // .into_iter()
+        // .filter(|addressindex| {
+        //     let is_empty = wtx
+        //         .prefix(
+        //             self.addressindex_to_txoutindexes.data(),
+        //             Slice::from(*addressindex),
+        //         )
+        //         .next()
+        //         .is_none();
+        //     is_empty
+        // })
+        // .try_for_each(|addressindex| -> color_eyre::Result<()> {
+        //     let addressindex_slice = Slice::from(addressindex);
+
+        //     let addressbytes = Addressbytes::from(
+        //         wtx.get(
+        //             self.addressindex_to_addressbytes.data(),
+        //             &addressindex_slice,
+        //         )?
+        //         .context("addressindex_to_address to have value")?,
+        //     );
+        //     wtx.remove(
+        //         self.addressbytes_prefix_to_addressindex.data(),
+        //         addressbytes.prefix(),
+        //     );
+        //     wtx.remove(
+        //         self.addressindex_to_addressbytes.data(),
+        //         &addressindex_slice,
+        //     );
+        //     wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
+
+        //     Ok(())
+        // })?;
+        //
+
+        // todo!("clear addresstxoutindexes_out")
+        // todo!("clear addresstxoutindexes_in")
+        // todo!("clear zero_txoutindexes")
+
+        // Ok(())
+    }
+
+    pub fn flush(&mut self, height: Height) -> io::Result<()> {
+        self.as_mut_slice()
+            .into_par_iter()
+            .try_for_each(|vec| vec.flush(height))
+    }
+
+    pub fn min_height(&self) -> color_eyre::Result<Option<Height>> {
+        Ok(self
+            .as_slice()
+            .into_iter()
+            .map(|vec| vec.height().unwrap_or_default())
+            .min())
+    }
+
+    pub fn as_slice(&self) -> [&dyn AnyBindexVec; 39] {
+        [
+            &self.addressindex_to_addresstype as &dyn AnyBindexVec,
+            &self.addressindex_to_addresstypeindex,
+            &self.addressindex_to_height,
+            &self.height_to_blockhash,
+            &self.height_to_first_addressindex,
+            &self.height_to_first_emptyindex,
+            &self.height_to_first_multisigindex,
+            &self.height_to_first_opreturnindex,
+            &self.height_to_first_pushonlyindex,
+            &self.height_to_first_txindex,
+            &self.height_to_first_txinindex,
+            &self.height_to_first_txoutindex,
+            &self.height_to_first_unknownindex,
+            &self.height_to_first_p2pk33index,
+            &self.height_to_first_p2pk65index,
+            &self.height_to_first_p2pkhindex,
+            &self.height_to_first_p2shindex,
+            &self.height_to_first_p2trindex,
+            &self.height_to_first_p2wpkhindex,
+            &self.height_to_first_p2wshindex,
+            &self.height_to_size,
+            &self.height_to_timestamp,
+            &self.height_to_weight,
+            &self.p2pk33index_to_p2pk33addressbytes,
+            &self.p2pk65index_to_p2pk65addressbytes,
+            &self.p2pkhindex_to_p2pkhaddressbytes,
+            &self.p2shindex_to_p2shaddressbytes,
+            &self.p2trindex_to_p2traddressbytes,
+            &self.p2wpkhindex_to_p2wpkhaddressbytes,
+            &self.p2wshindex_to_p2wshaddressbytes,
+            &self.txindex_to_first_txinindex,
+            &self.txindex_to_first_txoutindex,
+            &self.txindex_to_height,
+            &self.txindex_to_locktime,
+            &self.txindex_to_txid,
+            &self.txindex_to_txversion,
+            &self.txinindex_to_txoutindex,
+            &self.txoutindex_to_addressindex,
+            &self.txoutindex_to_amount,
+        ]
+    }
+
+    pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyBindexVec + Send + Sync); 39] {
+        [
+            &mut self.addressindex_to_addresstype as &mut (dyn AnyBindexVec + Send + Sync),
+            &mut self.addressindex_to_addresstypeindex,
+            &mut self.addressindex_to_height,
+            &mut self.height_to_blockhash,
+            &mut self.height_to_first_addressindex,
+            &mut self.height_to_first_emptyindex,
+            &mut self.height_to_first_multisigindex,
+            &mut self.height_to_first_opreturnindex,
+            &mut self.height_to_first_pushonlyindex,
+            &mut self.height_to_first_txindex,
+            &mut self.height_to_first_txinindex,
+            &mut self.height_to_first_txoutindex,
+            &mut self.height_to_first_unknownindex,
+            &mut self.height_to_first_p2pk33index,
+            &mut self.height_to_first_p2pk65index,
+            &mut self.height_to_first_p2pkhindex,
+            &mut self.height_to_first_p2shindex,
+            &mut self.height_to_first_p2trindex,
+            &mut self.height_to_first_p2wpkhindex,
+            &mut self.height_to_first_p2wshindex,
+            &mut self.height_to_size,
+            &mut self.height_to_timestamp,
+            &mut self.height_to_weight,
+            &mut self.p2pk33index_to_p2pk33addressbytes,
+            &mut self.p2pk65index_to_p2pk65addressbytes,
+            &mut self.p2pkhindex_to_p2pkhaddressbytes,
+            &mut self.p2shindex_to_p2shaddressbytes,
+            &mut self.p2trindex_to_p2traddressbytes,
+            &mut self.p2wpkhindex_to_p2wpkhaddressbytes,
+            &mut self.p2wshindex_to_p2wshaddressbytes,
+            &mut self.txindex_to_first_txinindex,
+            &mut self.txindex_to_first_txoutindex,
+            &mut self.txindex_to_height,
+            &mut self.txindex_to_locktime,
+            &mut self.txindex_to_txid,
+            &mut self.txindex_to_txversion,
+            &mut self.txinindex_to_txoutindex,
+            &mut self.txoutindex_to_addressindex,
+            &mut self.txoutindex_to_amount,
+        ]
+    }
+}