global: move addressindex_to_outputindex stores from computer to indexer

crates/brk_store/examples/main.rs

@@ -6,10 +6,7 @@ use brk_store::{AnyStore, Store};
 fn main() -> Result<()> {
     let p = Path::new("./examples/_fjall");
 
-    let keyspace = brk_store::open_keyspace(p)?;
-
-    let mut store: Store<Dollars, Sats> =
-        brk_store::Store::import(&keyspace, p, "n", Version::ZERO, None)?;
+    let mut store: Store<Dollars, Sats> = brk_store::Store::import(p, "n", Version::ZERO, None)?;
 
     store.insert_if_needed(Dollars::from(10.0), Sats::FIFTY_BTC, Height::ZERO);
 

crates/brk_store/src/lib.rs

@@ -7,14 +7,15 @@ use std::{
     borrow::Cow,
     collections::{BTreeMap, BTreeSet},
     fmt::Debug,
-    mem,
+    fs, mem,
     path::Path,
 };
 
 use brk_core::{Height, Result, Version};
 use byteview::ByteView;
 use fjall::{
-    PartitionCreateOptions, ReadTransaction, TransactionalKeyspace, TransactionalPartitionHandle,
+    PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace,
+    TransactionalPartitionHandle,
 };
 
 mod meta;
@@ -28,6 +29,7 @@ pub struct Store<Key, Value> {
     meta: StoreMeta,
     name: &'static str,
     keyspace: TransactionalKeyspace,
+    // Arc it
     partition: Option<TransactionalPartitionHandle>,
     rtx: ReadTransaction,
     puts: BTreeMap<Key, Value>,
@@ -40,12 +42,6 @@ const DEFAULT_BLOOM_FILTER_BITS: Option<u8> = Some(5);
 // const CHECK_COLLISIONS: bool = true;
 const MAJOR_FJALL_VERSION: Version = Version::TWO;
 
-pub fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
-    fjall::Config::new(path.join("fjall"))
-        .max_write_buffer_size(32 * 1024 * 1024)
-        .open_transactional()
-}
-
 impl<'a, K, V> Store<K, V>
 where
     K: Debug + Clone + From<ByteView> + Ord + 'a,
@@ -53,18 +49,32 @@ where
     ByteView: From<K> + From<&'a K> + From<V>,
 {
     pub fn import(
-        keyspace: &TransactionalKeyspace,
-        path: &Path,
+        path_: &Path,
         name: &str,
         version: Version,
         bloom_filter_bits: Option<Option<u8>>,
     ) -> Result<Self> {
+        let path = path_.join(name);
+
+        fs::create_dir_all(&path)?;
+
+        let keyspace = match fjall::Config::new(path.join("fjall"))
+            .max_write_buffer_size(32 * 1024 * 1024)
+            .open_transactional()
+        {
+            Ok(keyspace) => keyspace,
+            Err(_) => {
+                fs::remove_dir_all(path)?;
+                return Self::import(path_, name, version, bloom_filter_bits);
+            }
+        };
+
         let (meta, partition) = StoreMeta::checked_open(
-            keyspace,
-            &path.join(format!("meta/{name}")),
+            &keyspace,
+            &path.join("meta"),
             MAJOR_FJALL_VERSION + version,
             || {
-                Self::open_partition_handle(keyspace, name, bloom_filter_bits).inspect_err(|e| {
+                Self::open_partition_handle(&keyspace, bloom_filter_bits).inspect_err(|e| {
                     eprintln!("{e}");
                     eprintln!("Delete {path:?} and try again");
                 })
@@ -98,6 +108,12 @@ where
         }
     }
 
+    pub fn is_empty(&self) -> Result<bool> {
+        self.rtx
+            .is_empty(self.partition.as_ref().unwrap())
+            .map_err(|e| e.into())
+    }
+
     // pub fn puts_first_key_value(&self) -> Option<(&K, &V)> {
     //     self.puts.first_key_value()
     // }
@@ -138,9 +154,9 @@ where
     }
 
     pub fn remove(&mut self, key: K) {
-        if self.is_empty() {
-            return;
-        }
+        // if self.is_empty()? {
+        //     return Ok(());
+        // }
 
         if !self.puts.is_empty() {
             unreachable!("Shouldn't reach this");
@@ -150,6 +166,8 @@ where
             dbg!(key, &self.meta.path());
             unreachable!();
         }
+
+        // Ok(())
     }
 
     // pub fn retain_or_del<F>(&mut self, retain: F)
@@ -167,12 +185,11 @@ where
 
     fn open_partition_handle(
         keyspace: &TransactionalKeyspace,
-        name: &str,
         bloom_filter_bits: Option<Option<u8>>,
     ) -> Result<TransactionalPartitionHandle> {
         keyspace
             .open_partition(
-                name,
+                "partition",
                 PartitionCreateOptions::default()
                     .bloom_filter_bits(bloom_filter_bits.unwrap_or(DEFAULT_BLOOM_FILTER_BITS))
                     .max_memtable_size(8 * 1024 * 1024)
@@ -191,7 +208,7 @@ where
             return Ok(());
         }
 
-        self.meta.export(self.len(), height)?;
+        self.meta.export(height)?;
 
         let mut wtx = self.keyspace.write_tx();
 
@@ -217,6 +234,8 @@ where
 
         wtx.commit()?;
 
+        self.keyspace.persist(PersistMode::SyncAll)?;
+
         self.rtx = self.keyspace.read_tx();
 
         Ok(())
@@ -249,11 +268,12 @@ where
 
         self.meta.reset();
 
-        let partition =
-            Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filter_bits)?;
+        let partition = Self::open_partition_handle(&self.keyspace, self.bloom_filter_bits)?;
 
         self.partition.replace(partition);
 
+        self.keyspace.persist(PersistMode::SyncAll)?;
+
         Ok(())
     }
 
@@ -265,21 +285,6 @@ where
         self.meta.height()
     }
 
-    fn len(&self) -> usize {
-        let len = self.meta.len() + self.puts.len() - self.dels.len();
-        if len > 18440000000000000000 {
-            dbg!((
-                len,
-                self.meta.path(),
-                self.meta.len(),
-                self.puts.len(),
-                &self.dels,
-            ));
-            unreachable!()
-        }
-        len
-    }
-
     fn has(&self, height: Height) -> bool {
         self.meta.has(height)
     }

crates/brk_store/src/meta.rs

@@ -3,7 +3,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use brk_core::{Result, Version, copy_first_8bytes};
+use brk_core::{Result, Version};
 use fjall::{TransactionalKeyspace, TransactionalPartitionHandle};
 
 use super::Height;
@@ -13,7 +13,6 @@ pub struct StoreMeta {
     pathbuf: PathBuf,
     version: Version,
     height: Option<Height>,
-    len: usize,
 }
 
 impl StoreMeta {
@@ -44,13 +43,10 @@ impl StoreMeta {
             partition = open_partition_handle()?;
         }
 
-        let len = Self::read_length_(path);
-
         let slf = Self {
             pathbuf: path.to_owned(),
             version,
             height: Height::try_from(Self::path_height_(path).as_path()).ok(),
-            len,
         };
 
         slf.version.write(&slf.path_version())?;
@@ -58,28 +54,17 @@ impl StoreMeta {
         Ok((slf, partition))
     }
 
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    // pub fn is_empty(&self) -> bool {
-    //     self.len() == 0
-    // }
-
     pub fn version(&self) -> Version {
         self.version
     }
 
-    pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
-        self.len = len;
-        self.write_length()?;
+    pub fn export(&mut self, height: Height) -> io::Result<()> {
         self.height = Some(height);
         height.write(&self.path_height())
     }
 
     pub fn reset(&mut self) {
         self.height.take();
-        self.len = 0
     }
 
     pub fn path(&self) -> &Path {
@@ -108,19 +93,4 @@ impl StoreMeta {
     fn path_height_(path: &Path) -> PathBuf {
         path.join("height")
     }
-
-    fn read_length_(path: &Path) -> usize {
-        fs::read(Self::path_length(path))
-            .map(|v| usize::from_ne_bytes(copy_first_8bytes(v.as_slice()).unwrap()))
-            .unwrap_or_default()
-    }
-    fn write_length(&self) -> io::Result<()> {
-        Self::write_length_(&self.pathbuf, self.len)
-    }
-    fn write_length_(path: &Path, len: usize) -> io::Result<()> {
-        fs::write(Self::path_length(path), len.to_ne_bytes())
-    }
-    fn path_length(path: &Path) -> PathBuf {
-        path.join("length")
-    }
 }

crates/brk_store/src/trait.rs

@@ -9,12 +9,6 @@ pub trait AnyStore {
 
     fn height(&self) -> Option<Height>;
 
-    fn len(&self) -> usize;
-
-    fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
     fn has(&self, height: Height) -> bool;
 
     fn needs(&self, height: Height) -> bool;