global: snapshot

2026-05-04 19:29:09 -07:00 · 2025-11-06 15:17:24 +01:00
parent df577ca7f5
commit 11b323ef00
9 changed files with 119 additions and 107 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -675,8 +675,8 @@ version = "0.0.111"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
- "brk_fjall 2.11.5",
- "brk_fjall 3.0.0-pre.4",
+ "brk_fjall",
+ "fjall",
 "jiff",
 "minreq",
 "sonic-rs",
@@ -713,20 +713,6 @@ dependencies = [
 "xxhash-rust",
 ]

-[[package]]
-name = "brk_fjall"
-version = "3.0.0-pre.4"
-dependencies = [
- "byteorder-lite",
- "byteview 0.8.0",
- "dashmap",
- "log",
- "lsm-tree 3.0.0-pre.4",
- "std-semaphore",
- "tempfile",
- "xxhash-rust",
-]
-
 [[package]]
 name = "brk_grouper"
 version = "0.0.111"
@@ -744,8 +730,7 @@ version = "0.0.111"
 dependencies = [
 "bitcoin",
 "brk_error",
- "brk_fjall 2.11.5",
- "brk_fjall 3.0.0-pre.4",
+ "brk_fjall",
 "brk_grouper",
 "brk_iterator",
 "brk_logger",
@@ -754,6 +739,7 @@ dependencies = [
 "brk_store",
 "brk_traversable",
 "brk_types",
+ "fjall",
 "log",
 "rand 0.9.2",
 "rayon",
@@ -1286,12 +1272,12 @@ name = "brk_store"
 version = "0.0.111"
 dependencies = [
 "brk_error",
- "brk_fjall 2.11.5",
- "brk_fjall 3.0.0-pre.4",
+ "brk_fjall",
 "brk_types",
 "byteview 0.6.1",
 "byteview 0.8.0",
 "candystore",
+ "fjall",
 "log",
 "parking_lot 0.12.5",
 "rustc-hash",
@@ -1335,7 +1321,7 @@ dependencies = [
 "allocative",
 "bitcoin",
 "brk_error",
- "byteview 0.6.1",
+ "byteview 0.8.0",
 "derive_deref",
 "itoa",
 "jiff",
@@ -2116,6 +2102,22 @@ version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"

+[[package]]
+name = "fjall"
+version = "3.0.0-pre.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf4746eb86124d4cd91564bc01c247b8c2c3ddf2d9fb3701ca16c86fdb5b538"
+dependencies = [
+ "byteorder-lite",
+ "byteview 0.8.0",
+ "dashmap",
+ "log",
+ "lsm-tree 3.0.0-pre.4",
+ "std-semaphore",
+ "tempfile",
+ "xxhash-rust",
+]
+
 [[package]]
 name = "flate2"
 version = "1.1.5"
@@ -4108,9 +4110,9 @@ dependencies = [

 [[package]]
 name = "rawdb"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c06cc46cc4b6da16c9981293906062230f822879c2c88367a40673db5e44414"
+checksum = "d07ae2ce76ce46d26972046ae67d1f65ddbf5bc1290d3abaee84b7ab159564c4"
 dependencies = [
 "libc",
 "memmap2",
@@ -5321,9 +5323,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"

 [[package]]
 name = "vecdb"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0cff5037d7e5d66344dff249c366a527b530ef3c723e04f4a90fa11d5f6a9fc"
+checksum = "d96acc025cc0039bf828cbf4efdcb485eb0eda88da7ec805fb6ec9159a340b73"
 dependencies = [
 "allocative",
 "ctrlc",
@@ -5340,9 +5342,9 @@ dependencies = [

 [[package]]
 name = "vecdb_derive"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f73046e065e63abb2ad8d1616a08ca8acf7d543f04ffb0222f6fc44e0d3eeec"
+checksum = "dae24e54786d92d4aa4b46098eed2594cd6ed51d2a47a36ff8ae3adfa232d958"
 dependencies = [
 "quote",
 "syn 2.0.109",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -52,14 +52,14 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" }
 brk_types = { version = "0.0.111", path = "crates/brk_types" }
 brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] }
 brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
-byteview = "=0.6.1"
-# byteview = "~0.8.0"
+# byteview = "=0.6.1"
+byteview = "~0.8.0"
 derive_deref = "1.1.1"
 fjall2 = { version = "2.11.5", package = "brk_fjall" }
 # fjall2 = { path = "../fjall2", package = "brk_fjall" }
 # fjall2 = { version = "2.11.2", package = "fjall" }
-# fjall3 = { version = "=3.0.0-pre.0", package = "fjall" }
-fjall3 = { path = "../fjall3", package = "brk_fjall" }
+fjall3 = { version = "=3.0.0-pre.4", package = "fjall" }
+# fjall3 = { path = "../fjall3", package = "brk_fjall" }
 # fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "bb15057500dce3115d7644d268b9deeaa895b431", package = "fjall" }
 jiff = "0.2.15"
 log = "0.4.28"
@@ -75,7 +75,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
 sonic-rs = "0.5.6"
 tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
 # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] }
-vecdb = { version = "0.3.7", features = ["derive"] }
+vecdb = { version = "0.3.8", features = ["derive"] }
 zerocopy = { version = "0.8.27", features = ["derive"] }

 [workspace.metadata.release]
--- a/crates/brk_computer/src/blks.rs
+++ b/crates/brk_computer/src/blks.rs
@@ -94,22 +94,16 @@ impl Vecs {
            .try_for_each(|block| -> Result<()> {
                let height = block.height();

-                self.height_to_position.forced_push_at(
-                    height,
-                    block.metadata().position(),
-                    exit,
-                )?;
+                self.height_to_position
+                    .forced_push(height, block.metadata().position(), exit)?;

                let txindex = height_to_first_txindex_iter.get_unwrap(height);

                block.tx_metadata().iter().enumerate().try_for_each(
                    |(index, metadata)| -> Result<()> {
                        let txindex = txindex + index;
-                        self.txindex_to_position.forced_push_at(
-                            txindex,
-                            metadata.position(),
-                            exit,
-                        )?;
+                        self.txindex_to_position
+                            .forced_push(txindex, metadata.position(), exit)?;
                        Ok(())
                    },
                )?;
--- a/crates/brk_computer/src/fetched.rs
+++ b/crates/brk_computer/src/fetched.rs
@@ -133,7 +133,7 @@ impl Vecs {
                prev.replace(ohlc.clone());

                self.dateindex_to_price_ohlc_in_cents
-                    .forced_push_at_(i, ohlc, exit)?;
+                    .forced_push_at(i, ohlc, exit)?;

                Ok(())
            })?;
--- a/crates/brk_computer/src/indexes.rs
+++ b/crates/brk_computer/src/indexes.rs
@@ -14,7 +14,7 @@ use brk_types::{
 };
 use vecdb::{
    AnyCloneableIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2, PAGE_SIZE,
-    StoredIndex, VecIterator,
+    StoredIndex, VecIteratorExtended,
 };

 const VERSION: Version = Version::ZERO;
@@ -117,15 +117,13 @@ impl Vecs {
            indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
            |index: TxInIndex, txinindex_to_outpoint_iter, txindex_to_first_txoutindex_iter| {
                txinindex_to_outpoint_iter
-                    .next_at(index.to_usize())
-                    .map(|(_, outpoint)| {
+                    .get_at(index.to_usize())
+                    .map(|outpoint| {
                        if outpoint.is_coinbase() {
                            return TxOutIndex::COINBASE;
                        }
                        txindex_to_first_txoutindex_iter
-                            .next_at(outpoint.txindex().to_usize())
-                            .unwrap()
-                            .1
+                            .get_unwrap_at(outpoint.txindex().to_usize())
                            + outpoint.vout()
                    })
            },
@@ -160,12 +158,12 @@ impl Vecs {
            |index: TxIndex, txindex_to_first_txinindex_iter, txinindex_to_txoutindex_iter| {
                let txindex = index.to_usize();
                txindex_to_first_txinindex_iter
-                    .next_at(txindex)
-                    .map(|(_, start)| {
+                    .get_at(txindex)
+                    .map(|start| {
                        let start = usize::from(start);
                        let end = txindex_to_first_txinindex_iter
-                            .next_at(txindex + 1)
-                            .map(|(_, v)| usize::from(v))
+                            .get_at(txindex + 1)
+                            .map(|v| usize::from(v))
                            .unwrap_or_else(|| txinindex_to_txoutindex_iter.len());
                        StoredU64::from((start..end).count())
                    })
@@ -180,12 +178,12 @@ impl Vecs {
            |index: TxIndex, txindex_to_first_txoutindex_iter, txoutindex_to_value_iter| {
                let txindex = index.to_usize();
                txindex_to_first_txoutindex_iter
-                    .next_at(txindex)
-                    .map(|(_, start)| {
+                    .get_at(txindex)
+                    .map(|start| {
                        let start = usize::from(start);
                        let end = txindex_to_first_txoutindex_iter
-                            .next_at(txindex + 1)
-                            .map(|(_, v)| usize::from(v))
+                            .get_at(txindex + 1)
+                            .map(|v| usize::from(v))
                            .unwrap_or_else(|| txoutindex_to_value_iter.len());
                        StoredU64::from((start..end).count())
                    })
@@ -558,7 +556,7 @@ impl Vecs {
                    prev_timestamp_fixed.replace(
                        height_to_timestamp_fixed_iter
                            .into_iter()
-                            .unsafe_get(prev_h),
+                            .get_unwrap(prev_h),
                    );
                }
                let timestamp_fixed =
@@ -585,7 +583,7 @@ impl Vecs {
        let starting_dateindex = self
            .height_to_dateindex
            .into_iter()
-            .get_inner(decremented_starting_height)
+            .get(decremented_starting_height)
            .unwrap_or_default();

        self.height_to_dateindex.compute_transform(
@@ -598,7 +596,7 @@ impl Vecs {
        let starting_dateindex = if let Some(dateindex) = self
            .height_to_dateindex
            .into_iter()
-            .get_inner(decremented_starting_height)
+            .get(decremented_starting_height)
        {
            starting_dateindex.min(dateindex)
        } else {
@@ -637,7 +635,7 @@ impl Vecs {
        let starting_weekindex = self
            .dateindex_to_weekindex
            .into_iter()
-            .get_inner(starting_dateindex)
+            .get(starting_dateindex)
            .unwrap_or_default();

        self.dateindex_to_weekindex.compute_range(
@@ -674,7 +672,7 @@ impl Vecs {
        let starting_difficultyepoch = self
            .height_to_difficultyepoch
            .into_iter()
-            .get_inner(decremented_starting_height)
+            .get(decremented_starting_height)
            .unwrap_or_default();

        self.height_to_difficultyepoch.compute_from_index(
@@ -710,7 +708,7 @@ impl Vecs {
        let starting_monthindex = self
            .dateindex_to_monthindex
            .into_iter()
-            .get_inner(starting_dateindex)
+            .get(starting_dateindex)
            .unwrap_or_default();

        self.dateindex_to_monthindex.compute_range(
@@ -747,7 +745,7 @@ impl Vecs {
        let starting_quarterindex = self
            .monthindex_to_quarterindex
            .into_iter()
-            .get_inner(starting_monthindex)
+            .get(starting_monthindex)
            .unwrap_or_default();

        self.monthindex_to_quarterindex.compute_from_index(
@@ -785,7 +783,7 @@ impl Vecs {
        let starting_semesterindex = self
            .monthindex_to_semesterindex
            .into_iter()
-            .get_inner(starting_monthindex)
+            .get(starting_monthindex)
            .unwrap_or_default();

        self.monthindex_to_semesterindex.compute_from_index(
@@ -823,7 +821,7 @@ impl Vecs {
        let starting_yearindex = self
            .monthindex_to_yearindex
            .into_iter()
-            .get_inner(starting_monthindex)
+            .get(starting_monthindex)
            .unwrap_or_default();

        self.monthindex_to_yearindex.compute_from_index(
@@ -858,7 +856,7 @@ impl Vecs {
        let starting_halvingepoch = self
            .height_to_halvingepoch
            .into_iter()
-            .get_inner(decremented_starting_height)
+            .get(decremented_starting_height)
            .unwrap_or_default();

        self.height_to_halvingepoch.compute_from_index(
@@ -886,7 +884,7 @@ impl Vecs {
        let starting_decadeindex = self
            .yearindex_to_decadeindex
            .into_iter()
-            .get_inner(starting_yearindex)
+            .get(starting_yearindex)
            .unwrap_or_default();

        self.yearindex_to_decadeindex.compute_from_index(
@@ -947,7 +945,7 @@ impl Indexes {
    pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
        self.indexes.height = height;
        self.dateindex =
-            DateIndex::try_from(indexes.height_to_date_fixed.into_iter().unsafe_get(height))
+            DateIndex::try_from(indexes.height_to_date_fixed.into_iter().get_unwrap(height))
                .unwrap();
        self.weekindex = WeekIndex::from(self.dateindex);
        self.monthindex = MonthIndex::from(self.dateindex);
--- a/crates/brk_computer/src/traits.rs
+++ b/crates/brk_computer/src/traits.rs
@@ -197,7 +197,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {

        stacks
            .iter()
-            .skip(index)
+            .skip(index.to_usize())
            .enumerate()
            .try_for_each(|(i, stack)| {
                let mut avg_price = Dollars::from(f64::NAN);
--- a/crates/brk_indexer/src/lib.rs
+++ b/crates/brk_indexer/src/lib.rs
@@ -18,20 +18,20 @@ use rayon::prelude::*;
 use rustc_hash::{FxHashMap, FxHashSet};
 use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended};
 mod indexes;
-mod stores_v2;
-// mod stores_v3;
+// mod stores_v2;
+mod stores_v3;
 mod vecs;

 pub use indexes::*;
-pub use stores_v2::*;
-// pub use stores_v3::*;
+// pub use stores_v2::*;
+pub use stores_v3::*;
 pub use vecs::*;

 // One version for all data sources
 // Increment on **change _OR_ addition**
 const VERSION: Version = Version::new(23);
 const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
-const COLLISIONS_CHECKED_UP_TO: Height = Height::new(920_000);
+const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0);

 #[derive(Clone)]
 pub struct Indexer {
--- a/crates/brk_logger/src/lib.rs
+++ b/crates/brk_logger/src/lib.rs
@@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
    });

    Builder::from_env(Env::default().default_filter_or(
-        "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
-        // "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
+        // "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
+        "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
    ))
    .format(move |buf, record| {
        let date_time = Timestamp::now()
--- a/crates/brk_store/src/fjall_v3/mod.rs
+++ b/crates/brk_store/src/fjall_v3/mod.rs
@@ -4,7 +4,10 @@ use brk_error::Result;
 use brk_types::{Height, Version};
 use byteview8::ByteView;
 use fjall3::{
-    Database, Keyspace, KeyspaceCreateOptions, ValueType,
+    Database,
+    Keyspace,
+    KeyspaceCreateOptions,
+    // ValueType,
    config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy},
 };

@@ -198,13 +201,28 @@ where
            )
            .collect::<Vec<_>>();
        items.sort_unstable();
-        batch.ingest(
-            items
-                .into_iter()
-                .map(|i| i.fjalled(&self.keyspace))
-                .collect::<Vec<_>>(),
-        );
-        batch.commit_keyspace(&self.keyspace)?;
+        items.into_iter().for_each(|item| match item {
+            Item::Value { key, value } => {
+                batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
+            }
+            Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)),
+        });
+        batch.commit()?;
+
+        // batch.ingest(
+        //     items
+        //         .into_iter()
+        //         .map(|i| i.fjalled(&self.keyspace))
+        //         .collect::<Vec<_>>(),
+        // );
+        // batch.commit_keyspace(&self.keyspace)?;
+        // batch.ingest(
+        //     items
+        //         .into_iter()
+        //         .map(|i| i.fjalled(&self.keyspace))
+        //         .collect::<Vec<_>>(),
+        // );
+        // batch.commit_keyspace(&self.keyspace)?;
        // batch.commit_keyspace(self.keyspace.inner())?;

        Ok(())
@@ -263,26 +281,26 @@ impl<K, V> Item<K, V> {
        }
    }

-    pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item
-    where
-        K: Into<ByteView>,
-        V: Into<ByteView>,
-    {
-        let keyspace_id = keyspace.id;
-        // let keyspace_id = keyspace.inner().id;
-        match self {
-            Item::Value { key, value } => fjall3::Item {
-                keyspace_id,
-                key: key.into().into(),
-                value: value.into().into(),
-                value_type: ValueType::Value,
-            },
-            Item::Tomb(key) => fjall3::Item {
-                keyspace_id,
-                key: key.into().into(),
-                value: [].into(),
-                value_type: ValueType::WeakTombstone,
-            },
-        }
-    }
+    //     pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item
+    //     where
+    //         K: Into<ByteView>,
+    //         V: Into<ByteView>,
+    //     {
+    //         let keyspace_id = keyspace.id;
+    //         // let keyspace_id = keyspace.inner().id;
+    //         match self {
+    //             Item::Value { key, value } => fjall3::Item {
+    //                 keyspace_id,
+    //                 key: key.into().into(),
+    //                 value: value.into().into(),
+    //                 value_type: ValueType::Value,
+    //             },
+    //             Item::Tomb(key) => fjall3::Item {
+    //                 keyspace_id,
+    //                 key: key.into().into(),
+    //                 value: [].into(),
+    //                 value_type: ValueType::WeakTombstone,
+    //             },
+    //         }
+    //     }
 }