diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e06d6fedc..5af17713b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.0/cargo-dist-installer.sh | sh" + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.2/cargo-dist-installer.sh | sh" - name: Cache dist uses: actions/upload-artifact@v4 with: @@ -217,8 +217,8 @@ jobs: - plan - build-local-artifacts - build-global-artifacts - # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) - if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} runs-on: "ubuntu-22.04" diff --git a/Cargo.lock b/Cargo.lock index 6e4e54bde..39e0a6ef4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4108,11 +4108,10 @@ dependencies = [ [[package]] name = "rawdb" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6601495283897e6193cbd70d65e4e7420995c59c0de3de953f297c2fa8d9dc7" +checksum = "c184e0f054f8b9018424fb226dbf122ceae050eb5f900e45d0809ff9fdc7b0d0" dependencies = [ - "allocative", "libc", "memmap2", "parking_lot 0.12.5", @@ -5331,9 +5330,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffa6180edb8fb521c9958d4d00a1cca1b5de396ee371f2f8e3ec54c21205eff" +checksum = "44fef339cc0a6103cea9df44c1133d8275585b90dbe2faa6e3f60e6d2bac47b6" dependencies = [ "allocative", "ctrlc", @@ -5350,9 +5349,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dffd7af8e5579122f79e4a2df3f62f5f315ef0d593721f585384b61bc722e2" +checksum = "c09556497d1695d5ba268ebeaa1d2547c2c597a60098fde60fe78028b463c4c1" dependencies = [ "quote", "syn 2.0.108", diff --git a/Cargo.toml b/Cargo.toml index 6c60af4a8..0f526a2e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] } sonic-rs = "0.5.6" tokio = { version = "1.48.0", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] } -vecdb = { version = "0.3.1", features = ["derive"] } +vecdb = { version = "0.3.4", features = ["derive"] } zerocopy = { version = "0.8.27", features = ["derive"] } [workspace.metadata.release] @@ -86,8 +86,8 @@ pre-release-commit-message = "release: v{{version}}" tag-message = "release: v{{version}}" [workspace.metadata.dist] -cargo-dist-version = "0.30.0" +cargo-dist-version = "0.30.2" ci = "github" -allow-dirty = ["ci"] +# allow-dirty = ["ci"] installers = [] targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"] diff --git a/crates/brk_error/src/lib.rs b/crates/brk_error/src/lib.rs index 10ad3735e..22c620e89 100644 --- a/crates/brk_error/src/lib.rs +++ b/crates/brk_error/src/lib.rs @@ -15,7 +15,7 @@ pub enum Error { FjallV2(fjall2::Error), FjallV3(fjall3::Error), VecDB(vecdb::Error), - SeqDB(vecdb::SeqDBError), + RawDB(vecdb::RawDBError), Minreq(minreq::Error), SystemTimeError(time::SystemTimeError), BitcoinConsensusEncode(bitcoin::consensus::encode::Error), @@ -106,10 +106,10 @@ impl From for Error { } } -impl From for Error { +impl From for Error { #[inline] - fn from(value: vecdb::SeqDBError) -> Self { - Self::SeqDB(value) + fn from(value: vecdb::RawDBError) -> Self { + Self::RawDB(value) } } @@ -183,7 +183,7 @@ impl fmt::Display for Error { Error::IO(error) => Display::fmt(&error, f), Error::Jiff(error) => Display::fmt(&error, f), Error::Minreq(error) => Display::fmt(&error, f), - Error::SeqDB(error) => Display::fmt(&error, f), + Error::RawDB(error) => Display::fmt(&error, f), Error::SonicRS(error) => Display::fmt(&error, f), Error::SystemTimeError(error) => Display::fmt(&error, f), Error::VecDB(error) => Display::fmt(&error, f), diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 8926ed03f..26e7a0c32 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -17,22 +17,22 @@ use rayon::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended}; mod indexes; -mod stores_redb; +// mod stores_redb; // mod stores_v2; -// mod stores_v3; +mod stores_v3; mod vecs; pub use indexes::*; -pub use stores_redb::*; +// pub use stores_redb::*; // pub use stores_v2::*; -// pub use stores_v3::*; +pub use stores_v3::*; pub use vecs::*; // One version for all data sources // Increment on **change _OR_ addition** const VERSION: Version = Version::new(23); const SNAPSHOT_BLOCK_RANGE: usize = 1_000; -const COLLISIONS_CHECKED_UP_TO: Height = Height::new(920_000); +const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0); #[derive(Clone)] pub struct Indexer { @@ -746,7 +746,7 @@ impl Indexer { } // let i = Instant::now(); - self.vecs.punch_holes()?; + self.vecs.compact()?; // info!("Punched holes in db in {}s", i.elapsed().as_secs()); Ok(starting_indexes) diff --git a/crates/brk_indexer/src/stores_v3.rs b/crates/brk_indexer/src/stores_v3.rs index 4876dd13d..27551b690 100644 --- a/crates/brk_indexer/src/stores_v3.rs +++ b/crates/brk_indexer/src/stores_v3.rs @@ -8,7 +8,7 @@ use brk_types::{ TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version, Vout, }; -use fjall3::{PersistMode, TxDatabase}; +use fjall3::{Database, PersistMode}; use log::info; use rayon::prelude::*; use vecdb::{AnyVec, GenericStoredVec, StoredIndex, VecIterator, VecIteratorExtended}; @@ -19,7 +19,7 @@ use super::Vecs; #[derive(Clone)] pub struct Stores { - pub database: TxDatabase, + pub database: Database, pub addressbyteshash_to_typeindex: Store, pub blockhashprefix_to_height: Store, @@ -37,7 +37,7 @@ impl Stores { fs::create_dir_all(&pathbuf)?; - let database = match brk_store::open_fjall2_database(path) { + let database = match brk_store::open_fjall3_database(path) { Ok(database) => database, Err(_) => { fs::remove_dir_all(path)?; diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index 10c1df7b6..aba0cc2fa 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -364,8 +364,8 @@ impl Vecs { .unwrap() } - pub fn punch_holes(&self) -> Result<()> { - self.db.punch_holes()?; + pub fn compact(&self) -> Result<()> { + self.db.compact()?; Ok(()) } diff --git a/crates/brk_store/src/fjall_v3/meta.rs b/crates/brk_store/src/fjall_v3/meta.rs index 2636f2952..843913bfb 100644 --- a/crates/brk_store/src/fjall_v3/meta.rs +++ b/crates/brk_store/src/fjall_v3/meta.rs @@ -5,7 +5,7 @@ use std::{ use brk_error::Result; use brk_types::Version; -use fjall3::{TxDatabase, TxKeyspace}; +use fjall3::{Database, Keyspace}; use super::Height; @@ -18,13 +18,13 @@ pub struct StoreMeta { impl StoreMeta { pub fn checked_open( - _database: &TxDatabase, + _database: &Database, path: &Path, version: Version, open_partition_handle: F, - ) -> Result<(Self, TxKeyspace)> + ) -> Result<(Self, Keyspace)> where - F: Fn() -> Result, + F: Fn() -> Result, { fs::create_dir_all(path)?; diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index ad9a19aa1..54cc08608 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -4,8 +4,8 @@ use brk_error::Result; use brk_types::{Height, Version}; use byteview8::ByteView; use fjall3::{ - KeyspaceCreateOptions, TxDatabase, TxKeyspace, ValueType, - config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry}, + Database, Keyspace, KeyspaceCreateOptions, ValueType, + config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy}, }; mod meta; @@ -19,16 +19,16 @@ use crate::any::AnyStore; pub struct StoreFjallV3 { meta: StoreMeta, name: &'static str, - database: TxDatabase, - keyspace: TxKeyspace, + database: Database, + keyspace: Keyspace, puts: FxHashMap, dels: FxHashSet, } const MAJOR_FJALL_VERSION: Version = Version::new(3); -pub fn open_fjall2_database(path: &Path) -> fjall3::Result { - TxDatabase::builder(path.join("fjall")) +pub fn open_fjall3_database(path: &Path) -> fjall3::Result { + Database::builder(path.join("fjall")) .max_write_buffer_size(32 * 1024 * 1024) .cache_size(1024 * 1024 * 1024) .open() @@ -41,12 +41,14 @@ where ByteView: From + From, { fn open_keyspace( - database: &TxDatabase, + database: &Database, name: &str, bloom_filters: Option, - ) -> Result { + ) -> Result { let mut options = KeyspaceCreateOptions::default() .manual_journal_persist(true) + .filter_block_pinning_policy(PinningPolicy::all(false)) + .index_block_pinning_policy(PinningPolicy::all(false)) .max_memtable_size(8 * 1024 * 1024); if bloom_filters.is_some_and(|b| !b) { @@ -61,7 +63,7 @@ where } pub fn import( - database: &TxDatabase, + database: &Database, path: &Path, name: &str, version: Version, @@ -112,10 +114,7 @@ where #[inline] pub fn is_empty(&self) -> Result { - self.database - .read_tx() - .is_empty(&self.keyspace) - .map_err(|e| e.into()) + self.keyspace.is_empty().map_err(|e| e.into()) } #[inline] @@ -178,7 +177,7 @@ where return Ok(()); } - let mut batch = self.database.inner().batch(); + let mut batch = self.database.batch(); let mut items = mem::take(&mut self.puts) .into_iter() .map(|(key, value)| Item::Value { key, value }) @@ -193,7 +192,7 @@ where .into_iter() .map(|i| i.fjall(&self.keyspace)) .collect::>(); - batch.commit_keyspace(self.keyspace.inner())?; + batch.commit_keyspace(&self.keyspace)?; // let mut wtx = self.database.write_tx(); @@ -266,20 +265,20 @@ impl Item { } } - pub fn fjall(self, keyspace: &TxKeyspace) -> fjall3::Item + pub fn fjall(self, keyspace: &Keyspace) -> fjall3::Item where K: Into, V: Into, { match self { Item::Value { key, value } => fjall3::Item { - keyspace_id: keyspace.inner().id, + keyspace_id: keyspace.id, key: key.into().into(), value: value.into().into(), value_type: ValueType::Value, }, Item::Tomb(key) => fjall3::Item { - keyspace_id: keyspace.inner().id, + keyspace_id: keyspace.id, key: key.into().into(), value: [].into(), value_type: ValueType::WeakTombstone,