Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 06:01:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-11-05 11:14:31 +01:00
parent 2ad55bf558
commit a2ba4d89f3
9 changed files with 49 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/release.yml
+3 -3
View File
@@ -64,7 +64,7 @@ jobs:
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.0/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.2/cargo-dist-installer.sh | sh"
- name: Cache dist
uses: actions/upload-artifact@v4
with:
@@ -217,8 +217,8 @@ jobs:
- plan
- build-local-artifacts
- build-global-artifacts
# Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: "ubuntu-22.04"
Cargo.lock
Generated
+6 -7
View File
@@ -4108,11 +4108,10 @@ dependencies = [
[[package]]
name = "rawdb"
version = "0.3.1"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6601495283897e6193cbd70d65e4e7420995c59c0de3de953f297c2fa8d9dc7"
checksum = "c184e0f054f8b9018424fb226dbf122ceae050eb5f900e45d0809ff9fdc7b0d0"
dependencies = [
"allocative",
"libc",
"memmap2",
"parking_lot 0.12.5",
@@ -5331,9 +5330,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vecdb"
version = "0.3.1"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cffa6180edb8fb521c9958d4d00a1cca1b5de396ee371f2f8e3ec54c21205eff"
checksum = "44fef339cc0a6103cea9df44c1133d8275585b90dbe2faa6e3f60e6d2bac47b6"
dependencies = [
"allocative",
"ctrlc",
@@ -5350,9 +5349,9 @@ dependencies = [
[[package]]
name = "vecdb_derive"
version = "0.3.1"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23dffd7af8e5579122f79e4a2df3f62f5f315ef0d593721f585384b61bc722e2"
checksum = "c09556497d1695d5ba268ebeaa1d2547c2c597a60098fde60fe78028b463c4c1"
dependencies = [
"quote",
"syn 2.0.108",
Cargo.toml
+3 -3
View File
@@ -76,7 +76,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
sonic-rs = "0.5.6"
tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] }
vecdb = { version = "0.3.1", features = ["derive"] }
vecdb = { version = "0.3.4", features = ["derive"] }
zerocopy = { version = "0.8.27", features = ["derive"] }
[workspace.metadata.release]
@@ -86,8 +86,8 @@ pre-release-commit-message = "release: v{{version}}"
tag-message = "release: v{{version}}"
[workspace.metadata.dist]
cargo-dist-version = "0.30.0"
cargo-dist-version = "0.30.2"
ci = "github"
allow-dirty = ["ci"]
# allow-dirty = ["ci"]
installers = []
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
crates/brk_error/src/lib.rs
+5 -5
View File
@@ -15,7 +15,7 @@ pub enum Error {
FjallV2(fjall2::Error),
FjallV3(fjall3::Error),
VecDB(vecdb::Error),
SeqDB(vecdb::SeqDBError),
RawDB(vecdb::RawDBError),
Minreq(minreq::Error),
SystemTimeError(time::SystemTimeError),
BitcoinConsensusEncode(bitcoin::consensus::encode::Error),
@@ -106,10 +106,10 @@ impl From<vecdb::Error> for Error {
}
}
impl From<vecdb::SeqDBError> for Error {
impl From<vecdb::RawDBError> for Error {
#[inline]
fn from(value: vecdb::SeqDBError) -> Self {
Self::SeqDB(value)
fn from(value: vecdb::RawDBError) -> Self {
Self::RawDB(value)
}
}
@@ -183,7 +183,7 @@ impl fmt::Display for Error {
Error::IO(error) => Display::fmt(&error, f),
Error::Jiff(error) => Display::fmt(&error, f),
Error::Minreq(error) => Display::fmt(&error, f),
Error::SeqDB(error) => Display::fmt(&error, f),
Error::RawDB(error) => Display::fmt(&error, f),
Error::SonicRS(error) => Display::fmt(&error, f),
Error::SystemTimeError(error) => Display::fmt(&error, f),
Error::VecDB(error) => Display::fmt(&error, f),
crates/brk_indexer/src/lib.rs
+6 -6
View File
@@ -17,22 +17,22 @@ use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended};
mod indexes;
mod stores_redb;
// mod stores_redb;
// mod stores_v2;
// mod stores_v3;
mod stores_v3;
mod vecs;
pub use indexes::*;
pub use stores_redb::*;
// pub use stores_redb::*;
// pub use stores_v2::*;
// pub use stores_v3::*;
pub use stores_v3::*;
pub use vecs::*;
// One version for all data sources
// Increment on **change _OR_ addition**
const VERSION: Version = Version::new(23);
const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(920_000);
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0);
#[derive(Clone)]
pub struct Indexer {
@@ -746,7 +746,7 @@ impl Indexer {
}
// let i = Instant::now();
self.vecs.punch_holes()?;
self.vecs.compact()?;
// info!("Punched holes in db in {}s", i.elapsed().as_secs());
Ok(starting_indexes)
crates/brk_indexer/src/stores_v3.rs
+3 -3
View File
@@ -8,7 +8,7 @@ use brk_types::{
TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version,
Vout,
};
use fjall3::{PersistMode, TxDatabase};
use fjall3::{Database, PersistMode};
use log::info;
use rayon::prelude::*;
use vecdb::{AnyVec, GenericStoredVec, StoredIndex, VecIterator, VecIteratorExtended};
@@ -19,7 +19,7 @@ use super::Vecs;
#[derive(Clone)]
pub struct Stores {
pub database: TxDatabase,
pub database: Database,
pub addressbyteshash_to_typeindex: Store<AddressBytesHash, TypeIndex>,
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
@@ -37,7 +37,7 @@ impl Stores {
fs::create_dir_all(&pathbuf)?;
let database = match brk_store::open_fjall2_database(path) {
let database = match brk_store::open_fjall3_database(path) {
Ok(database) => database,
Err(_) => {
fs::remove_dir_all(path)?;
crates/brk_indexer/src/vecs.rs
+2 -2
View File
@@ -364,8 +364,8 @@ impl Vecs {
.unwrap()
}
pub fn punch_holes(&self) -> Result<()> {
self.db.punch_holes()?;
pub fn compact(&self) -> Result<()> {
self.db.compact()?;
Ok(())
}
crates/brk_store/src/fjall_v3/meta.rs
+4 -4
View File
@@ -5,7 +5,7 @@ use std::{
use brk_error::Result;
use brk_types::Version;
use fjall3::{TxDatabase, TxKeyspace};
use fjall3::{Database, Keyspace};
use super::Height;
@@ -18,13 +18,13 @@ pub struct StoreMeta {
impl StoreMeta {
pub fn checked_open<F>(
_database: &TxDatabase,
_database: &Database,
path: &Path,
version: Version,
open_partition_handle: F,
) -> Result<(Self, TxKeyspace)>
) -> Result<(Self, Keyspace)>
where
F: Fn() -> Result<TxKeyspace>,
F: Fn() -> Result<Keyspace>,
{
fs::create_dir_all(path)?;
crates/brk_store/src/fjall_v3/mod.rs
+17 -18
View File
@@ -4,8 +4,8 @@ use brk_error::Result;
use brk_types::{Height, Version};
use byteview8::ByteView;
use fjall3::{
KeyspaceCreateOptions, TxDatabase, TxKeyspace, ValueType,
config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry},
Database, Keyspace, KeyspaceCreateOptions, ValueType,
config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy},
};
mod meta;
@@ -19,16 +19,16 @@ use crate::any::AnyStore;
pub struct StoreFjallV3<Key, Value> {
meta: StoreMeta,
name: &'static str,
database: TxDatabase,
keyspace: TxKeyspace,
database: Database,
keyspace: Keyspace,
puts: FxHashMap<Key, Value>,
dels: FxHashSet<Key>,
}
const MAJOR_FJALL_VERSION: Version = Version::new(3);
pub fn open_fjall2_database(path: &Path) -> fjall3::Result<TxDatabase> {
TxDatabase::builder(path.join("fjall"))
pub fn open_fjall3_database(path: &Path) -> fjall3::Result<Database> {
Database::builder(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.cache_size(1024 * 1024 * 1024)
.open()
@@ -41,12 +41,14 @@ where
ByteView: From<K> + From<V>,
{
fn open_keyspace(
database: &TxDatabase,
database: &Database,
name: &str,
bloom_filters: Option<bool>,
) -> Result<TxKeyspace> {
) -> Result<Keyspace> {
let mut options = KeyspaceCreateOptions::default()
.manual_journal_persist(true)
.filter_block_pinning_policy(PinningPolicy::all(false))
.index_block_pinning_policy(PinningPolicy::all(false))
.max_memtable_size(8 * 1024 * 1024);
if bloom_filters.is_some_and(|b| !b) {
@@ -61,7 +63,7 @@ where
}
pub fn import(
database: &TxDatabase,
database: &Database,
path: &Path,
name: &str,
version: Version,
@@ -112,10 +114,7 @@ where
#[inline]
pub fn is_empty(&self) -> Result<bool> {
self.database
.read_tx()
.is_empty(&self.keyspace)
.map_err(|e| e.into())
self.keyspace.is_empty().map_err(|e| e.into())
}
#[inline]
@@ -178,7 +177,7 @@ where
return Ok(());
}
let mut batch = self.database.inner().batch();
let mut batch = self.database.batch();
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
@@ -193,7 +192,7 @@ where
.into_iter()
.map(|i| i.fjall(&self.keyspace))
.collect::<Vec<_>>();
batch.commit_keyspace(self.keyspace.inner())?;
batch.commit_keyspace(&self.keyspace)?;
// let mut wtx = self.database.write_tx();
@@ -266,20 +265,20 @@ impl<K, V> Item<K, V> {
}
}
pub fn fjall(self, keyspace: &TxKeyspace) -> fjall3::Item
pub fn fjall(self, keyspace: &Keyspace) -> fjall3::Item
where
K: Into<ByteView>,
V: Into<ByteView>,
{
match self {
Item::Value { key, value } => fjall3::Item {
keyspace_id: keyspace.inner().id,
keyspace_id: keyspace.id,
key: key.into().into(),
value: value.into().into(),
value_type: ValueType::Value,
},
Item::Tomb(key) => fjall3::Item {
keyspace_id: keyspace.inner().id,
keyspace_id: keyspace.id,
key: key.into().into(),
value: [].into(),
value_type: ValueType::WeakTombstone,