mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 14:11:56 -07:00
indexer: snapshot
This commit is contained in:
nym21
Generated
+17
-26
@@ -1321,7 +1321,7 @@ dependencies = [
|
||||
"allocative",
|
||||
"bitcoin",
|
||||
"brk_error",
|
||||
"byteview 0.8.0",
|
||||
"byteview 0.6.1",
|
||||
"derive_deref",
|
||||
"itoa",
|
||||
"jiff",
|
||||
@@ -1448,9 +1448,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.44"
|
||||
version = "1.2.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3"
|
||||
checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -2670,24 +2670,24 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
||||
|
||||
[[package]]
|
||||
name = "jiff"
|
||||
version = "0.2.15"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
|
||||
checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35"
|
||||
dependencies = [
|
||||
"jiff-static",
|
||||
"jiff-tzdb-platform",
|
||||
"log",
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde",
|
||||
"windows-sys 0.59.0",
|
||||
"serde_core",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jiff-static"
|
||||
version = "0.2.15"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
|
||||
checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3982,9 +3982,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.41"
|
||||
version = "1.0.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"
|
||||
checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -4110,9 +4110,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rawdb"
|
||||
version = "0.3.8"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d07ae2ce76ce46d26972046ae67d1f65ddbf5bc1290d3abaee84b7ab159564c4"
|
||||
checksum = "f79d5eea2a18c2cf7318e327bb3ead546514ea3bbea5f42c4e0cc0a89df2b46d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memmap2",
|
||||
@@ -5323,9 +5323,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
|
||||
|
||||
[[package]]
|
||||
name = "vecdb"
|
||||
version = "0.3.8"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d96acc025cc0039bf828cbf4efdcb485eb0eda88da7ec805fb6ec9159a340b73"
|
||||
checksum = "7393aaf472d72417bb7b3366addc225ae4b65f64ceef6ec975dd2891e18b08cf"
|
||||
dependencies = [
|
||||
"allocative",
|
||||
"ctrlc",
|
||||
@@ -5342,9 +5342,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "vecdb_derive"
|
||||
version = "0.3.8"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dae24e54786d92d4aa4b46098eed2594cd6ed51d2a47a36ff8ae3adfa232d958"
|
||||
checksum = "323a1fc669d513d1ad83a2fb1d1a589dfb502c9c626993a3313a834361bc64b5"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.109",
|
||||
@@ -5594,15 +5594,6 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
|
||||
+4
-4
@@ -52,8 +52,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" }
|
||||
brk_types = { version = "0.0.111", path = "crates/brk_types" }
|
||||
brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] }
|
||||
brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
|
||||
# byteview = "=0.6.1"
|
||||
byteview = "~0.8.0"
|
||||
byteview = "=0.6.1"
|
||||
# byteview = "~0.8.0"
|
||||
derive_deref = "1.1.1"
|
||||
fjall2 = { version = "2.11.5", package = "brk_fjall" }
|
||||
# fjall2 = { path = "../fjall2", package = "brk_fjall" }
|
||||
@@ -61,7 +61,7 @@ fjall2 = { version = "2.11.5", package = "brk_fjall" }
|
||||
fjall3 = { version = "=3.0.0-pre.4", package = "fjall" }
|
||||
# fjall3 = { path = "../fjall3", package = "brk_fjall" }
|
||||
# fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "bb15057500dce3115d7644d268b9deeaa895b431", package = "fjall" }
|
||||
jiff = "0.2.15"
|
||||
jiff = "0.2.16"
|
||||
log = "0.4.28"
|
||||
minreq = { version = "2.14.1", features = ["https", "serde_json"] }
|
||||
parking_lot = "0.12.5"
|
||||
@@ -75,7 +75,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
|
||||
sonic-rs = "0.5.6"
|
||||
tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
|
||||
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] }
|
||||
vecdb = { version = "0.3.8", features = ["derive"] }
|
||||
vecdb = { version = "0.3.11", features = ["derive"] }
|
||||
zerocopy = { version = "0.8.27", features = ["derive"] }
|
||||
|
||||
[workspace.metadata.release]
|
||||
|
||||
@@ -18,13 +18,13 @@ use rayon::prelude::*;
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended};
|
||||
mod indexes;
|
||||
// mod stores_v2;
|
||||
mod stores_v3;
|
||||
mod stores_v2;
|
||||
// mod stores_v3;
|
||||
mod vecs;
|
||||
|
||||
pub use indexes::*;
|
||||
// pub use stores_v2::*;
|
||||
pub use stores_v3::*;
|
||||
pub use stores_v2::*;
|
||||
// pub use stores_v3::*;
|
||||
pub use vecs::*;
|
||||
|
||||
// One version for all data sources
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_store::{AnyStore, StoreFjallV2 as Store};
|
||||
use brk_store::{AnyStore, Mode, StoreFjallV2 as Store, Type};
|
||||
use brk_types::{
|
||||
AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint,
|
||||
AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
|
||||
TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout,
|
||||
};
|
||||
use fjall2::{PersistMode, TransactionalKeyspace};
|
||||
use fjall2::{Keyspace, PersistMode};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{AnyVec, GenericStoredVec, StoredIndex, VecIterator, VecIteratorExtended};
|
||||
|
||||
@@ -17,7 +17,7 @@ use super::Vecs;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Stores {
|
||||
pub keyspace: TransactionalKeyspace,
|
||||
pub keyspace: Keyspace,
|
||||
|
||||
pub addressbyteshash_to_typeindex: Store<AddressBytesHash, TypeIndex>,
|
||||
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
|
||||
@@ -48,23 +48,47 @@ impl Stores {
|
||||
Ok(Self {
|
||||
keyspace: keyspace.clone(),
|
||||
|
||||
height_to_coinbase_tag: Store::import(keyspace_ref, path, "h2c", version, true)?,
|
||||
addressbyteshash_to_typeindex: Store::import(keyspace_ref, path, "a2t", version, true)?,
|
||||
blockhashprefix_to_height: Store::import(keyspace_ref, path, "b2h", version, true)?,
|
||||
txidprefix_to_txindex: Store::import(keyspace_ref, path, "t2t", version, true)?,
|
||||
height_to_coinbase_tag: Store::import(
|
||||
keyspace_ref,
|
||||
path,
|
||||
"h2c",
|
||||
version,
|
||||
Mode::UniquePushOnly(Type::Sequential),
|
||||
)?,
|
||||
addressbyteshash_to_typeindex: Store::import(
|
||||
keyspace_ref,
|
||||
path,
|
||||
"a2t",
|
||||
version,
|
||||
Mode::UniquePushOnly(Type::Random),
|
||||
)?,
|
||||
blockhashprefix_to_height: Store::import(
|
||||
keyspace_ref,
|
||||
path,
|
||||
"b2h",
|
||||
version,
|
||||
Mode::UniquePushOnly(Type::Random),
|
||||
)?,
|
||||
txidprefix_to_txindex: Store::import(
|
||||
keyspace_ref,
|
||||
path,
|
||||
"t2t",
|
||||
version,
|
||||
Mode::UniquePushOnly(Type::Random),
|
||||
)?,
|
||||
addresstype_to_addressindex_and_txindex: Store::import(
|
||||
keyspace_ref,
|
||||
path,
|
||||
"aat",
|
||||
version,
|
||||
false,
|
||||
Mode::VecLike,
|
||||
)?,
|
||||
addresstype_to_addressindex_and_unspentoutpoint: Store::import(
|
||||
keyspace_ref,
|
||||
path,
|
||||
"aau",
|
||||
version,
|
||||
false,
|
||||
Mode::VecLike,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_store::{AnyStore, StoreFjallV3 as Store};
|
||||
use brk_store::{AnyStore, Kind3, Mode3, StoreFjallV3 as Store};
|
||||
use brk_types::{
|
||||
AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint,
|
||||
AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
|
||||
@@ -24,8 +24,8 @@ pub struct Stores {
|
||||
pub height_to_coinbase_tag: Store<Height, StoredString>,
|
||||
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
|
||||
pub addresstype_to_addressindex_and_txindex: Store<AddressTypeAddressIndexTxIndex, Unit>,
|
||||
pub addresstype_to_addressindex_and_unspentoutpoint:
|
||||
Store<AddressTypeAddressIndexOutPoint, Unit>,
|
||||
// pub addresstype_to_addressindex_and_unspentoutpoint:
|
||||
// Store<AddressTypeAddressIndexOutPoint, Unit>,
|
||||
}
|
||||
|
||||
impl Stores {
|
||||
@@ -53,49 +53,49 @@ impl Stores {
|
||||
path,
|
||||
"height_to_coinbase_tag",
|
||||
version,
|
||||
true,
|
||||
true,
|
||||
Mode3::PushOnly,
|
||||
Kind3::Sequential,
|
||||
)?,
|
||||
addressbyteshash_to_typeindex: Store::import(
|
||||
database_ref,
|
||||
path,
|
||||
"addressbyteshash_to_typeindex",
|
||||
version,
|
||||
true,
|
||||
false,
|
||||
Mode3::PushOnly,
|
||||
Kind3::Random,
|
||||
)?,
|
||||
blockhashprefix_to_height: Store::import(
|
||||
database_ref,
|
||||
path,
|
||||
"blockhashprefix_to_height",
|
||||
version,
|
||||
true,
|
||||
false,
|
||||
Mode3::PushOnly,
|
||||
Kind3::Random,
|
||||
)?,
|
||||
txidprefix_to_txindex: Store::import(
|
||||
database_ref,
|
||||
path,
|
||||
"txidprefix_to_txindex",
|
||||
version,
|
||||
true,
|
||||
false,
|
||||
Mode3::PushOnly,
|
||||
Kind3::Random,
|
||||
)?,
|
||||
addresstype_to_addressindex_and_txindex: Store::import(
|
||||
database_ref,
|
||||
path,
|
||||
"addresstype_to_addressindex_and_txindex",
|
||||
version,
|
||||
false,
|
||||
false,
|
||||
)?,
|
||||
addresstype_to_addressindex_and_unspentoutpoint: Store::import(
|
||||
database_ref,
|
||||
path,
|
||||
"addresstype_to_addressindex_and_unspentoutpoint",
|
||||
version,
|
||||
false,
|
||||
false,
|
||||
Mode3::PushOnly,
|
||||
Kind3::Vec,
|
||||
)?,
|
||||
// addresstype_to_addressindex_and_unspentoutpoint: Store::import(
|
||||
// database_ref,
|
||||
// path,
|
||||
// "addresstype_to_addressindex_and_unspentoutpoint",
|
||||
// version,
|
||||
// Mode3::Any,
|
||||
// Kind3::Vec,
|
||||
// )?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ impl Stores {
|
||||
&mut self.height_to_coinbase_tag,
|
||||
&mut self.txidprefix_to_txindex,
|
||||
&mut self.addresstype_to_addressindex_and_txindex,
|
||||
&mut self.addresstype_to_addressindex_and_unspentoutpoint,
|
||||
// &mut self.addresstype_to_addressindex_and_unspentoutpoint,
|
||||
]
|
||||
.into_par_iter() // Changed from par_iter_mut()
|
||||
.try_for_each(|store| store.commit(height))?;
|
||||
@@ -134,7 +134,7 @@ impl Stores {
|
||||
&self.height_to_coinbase_tag,
|
||||
&self.txidprefix_to_txindex,
|
||||
&self.addresstype_to_addressindex_and_txindex,
|
||||
&self.addresstype_to_addressindex_and_unspentoutpoint,
|
||||
// &self.addresstype_to_addressindex_and_unspentoutpoint,
|
||||
]
|
||||
.into_iter()
|
||||
}
|
||||
@@ -149,9 +149,9 @@ impl Stores {
|
||||
&& self.txidprefix_to_txindex.is_empty()?
|
||||
&& self.height_to_coinbase_tag.is_empty()?
|
||||
&& self.addresstype_to_addressindex_and_txindex.is_empty()?
|
||||
&& self
|
||||
.addresstype_to_addressindex_and_unspentoutpoint
|
||||
.is_empty()?
|
||||
// && self
|
||||
// .addresstype_to_addressindex_and_unspentoutpoint
|
||||
// .is_empty()?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
@@ -351,13 +351,13 @@ impl Stores {
|
||||
);
|
||||
let outpoint = OutPoint::new(txindex, vout);
|
||||
|
||||
self.addresstype_to_addressindex_and_unspentoutpoint.remove(
|
||||
AddressTypeAddressIndexOutPoint::from((
|
||||
addresstype,
|
||||
addressindex,
|
||||
outpoint,
|
||||
)),
|
||||
);
|
||||
// self.addresstype_to_addressindex_and_unspentoutpoint.remove(
|
||||
// AddressTypeAddressIndexOutPoint::from((
|
||||
// addresstype,
|
||||
// addressindex,
|
||||
// outpoint,
|
||||
// )),
|
||||
// );
|
||||
});
|
||||
|
||||
// Add back outputs that were spent after the rollback point
|
||||
@@ -394,14 +394,14 @@ impl Stores {
|
||||
)),
|
||||
);
|
||||
|
||||
self.addresstype_to_addressindex_and_unspentoutpoint.insert(
|
||||
AddressTypeAddressIndexOutPoint::from((
|
||||
addresstype,
|
||||
addressindex,
|
||||
outpoint,
|
||||
)),
|
||||
Unit,
|
||||
);
|
||||
// self.addresstype_to_addressindex_and_unspentoutpoint.insert(
|
||||
// AddressTypeAddressIndexOutPoint::from((
|
||||
// addresstype,
|
||||
// addressindex,
|
||||
// outpoint,
|
||||
// )),
|
||||
// Unit,
|
||||
// );
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
|
||||
});
|
||||
|
||||
Builder::from_env(Env::default().default_filter_or(
|
||||
// "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
|
||||
"debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
|
||||
"info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
|
||||
// "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
|
||||
))
|
||||
.format(move |buf, record| {
|
||||
let date_time = Timestamp::now()
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::Version;
|
||||
use fjall2::{PersistMode, TransactionalKeyspace, TransactionalPartitionHandle};
|
||||
use fjall2::{Keyspace, PartitionHandle, PersistMode};
|
||||
|
||||
use super::Height;
|
||||
|
||||
@@ -18,13 +18,13 @@ pub struct StoreMeta {
|
||||
|
||||
impl StoreMeta {
|
||||
pub fn checked_open<F>(
|
||||
keyspace: &TransactionalKeyspace,
|
||||
keyspace: &Keyspace,
|
||||
path: &Path,
|
||||
version: Version,
|
||||
open_partition_handle: F,
|
||||
) -> Result<(Self, TransactionalPartitionHandle)>
|
||||
) -> Result<(Self, PartitionHandle)>
|
||||
where
|
||||
F: Fn() -> Result<TransactionalPartitionHandle>,
|
||||
F: Fn() -> Result<PartitionHandle>,
|
||||
{
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
|
||||
@@ -3,10 +3,7 @@ use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path};
|
||||
use brk_error::Result;
|
||||
use brk_types::{Height, Version};
|
||||
use byteview6::ByteView;
|
||||
use fjall2::{
|
||||
InnerItem, PartitionCreateOptions, TransactionalKeyspace, TransactionalPartitionHandle,
|
||||
ValueType,
|
||||
};
|
||||
use fjall2::{InnerItem, Keyspace, PartitionCreateOptions, PartitionHandle, ValueType};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
|
||||
use crate::any::AnyStore;
|
||||
@@ -19,16 +16,17 @@ use meta::*;
|
||||
pub struct StoreFjallV2<Key, Value> {
|
||||
meta: StoreMeta,
|
||||
name: &'static str,
|
||||
keyspace: TransactionalKeyspace,
|
||||
partition: TransactionalPartitionHandle,
|
||||
keyspace: Keyspace,
|
||||
partition: PartitionHandle,
|
||||
puts: FxHashMap<Key, Value>,
|
||||
dels: FxHashSet<Key>,
|
||||
mode: Mode,
|
||||
}
|
||||
|
||||
const MAJOR_FJALL_VERSION: Version = Version::TWO;
|
||||
|
||||
pub fn open_keyspace(path: &Path) -> fjall2::Result<TransactionalKeyspace> {
|
||||
fjall2::Config::new(path.join("fjall")).open_transactional()
|
||||
pub fn open_keyspace(path: &Path) -> fjall2::Result<Keyspace> {
|
||||
fjall2::Config::new(path.join("fjall")).open()
|
||||
}
|
||||
|
||||
impl<K, V> StoreFjallV2<K, V>
|
||||
@@ -38,14 +36,14 @@ where
|
||||
ByteView: From<K> + From<V>,
|
||||
{
|
||||
fn open_partition_handle(
|
||||
keyspace: &TransactionalKeyspace,
|
||||
keyspace: &Keyspace,
|
||||
name: &str,
|
||||
bloom_filters: bool,
|
||||
) -> Result<TransactionalPartitionHandle> {
|
||||
mode: Mode,
|
||||
) -> Result<PartitionHandle> {
|
||||
let mut options = PartitionCreateOptions::default().manual_journal_persist(true);
|
||||
|
||||
if bloom_filters {
|
||||
options = options.bloom_filter_bits(Some(5));
|
||||
if mode.is_unique_push_only() {
|
||||
options = options.bloom_filter_bits(Some(7));
|
||||
} else {
|
||||
options = options.bloom_filter_bits(None);
|
||||
}
|
||||
@@ -54,11 +52,11 @@ where
|
||||
}
|
||||
|
||||
pub fn import(
|
||||
keyspace: &TransactionalKeyspace,
|
||||
keyspace: &Keyspace,
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
bloom_filters: bool,
|
||||
mode: Mode,
|
||||
) -> Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
@@ -67,7 +65,7 @@ where
|
||||
&path.join(format!("meta/{name}")),
|
||||
MAJOR_FJALL_VERSION + version,
|
||||
|| {
|
||||
Self::open_partition_handle(keyspace, name, bloom_filters).inspect_err(|e| {
|
||||
Self::open_partition_handle(keyspace, name, mode).inspect_err(|e| {
|
||||
eprintln!("{e}");
|
||||
eprintln!("Delete {path:?} and try again");
|
||||
})
|
||||
@@ -81,6 +79,7 @@ where
|
||||
partition,
|
||||
puts: FxHashMap::default(),
|
||||
dels: FxHashSet::default(),
|
||||
mode,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -99,16 +98,12 @@ where
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> Result<bool> {
|
||||
self.keyspace
|
||||
.read_tx()
|
||||
.is_empty(&self.partition)
|
||||
.map_err(|e| e.into())
|
||||
self.partition.is_empty().map_err(|e| e.into())
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
|
||||
self.keyspace
|
||||
.read_tx()
|
||||
.iter(&self.partition)
|
||||
self.partition
|
||||
.iter()
|
||||
.map(|res| res.unwrap())
|
||||
.map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v))))
|
||||
}
|
||||
@@ -177,6 +172,18 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// if self.mode.is_unique_push_only() {
|
||||
// if !self.dels.is_empty() {
|
||||
// unreachable!();
|
||||
// }
|
||||
// let mut puts = mem::take(&mut self.puts).into_iter().collect::<Vec<_>>();
|
||||
// puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2));
|
||||
// dbg!(&puts);
|
||||
// self.partition.ingest(
|
||||
// puts.into_iter()
|
||||
// .map(|(k, v)| (ByteView::from(k), ByteView::from(v))),
|
||||
// )?;
|
||||
// } else {
|
||||
let mut items = mem::take(&mut self.puts)
|
||||
.into_iter()
|
||||
.map(|(key, value)| Item::Value { key, value })
|
||||
@@ -188,10 +195,11 @@ where
|
||||
.collect::<Vec<_>>();
|
||||
items.sort_unstable();
|
||||
|
||||
self.keyspace.inner().batch().commit_partition(
|
||||
self.partition.inner(),
|
||||
self.keyspace.batch().commit_partition(
|
||||
&self.partition,
|
||||
items.into_iter().map(InnerItem::from).collect::<Vec<_>>(),
|
||||
)?;
|
||||
// }
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -271,3 +279,25 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Mode {
|
||||
VecLike,
|
||||
UniquePushOnly(Type),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Type {
|
||||
Random,
|
||||
Sequential,
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
pub fn is_vec_like(&self) -> bool {
|
||||
matches!(*self, Self::VecLike)
|
||||
}
|
||||
|
||||
pub fn is_unique_push_only(&self) -> bool {
|
||||
matches!(*self, Self::UniquePushOnly(_))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ pub struct StoreFjallV3<Key, Value> {
|
||||
keyspace: Keyspace,
|
||||
puts: FxHashMap<Key, Value>,
|
||||
dels: FxHashSet<Key>,
|
||||
mode: Mode3,
|
||||
kind: Kind3,
|
||||
}
|
||||
|
||||
const MAJOR_FJALL_VERSION: Version = Version::new(3);
|
||||
@@ -47,8 +49,8 @@ where
|
||||
path: &Path,
|
||||
name: &str,
|
||||
version: Version,
|
||||
bloom_filters: bool,
|
||||
sequential: bool,
|
||||
mode: Mode3,
|
||||
kind: Kind3,
|
||||
) -> Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
@@ -57,7 +59,7 @@ where
|
||||
&path.join(format!("meta/{name}")),
|
||||
MAJOR_FJALL_VERSION + version,
|
||||
|| {
|
||||
Self::open_keyspace(database, name, bloom_filters, sequential).inspect_err(|e| {
|
||||
Self::open_keyspace(database, name, mode, kind).inspect_err(|e| {
|
||||
eprintln!("{e}");
|
||||
eprintln!("Delete {path:?} and try again");
|
||||
})
|
||||
@@ -71,29 +73,31 @@ where
|
||||
keyspace,
|
||||
puts: FxHashMap::default(),
|
||||
dels: FxHashSet::default(),
|
||||
mode,
|
||||
kind,
|
||||
})
|
||||
}
|
||||
|
||||
fn open_keyspace(
|
||||
database: &Database,
|
||||
name: &str,
|
||||
bloom_filters: bool,
|
||||
sequential: bool,
|
||||
_mode: Mode3,
|
||||
kind: Kind3,
|
||||
) -> Result<Keyspace> {
|
||||
let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true);
|
||||
|
||||
if bloom_filters {
|
||||
if kind.is_not_vec() {
|
||||
options = options.filter_policy(FilterPolicy::new(&[
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.5)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(5.0)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(2.5)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(8.0)),
|
||||
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.0)),
|
||||
]));
|
||||
} else {
|
||||
options = options.filter_policy(FilterPolicy::disabled());
|
||||
}
|
||||
|
||||
if sequential {
|
||||
if kind.is_sequential() {
|
||||
options = options
|
||||
.filter_block_pinning_policy(PinningPolicy::all(false))
|
||||
.index_block_pinning_policy(PinningPolicy::all(false));
|
||||
@@ -109,11 +113,6 @@ where
|
||||
{
|
||||
if let Some(v) = self.puts.get(key) {
|
||||
Ok(Some(Cow::Borrowed(v)))
|
||||
// } else if let Some(slice) = self
|
||||
// .database
|
||||
// .read_tx()
|
||||
// .get(&self.keyspace, ByteView::from(key))?
|
||||
// {
|
||||
} else if let Some(slice) = self.keyspace.get(ByteView::from(key))? {
|
||||
Ok(Some(Cow::Owned(V::from(ByteView::from(slice)))))
|
||||
} else {
|
||||
@@ -123,9 +122,6 @@ where
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> Result<bool> {
|
||||
// self.database
|
||||
// .read_tx()
|
||||
// .is_empty(&self.keyspace)
|
||||
self.keyspace.is_empty().map_err(|e| e.into())
|
||||
}
|
||||
|
||||
@@ -189,25 +185,38 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut batch = self.database.batch();
|
||||
// let mut batch = self.database.inner().batch();
|
||||
let mut items = mem::take(&mut self.puts)
|
||||
.into_iter()
|
||||
.map(|(key, value)| Item::Value { key, value })
|
||||
.chain(
|
||||
mem::take(&mut self.dels)
|
||||
.into_iter()
|
||||
.map(|key| Item::Tomb(key)),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
items.sort_unstable();
|
||||
items.into_iter().for_each(|item| match item {
|
||||
Item::Value { key, value } => {
|
||||
batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
|
||||
if self.mode.is_push_only() {
|
||||
if !self.dels.is_empty() {
|
||||
unreachable!();
|
||||
}
|
||||
Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)),
|
||||
});
|
||||
batch.commit()?;
|
||||
let mut puts = mem::take(&mut self.puts).into_iter().collect::<Vec<_>>();
|
||||
puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2));
|
||||
// dbg!(&puts);
|
||||
self.keyspace.ingest(
|
||||
puts.into_iter()
|
||||
.map(|(k, v)| (ByteView::from(k), ByteView::from(v))),
|
||||
)?;
|
||||
} else {
|
||||
let mut batch = self.database.batch();
|
||||
// let mut batch = self.database.inner().batch();
|
||||
let mut items = mem::take(&mut self.puts)
|
||||
.into_iter()
|
||||
.map(|(key, value)| Item::Value { key, value })
|
||||
.chain(
|
||||
mem::take(&mut self.dels)
|
||||
.into_iter()
|
||||
.map(|key| Item::Tomb(key)),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
items.sort_unstable();
|
||||
items.into_iter().for_each(|item| match item {
|
||||
Item::Value { key, value } => {
|
||||
batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
|
||||
}
|
||||
Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)),
|
||||
});
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
// batch.ingest(
|
||||
// items
|
||||
@@ -304,3 +313,40 @@ impl<K, V> Item<K, V> {
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Mode3 {
|
||||
Any,
|
||||
PushOnly,
|
||||
}
|
||||
|
||||
impl Mode3 {
|
||||
pub fn is_any(&self) -> bool {
|
||||
matches!(*self, Self::Any)
|
||||
}
|
||||
|
||||
pub fn is_push_only(&self) -> bool {
|
||||
matches!(*self, Self::PushOnly)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Kind3 {
|
||||
Random,
|
||||
Sequential,
|
||||
Vec,
|
||||
}
|
||||
|
||||
impl Kind3 {
|
||||
pub fn is_sequential(&self) -> bool {
|
||||
matches!(*self, Self::Sequential)
|
||||
}
|
||||
|
||||
pub fn is_random(&self) -> bool {
|
||||
matches!(*self, Self::Random)
|
||||
}
|
||||
|
||||
pub fn is_not_vec(&self) -> bool {
|
||||
!matches!(*self, Self::Vec)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ use byteview::ByteView;
|
||||
use derive_deref::Deref;
|
||||
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::copy_first_8bytes;
|
||||
|
||||
use super::AddressBytes;
|
||||
|
||||
#[derive(
|
||||
@@ -31,7 +33,7 @@ impl From<&AddressBytes> for AddressBytesHash {
|
||||
impl From<ByteView> for AddressBytesHash {
|
||||
#[inline]
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
Self(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +47,6 @@ impl From<AddressBytesHash> for ByteView {
|
||||
impl From<&AddressBytesHash> for ByteView {
|
||||
#[inline]
|
||||
fn from(value: &AddressBytesHash) -> Self {
|
||||
Self::new(value.as_bytes())
|
||||
Self::new(&value.0.to_be_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,14 +40,7 @@ impl From<&BlockHash> for BlockHashPrefix {
|
||||
impl From<ByteView> for BlockHashPrefix {
|
||||
#[inline]
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&BlockHashPrefix> for ByteView {
|
||||
#[inline]
|
||||
fn from(value: &BlockHashPrefix) -> Self {
|
||||
Self::new(value.as_bytes())
|
||||
Self(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,3 +50,10 @@ impl From<BlockHashPrefix> for ByteView {
|
||||
Self::from(&value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&BlockHashPrefix> for ByteView {
|
||||
#[inline]
|
||||
fn from(value: &BlockHashPrefix) -> Self {
|
||||
Self::from(value.to_be_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,14 +40,7 @@ impl From<&Txid> for TxidPrefix {
|
||||
impl From<ByteView> for TxidPrefix {
|
||||
#[inline]
|
||||
fn from(value: ByteView) -> Self {
|
||||
Self::read_from_bytes(&value).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&TxidPrefix> for ByteView {
|
||||
#[inline]
|
||||
fn from(value: &TxidPrefix) -> Self {
|
||||
Self::new(value.as_bytes())
|
||||
Self(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +51,13 @@ impl From<TxidPrefix> for ByteView {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&TxidPrefix> for ByteView {
|
||||
#[inline]
|
||||
fn from(value: &TxidPrefix) -> Self {
|
||||
Self::from(value.to_be_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<[u8; 8]> for TxidPrefix {
|
||||
#[inline]
|
||||
fn from(value: [u8; 8]) -> Self {
|
||||
|
||||
Reference in New Issue
Block a user