mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-08 14:11:56 -07:00
bindex: retrying fjall
This commit is contained in:
nym21
Generated
+266
-2
@@ -75,7 +75,9 @@ dependencies = [
|
||||
"color-eyre",
|
||||
"derive_deref",
|
||||
"exit",
|
||||
"fjall",
|
||||
"jiff",
|
||||
"rapidhash",
|
||||
"rayon",
|
||||
"snkrj",
|
||||
"storable_vec",
|
||||
@@ -317,6 +319,16 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-skiplist"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.21"
|
||||
@@ -333,6 +345,20 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"hashbrown",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core 0.9.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_deref"
|
||||
version = "1.1.1"
|
||||
@@ -355,12 +381,46 @@ dependencies = [
|
||||
"syn 2.0.96",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "double-ended-peekable"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
|
||||
|
||||
[[package]]
|
||||
name = "enum_dispatch"
|
||||
version = "0.3.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.96",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exit"
|
||||
version = "0.1.0"
|
||||
@@ -378,6 +438,28 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "fjall"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80910a26e4fb5e5393ff64d293602ac1ade56cf4d14d244c02a7d4ddcd5f10bc"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"dashmap",
|
||||
"log",
|
||||
"lsm-tree",
|
||||
"path-absolutize",
|
||||
"std-semaphore",
|
||||
"tempfile",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
@@ -405,6 +487,18 @@ version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "guardian"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "493913a18c0d7bebb75127a26a432162c59edbe06f6cf712001e3e769345e8b5"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
|
||||
[[package]]
|
||||
name = "hex-conservative"
|
||||
version = "0.2.1"
|
||||
@@ -503,6 +597,12 @@ version = "0.2.169"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.12"
|
||||
@@ -519,6 +619,35 @@ version = "0.4.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
|
||||
|
||||
[[package]]
|
||||
name = "lsm-tree"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d858efa63a32a286a06aa3968f3ed09811d556697e9f9804aa72ea679ed3b83"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crossbeam-skiplist",
|
||||
"double-ended-peekable",
|
||||
"enum_dispatch",
|
||||
"guardian",
|
||||
"log",
|
||||
"lz4_flex",
|
||||
"path-absolutize",
|
||||
"quick_cache",
|
||||
"rustc-hash",
|
||||
"self_cell",
|
||||
"tempfile",
|
||||
"value-log",
|
||||
"varint-rs",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
@@ -534,6 +663,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "min-max-heap"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.4"
|
||||
@@ -595,7 +730,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
"parking_lot_core 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -607,11 +742,42 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"redox_syscall 0.2.16",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall 0.5.8",
|
||||
"smallvec",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "path-absolutize"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
|
||||
dependencies = [
|
||||
"path-dedot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "path-dedot"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.16"
|
||||
@@ -651,6 +817,16 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d7c94f8935a9df96bb6380e8592c70edf497a643f94bd23b2f76b399385dbf4"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.38"
|
||||
@@ -690,6 +866,12 @@ dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rapidhash"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c3cf30fb8a0540827f3e22514836533403dece9ac517631160a1fb36d0dd5d6"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.10.0"
|
||||
@@ -719,12 +901,40 @@ dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
|
||||
dependencies = [
|
||||
"bitflags 2.8.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags 2.8.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.18"
|
||||
@@ -779,6 +989,12 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "self_cell"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.217"
|
||||
@@ -839,6 +1055,12 @@ dependencies = [
|
||||
"sanakirja",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "std-semaphore"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
|
||||
|
||||
[[package]]
|
||||
name = "storable_vec"
|
||||
version = "0.1.2"
|
||||
@@ -890,6 +1112,20 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
@@ -973,6 +1209,28 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
|
||||
|
||||
[[package]]
|
||||
name = "value-log"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f90495556d09c3026f7f3897f8a7db59c8c701082e32dcf58c2319062ae1eb0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"log",
|
||||
"min-max-heap",
|
||||
"path-absolutize",
|
||||
"quick_cache",
|
||||
"rustc-hash",
|
||||
"tempfile",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "varint-rs"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
@@ -1074,6 +1332,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
|
||||
@@ -9,6 +9,7 @@ biter = { path = "../biter" }
|
||||
color-eyre = "0.6.3"
|
||||
derive_deref = "1.1.1"
|
||||
exit = { path = "../exit" }
|
||||
fjall = "2.5.0"
|
||||
jiff = "0.1.24"
|
||||
rapidhash = "1.3.0"
|
||||
rayon = "1.10.0"
|
||||
|
||||
+54
-38
@@ -3,7 +3,8 @@ use std::{
|
||||
io::{Read, Write},
|
||||
path::Path,
|
||||
str::FromStr,
|
||||
thread::{self},
|
||||
thread::{self, sleep},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use biter::{
|
||||
@@ -12,12 +13,13 @@ use biter::{
|
||||
};
|
||||
use color_eyre::eyre::{eyre, ContextCompat};
|
||||
use exit::Exit;
|
||||
use fjall::{PersistMode, ReadTransaction, TransactionalKeyspace};
|
||||
use rayon::prelude::*;
|
||||
|
||||
mod storage;
|
||||
mod structs;
|
||||
|
||||
use storage::{Stores, Vecs};
|
||||
use storage::{Partitions, Vecs};
|
||||
use structs::{
|
||||
Addressbytes, AddressbytesPrefix, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp,
|
||||
TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout,
|
||||
@@ -35,17 +37,17 @@ impl Indexer {
|
||||
|
||||
let mut vecs = Vecs::import(&indexes_dir.join("vecs"))?;
|
||||
|
||||
let open_stores = || Stores::open(&indexes_dir.join("stores"));
|
||||
let stores = open_stores()?;
|
||||
let keyspace = fjall::Config::new(indexes_dir.join("fjall")).open_transactional()?;
|
||||
let mut parts = Partitions::import(&keyspace, &exit)?;
|
||||
let rtx = keyspace.read_tx();
|
||||
|
||||
let mut height = vecs
|
||||
.min_height()
|
||||
.unwrap_or_default()
|
||||
.min(stores.min_height())
|
||||
.min(parts.min_height())
|
||||
.and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
|
||||
.map(Height::from)
|
||||
.unwrap_or_default();
|
||||
// let mut height = Height::default();
|
||||
|
||||
let mut txindex_global = vecs.height_to_first_txindex.get_or_default(height)?;
|
||||
let mut txinindex_global = vecs.height_to_first_txinindex.get_or_default(height)?;
|
||||
@@ -64,31 +66,37 @@ impl Indexer {
|
||||
let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
|
||||
let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
|
||||
|
||||
let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> {
|
||||
let export = |keyspace: &TransactionalKeyspace,
|
||||
rtx: ReadTransaction,
|
||||
parts: &mut Partitions,
|
||||
vecs: &mut Vecs,
|
||||
height: Height|
|
||||
-> color_eyre::Result<()> {
|
||||
println!("Exporting...");
|
||||
|
||||
if height >= Height::from(400_000_u32) {
|
||||
pause();
|
||||
// println!("Flushing vecs...");
|
||||
}
|
||||
drop(rtx);
|
||||
|
||||
exit.block();
|
||||
|
||||
thread::scope(|scope| -> color_eyre::Result<()> {
|
||||
let vecs_handle = scope.spawn(|| vecs.flush(height));
|
||||
let stores_handle = scope.spawn(|| stores.export(height));
|
||||
|
||||
parts.write(keyspace, height)?;
|
||||
keyspace.persist(PersistMode::SyncAll)?;
|
||||
|
||||
vecs_handle.join().unwrap()?;
|
||||
stores_handle.join().unwrap()?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
exit.unblock();
|
||||
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let mut stores_opt = Some(stores);
|
||||
// let mut stores_opt = Some(stores);
|
||||
let mut rtx_opt = Some(rtx);
|
||||
|
||||
biter::new(bitcoin_dir, Some(height.into()), Some(500_000), rpc)
|
||||
biter::new(bitcoin_dir, Some(height.into()), None, rpc)
|
||||
.iter()
|
||||
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
|
||||
println!("Processing block {_height}...");
|
||||
@@ -96,27 +104,28 @@ impl Indexer {
|
||||
height = Height::from(_height);
|
||||
let timestamp = Timestamp::try_from(block.header.time)?;
|
||||
|
||||
let mut stores = stores_opt.take().context("option should have wtx")?;
|
||||
// let mut stores = stores_opt.take().context("option should have store")?;
|
||||
let rtx = rtx_opt.take().context("option should have rtx")?;
|
||||
|
||||
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
|
||||
if &blockhash != saved_blockhash.as_ref() {
|
||||
todo!("Rollback not implemented");
|
||||
// parts.rollback_from(&mut wtx, height, &exit)?;
|
||||
// parts.rollback_from(&mut rtx, height, &exit)?;
|
||||
}
|
||||
}
|
||||
|
||||
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
|
||||
|
||||
if stores
|
||||
if parts
|
||||
.blockhash_prefix_to_height
|
||||
.get(&blockhash_prefix)?
|
||||
.is_some_and(|prev_height| *prev_height != height)
|
||||
.get(&rtx, &blockhash_prefix)?
|
||||
.is_some_and(|prev_height| prev_height != height)
|
||||
{
|
||||
dbg!(blockhash);
|
||||
return Err(eyre!("Collision, expect prefix to need be set yet"));
|
||||
}
|
||||
|
||||
stores
|
||||
parts
|
||||
.blockhash_prefix_to_height
|
||||
.insert_if_needed(blockhash_prefix, height, height);
|
||||
|
||||
@@ -194,9 +203,9 @@ impl Indexer {
|
||||
let txid_prefix = TxidPrefix::try_from(&txid)?;
|
||||
|
||||
let prev_txindex_slice_opt =
|
||||
if check_collisions && stores.txid_prefix_to_txindex.needs(height) {
|
||||
if check_collisions && parts.txid_prefix_to_txindex.needs(height) {
|
||||
// Should only find collisions for two txids (duplicates), see below
|
||||
stores.txid_prefix_to_txindex.get(&txid_prefix)?.cloned()
|
||||
parts.txid_prefix_to_txindex.get(&rtx, &txid_prefix)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -235,14 +244,14 @@ impl Indexer {
|
||||
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
|
||||
}
|
||||
|
||||
let prev_txindex = if let Some(txindex) = stores
|
||||
let prev_txindex = if let Some(txindex) = parts
|
||||
.txid_prefix_to_txindex
|
||||
.get(&TxidPrefix::try_from(&outpoint.txid)?)?
|
||||
.get(&rtx, &TxidPrefix::try_from(&outpoint.txid)?)?
|
||||
.and_then(|txindex| {
|
||||
// Checking if not finding txindex from the future
|
||||
(txindex < &txindex_global).then_some(txindex)
|
||||
(txindex < txindex_global).then_some(txindex)
|
||||
}) {
|
||||
*txindex
|
||||
txindex
|
||||
} else {
|
||||
// dbg!(txindex_global + block_txindex, txindex, txin, vin);
|
||||
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
|
||||
@@ -312,11 +321,10 @@ impl Indexer {
|
||||
});
|
||||
|
||||
let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
|
||||
stores
|
||||
parts
|
||||
.addressbytes_prefix_to_addressindex
|
||||
.get(&AddressbytesPrefix::from((addressbytes, addresstype)))
|
||||
.get(&rtx, &AddressbytesPrefix::from((addressbytes, addresstype)))
|
||||
.unwrap()
|
||||
.cloned()
|
||||
// Checking if not in the future
|
||||
.and_then(|addressindex_local| {
|
||||
(addressindex_local < addressindex_global).then_some(addressindex_local)
|
||||
@@ -346,7 +354,7 @@ impl Indexer {
|
||||
|
||||
if (vecs.addressindex_to_addresstype.hasnt(addressindex)
|
||||
&& addresstype != prev_addresstype)
|
||||
|| (stores.addressbytes_prefix_to_addressindex.needs(height)
|
||||
|| (parts.addressbytes_prefix_to_addressindex.needs(height)
|
||||
&& prev_addressbytes != addressbytes)
|
||||
{
|
||||
let txid = tx.compute_txid();
|
||||
@@ -494,9 +502,9 @@ impl Indexer {
|
||||
let addressbytes_prefix = addressbytes_prefix.unwrap();
|
||||
|
||||
already_added_addressbytes_prefix
|
||||
.insert(addressbytes_prefix.clone(), addressindex);
|
||||
.insert(addressbytes_prefix, addressindex);
|
||||
|
||||
stores.addressbytes_prefix_to_addressindex.insert_if_needed(
|
||||
parts.addressbytes_prefix_to_addressindex.insert_if_needed(
|
||||
addressbytes_prefix,
|
||||
addressindex,
|
||||
height,
|
||||
@@ -580,7 +588,7 @@ impl Indexer {
|
||||
|
||||
match prev_txindex_opt {
|
||||
None => {
|
||||
stores
|
||||
parts
|
||||
.txid_prefix_to_txindex
|
||||
.insert_if_needed(txid_prefix, txindex, height);
|
||||
}
|
||||
@@ -646,16 +654,24 @@ impl Indexer {
|
||||
|
||||
let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active();
|
||||
if should_snapshot {
|
||||
export(stores, &mut vecs, height)?;
|
||||
stores_opt.replace(open_stores()?);
|
||||
export(&keyspace, rtx, &mut parts, &mut vecs, height)?;
|
||||
rtx_opt.replace(keyspace.read_tx());
|
||||
} else {
|
||||
stores_opt.replace(stores);
|
||||
rtx_opt.replace(rtx);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
export(stores_opt.take().context("option should have wtx")?, &mut vecs, height)?;
|
||||
export(
|
||||
&keyspace,
|
||||
rtx_opt.take().context("option should have wtx")?,
|
||||
&mut parts,
|
||||
&mut vecs,
|
||||
height,
|
||||
)?;
|
||||
|
||||
sleep(Duration::from_millis(100));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
use std::{
|
||||
ops::{Deref, DerefMut},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use canopydb::{Database as CanopyDatabase, DbOptions, Error, WriteTransaction};
|
||||
|
||||
use super::Environment;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Database {
|
||||
db: CanopyDatabase,
|
||||
// pub wtx: WriteTransaction,
|
||||
}
|
||||
impl Deref for Database {
|
||||
type Target = CanopyDatabase;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.db
|
||||
}
|
||||
}
|
||||
impl DerefMut for Database {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.db
|
||||
}
|
||||
}
|
||||
|
||||
impl Database {
|
||||
pub fn new(environment: &Environment, name: &str) -> color_eyre::Result<Self> {
|
||||
let mut options = DbOptions::default();
|
||||
options.use_wal = false;
|
||||
options.checkpoint_interval = Duration::from_secs(u64::MAX);
|
||||
options.checkpoint_target_size = usize::MAX;
|
||||
options.throttle_memory_limit = usize::MAX;
|
||||
options.stall_memory_limit = usize::MAX;
|
||||
options.write_txn_memory_limit = usize::MAX;
|
||||
|
||||
let db = environment.get_or_create_database_with(name, options)?;
|
||||
|
||||
Ok(Self {
|
||||
// wtx: db.begin_write()?,
|
||||
db,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn flush(&mut self) -> Result<(), Error> {
|
||||
// drop(blockhash_prefix_to_height_tree);
|
||||
// blockhash_prefix_to_height_tx_opt.take().map(|tx| tx.commit());
|
||||
self.checkpoint()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
use std::path::Path;
|
||||
|
||||
use canopydb::{EnvOptions, Environment as CanopyEnvironment};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct Environment(CanopyEnvironment);
|
||||
|
||||
impl Environment {
|
||||
pub fn new(path: &Path) -> color_eyre::Result<Self> {
|
||||
let mut options = EnvOptions::new(path);
|
||||
// options.use_mmap = true;
|
||||
options.disable_fsync = true;
|
||||
options.wal_new_file_on_checkpoint = false;
|
||||
options.wal_background_sync_interval = None;
|
||||
options.wal_write_batch_memory_limit = usize::MAX;
|
||||
|
||||
Ok(Self(CanopyEnvironment::with_options(options)?))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod database;
|
||||
mod environment;
|
||||
// mod transaction;
|
||||
mod tree;
|
||||
|
||||
pub use database::*;
|
||||
pub use environment::*;
|
||||
// pub use transaction::*;
|
||||
pub use tree::*;
|
||||
@@ -0,0 +1,19 @@
|
||||
use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
|
||||
|
||||
use super::{Database, Tree};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Transaction<'a, K, V> {
|
||||
tx: WriteTransaction,
|
||||
tree: Tree<'a, K, V>,
|
||||
}
|
||||
|
||||
impl<'a, K, V> Transaction<'a, K, V> {
|
||||
pub fn new(db: &Database) -> color_eyre::Result<Self> {
|
||||
let tx = db.begin_write()?;
|
||||
|
||||
let tree = Tree::new(&tx)?;
|
||||
|
||||
Ok(Self { tx, tree })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
marker::PhantomData,
|
||||
ops::{Deref, DerefMut},
|
||||
};
|
||||
|
||||
use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
|
||||
use color_eyre::eyre::eyre;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Tree<'a, K, V> {
|
||||
tree: CanopyTree<'a>,
|
||||
k: PhantomData<K>,
|
||||
v: PhantomData<V>,
|
||||
}
|
||||
impl<'a, K, V> Deref for Tree<'a, K, V> {
|
||||
type Target = CanopyTree<'a>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.tree
|
||||
}
|
||||
}
|
||||
impl<'a, K, V> DerefMut for Tree<'a, K, V> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.tree
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, K, V> Tree<'a, K, V>
|
||||
where
|
||||
K: Debug + Sized,
|
||||
V: Debug + Sized + Clone + Copy,
|
||||
{
|
||||
const SIZE_OF_K: usize = size_of::<K>();
|
||||
const SIZE_OF_V: usize = size_of::<V>();
|
||||
|
||||
pub fn new(tx: &'a WriteTransaction) -> color_eyre::Result<Self> {
|
||||
let mut options = TreeOptions::new();
|
||||
options.compress_overflow_values = None;
|
||||
options.fixed_key_len = size_of::<K>() as i8;
|
||||
options.fixed_value_len = size_of::<V>() as i8;
|
||||
|
||||
Ok(Self {
|
||||
tree: tx.get_or_create_tree_with(b"tree", options)?,
|
||||
k: PhantomData,
|
||||
v: PhantomData,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &K) -> color_eyre::Result<Option<V>> {
|
||||
let slice = self.tree.get(Self::key_as_slice(key))?;
|
||||
|
||||
if slice.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let slice = slice.unwrap();
|
||||
|
||||
let (prefix, shorts, suffix) = unsafe { slice.align_to::<V>() };
|
||||
|
||||
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
|
||||
dbg!(&key, &prefix, &shorts, &suffix);
|
||||
return Err(eyre!("align_to issue"));
|
||||
}
|
||||
|
||||
Ok(Some(shorts[0]))
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, key: &K, value: &V) -> Result<(), canopydb::Error> {
|
||||
self.tree
|
||||
.insert(Self::key_as_slice(key), Self::value_as_slice(value))
|
||||
}
|
||||
|
||||
fn key_as_slice(key: &K) -> &[u8] {
|
||||
let data: *const K = key;
|
||||
let data: *const u8 = data as *const u8;
|
||||
unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_K) }
|
||||
}
|
||||
|
||||
fn value_as_slice(value: &V) -> &[u8] {
|
||||
let data: *const V = value;
|
||||
let data: *const u8 = data as *const u8;
|
||||
unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_V) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
use std::{collections::BTreeMap, mem};
|
||||
|
||||
use exit::Exit;
|
||||
use fjall::{
|
||||
PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
|
||||
TransactionalPartitionHandle, TxKeyspace, WriteTransaction,
|
||||
};
|
||||
|
||||
use crate::structs::{Height, Version};
|
||||
|
||||
pub struct Partition<Key, Value> {
|
||||
version: Version,
|
||||
data: TransactionalPartitionHandle,
|
||||
meta: TransactionalPartitionHandle,
|
||||
height: Option<Height>,
|
||||
puts: BTreeMap<Key, Value>,
|
||||
}
|
||||
|
||||
impl<Key, Value> Partition<Key, Value>
|
||||
where
|
||||
Key: Into<Slice> + Ord,
|
||||
Value: Into<Slice> + TryFrom<Slice> + Clone,
|
||||
{
|
||||
pub const VERSION: &str = "version";
|
||||
pub const HEIGHT: &str = "height";
|
||||
|
||||
pub fn import(
|
||||
keyspace: &TransactionalKeyspace,
|
||||
name: &str,
|
||||
version: Version,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let data = Self::open_data(keyspace, name)?;
|
||||
let meta = Self::open_meta(keyspace, name)?;
|
||||
|
||||
let height = if let Some(slice) = meta.get(Self::HEIGHT)? {
|
||||
Some(Height::try_from(slice)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut this = Self {
|
||||
version,
|
||||
height,
|
||||
data,
|
||||
meta,
|
||||
puts: BTreeMap::new(),
|
||||
};
|
||||
|
||||
if let Some(slice) = this.meta.get(Self::VERSION)? {
|
||||
if version != Version::try_from(slice)? {
|
||||
this = this.reset(keyspace, name, exit)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(this)
|
||||
}
|
||||
|
||||
fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
|
||||
keyspace.open_partition(&format!("{name}-data"), Self::create_options())
|
||||
}
|
||||
|
||||
fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
|
||||
keyspace.open_partition(&format!("{name}-meta"), Self::create_options())
|
||||
}
|
||||
|
||||
fn create_options() -> PartitionCreateOptions {
|
||||
PartitionCreateOptions::default().manual_journal_persist(true)
|
||||
}
|
||||
|
||||
pub fn has(&self, height: Height) -> bool {
|
||||
self.height.is_some_and(|self_height| self_height >= height)
|
||||
}
|
||||
pub fn needs(&self, height: Height) -> bool {
|
||||
!self.has(height)
|
||||
}
|
||||
|
||||
pub fn get<'a>(&self, rtx: &ReadTransaction, key: &'a Key) -> color_eyre::Result<Option<Value>>
|
||||
where
|
||||
fjall::Slice: std::convert::From<&'a Key>,
|
||||
<Value as std::convert::TryFrom<fjall::Slice>>::Error: std::error::Error + Send + Sync,
|
||||
<Value as std::convert::TryFrom<fjall::Slice>>::Error: 'static,
|
||||
{
|
||||
if let Some(v) = self.puts.get(key) {
|
||||
return Ok(Some(v.clone()));
|
||||
}
|
||||
|
||||
if let Some(slice) = rtx.get(&self.data, Slice::from(key))? {
|
||||
let v_res = Value::try_from(slice);
|
||||
let v = v_res?;
|
||||
Ok(Some(v))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn insert_if_needed(&mut self, key: Key, value: Value, height: Height) {
|
||||
if self.needs(height) {
|
||||
self.puts.insert(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_meta(&self, wtx: &mut WriteTransaction, height: Height) {
|
||||
wtx.insert(&self.meta, Self::VERSION, self.version());
|
||||
wtx.insert(&self.meta, Self::HEIGHT, height);
|
||||
}
|
||||
|
||||
pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> Result<()> {
|
||||
if self.has(height) && self.puts.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut wtx = keyspace.write_tx();
|
||||
mem::take(&mut self.puts)
|
||||
.into_iter()
|
||||
.for_each(|(key, value)| wtx.insert(&self.data, key, value));
|
||||
self.update_meta(&mut wtx, height);
|
||||
wtx.commit()
|
||||
}
|
||||
|
||||
pub fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
|
||||
fn reset(mut self, keyspace: &TransactionalKeyspace, name: &str, exit: &Exit) -> Result<Self> {
|
||||
exit.block();
|
||||
|
||||
keyspace.delete_partition(self.data)?;
|
||||
keyspace.delete_partition(self.meta)?;
|
||||
|
||||
keyspace.persist(PersistMode::SyncAll)?;
|
||||
|
||||
self.data = Self::open_data(keyspace, name)?;
|
||||
self.meta = Self::open_meta(keyspace, name)?;
|
||||
self.height = None;
|
||||
|
||||
exit.unblock();
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<&Height> {
|
||||
self.height.as_ref()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
use std::thread;
|
||||
|
||||
use crate::{structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
|
||||
|
||||
mod base;
|
||||
use base::*;
|
||||
use exit::Exit;
|
||||
use fjall::{TransactionalKeyspace, TxKeyspace};
|
||||
|
||||
pub struct Partitions {
|
||||
pub addressbytes_prefix_to_addressindex: Partition<AddressbytesPrefix, Addressindex>,
|
||||
pub blockhash_prefix_to_height: Partition<BlockHashPrefix, Height>,
|
||||
pub txid_prefix_to_txindex: Partition<TxidPrefix, Txindex>,
|
||||
}
|
||||
|
||||
impl Partitions {
|
||||
pub fn import(keyspace: &TransactionalKeyspace, exit: &Exit) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
addressbytes_prefix_to_addressindex: Partition::import(
|
||||
keyspace,
|
||||
"addressbytes_prefix_to_addressindex",
|
||||
Version::from(1),
|
||||
exit,
|
||||
)?,
|
||||
blockhash_prefix_to_height: Partition::import(
|
||||
keyspace,
|
||||
"blockhash_prefix_to_height",
|
||||
Version::from(1),
|
||||
exit,
|
||||
)?,
|
||||
txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?,
|
||||
})
|
||||
}
|
||||
|
||||
// pub fn rollback_from(
|
||||
// &mut self,
|
||||
// _wtx: &mut WriteTransaction,
|
||||
// _height: Height,
|
||||
// _exit: &Exit,
|
||||
// ) -> color_eyre::Result<()> {
|
||||
// panic!();
|
||||
// let mut txindex = None;
|
||||
|
||||
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (height_slice, slice_blockhash) = slice?;
|
||||
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
|
||||
|
||||
// wtx.remove(self.height_to_blockhash.data(), height_slice);
|
||||
|
||||
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
|
||||
|
||||
// if txindex.is_none() {
|
||||
// txindex.replace(
|
||||
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
|
||||
// .context("for height to have first txindex")?,
|
||||
// );
|
||||
// }
|
||||
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
|
||||
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txindex = txindex.context("txindex to not be none by now")?;
|
||||
|
||||
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (slice_txindex, slice_txid) = slice?;
|
||||
// let txindex = Txindex::from(slice_txindex);
|
||||
// let txid = Txid::from_slice(&slice_txid)?;
|
||||
|
||||
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txoutindex = Txoutindex::from(txindex);
|
||||
|
||||
// let mut addressindexes = BTreeSet::new();
|
||||
|
||||
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (txoutindex_slice, _) = slice?;
|
||||
|
||||
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
|
||||
|
||||
// if let Some(addressindex_slice) =
|
||||
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
|
||||
// {
|
||||
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
|
||||
|
||||
// let addressindex = Addressindex::from(addressindex_slice);
|
||||
// addressindexes.insert(addressindex);
|
||||
|
||||
// let txoutindex = Txoutindex::from(txoutindex_slice);
|
||||
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
|
||||
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(addresstxoutindex),
|
||||
// );
|
||||
// }
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// addressindexes
|
||||
// .into_iter()
|
||||
// .filter(|addressindex| {
|
||||
// let is_empty = wtx
|
||||
// .prefix(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(*addressindex),
|
||||
// )
|
||||
// .next()
|
||||
// .is_none();
|
||||
// is_empty
|
||||
// })
|
||||
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
|
||||
// let addressindex_slice = Slice::from(addressindex);
|
||||
|
||||
// let addressbytes = Addressbytes::from(
|
||||
// wtx.get(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// )?
|
||||
// .context("addressindex_to_address to have value")?,
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressbytes_prefix_to_addressindex.data(),
|
||||
// addressbytes.prefix(),
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// );
|
||||
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
//
|
||||
|
||||
// todo!("clear addresstxoutindexes_out")
|
||||
// todo!("clear addresstxoutindexes_in")
|
||||
// todo!("clear zero_txoutindexes")
|
||||
// todo!("clear txindexvout_to_txoutindex")
|
||||
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
pub fn min_height(&self) -> Option<Height> {
|
||||
[
|
||||
self.addressbytes_prefix_to_addressindex.height(),
|
||||
self.blockhash_prefix_to_height.height(),
|
||||
self.txid_prefix_to_txindex.height(),
|
||||
]
|
||||
.into_iter()
|
||||
.min()
|
||||
.flatten()
|
||||
.cloned()
|
||||
}
|
||||
|
||||
pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> fjall::Result<()> {
|
||||
thread::scope(|scope| {
|
||||
let addressbytes_prefix_to_addressindex_write_handle =
|
||||
scope.spawn(|| self.addressbytes_prefix_to_addressindex.write(keyspace, height));
|
||||
let blockhash_prefix_to_height_write_handle =
|
||||
scope.spawn(|| self.blockhash_prefix_to_height.write(keyspace, height));
|
||||
let txid_prefix_to_txindex_write_handle =
|
||||
scope.spawn(|| self.txid_prefix_to_txindex.write(keyspace, height));
|
||||
|
||||
addressbytes_prefix_to_addressindex_write_handle.join().unwrap()?;
|
||||
blockhash_prefix_to_height_write_handle.join().unwrap()?;
|
||||
txid_prefix_to_txindex_write_handle.join().unwrap()?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
// pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
|
||||
// self.addressbytes_prefix_to_addressindex.update_meta(wtx, height);
|
||||
// self.blockhash_prefix_to_height.update_meta(wtx, height);
|
||||
// self.txid_prefix_to_txindex.update_meta(wtx, height);
|
||||
// }
|
||||
|
||||
// pub fn export(self, height: Height) -> Result<(), snkrj::Error> {
|
||||
// thread::scope(|scope| {
|
||||
// vec![
|
||||
// scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
|
||||
// scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
|
||||
// scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
|
||||
// ]
|
||||
// .into_iter()
|
||||
// .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() })
|
||||
// })
|
||||
// }
|
||||
}
|
||||
@@ -1,5 +1,9 @@
|
||||
mod stores;
|
||||
mod vecs;
|
||||
// mod canopy;
|
||||
mod fjall;
|
||||
// mod sanakirja;
|
||||
mod storable_vec;
|
||||
|
||||
pub use stores::*;
|
||||
pub use vecs::*;
|
||||
// pub use canopy::*;
|
||||
pub use fjall::*;
|
||||
// pub use sanakirja::*;
|
||||
pub use storable_vec::*;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
use storable_vec::UnsafeSizedSerDe;
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
|
||||
pub struct Addressindex(u32);
|
||||
@@ -48,3 +49,15 @@ impl From<Addressindex> for usize {
|
||||
value.0 as usize
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<fjall::Slice> for Addressindex {
|
||||
type Error = storable_vec::Error;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<Addressindex> for fjall::Slice {
|
||||
fn from(value: Addressindex) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,3 +119,15 @@ impl TryFrom<&rpc::Client> for Height {
|
||||
Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<fjall::Slice> for Height {
|
||||
type Error = storable_vec::Error;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<Height> for fjall::Slice {
|
||||
fn from(value: Height) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,10 +3,11 @@ use std::hash::Hasher;
|
||||
use biter::bitcoin::{BlockHash, Txid};
|
||||
use derive_deref::Deref;
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
use storable_vec::UnsafeSizedSerDe;
|
||||
|
||||
use super::{Addressbytes, Addresstype, SliceExtended};
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct AddressbytesPrefix([u8; 8]);
|
||||
direct_repr!(AddressbytesPrefix);
|
||||
impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix {
|
||||
@@ -23,8 +24,24 @@ impl From<[u8; 8]> for AddressbytesPrefix {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
impl TryFrom<fjall::Slice> for AddressbytesPrefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<&AddressbytesPrefix> for fjall::Slice {
|
||||
fn from(value: &AddressbytesPrefix) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
impl From<AddressbytesPrefix> for fjall::Slice {
|
||||
fn from(value: AddressbytesPrefix) -> Self {
|
||||
Self::from(&value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct BlockHashPrefix([u8; 8]);
|
||||
direct_repr!(BlockHashPrefix);
|
||||
impl TryFrom<&BlockHash> for BlockHashPrefix {
|
||||
@@ -33,8 +50,24 @@ impl TryFrom<&BlockHash> for BlockHashPrefix {
|
||||
Ok(Self((&value[..]).read_8x_u8()?))
|
||||
}
|
||||
}
|
||||
impl TryFrom<fjall::Slice> for BlockHashPrefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<&BlockHashPrefix> for fjall::Slice {
|
||||
fn from(value: &BlockHashPrefix) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
impl From<BlockHashPrefix> for fjall::Slice {
|
||||
fn from(value: BlockHashPrefix) -> Self {
|
||||
Self::from(&value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct TxidPrefix([u8; 8]);
|
||||
direct_repr!(TxidPrefix);
|
||||
impl TryFrom<&Txid> for TxidPrefix {
|
||||
@@ -43,3 +76,19 @@ impl TryFrom<&Txid> for TxidPrefix {
|
||||
Ok(Self((&value[..]).read_8x_u8()?))
|
||||
}
|
||||
}
|
||||
impl TryFrom<fjall::Slice> for TxidPrefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<&TxidPrefix> for fjall::Slice {
|
||||
fn from(value: &TxidPrefix) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
impl From<TxidPrefix> for fjall::Slice {
|
||||
fn from(value: TxidPrefix) -> Self {
|
||||
Self::from(&value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::ops::{Add, AddAssign};
|
||||
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
use storable_vec::UnsafeSizedSerDe;
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
|
||||
pub struct Txindex(u32);
|
||||
@@ -57,3 +58,15 @@ impl From<Txindex> for usize {
|
||||
value.0 as usize
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<fjall::Slice> for Txindex {
|
||||
type Error = storable_vec::Error;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<Txindex> for fjall::Slice {
|
||||
fn from(value: Txindex) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,3 +23,15 @@ impl TryFrom<&Path> for Version {
|
||||
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<fjall::Slice> for Version {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
|
||||
Ok(*Self::unsafe_try_from_slice(&value)?)
|
||||
}
|
||||
}
|
||||
impl From<Version> for fjall::Slice {
|
||||
fn from(value: Version) -> Self {
|
||||
Self::new(value.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
+16
-8
@@ -27,7 +27,7 @@ use memmap2::{Mmap, MmapOptions};
|
||||
pub struct StorableVec<I, T> {
|
||||
pathbuf: PathBuf,
|
||||
file: File,
|
||||
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed to reduce the size of the lock (24 > 16)
|
||||
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
|
||||
disk_len: usize,
|
||||
pushed: Vec<T>,
|
||||
// updated: BTreeMap<usize, T>,
|
||||
@@ -40,6 +40,7 @@ pub struct StorableVec<I, T> {
|
||||
const MAX_PAGE_SIZE: usize = 4 * 4096;
|
||||
const ONE_MB: usize = 1000 * 1024;
|
||||
const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
|
||||
// const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
|
||||
|
||||
impl<I, T> StorableVec<I, T>
|
||||
where
|
||||
@@ -50,8 +51,7 @@ where
|
||||
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
|
||||
/// In bytes
|
||||
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
|
||||
pub const CACHE_LENGTH: usize = usize::MAX;
|
||||
// pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
|
||||
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
|
||||
|
||||
pub fn import(path: &Path) -> Result<Self, io::Error> {
|
||||
let file = Self::open_file(path)?;
|
||||
@@ -78,13 +78,18 @@ where
|
||||
// self.cache.clear();
|
||||
// self.cache.resize_with(len, Default::default);
|
||||
|
||||
// par_iter_mut ?
|
||||
self.cache.iter_mut().for_each(|lock| {
|
||||
lock.take();
|
||||
});
|
||||
|
||||
let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
|
||||
self.cache
|
||||
.resize_with(Self::CACHE_LENGTH.min(len), Default::default);
|
||||
self.cache.shrink_to_fit();
|
||||
let len = Self::CACHE_LENGTH.min(len);
|
||||
|
||||
if self.cache.len() != len {
|
||||
self.cache.resize_with(len, Default::default);
|
||||
self.cache.shrink_to_fit();
|
||||
}
|
||||
}
|
||||
|
||||
fn open_file(path: &Path) -> Result<File, io::Error> {
|
||||
@@ -222,7 +227,7 @@ where
|
||||
let len = self.len();
|
||||
match len.cmp(&index) {
|
||||
Ordering::Greater => {
|
||||
// dbg!(len, index);
|
||||
// dbg!(len, index, &self.pathbuf);
|
||||
// panic!();
|
||||
Ok(())
|
||||
}
|
||||
@@ -230,7 +235,10 @@ where
|
||||
self.push(value);
|
||||
Ok(())
|
||||
}
|
||||
Ordering::Less => Err(Error::IndexTooHigh),
|
||||
Ordering::Less => {
|
||||
dbg!(index, value);
|
||||
Err(Error::IndexTooHigh)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user