diff --git a/Cargo.lock b/Cargo.lock index 93e87f14b..55cedc130 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,7 @@ checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" name = "bindex" version = "0.1.0" dependencies = [ + "bitcoin", "bitcoin_hashes 0.16.0", "biter", "color-eyre", @@ -79,8 +80,8 @@ dependencies = [ "jiff", "rapidhash", "rayon", - "snkrj", "storable_vec", + "unsafe_slice_serde", ] [[package]] @@ -92,7 +93,7 @@ dependencies = [ "base58ck", "bech32", "bitcoin-internals 0.3.0", - "bitcoin-io 0.1.2", + "bitcoin-io 0.1.3", "bitcoin-units", "bitcoin_hashes 0.14.0", "hex-conservative 0.2.1", @@ -118,9 +119,9 @@ checksum = "2b854212e29b96c8f0fe04cab11d57586c8f3257de0d146c76cb3b42b3eb9118" [[package]] name = "bitcoin-io" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56" +checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" [[package]] name = "bitcoin-io" @@ -147,7 +148,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" dependencies = [ - "bitcoin-io 0.1.2", + "bitcoin-io 0.1.3", "hex-conservative 0.2.1", "serde", ] @@ -193,18 +194,12 @@ dependencies = [ "bitcoin", "bitcoincore-rpc", "crossbeam", - "derived-deref", + "derive_deref", "rayon", "serde", "serde_json", ] -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.8.0" @@ -216,6 +211,7 @@ name = "bomputer" version = "0.1.0" dependencies = [ "bindex", + "biter", "color-eyre", "derive_deref", "exit", @@ -231,9 +227,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" -version = "1.2.7" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "shlex", ] @@ -320,9 +316,9 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] @@ -364,7 +360,7 @@ dependencies = [ "hashbrown", "lock_api", "once_cell", - "parking_lot_core 0.9.10", + "parking_lot_core", ] [[package]] @@ -378,17 +374,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derived-deref" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "double-ended-peekable" version = "0.1.0" @@ -468,16 +453,6 @@ dependencies = [ "xxhash-rust", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "getrandom" version = "0.2.15" @@ -537,15 +512,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "itoa" version = "1.0.14" @@ -554,9 +520,9 @@ checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jiff" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85348106ab244d90fe2d70faad939b71c5dad1258e5da9116e176064fc6c078" +checksum = "c607c728e28764fecde611a2764a3a5db19ae21dcec46f292244f5cc5c085a81" dependencies = [ "jiff-tzdb-platform", "log", @@ -688,9 +654,9 @@ dependencies = [ [[package]] name = "minreq" -version = "2.12.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0" +checksum = "36a8e50e917e18a37d500d27d40b7bc7d127e71c0c94fb2d83f43b4afd308390" dependencies = [ "log", "serde", @@ -703,7 +669,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.8.0", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -730,31 +696,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", -] - [[package]] name = "parking_lot_core" version = "0.9.10" @@ -763,7 +704,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.8", + "redox_syscall", "smallvec", "windows-targets", ] @@ -900,22 +841,13 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.8.0", + "bitflags", ] [[package]] @@ -936,7 +868,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -945,30 +877,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "sanakirja" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11" -dependencies = [ - "fs2", - "log", - "memmap2", - "parking_lot", - "sanakirja-core", - "serde", - "thiserror", -] - -[[package]] -name = "sanakirja-core" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "scopeguard" @@ -1025,9 +936,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", "memchr", @@ -1056,13 +967,6 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -[[package]] -name = "snkrj" -version = "0.1.1" -dependencies = [ - "sanakirja", -] - [[package]] name = "std-semaphore" version = "0.1.0" @@ -1074,6 +978,7 @@ name = "storable_vec" version = "0.1.2" dependencies = [ "memmap2", + "unsafe_slice_serde", ] [[package]] @@ -1134,26 +1039,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "thread_local" version = "1.1.8" @@ -1186,9 +1071,9 @@ dependencies = [ [[package]] name = "tracing-error" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" dependencies = [ "tracing", "tracing-subscriber", @@ -1196,9 +1081,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "sharded-slab", "thread_local", @@ -1207,15 +1092,19 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "unsafe_slice_serde" +version = "0.1.0" [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-log" @@ -1245,28 +1134,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index 8a7b88571..daf8e4a05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,52 +1,25 @@ -# [package] -# name = "kibo_money" -# version = "0.6.0" -# edition = "2021" - [workspace] members = [ - "bindex", - "biter", - "bomputer", + "computer", "exit", - # "server", - "snkrj", - "struct_iterable", + "indexer", + "iterator", "storable_vec", + "struct_iterable", + "unsafe_slice_serde", ] resolver = "2" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -# [dependencies] -# allocative = "0.3.4" -# axum = "0.7.9" -# bincode = { git = "https://github.com/bincode-org/bincode.git", features = [ -# "serde", -# ] } -# bitcoin_hashes = { version = "0.15.0" } -# biter = { path = "./src/crates/biter" } -# chrono = { version = "0.4.39", features = ["serde"] } -# clap = { version = "4.5.26", features = ["derive"] } -# color-eyre = "0.6.3" -# ctrlc = { version = "3.4.5", features = ["termination"] } -# derive_deref = "1.1.1" -# env_logger = "0.11.6" -# inferno = "0.12.1" -# itertools = "0.13.0" -# log = { version = "0.4.25", features = ["std", "serde"] } -# ordered-float = "4.6.0" -# rayon = "1.10.0" -# regex = "1.11.1" -# reqwest = { version = "0.12.12", features = ["blocking", "json"] } -# rlimit = "0.10.2" -# snkrj = { path = "./src/crates/snkrj" } -# serde = { version = "1.0.217", features = ["derive"] } -# serde_json = "1.0.135" -# struct_iterable = { path = "./src/crates/iterable" } -# swc = "9.0.2" -# swc_common = "5.0.0" -# tokio = { version = "1.43.0", features = ["full"] } -# toml = "0.8.19" -# tower-http = { version = "0.6.2", features = ["compression-full"] } -# zstd = "0.13.2" +[workspace.dependencies] +bindex = { path = "indexer" } +bitcoin = { version = "0.32.5", features = ["serde"] } +biter = { path = "iterator" } +bomputer = { path = "computer" } +color-eyre = "0.6.3" +derive_deref = "1.1.1" +exit = { path = "exit" } +jiff = "0.1.28" +rayon = "1.10.0" +storable_vec = { path = "storable_vec" } +struct_iterable = { path = "struct_iterable" } +unsafe_slice_serde = { path = "unsafe_slice_serde" } diff --git a/LICENSE.md b/LICENSE.md index 819632a70..4827b5007 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 kibō.money +Copyright (c) 2025 kibō.money Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/bindex/Cargo.toml b/bindex/Cargo.toml deleted file mode 100644 index f07cd8537..000000000 --- a/bindex/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "bindex" -version = "0.1.0" -edition = "2021" - -[dependencies] -bitcoin_hashes = "0.16.0" -biter = { path = "../biter" } -color-eyre = "0.6.3" -derive_deref = "1.1.1" -exit = { path = "../exit" } -fjall = "2.5.0" -jiff = "0.1.27" -rapidhash = "1.3.0" -rayon = "1.10.0" -snkrj = { path = "../snkrj" } -storable_vec = { path = "../storable_vec" } diff --git a/bomputer/Cargo.toml b/bomputer/Cargo.toml deleted file mode 100644 index cd8dcb5ea..000000000 --- a/bomputer/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "bomputer" -version = "0.1.0" -edition = "2021" - -[dependencies] -bindex = { path = "../bindex" } -color-eyre = "0.6.3" -derive_deref = "1.1.1" -exit = { path = "../exit" } -jiff = "0.1.27" -rayon = "1.10.0" diff --git a/computer/Cargo.toml b/computer/Cargo.toml new file mode 100644 index 000000000..a8f285300 --- /dev/null +++ b/computer/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "bomputer" +version = "0.1.0" +edition = "2021" + +[dependencies] +biter = { workspace = true } +bindex = { workspace = true } +color-eyre = { workspace = true } +derive_deref = { workspace = true } +exit = { workspace = true } +jiff = { workspace = true } +rayon = { workspace = true } diff --git a/bomputer/src/lib.rs b/computer/src/lib.rs similarity index 94% rename from bomputer/src/lib.rs rename to computer/src/lib.rs index a1e960b65..ad59cc0b5 100644 --- a/bomputer/src/lib.rs +++ b/computer/src/lib.rs @@ -1,6 +1,7 @@ use std::path::Path; -use bindex::{biter::rpc, Indexer}; +use bindex::Indexer; +use biter::rpc; use exit::Exit; mod structs; diff --git a/bomputer/src/main.rs b/computer/src/main.rs similarity index 100% rename from bomputer/src/main.rs rename to computer/src/main.rs diff --git a/bomputer/src/structs/date.rs b/computer/src/structs/date.rs similarity index 100% rename from bomputer/src/structs/date.rs rename to computer/src/structs/date.rs diff --git a/bomputer/src/structs/feerate.rs b/computer/src/structs/feerate.rs similarity index 100% rename from bomputer/src/structs/feerate.rs rename to computer/src/structs/feerate.rs diff --git a/bomputer/src/structs/mod.rs b/computer/src/structs/mod.rs similarity index 100% rename from bomputer/src/structs/mod.rs rename to computer/src/structs/mod.rs diff --git a/bindex/.gitignore b/indexer/.gitignore similarity index 100% rename from bindex/.gitignore rename to indexer/.gitignore diff --git a/bindex/Cargo.lock b/indexer/Cargo.lock similarity index 100% rename from bindex/Cargo.lock rename to indexer/Cargo.lock diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml new file mode 100644 index 000000000..6773c2618 --- /dev/null +++ b/indexer/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bindex" +version = "0.1.0" +edition = "2021" + +[dependencies] +bitcoin = { workspace = true } +bitcoin_hashes = "0.16.0" +biter = { workspace = true } +color-eyre = { workspace = true } +derive_deref = { workspace = true } +exit = { workspace = true } +fjall = "2.5.0" +jiff = { workspace = true } +rapidhash = "1.3.0" +rayon = { workspace = true } +# snkrj = { workspace = true } +storable_vec = { workspace = true } +unsafe_slice_serde = { workspace = true } diff --git a/bindex/rustfmt.toml b/indexer/rustfmt.toml similarity index 100% rename from bindex/rustfmt.toml rename to indexer/rustfmt.toml diff --git a/bindex/src/lib.rs b/indexer/src/lib.rs similarity index 93% rename from bindex/src/lib.rs rename to indexer/src/lib.rs index 5a4bbb159..b286b2866 100644 --- a/bindex/src/lib.rs +++ b/indexer/src/lib.rs @@ -9,7 +9,7 @@ use std::{ pub use biter::*; -use biter::bitcoin::{Transaction, TxIn, TxOut, Txid}; +use bitcoin::{Transaction, TxIn, TxOut, Txid}; use color_eyre::eyre::{eyre, ContextCompat}; use exit::Exit; use rayon::prelude::*; @@ -21,8 +21,8 @@ pub use biter; use storage::{Fjalls, StorableVecs}; pub use structs::{ - Addressbytes, AddressbytesPrefix, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp, - TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout, + AddressHash, Addressbytes, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp, TxidPrefix, + Txindex, Txinindex, Txoutindex, Vin, Vout, }; const UNSAFE_BLOCKS: u32 = 100; @@ -89,10 +89,7 @@ impl Indexer { Ok(()) }; - // // let mut stores_opt = Some(stores); - // let mut rtx_opt = Some(rtx); - - biter::new(bitcoin_dir, Some(height.into()), Some(400_000), rpc) + biter::new(bitcoin_dir, Some(height.into()), None, rpc) .iter() .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { println!("Processing block {_height}..."); @@ -100,9 +97,6 @@ impl Indexer { height = Height::from(_height); let timestamp = Timestamp::try_from(block.header.time)?; - // let mut stores = stores_opt.take().context("option should have store")?; - // let rtx = rtx_opt.take().context("option should have rtx")?; - if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? { if &blockhash != saved_blockhash.as_ref() { todo!("Rollback not implemented"); @@ -115,7 +109,7 @@ impl Indexer { if parts .blockhash_prefix_to_height .get(&blockhash_prefix)? - .is_some_and(|prev_height| prev_height != height) + .is_some_and(|prev_height| *prev_height != height) { dbg!(blockhash); return Err(eyre!("Collision, expect prefix to need be set yet")); @@ -198,15 +192,15 @@ impl Indexer { let txid_prefix = TxidPrefix::try_from(&txid)?; - let prev_txindex_slice_opt = + let prev_txindex_opt = if check_collisions && parts.txid_prefix_to_txindex.needs(height) { // Should only find collisions for two txids (duplicates), see below - parts.txid_prefix_to_txindex.get(&txid_prefix)? + parts.txid_prefix_to_txindex.get(&txid_prefix)?.map(|v| *v) } else { None }; - Ok((txid_prefix, (tx, txid, Txindex::from(index), prev_txindex_slice_opt))) + Ok((txid_prefix, (tx, txid, Txindex::from(index), prev_txindex_opt))) }) .try_fold(BTreeMap::new, |mut map, tuple| { let (key, value) = tuple?; @@ -243,6 +237,7 @@ impl Indexer { let prev_txindex = if let Some(txindex) = parts .txid_prefix_to_txindex .get(&TxidPrefix::try_from(&outpoint.txid)?)? + .map(|v| *v) .and_then(|txindex| { // Checking if not finding txindex from the future (txindex < txindex_global).then_some(txindex) @@ -318,9 +313,10 @@ impl Indexer { let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| { parts - .addressbytes_prefix_to_addressindex - .get(&AddressbytesPrefix::from((addressbytes, addresstype))) + .addresshash_to_addressindex + .get(&AddressHash::from((addressbytes, addresstype))) .unwrap() + .map(|v| *v) // Checking if not in the future .and_then(|addressindex_local| { (addressindex_local < addressindex_global).then_some(addressindex_local) @@ -350,7 +346,7 @@ impl Indexer { if (vecs.addressindex_to_addresstype.hasnt(addressindex) && addresstype != prev_addresstype) - || (parts.addressbytes_prefix_to_addressindex.needs(height) + || (parts.addresshash_to_addressindex.needs(height) && prev_addressbytes != addressbytes) { let txid = tx.compute_txid(); @@ -367,8 +363,8 @@ impl Indexer { addressindex, addresstypeindex, txout, - AddressbytesPrefix::from((addressbytes, addresstype)), - AddressbytesPrefix::from((prev_addressbytes, prev_addresstype)) + AddressHash::from((addressbytes, addresstype)), + AddressHash::from((prev_addressbytes, prev_addresstype)) ); panic!() } @@ -434,7 +430,7 @@ impl Indexer { Txoutindex, > = BTreeMap::new(); - let mut already_added_addressbytes_prefix: BTreeMap = BTreeMap::new(); + let mut already_added_addresshash: BTreeMap = BTreeMap::new(); txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt .into_iter() @@ -454,15 +450,15 @@ impl Indexer { let mut addressindex = addressindex_global; - let mut addressbytes_prefix = None; + let mut addresshash = None; if let Some(addressindex_local) = addressindex_opt.or_else(|| { addressbytes_res.as_ref().ok().and_then(|addressbytes| { // Check if address was first seen before in this iterator // Example: https://mempool.space/address/046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0c - addressbytes_prefix.replace(AddressbytesPrefix::from((addressbytes, addresstype))); - already_added_addressbytes_prefix - .get(addressbytes_prefix.as_ref().unwrap()) + addresshash.replace(AddressHash::from((addressbytes, addresstype))); + already_added_addresshash + .get(addresshash.as_ref().unwrap()) .cloned() }) }) { @@ -495,13 +491,13 @@ impl Indexer { .push_if_needed(addressindex, height)?; if let Ok(addressbytes) = addressbytes_res { - let addressbytes_prefix = addressbytes_prefix.unwrap(); + let addresshash = addresshash.unwrap(); - already_added_addressbytes_prefix - .insert(addressbytes_prefix, addressindex); + already_added_addresshash + .insert(addresshash, addressindex); - parts.addressbytes_prefix_to_addressindex.insert_if_needed( - addressbytes_prefix, + parts.addresshash_to_addressindex.insert_if_needed( + addresshash, addressindex, height, ); @@ -520,7 +516,7 @@ impl Indexer { }, )?; - drop(already_added_addressbytes_prefix); + drop(already_added_addresshash); input_source_vec .into_iter() diff --git a/bindex/src/main.rs b/indexer/src/main.rs similarity index 100% rename from bindex/src/main.rs rename to indexer/src/main.rs diff --git a/bindex/src/storage/canopies/database.rs b/indexer/src/storage/canopies/database.rs similarity index 100% rename from bindex/src/storage/canopies/database.rs rename to indexer/src/storage/canopies/database.rs diff --git a/bindex/src/storage/canopies/environment.rs b/indexer/src/storage/canopies/environment.rs similarity index 100% rename from bindex/src/storage/canopies/environment.rs rename to indexer/src/storage/canopies/environment.rs diff --git a/bindex/src/storage/canopies/mod.rs b/indexer/src/storage/canopies/mod.rs similarity index 100% rename from bindex/src/storage/canopies/mod.rs rename to indexer/src/storage/canopies/mod.rs diff --git a/bindex/src/storage/canopies/transaction.rs b/indexer/src/storage/canopies/transaction.rs similarity index 100% rename from bindex/src/storage/canopies/transaction.rs rename to indexer/src/storage/canopies/transaction.rs diff --git a/bindex/src/storage/canopies/tree.rs b/indexer/src/storage/canopies/tree.rs similarity index 100% rename from bindex/src/storage/canopies/tree.rs rename to indexer/src/storage/canopies/tree.rs diff --git a/bindex/src/storage/fjalls/base.rs b/indexer/src/storage/fjalls/base.rs similarity index 83% rename from bindex/src/storage/fjalls/base.rs rename to indexer/src/storage/fjalls/base.rs index ca69556f1..42a575e4a 100644 --- a/bindex/src/storage/fjalls/base.rs +++ b/indexer/src/storage/fjalls/base.rs @@ -1,10 +1,11 @@ -use std::{collections::BTreeMap, mem, path::Path}; +use std::{collections::BTreeMap, error, mem, path::Path}; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace, TransactionalPartitionHandle, }; -use storable_vec::UnsafeSizedSerDe; +use storable_vec::Value; +use unsafe_slice_serde::UnsafeSliceSerde; use crate::structs::{Height, Version}; @@ -18,10 +19,11 @@ pub struct Partition { puts: BTreeMap, } -impl Partition +impl Partition where - Key: Into + Ord, - Value: Into + TryFrom + Clone, + K: Into + Ord, + V: Into + TryFrom, + >::Error: error::Error + Send + Sync + 'static, { pub fn import(path: &Path, version: Version) -> color_eyre::Result { let meta = Meta::checked_open(path, version)?; @@ -49,20 +51,17 @@ where }) } - pub fn get(&self, key: &Key) -> color_eyre::Result> - where - >::Error: std::error::Error + Send + Sync + 'static, - { + pub fn get(&self, key: &K) -> color_eyre::Result>> { if let Some(v) = self.puts.get(key) { - Ok(Some(v.clone())) + Ok(Some(Value::Ref(v))) } else if let Some(slice) = self.rtx.get(&self.part, key.unsafe_as_slice())? { - Ok(Some(Value::try_from(slice)?)) + Ok(Some(Value::Owned(V::try_from(slice)?))) } else { Ok(None) } } - pub fn insert_if_needed(&mut self, key: Key, value: Value, height: Height) { + pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) { if self.needs(height) { self.puts.insert(key, value); } diff --git a/bindex/src/storage/fjalls/meta.rs b/indexer/src/storage/fjalls/meta.rs similarity index 98% rename from bindex/src/storage/fjalls/meta.rs rename to indexer/src/storage/fjalls/meta.rs index 2431a0fe4..c26a19033 100644 --- a/bindex/src/storage/fjalls/meta.rs +++ b/indexer/src/storage/fjalls/meta.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use storable_vec::UnsafeSizedSerDe; +use unsafe_slice_serde::UnsafeSliceSerde; use super::{Height, Version}; diff --git a/bindex/src/storage/fjalls/mod.rs b/indexer/src/storage/fjalls/mod.rs similarity index 91% rename from bindex/src/storage/fjalls/mod.rs rename to indexer/src/storage/fjalls/mod.rs index 3d17bb007..2a952a7e4 100644 --- a/bindex/src/storage/fjalls/mod.rs +++ b/indexer/src/storage/fjalls/mod.rs @@ -1,6 +1,6 @@ use std::{path::Path, thread}; -use crate::{structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex}; +use crate::{structs::Version, AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex}; mod base; mod meta; @@ -9,7 +9,7 @@ use base::*; use meta::*; pub struct Fjalls { - pub addressbytes_prefix_to_addressindex: Partition, + pub addresshash_to_addressindex: Partition, pub blockhash_prefix_to_height: Partition, pub txid_prefix_to_txindex: Partition, } @@ -17,8 +17,8 @@ pub struct Fjalls { impl Fjalls { pub fn import(path: &Path) -> color_eyre::Result { Ok(Self { - addressbytes_prefix_to_addressindex: Partition::import( - &path.join("addressbytes_prefix_to_addressindex"), + addresshash_to_addressindex: Partition::import( + &path.join("addresshash_to_addressindex"), Version::from(1), )?, blockhash_prefix_to_height: Partition::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?, @@ -147,7 +147,7 @@ impl Fjalls { pub fn min_height(&self) -> Option { [ - self.addressbytes_prefix_to_addressindex.height(), + self.addresshash_to_addressindex.height(), self.blockhash_prefix_to_height.height(), self.txid_prefix_to_txindex.height(), ] @@ -159,13 +159,13 @@ impl Fjalls { pub fn commit(&mut self, height: Height) -> fjall::Result<()> { thread::scope(|scope| { - let addressbytes_prefix_to_addressindex_commit_handle = - scope.spawn(|| self.addressbytes_prefix_to_addressindex.commit(height)); + let addresshash_to_addressindex_commit_handle = + scope.spawn(|| self.addresshash_to_addressindex.commit(height)); let blockhash_prefix_to_height_commit_handle = scope.spawn(|| self.blockhash_prefix_to_height.commit(height)); let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height)); - addressbytes_prefix_to_addressindex_commit_handle.join().unwrap()?; + addresshash_to_addressindex_commit_handle.join().unwrap()?; blockhash_prefix_to_height_commit_handle.join().unwrap()?; txid_prefix_to_txindex_commit_handle.join().unwrap()?; diff --git a/bindex/src/storage/mod.rs b/indexer/src/storage/mod.rs similarity index 100% rename from bindex/src/storage/mod.rs rename to indexer/src/storage/mod.rs diff --git a/bindex/src/storage/sanakirjas/meta.rs b/indexer/src/storage/sanakirjas/meta.rs similarity index 100% rename from bindex/src/storage/sanakirjas/meta.rs rename to indexer/src/storage/sanakirjas/meta.rs diff --git a/bindex/src/storage/sanakirjas/mod.rs b/indexer/src/storage/sanakirjas/mod.rs similarity index 100% rename from bindex/src/storage/sanakirjas/mod.rs rename to indexer/src/storage/sanakirjas/mod.rs diff --git a/bindex/src/storage/sanakirjas/multi.rs b/indexer/src/storage/sanakirjas/multi.rs similarity index 100% rename from bindex/src/storage/sanakirjas/multi.rs rename to indexer/src/storage/sanakirjas/multi.rs diff --git a/bindex/src/storage/sanakirjas/unique.rs b/indexer/src/storage/sanakirjas/unique.rs similarity index 100% rename from bindex/src/storage/sanakirjas/unique.rs rename to indexer/src/storage/sanakirjas/unique.rs diff --git a/bindex/src/storage/storable_vecs/base.rs b/indexer/src/storage/storable_vecs/base.rs similarity index 100% rename from bindex/src/storage/storable_vecs/base.rs rename to indexer/src/storage/storable_vecs/base.rs diff --git a/bindex/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs similarity index 100% rename from bindex/src/storage/storable_vecs/mod.rs rename to indexer/src/storage/storable_vecs/mod.rs diff --git a/bindex/src/structs/addressbytes.rs b/indexer/src/structs/addressbytes.rs similarity index 100% rename from bindex/src/structs/addressbytes.rs rename to indexer/src/structs/addressbytes.rs diff --git a/bindex/src/structs/addressindex.rs b/indexer/src/structs/addressindex.rs similarity index 87% rename from bindex/src/structs/addressindex.rs rename to indexer/src/structs/addressindex.rs index eab1f7c97..2780cf4d5 100644 --- a/bindex/src/structs/addressindex.rs +++ b/indexer/src/structs/addressindex.rs @@ -1,10 +1,10 @@ use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; -use storable_vec::UnsafeSizedSerDe; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Addressindex(u32); -direct_repr!(Addressindex); +// direct_repr!(Addressindex); impl Addressindex { pub const BYTES: usize = size_of::(); @@ -51,7 +51,7 @@ impl From for usize { } impl TryFrom for Addressindex { - type Error = storable_vec::Error; + type Error = unsafe_slice_serde::Error; fn try_from(value: fjall::Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } diff --git a/bindex/src/structs/addresstype.rs b/indexer/src/structs/addresstype.rs similarity index 100% rename from bindex/src/structs/addresstype.rs rename to indexer/src/structs/addresstype.rs diff --git a/bindex/src/structs/addresstypeindex.rs b/indexer/src/structs/addresstypeindex.rs similarity index 92% rename from bindex/src/structs/addresstypeindex.rs rename to indexer/src/structs/addresstypeindex.rs index 5b126b31d..7fe615080 100644 --- a/bindex/src/structs/addresstypeindex.rs +++ b/indexer/src/structs/addresstypeindex.rs @@ -1,9 +1,9 @@ use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Addresstypeindex(u32); -direct_repr!(Addresstypeindex); +// direct_repr!(Addresstypeindex); impl Addresstypeindex { pub fn decremented(self) -> Self { diff --git a/bindex/src/structs/amount.rs b/indexer/src/structs/amount.rs similarity index 96% rename from bindex/src/structs/amount.rs rename to indexer/src/structs/amount.rs index 9f4320b8a..f6ac571bc 100644 --- a/bindex/src/structs/amount.rs +++ b/indexer/src/structs/amount.rs @@ -5,13 +5,13 @@ use std::{ use biter::bitcoin; use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Height; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Amount(bitcoin::Amount); -direct_repr!(Amount); +// direct_repr!(Amount); impl Amount { pub const ZERO: Self = Self(bitcoin::Amount::ZERO); diff --git a/bindex/src/structs/prefix.rs b/indexer/src/structs/compressed.rs similarity index 80% rename from bindex/src/structs/prefix.rs rename to indexer/src/structs/compressed.rs index cc2e2e15e..4aab25aa1 100644 --- a/bindex/src/structs/prefix.rs +++ b/indexer/src/structs/compressed.rs @@ -2,15 +2,15 @@ use std::hash::Hasher; use biter::bitcoin::{BlockHash, Txid}; use derive_deref::Deref; -use snkrj::{direct_repr, Storable, UnsizedStorable}; -use storable_vec::UnsafeSizedSerDe; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use unsafe_slice_serde::UnsafeSliceSerde; use super::{Addressbytes, Addresstype, SliceExtended}; #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct AddressbytesPrefix([u8; 8]); -direct_repr!(AddressbytesPrefix); -impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix { +pub struct AddressHash([u8; 8]); +// direct_repr!(AddressHash); +impl From<(&Addressbytes, Addresstype)> for AddressHash { fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self { let mut hasher = rapidhash::RapidHasher::default(); hasher.write(addressbytes.as_slice()); @@ -19,31 +19,31 @@ impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix { Self(slice) } } -impl From<[u8; 8]> for AddressbytesPrefix { +impl From<[u8; 8]> for AddressHash { fn from(value: [u8; 8]) -> Self { Self(value) } } -impl TryFrom for AddressbytesPrefix { +impl TryFrom for AddressHash { type Error = color_eyre::Report; fn try_from(value: fjall::Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From<&AddressbytesPrefix> for fjall::Slice { - fn from(value: &AddressbytesPrefix) -> Self { +impl From<&AddressHash> for fjall::Slice { + fn from(value: &AddressHash) -> Self { Self::new(value.unsafe_as_slice()) } } -impl From for fjall::Slice { - fn from(value: AddressbytesPrefix) -> Self { +impl From for fjall::Slice { + fn from(value: AddressHash) -> Self { Self::from(&value) } } #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockHashPrefix([u8; 8]); -direct_repr!(BlockHashPrefix); +// direct_repr!(BlockHashPrefix); impl TryFrom<&BlockHash> for BlockHashPrefix { type Error = color_eyre::Report; fn try_from(value: &BlockHash) -> Result { @@ -69,7 +69,7 @@ impl From for fjall::Slice { #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct TxidPrefix([u8; 8]); -direct_repr!(TxidPrefix); +// direct_repr!(TxidPrefix); impl TryFrom<&Txid> for TxidPrefix { type Error = color_eyre::Report; fn try_from(value: &Txid) -> Result { diff --git a/bindex/src/structs/height.rs b/indexer/src/structs/height.rs similarity index 94% rename from bindex/src/structs/height.rs rename to indexer/src/structs/height.rs index 717a1cde5..1784fc206 100644 --- a/bindex/src/structs/height.rs +++ b/indexer/src/structs/height.rs @@ -6,12 +6,12 @@ use std::{ use biter::rpc::{self, RpcApi}; use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; -use storable_vec::UnsafeSizedSerDe; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct Height(u32); -direct_repr!(Height); +// direct_repr!(Height); impl Height { pub fn write(&self, path: &Path) -> Result<(), io::Error> { @@ -121,7 +121,7 @@ impl TryFrom<&rpc::Client> for Height { } impl TryFrom for Height { - type Error = storable_vec::Error; + type Error = unsafe_slice_serde::Error; fn try_from(value: fjall::Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } diff --git a/bindex/src/structs/mod.rs b/indexer/src/structs/mod.rs similarity index 92% rename from bindex/src/structs/mod.rs rename to indexer/src/structs/mod.rs index 7f5daac0a..bb92346ed 100644 --- a/bindex/src/structs/mod.rs +++ b/indexer/src/structs/mod.rs @@ -3,8 +3,8 @@ mod addressindex; mod addresstype; mod addresstypeindex; mod amount; +mod compressed; mod height; -mod prefix; mod slice; mod timestamp; mod txindex; @@ -19,8 +19,8 @@ pub use addressindex::*; pub use addresstype::*; pub use addresstypeindex::*; pub use amount::*; +pub use compressed::*; pub use height::*; -pub use prefix::*; pub use slice::*; pub use timestamp::*; pub use txindex::*; diff --git a/bindex/src/structs/slice.rs b/indexer/src/structs/slice.rs similarity index 100% rename from bindex/src/structs/slice.rs rename to indexer/src/structs/slice.rs diff --git a/bindex/src/structs/timestamp.rs b/indexer/src/structs/timestamp.rs similarity index 100% rename from bindex/src/structs/timestamp.rs rename to indexer/src/structs/timestamp.rs diff --git a/bindex/src/structs/txindex.rs b/indexer/src/structs/txindex.rs similarity index 89% rename from bindex/src/structs/txindex.rs rename to indexer/src/structs/txindex.rs index 4b2b781e6..974565a02 100644 --- a/bindex/src/structs/txindex.rs +++ b/indexer/src/structs/txindex.rs @@ -1,12 +1,12 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; -use storable_vec::UnsafeSizedSerDe; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txindex(u32); -direct_repr!(Txindex); +// direct_repr!(Txindex); impl Txindex { pub fn incremented(self) -> Self { @@ -60,7 +60,7 @@ impl From for usize { } impl TryFrom for Txindex { - type Error = storable_vec::Error; + type Error = unsafe_slice_serde::Error; fn try_from(value: fjall::Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } diff --git a/bindex/src/structs/txinindex.rs b/indexer/src/structs/txinindex.rs similarity index 93% rename from bindex/src/structs/txinindex.rs rename to indexer/src/structs/txinindex.rs index 2c6bb7ad1..03807b5c7 100644 --- a/bindex/src/structs/txinindex.rs +++ b/indexer/src/structs/txinindex.rs @@ -1,13 +1,13 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Vout; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txinindex(u64); -direct_repr!(Txinindex); +// direct_repr!(Txinindex); impl Txinindex { pub fn incremented(self) -> Self { diff --git a/bindex/src/structs/txoutindex.rs b/indexer/src/structs/txoutindex.rs similarity index 93% rename from bindex/src/structs/txoutindex.rs rename to indexer/src/structs/txoutindex.rs index 91740bcb1..5ab2e7ccf 100644 --- a/bindex/src/structs/txoutindex.rs +++ b/indexer/src/structs/txoutindex.rs @@ -1,13 +1,13 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; +// use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Vout; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txoutindex(u64); -direct_repr!(Txoutindex); +// direct_repr!(Txoutindex); impl Txoutindex { pub const COINBASE: Self = Self(u64::MAX); diff --git a/bindex/src/structs/version.rs b/indexer/src/structs/version.rs similarity index 95% rename from bindex/src/structs/version.rs rename to indexer/src/structs/version.rs index 4faa4ec0d..3f01cb414 100644 --- a/bindex/src/structs/version.rs +++ b/indexer/src/structs/version.rs @@ -1,6 +1,6 @@ use std::{fs, io, path::Path}; -use storable_vec::UnsafeSizedSerDe; +use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Version(u32); diff --git a/bindex/src/structs/vin.rs b/indexer/src/structs/vin.rs similarity index 100% rename from bindex/src/structs/vin.rs rename to indexer/src/structs/vin.rs diff --git a/bindex/src/structs/vout.rs b/indexer/src/structs/vout.rs similarity index 100% rename from bindex/src/structs/vout.rs rename to indexer/src/structs/vout.rs diff --git a/biter/CHANGELOG.md b/iterator/CHANGELOG.md similarity index 100% rename from biter/CHANGELOG.md rename to iterator/CHANGELOG.md diff --git a/biter/Cargo.lock b/iterator/Cargo.lock similarity index 100% rename from biter/Cargo.lock rename to iterator/Cargo.lock diff --git a/biter/Cargo.toml b/iterator/Cargo.toml similarity index 81% rename from biter/Cargo.toml rename to iterator/Cargo.toml index 6bb8bc3f6..853d673db 100644 --- a/biter/Cargo.toml +++ b/iterator/Cargo.toml @@ -9,11 +9,11 @@ categories = ["cryptography::cryptocurrencies", "encoding"] edition = "2021" [dependencies] -bitcoin = { version = "0.32.5", features = ["serde"] } -rayon = "1.10.0" +bitcoin = { workspace = true } +rayon = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } serde = { version = "1.0.217", features = ["derive"] } -serde_json = "1.0.135" -derived-deref = "2.1.0" +serde_json = "1.0.137" +derive_deref = { workspace = true } bitcoincore-rpc = "0.19.0" # tokio = { version = "1.39.2", features = ["rt-multi-thread"] } diff --git a/biter/LICENSE.md b/iterator/LICENSE.md similarity index 100% rename from biter/LICENSE.md rename to iterator/LICENSE.md diff --git a/biter/README.md b/iterator/README.md similarity index 100% rename from biter/README.md rename to iterator/README.md diff --git a/biter/src/blk_index_to_blk_path.rs b/iterator/src/blk_index_to_blk_path.rs similarity index 97% rename from biter/src/blk_index_to_blk_path.rs rename to iterator/src/blk_index_to_blk_path.rs index 17cf2da39..e365018c8 100644 --- a/biter/src/blk_index_to_blk_path.rs +++ b/iterator/src/blk_index_to_blk_path.rs @@ -4,7 +4,7 @@ use std::{ path::{Path, PathBuf}, }; -use derived_deref::{Deref, DerefMut}; +use derive_deref::{Deref, DerefMut}; const BLK: &str = "blk"; const DAT: &str = ".dat"; diff --git a/biter/src/blk_index_to_blk_recap.rs b/iterator/src/blk_index_to_blk_recap.rs similarity index 80% rename from biter/src/blk_index_to_blk_recap.rs rename to iterator/src/blk_index_to_blk_recap.rs index b819448a6..f080b6d54 100644 --- a/biter/src/blk_index_to_blk_recap.rs +++ b/iterator/src/blk_index_to_blk_recap.rs @@ -6,16 +6,13 @@ use std::{ path::{Path, PathBuf}, }; -use derived_deref::{Deref, DerefMut}; - use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock}; const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30; -#[derive(Deref, DerefMut, Debug)] +#[derive(Debug)] pub struct BlkIndexToBlkRecap { path: PathBuf, - #[target] tree: BTreeMap, last_safe_height: Option, } @@ -47,37 +44,38 @@ impl BlkIndexToBlkRecap { } pub fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) { - let mut unprocessed_keys = self.keys().copied().collect::>(); + let mut unprocessed_keys = self.tree.keys().copied().collect::>(); blocks_dir.iter().for_each(|(blk_index, blk_path)| { unprocessed_keys.remove(blk_index); - if let Some(blk_recap) = self.get(blk_index) { + if let Some(blk_recap) = self.tree.get(blk_index) { if blk_recap.has_different_modified_time(blk_path) { - self.remove(blk_index); + self.tree.remove(blk_index); } } }); unprocessed_keys.into_iter().for_each(|blk_index| { - self.remove(&blk_index); + self.tree.remove(&blk_index); }); - self.last_safe_height = self.iter().map(|(_, recap)| recap.height()).max(); + self.last_safe_height = self.tree.values().map(|recap| recap.height()).max(); } pub fn get_start_recap(&self, start: Option) -> Option<(usize, BlkRecap)> { if let Some(start) = start { - let (last_key, last_value) = self.last_key_value()?; + let (last_key, last_value) = self.tree.last_key_value()?; if last_value.height() < start { return Some((*last_key, *last_value)); } else if let Some((blk_index, _)) = self + .tree .iter() .find(|(_, blk_recap)| blk_recap.is_younger_than(start)) { if *blk_index != 0 { let blk_index = *blk_index - 1; - return Some((blk_index, *self.get(&blk_index).unwrap())); + return Some((blk_index, *self.tree.get(&blk_index).unwrap())); } } } @@ -88,13 +86,14 @@ impl BlkIndexToBlkRecap { pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) { let blk_index = blk_metadata_and_block.blk_metadata.index; - if let Some(last_entry) = self.last_entry() { + if let Some(last_entry) = self.tree.last_entry() { match last_entry.key().cmp(&blk_index) { Ordering::Greater => { last_entry.remove_entry(); } Ordering::Less => { - self.insert(blk_index, BlkRecap::from(height, blk_metadata_and_block)); + self.tree + .insert(blk_index, BlkRecap::from(height, blk_metadata_and_block)); } Ordering::Equal => {} }; @@ -104,7 +103,8 @@ impl BlkIndexToBlkRecap { unreachable!(); } - self.insert(blk_index, BlkRecap::first(blk_metadata_and_block)); + self.tree + .insert(blk_index, BlkRecap::first(blk_metadata_and_block)); } if self diff --git a/biter/src/blk_metadata.rs b/iterator/src/blk_metadata.rs similarity index 100% rename from biter/src/blk_metadata.rs rename to iterator/src/blk_metadata.rs diff --git a/biter/src/blk_metadata_and_block.rs b/iterator/src/blk_metadata_and_block.rs similarity index 100% rename from biter/src/blk_metadata_and_block.rs rename to iterator/src/blk_metadata_and_block.rs diff --git a/biter/src/blk_recap.rs b/iterator/src/blk_recap.rs similarity index 100% rename from biter/src/blk_recap.rs rename to iterator/src/blk_recap.rs diff --git a/biter/src/lib.rs b/iterator/src/lib.rs similarity index 100% rename from biter/src/lib.rs rename to iterator/src/lib.rs diff --git a/biter/src/main.rs b/iterator/src/main.rs similarity index 100% rename from biter/src/main.rs rename to iterator/src/main.rs diff --git a/biter/src/utils.rs b/iterator/src/utils.rs similarity index 100% rename from biter/src/utils.rs rename to iterator/src/utils.rs diff --git a/storable_vec/Cargo.toml b/storable_vec/Cargo.toml index 2cb9ff888..61cb279ee 100644 --- a/storable_vec/Cargo.toml +++ b/storable_vec/Cargo.toml @@ -9,3 +9,4 @@ edition = "2021" [dependencies] memmap2 = "0.9.5" +unsafe_slice_serde = { workspace = true } diff --git a/storable_vec/src/lib.rs b/storable_vec/src/lib.rs index ad5b5de24..9924c827a 100644 --- a/storable_vec/src/lib.rs +++ b/storable_vec/src/lib.rs @@ -7,10 +7,14 @@ use std::{ mem, ops::{Deref, Range}, path::{Path, PathBuf}, - sync::OnceLock, + sync::{ + // atomic::{AtomicUsize, Ordering as AtomicOrdering}, + OnceLock, + }, }; use memmap2::{Mmap, MmapOptions}; +use unsafe_slice_serde::UnsafeSliceSerde; /// /// A very small, fast, efficient and simple storable Vec @@ -33,6 +37,8 @@ pub struct StorableVec { // updated: BTreeMap, // inserted: BTreeMap, // removed: BTreeSet, + // min: AtomicUsize, + // opened_mmaps: AtomicUsize, phantom: PhantomData, } @@ -66,6 +72,8 @@ where // inserted: BTreeMap::new(), // removed: BTreeSet::new(), phantom: PhantomData, + // min: AtomicUsize::new(usize::MAX), + // opened_mmaps: AtomicUsize::new(0), }; this.reset_cache(); @@ -159,6 +167,12 @@ where .checked_sub(self.cache.len()) .unwrap_or_default(); + // let min_open_page = self.min.load(AtomicOrdering::SeqCst); + + // if self.min.load(AtomicOrdering::SeqCst) { + // self.min.set(value) + // } + if page_index >= min_page_index { let mmap = &**self .cache @@ -178,16 +192,19 @@ where let slice = &mmap[range]; - Ok(Some(Value::Ref(T::unsafe_try_from_slice(slice)?))) + Ok(Some(Value::Ref( + T::unsafe_try_from_slice(slice).map_err(Error::UnsafeSliceSerde)?, + ))) } else { - let mut file = Self::open_file(&self.pathbuf).unwrap(); + let mut file = Self::open_file(&self.pathbuf).map_err(Error::IO)?; - file.seek(SeekFrom::Start(byte_index as u64)).unwrap(); + file.seek(SeekFrom::Start(byte_index as u64)) + .map_err(Error::IO)?; let mut buf = vec![0; Self::SIZE_OF_T]; - file.read_exact(&mut buf).unwrap(); + file.read_exact(&mut buf).map_err(Error::IO)?; - let value = T::unsafe_try_from_slice(&buf[..])?; + let value = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?; Ok(Some(Value::Owned(value.to_owned()))) } @@ -387,37 +404,13 @@ where } } -pub trait UnsafeSizedSerDe -where - Self: Sized, -{ - const SIZE: usize = size_of::(); - - fn unsafe_try_from_slice(slice: &[u8]) -> Result<&Self> { - let (prefix, shorts, suffix) = unsafe { slice.align_to::() }; - - if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() { - // dbg!(&slice, &prefix, &shorts, &suffix); - return Err(Error::FailedToAlignToSelf); - } - - Ok(&shorts[0]) - } - - fn unsafe_as_slice(&self) -> &[u8] { - let data: *const Self = self; - let data: *const u8 = data as *const u8; - unsafe { std::slice::from_raw_parts(data, Self::SIZE) } - } -} -impl UnsafeSizedSerDe for T {} - pub type Result = std::result::Result; #[derive(Debug)] pub enum Error { MmapsVecIsTooSmall, - FailedToAlignToSelf, + IO(io::Error), + UnsafeSliceSerde(unsafe_slice_serde::Error), IndexTooHigh, ExpectFileToHaveIndex, ExpectVecToHaveIndex, @@ -427,7 +420,8 @@ impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"), - Error::FailedToAlignToSelf => write!(f, "Failed to align_to for T"), + Error::IO(error) => Debug::fmt(&error, f), + Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f), Error::IndexTooHigh => write!(f, "Index too high"), Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"), Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"), diff --git a/struct_iterable/struct_iterable_derive/Cargo.toml b/struct_iterable/struct_iterable_derive/Cargo.toml index 95d4be924..c47efa491 100644 --- a/struct_iterable/struct_iterable_derive/Cargo.toml +++ b/struct_iterable/struct_iterable_derive/Cargo.toml @@ -11,7 +11,7 @@ proc-macro = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -syn = "2.0.85" -quote = "1.0.37" -proc-macro2 = "1.0.89" +syn = "2.0.96" +quote = "1.0.38" +proc-macro2 = "1.0.93" struct_iterable_internal = { path = "../struct_iterable_internal" } diff --git a/unsafe_slice_serde/Cargo.toml b/unsafe_slice_serde/Cargo.toml new file mode 100644 index 000000000..60a15b1ff --- /dev/null +++ b/unsafe_slice_serde/Cargo.toml @@ -0,0 +1,3 @@ +[package] +name = "unsafe_slice_serde" +version = "0.1.0" diff --git a/unsafe_slice_serde/src/lib.rs b/unsafe_slice_serde/src/lib.rs new file mode 100644 index 000000000..02f7b5fb8 --- /dev/null +++ b/unsafe_slice_serde/src/lib.rs @@ -0,0 +1,42 @@ +use std::{fmt, slice}; + +pub trait UnsafeSliceSerde +where + Self: Sized, +{ + const SIZE: usize = size_of::(); + + fn unsafe_try_from_slice(slice: &[u8]) -> Result<&Self> { + let (prefix, shorts, suffix) = unsafe { slice.align_to::() }; + + if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() { + // dbg!(&slice, &prefix, &shorts, &suffix); + return Err(Error::FailedToAlignToSelf); + } + + Ok(&shorts[0]) + } + + fn unsafe_as_slice(&self) -> &[u8] { + let data: *const Self = self; + let data: *const u8 = data as *const u8; + unsafe { slice::from_raw_parts(data, Self::SIZE) } + } +} +impl UnsafeSliceSerde for T {} + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum Error { + FailedToAlignToSelf, +} +impl fmt::Display for Error { + // This trait requires `fmt` with this exact signature. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::FailedToAlignToSelf => write!(f, "Failed to align_to for T"), + } + } +} +impl std::error::Error for Error {}