diff --git a/Cargo.lock b/Cargo.lock index 96a8b7669..6222c5d05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,7 +275,7 @@ dependencies = [ "serde", "serde_bytes", "storable_vec", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -368,7 +368,7 @@ dependencies = [ "rayon", "serde", "serde_json", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -393,7 +393,7 @@ dependencies = [ "fjall", "hodor", "storable_vec", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -409,7 +409,7 @@ dependencies = [ "serde", "serde_json", "storable_vec", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -456,9 +456,9 @@ checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "byteview" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f101f5d5ac7c27fa7e664bbdc2697b2627f24c701e1309564840c67554dbcc" +checksum = "7a4516a8561bff0598c45512f90ee04ed62cee2cb36839e650a0a0704d5f741f" [[package]] name = "castaway" @@ -471,9 +471,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.14" +version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ "jobserver", "libc", @@ -758,9 +758,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "fjall" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c240d783ba894f1ce20a7337c4eabb7814920b4bfd1ae74da1b17af3c7dfc19a" +checksum = "9a7107ac1ad7c38b8a32c8bfe579bdb24bb1373b80c8e0b168f36b092647f917" dependencies = [ "byteorder", "byteview", @@ -1123,15 +1123,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "lsm-tree" -version = "2.6.3" +version = "2.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b7d9dc8488e02ab7be88626cdbd3a6ee4690d7ffa6a2ad460da9e52105e393" +checksum = "614a43954a8414dcca688f9d4c91f727860b55ffd9a029438b18f234ed33330f" dependencies = [ "byteorder", "crossbeam-skiplist", @@ -1309,9 +1309,9 @@ checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" [[package]] name = "oxc" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9f57beb511c2a1848baf900782d3975f89dbf75fc47ad2207147acf95523ab0" +checksum = "fd6be05b94a99c886e3c8f79c0330e746df6fe27f8440b498ea7a5eb2fee2f67" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1352,9 +1352,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6d450673da14c60c6946deb0c06d68377d2d6f2ee41ad5b462fa9894001560" +checksum = "bf460405a383f3f7ac134a0e06afdb3525c8fa8f119453167f179ea6b12faaf6" dependencies = [ "allocator-api2", "bumpalo", @@ -1365,9 +1365,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692c74b135e66d3fa13312d886f4a307131a5bd2fd76e5a4137825c6161b56a5" +checksum = "b273094a3e96e84d3d8ce82190fd7113f70bb2a8dcb652c78b46df79fdd156cc" dependencies = [ "bitflags", "cow-utils", @@ -1382,9 +1382,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4759d0523a72576036df640954ac9c4ce995f39f290cfc723221cc45ac7b11a" +checksum = "ab1dff20655433e64452ee12a9c87dba5d307a47fc8e155b4e37d746d66b37dd" dependencies = [ "proc-macro2", "quote", @@ -1393,9 +1393,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58e5838966f971c24e8be8c34fb901e6e8a48211ea1db6d66c76fbb77b58d6c" +checksum = "f289d7b40c97d1b7d2a8cf1528cd5af0f3f4e832dffdec20b6485b9d6240de0b" dependencies = [ "bitflags", "itertools", @@ -1408,9 +1408,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e298be4a113f57749ed51a38dc253671eaed418a5de6dbd2bf29ff34c4f595" +checksum = "8aa20c10288eacf1a8ec93981198c1428c7c9d0436ae7e0f0037a02e87535c94" dependencies = [ "assert-unchecked", "bitflags", @@ -1430,9 +1430,9 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e87ee74753f20549e9d6bcc00e956f5ab67fde4a23e383fd4f7d8ad752d71d3" +checksum = "e902c119eca30d0016d3d2aed31052d6e3ef1f594d71ca3fe193286771a8088d" dependencies = [ "assert-unchecked", "ropey", @@ -1440,9 +1440,9 @@ dependencies = [ [[package]] name = "oxc_diagnostics" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27133a883a4f5d9796259a3ca6ab28ba8c097ee32cd58fc794688e205f66727" +checksum = "640a5f40eb0e09725a5e9c90e88727272c999d9805403747e2140cab2a742b6e" dependencies = [ "cow-utils", "oxc-miette", @@ -1450,9 +1450,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a7535550ce59cc7373d60d13e5fb32eee1817ba9dcfa3df7a1ae96bdb0945e" +checksum = "87543a59fd0d2d27102ccd8eee9f954564f7e82124992cc21975bbbe75851ce4" dependencies = [ "cow-utils", "num-bigint", @@ -1464,9 +1464,12 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a817a58818b63a5b26a15527e9c7f09c93ba1a95f570ae75e5e84bd3dee047de" +checksum = "c1efeb3bb8ce6bced1e382e49d3978a2e5067cdc3881584f1cdef5aaf52d28b4" +dependencies = [ + "itoa", +] [[package]] name = "oxc_index" @@ -1476,9 +1479,9 @@ checksum = "5eca5d9726cd0a6e433debe003b7bc88b2ecad0bb6109f0cef7c55e692139a34" [[package]] name = "oxc_mangler" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40dc09dd18e8c7cfaa85ef14f2634b81439de7ac900d579dcc3d6885c0847895" +checksum = "74396fefe8eb5a62b2d79023b468f3012d0daa25b4a6fdff64816ade625a6dfc" dependencies = [ "fixedbitset", "itertools", @@ -1492,9 +1495,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8987515e26c6765bf10c12073cd1d135bd99f531fcb2bcae399b8590e3dd44" +checksum = "1fc9099e8cb43a9362ffde2cfcb5d69bbbf87f0303c36d6209379b4f88f3d866" dependencies = [ "cow-utils", "oxc_allocator", @@ -1513,9 +1516,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6410afa5ad3cde657f26803f828726b8d3b475131299faf5c7948299892a560c" +checksum = "d0f5195b60a980baf22e34c801c8a37901ee21149e523d2a2fc68111991dd0b6" dependencies = [ "assert-unchecked", "bitflags", @@ -1536,9 +1539,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9711b8c74f28de9a295bc03baf7d61abf47af35556156e49f3a44fa572194325" +checksum = "c6b7199022f1e8a071a118737884cdf75bd3ba869471fd500f00dd4612a4faeb" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -1552,9 +1555,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e0eac139ab28789789f993fa4955c386454d7c34c1adfe44448c1cb7769cf0" +checksum = "55da371ce5345676e602719ce048eb2343b3e7b1798c6275e06ea431917340af" dependencies = [ "assert-unchecked", "itertools", @@ -1574,9 +1577,9 @@ dependencies = [ [[package]] name = "oxc_sourcemap" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989c3fa8bdd55ced0ad4a0b2d587dfd11613364c9db417889bdb09c217463766" +checksum = "dcb048fad7eee078b23dc3f3be7aba94eeba596c892c9255fc2646fba232a2ec" dependencies = [ "base64-simd", "cfg-if", @@ -1588,9 +1591,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c7d6b92b30eea1fdc8907c6dfe8525b75a815024d6567628697789076fb7e1" +checksum = "b3cbaec227f5bb91e52a04f9028915973cbb7522b123b41af2d3189eb1ce41d8" dependencies = [ "compact_str", "oxc-miette", @@ -1601,9 +1604,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540bf5b53cb0cd81dbdf07b25834a5064b762e6a330ca35d0d424a581a442b8" +checksum = "91bd42fae0c0e6212ea704ed533df47748002abc0af5496911fdb09735028a77" dependencies = [ "assert-unchecked", "bitflags", @@ -1622,9 +1625,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9583a428473921f6faa890fd9b564d8df59832d6638b03f4a30779a15d22e612" +checksum = "7c3558bebe882232c4c6fd5f28f08ce58586970c56b486a694b29c59b8b976a8" dependencies = [ "compact_str", "itoa", @@ -1839,9 +1842,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3cf30fb8a0540827f3e22514836533403dece9ac517631160a1fb36d0dd5d6" +checksum = "9813f789f95ee4fe6b4d01834404d7cccacbc3f6c029343af910b3c2835eb9f1" [[package]] name = "rayon" @@ -1865,9 +1868,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" dependencies = [ "bitflags", ] @@ -1903,9 +1906,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.9" +version = "0.17.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24" +checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839" dependencies = [ "cc", "cfg-if", @@ -2050,9 +2053,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" dependencies = [ "serde_derive", ] @@ -2068,9 +2071,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", @@ -2079,9 +2082,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", @@ -2189,7 +2192,7 @@ dependencies = [ "rayon", "serde", "serde_json", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -2462,9 +2465,9 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-log" -version = "1.5.3" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdbb3c254a6def3cbc8afcb523b88c4a3b424297db8207aa66cde5fa84edddf6" +checksum = "f2398750880b2b9770afbd1a3e299b9e859d6143c299867eb35fdf484b7625d3" dependencies = [ "byteorder", "byteview", @@ -2619,11 +2622,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.18" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" +checksum = "dde3bb8c68a8f3f1ed4ac9221aad6b10cece3e60a8e2ea54a6a2dec806d0084c" dependencies = [ - "zerocopy-derive 0.8.18", + "zerocopy-derive 0.8.20", ] [[package]] @@ -2639,9 +2642,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.18" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +checksum = "eea57037071898bf96a6da35fd626f4f27e9cee3ead2a6c703cf09d472b2e700" dependencies = [ "proc-macro2", "quote", @@ -2650,27 +2653,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.14+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 0cf5cc79f..c550b953e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,22 +19,21 @@ bitcoin = { version = "0.32.5", features = ["serde"] } color-eyre = "0.6.3" computer = { version = "0", path = "computer", package = "bomputer" } derive_deref = "1.1.1" -fjall = "2.6.3" +fjall = "2.6.4" hodor = { version = "0", path = "hodor" } indexer = { version = "0", path = "indexer", package = "bindexer" } iterator = { version = "0", path = "iterator", package = "biterator", features = [ - "fjall", - "zerocopy", + "bytes", ] } jiff = "0.2.1" -log = { version = "0.4.25" } +log = { version = "0.4.26" } logger = { version = "0", path = "logger", package = "cl0g" } minreq = { version = "2.13.2", features = ["https", "serde_json"] } pricer = { version = "0", path = "pricer", package = "bricer" } rayon = "1.10.0" -serde = { version = "1.0.217", features = ["derive"] } +serde = { version = "1.0.218", features = ["derive"] } serde_bytes = "0.11.15" -serde_json = { version = "1.0.138", features = ["float_roundtrip"] } +serde_json = { version = "1.0.139", features = ["float_roundtrip"] } server = { version = "0", path = "server", package = "berver" } storable_vec = { version = "0", path = "storable_vec", features = ["json"] } -zerocopy = { version = "0.8.18", features = ["derive"] } +zerocopy = { version = "0.8.20", features = ["derive"] } diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 6c3325105..5ae90ad97 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -14,7 +14,7 @@ hodor = { workspace = true } iterator = { workspace = true } jiff = { workspace = true } logger = { workspace = true } -rapidhash = "1.3.0" +rapidhash = "1.4.0" rayon = { workspace = true } rlimit = { version = "0.10.2" } serde = { workspace = true } diff --git a/iterator/Cargo.toml b/iterator/Cargo.toml index 1b95676f4..014a1c029 100644 --- a/iterator/Cargo.toml +++ b/iterator/Cargo.toml @@ -9,16 +9,15 @@ edition = { workspace = true } license = { workspace = true } [features] -fjall = ["dep:fjall"] -zerocopy = ["dep:zerocopy"] +bytes = ["dep:fjall", "dep:zerocopy"] [dependencies] bitcoin = { workspace = true } -rayon = { workspace = true } +bitcoincore-rpc = "0.19.0" crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } +derive_deref = { workspace = true } fjall = { workspace = true, optional = true } +rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -derive_deref = { workspace = true } -bitcoincore-rpc = "0.19.0" zerocopy = { workspace = true, optional = true } diff --git a/iterator/src/blk_index_to_blk_path.rs b/iterator/src/blk_index_to_blk_path.rs index e365018c8..8c95acb7e 100644 --- a/iterator/src/blk_index_to_blk_path.rs +++ b/iterator/src/blk_index_to_blk_path.rs @@ -10,7 +10,7 @@ const BLK: &str = "blk"; const DAT: &str = ".dat"; #[derive(Debug, Deref, DerefMut)] -pub struct BlkIndexToBlkPath(BTreeMap); +pub struct BlkIndexToBlkPath(BTreeMap); impl BlkIndexToBlkPath { pub fn scan(data_dir: &Path) -> Self { @@ -35,7 +35,7 @@ impl BlkIndexToBlkPath { let file_name = path.file_name().unwrap().to_str().unwrap(); let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())] - .parse::() + .parse::() .unwrap(); (blk_index, path) diff --git a/iterator/src/blk_index_to_blk_recap.rs b/iterator/src/blk_index_to_blk_recap.rs index a03025a67..df20ae43c 100644 --- a/iterator/src/blk_index_to_blk_recap.rs +++ b/iterator/src/blk_index_to_blk_recap.rs @@ -1,29 +1,23 @@ use std::{ - cmp::Ordering, collections::{BTreeMap, BTreeSet}, - fs::{self, File}, + fs::File, io::{BufReader, BufWriter}, path::{Path, PathBuf}, }; -use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock, Height}; - -const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30; +use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, Height}; #[derive(Debug)] pub struct BlkIndexToBlkRecap { - path: PathBuf, - tree: BTreeMap, - last_safe_height: Option, + pub path: PathBuf, + pub tree: BTreeMap, } impl BlkIndexToBlkRecap { - pub fn import(blocks_dir: &BlkIndexToBlkPath, data_dir: &Path) -> Self { + pub fn import(data_dir: &Path, blk_index_to_blk_path: &BlkIndexToBlkPath, start: Option) -> (Self, u16) { let path = data_dir.join("blk_index_to_blk_recap.json"); let tree = { - fs::create_dir_all(data_dir).unwrap(); - if let Ok(file) = File::open(&path) { let reader = BufReader::new(file); serde_json::from_reader(reader).unwrap_or_default() @@ -32,90 +26,62 @@ impl BlkIndexToBlkRecap { } }; - // dbg!(&tree); + let mut slf = Self { path, tree }; - let mut this = Self { - path, - tree, - last_safe_height: None, - }; + let min_removed = slf.clean_outdated(blk_index_to_blk_path); - this.clean_outdated(blocks_dir); + let blk_index = slf.get_start_recap(min_removed, start); - this + (slf, blk_index) } - fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) { - self.tree.pop_last(); + fn clean_outdated(&mut self, blk_index_to_blk_path: &BlkIndexToBlkPath) -> Option { + let mut min_removed_blk_index: Option = None; let mut unprocessed_keys = self.tree.keys().copied().collect::>(); - blocks_dir.iter().for_each(|(blk_index, blk_path)| { + blk_index_to_blk_path.iter().for_each(|(blk_index, blk_path)| { unprocessed_keys.remove(blk_index); if let Some(blk_recap) = self.tree.get(blk_index) { if blk_recap.has_different_modified_time(blk_path) { - self.tree.remove(blk_index); + self.tree.remove(blk_index).unwrap(); + if min_removed_blk_index.map_or(true, |_blk_index| *blk_index < _blk_index) { + min_removed_blk_index.replace(*blk_index); + } } } }); unprocessed_keys.into_iter().for_each(|blk_index| { - self.tree.remove(&blk_index); + self.tree.remove(&blk_index).unwrap(); + if min_removed_blk_index.map_or(true, |_blk_index| blk_index < _blk_index) { + min_removed_blk_index.replace(blk_index); + } }); - self.last_safe_height = self.tree.values().map(|recap| recap.height()).max(); + min_removed_blk_index } - pub fn get_start_recap(&mut self, start: Option) -> Option<(usize, BlkRecap)> { - if let Some(start) = start { - let (last_key, last_value) = self.tree.last_key_value()?; + pub fn get_start_recap(&mut self, min_removed: Option, start: Option) -> u16 { + if start.is_none() { + return 0; + } - dbg!((last_key, last_value)); + let height = start.unwrap(); - if last_value.height() < start { - return Some((*last_key, *last_value)); - } else if let Some((blk_index, _)) = - self.tree.iter().find(|(_, blk_recap)| blk_recap.is_younger_than(start)) - { - if *blk_index != 0 { - // Temporary fix, need to rethink the whole thing - let blk_index = (*blk_index).checked_sub(1).unwrap_or_default(); - return Some((blk_index, *self.tree.get(&blk_index).unwrap())); - } + let mut start = 0; + + if let Some(found) = self.tree.iter().find(|(_, recap)| recap.max_height >= height) { + start = *found.0; + } + + if let Some(min_removed) = min_removed { + if start > min_removed { + start = min_removed; } } - None - } - - pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: Height) { - let blk_index = blk_metadata_and_block.blk_metadata.index; - - if let Some(last_entry) = self.tree.last_entry() { - match last_entry.key().cmp(&blk_index) { - Ordering::Greater => { - last_entry.remove_entry(); - } - Ordering::Less => { - self.tree - .insert(blk_index, BlkRecap::from(height, blk_metadata_and_block)); - } - Ordering::Equal => {} - }; - } else { - if blk_index != 0 || height != 0 { - // dbg!(blk_index, height); - unreachable!(); - } - - self.tree.insert(blk_index, BlkRecap::first(blk_metadata_and_block)); - } - - if self.last_safe_height.map_or(true, |safe_height| height >= safe_height) - && (height % TARGET_BLOCKS_PER_MONTH) == 0 - { - self.export(); - } + start } pub fn export(&self) { @@ -124,6 +90,6 @@ impl BlkIndexToBlkRecap { panic!("No such file or directory") }); - serde_json::to_writer_pretty(&mut BufWriter::new(file), &self.tree).unwrap(); + serde_json::to_writer(&mut BufWriter::new(file), &self.tree).unwrap(); } } diff --git a/iterator/src/blk_metadata.rs b/iterator/src/blk_metadata.rs index 4f07a98a2..2d8614b6b 100644 --- a/iterator/src/blk_metadata.rs +++ b/iterator/src/blk_metadata.rs @@ -4,12 +4,12 @@ use crate::path_to_modified_time; #[derive(Debug, Clone, Copy)] pub struct BlkMetadata { - pub index: usize, + pub index: u16, pub modified_time: u64, } impl BlkMetadata { - pub fn new(index: usize, path: &Path) -> Self { + pub fn new(index: u16, path: &Path) -> Self { Self { index, modified_time: path_to_modified_time(path), diff --git a/iterator/src/blk_metadata_and_block.rs b/iterator/src/blk_metadata_and_block.rs index 8b5bb288b..d65966b40 100644 --- a/iterator/src/blk_metadata_and_block.rs +++ b/iterator/src/blk_metadata_and_block.rs @@ -3,12 +3,12 @@ use bitcoin::Block; use crate::BlkMetadata; #[derive(Debug)] -pub struct BlkMetadataAndBlock { +pub struct BlkIndexAndBlock { pub blk_metadata: BlkMetadata, pub block: Block, } -impl BlkMetadataAndBlock { +impl BlkIndexAndBlock { pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self { Self { blk_metadata, block } } diff --git a/iterator/src/blk_recap.rs b/iterator/src/blk_recap.rs index 487ffc2ed..cd2fee31a 100644 --- a/iterator/src/blk_recap.rs +++ b/iterator/src/blk_recap.rs @@ -1,50 +1,21 @@ use std::path::Path; -use bitcoin::{hashes::Hash, BlockHash}; use serde::{Deserialize, Serialize}; -use crate::{path_to_modified_time, BlkMetadataAndBlock, Height}; +use crate::{path_to_modified_time, Height}; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[repr(C)] pub struct BlkRecap { - min_continuous_height: Height, - min_continuous_prev_hash: BlockHash, - modified_time: u64, + pub max_height: Height, + pub modified_time: u64, } impl BlkRecap { - pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self { - Self { - min_continuous_height: Height::default(), - min_continuous_prev_hash: BlockHash::all_zeros(), - modified_time: blk_metadata_and_block.blk_metadata.modified_time, - } - } - - pub fn from(height: Height, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self { - Self { - min_continuous_height: height, - min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash, - modified_time: blk_metadata_and_block.blk_metadata.modified_time, - } - } - pub fn has_different_modified_time(&self, blk_path: &Path) -> bool { if self.modified_time != path_to_modified_time(blk_path) { dbg!(self.modified_time, path_to_modified_time(blk_path)); } self.modified_time != path_to_modified_time(blk_path) } - - pub fn is_younger_than(&self, height: Height) -> bool { - self.min_continuous_height > height - } - - pub fn height(&self) -> Height { - self.min_continuous_height - } - - pub fn prev_hash(&self) -> &BlockHash { - &self.min_continuous_prev_hash - } } diff --git a/iterator/src/error.rs b/iterator/src/error.rs index 41bd962f0..0f670f987 100644 --- a/iterator/src/error.rs +++ b/iterator/src/error.rs @@ -8,7 +8,6 @@ pub type Result = std::result::Result; #[derive(Debug)] pub enum Error { IO(io::Error), - #[cfg(feature = "zerocopy")] ZeroCopyError, } @@ -18,7 +17,7 @@ impl From for Error { } } -#[cfg(feature = "zerocopy")] +#[cfg(feature = "bytes")] impl From> for Error { fn from(_: zerocopy::error::SizeError) -> Self { Self::ZeroCopyError @@ -29,7 +28,6 @@ impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Error::IO(error) => Debug::fmt(&error, f), - #[cfg(feature = "zerocopy")] Error::ZeroCopyError => write!(f, "Zero copy convert error"), } } diff --git a/iterator/src/height.rs b/iterator/src/height.rs index b646bbc3b..8df4c97f9 100644 --- a/iterator/src/height.rs +++ b/iterator/src/height.rs @@ -5,20 +5,20 @@ use std::{ use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; - -#[cfg(feature = "zerocopy")] +#[cfg(feature = "bytes")] use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::rpc::{self, RpcApi}; #[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] -#[cfg_attr(feature = "zerocopy", derive(FromBytes, Immutable, IntoBytes, KnownLayout))] +#[cfg_attr(feature = "bytes", derive(FromBytes, Immutable, IntoBytes, KnownLayout,))] pub struct Height(u32); impl Height { - const ZERO: Self = Height(0); + pub const ZERO: Self = Height(0); + pub const MAX: Self = Height(u32::MAX); - #[cfg(feature = "zerocopy")] + #[cfg(feature = "bytes")] pub fn write(&self, path: &std::path::Path) -> Result<(), std::io::Error> { std::fs::write(path, self.as_bytes()) } @@ -163,7 +163,7 @@ impl From for bitcoin::locktime::absolute::Height { } } -#[cfg(feature = "zerocopy")] +#[cfg(feature = "bytes")] impl TryFrom<&std::path::Path> for Height { type Error = crate::Error; fn try_from(value: &std::path::Path) -> Result { @@ -171,14 +171,14 @@ impl TryFrom<&std::path::Path> for Height { } } -#[cfg(feature = "fjall")] +#[cfg(feature = "bytes")] impl TryFrom for Height { type Error = crate::Error; fn try_from(value: fjall::Slice) -> Result { Ok(Self::read_from_bytes(&value)?) } } -#[cfg(feature = "fjall")] +#[cfg(feature = "bytes")] impl From for fjall::Slice { fn from(value: Height) -> Self { Self::new(value.as_bytes()) diff --git a/iterator/src/lib.rs b/iterator/src/lib.rs index c6811ffc4..ac303f0c2 100644 --- a/iterator/src/lib.rs +++ b/iterator/src/lib.rs @@ -1,5 +1,6 @@ use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + cmp::Ordering, + collections::BTreeMap, fs::{self}, ops::ControlFlow, path::Path, @@ -8,12 +9,12 @@ use std::{ use bitcoin::{ consensus::{Decodable, ReadExt}, - hashes::Hash, io::{Cursor, Read}, Block, BlockHash, }; use bitcoincore_rpc::RpcApi; use blk_index_to_blk_path::*; +use blk_recap::BlkRecap; use crossbeam::channel::{bounded, Receiver}; use rayon::prelude::*; @@ -41,41 +42,9 @@ const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217]; const BOUND_CAP: usize = 100; /// -/// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order. +/// Returns a crossbeam channel receiver that receives `(Height, Block, BlockHash)` tuples from an **inclusive** range (`start` and `end`) /// -/// # Arguments -/// -/// * `data_dir` - Path to the Bitcoin data directory -/// * `start` - Inclusive starting height of the blocks received, `None` for 0 -/// * `end` - Inclusive ending height of the blocks received, `None` for the last one -/// * `rpc` - RPC client to filter out forks -/// -/// # Example -/// -/// ```rust -/// use std::path::Path; -/// -/// use bitcoincore_rpc::{Auth, Client}; -/// -/// let i = std::time::Instant::now(); -/// -/// let data_dir = Path::new("../../bitcoin"); -/// let url = "http://localhost:8332"; -/// let cookie = Path::new(data_dir).join(".cookie"); -/// let auth = Auth::CookieFile(cookie); -/// let rpc = Client::new(url, auth).unwrap(); -/// -/// let start = Some(850_000); -/// let end = None; -/// -/// biter::new(data_dir, start, end, rpc) -/// .iter() -/// .for_each(|(height, _block, hash)| { -/// println!("{height}: {hash}"); -/// }); -/// -/// dbg!(i.elapsed()); -/// ``` +/// For an example checkout `iterator/main.rs` /// pub fn new( data_dir: &Path, @@ -89,17 +58,15 @@ pub fn new( let blk_index_to_blk_path = BlkIndexToBlkPath::scan(data_dir); - let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blk_index_to_blk_path, data_dir); - - let start_recap = blk_index_to_blk_recap.get_start_recap(start); - let starting_blk_index = start_recap.as_ref().map_or(0, |(index, _)| *index); + let (mut blk_index_to_blk_recap, blk_index) = BlkIndexToBlkRecap::import(data_dir, &blk_index_to_blk_path, start); thread::spawn(move || { blk_index_to_blk_path - .iter() - .filter(|(blk_index, _)| **blk_index >= starting_blk_index) + .range(blk_index..) .try_for_each(move |(blk_index, blk_path)| { - let blk_metadata = BlkMetadata::new(*blk_index, blk_path.as_path()); + let blk_index = *blk_index; + + let blk_metadata = BlkMetadata::new(blk_index, blk_path.as_path()); let blk_bytes = fs::read(blk_path).unwrap(); let blk_bytes_len = blk_bytes.len() as u64; @@ -128,22 +95,19 @@ pub fn new( } } - let block_size = cursor.read_u32().unwrap(); + let len = cursor.read_u32().unwrap(); - let mut raw_block = vec![0u8; block_size as usize]; + let mut bytes = vec![0u8; len as usize]; - cursor.read_exact(&mut raw_block).unwrap(); + cursor.read_exact(&mut bytes).unwrap(); - if send_block_reader - .send((blk_metadata, BlockState::Raw(raw_block))) - .is_err() - { + if send_block_reader.send((blk_metadata, BlockState::Raw(bytes))).is_err() { return ControlFlow::Break(()); } } ControlFlow::Continue(()) - }) + }); }); thread::spawn(move || { @@ -152,16 +116,7 @@ pub fn new( let drain_and_send = |bulk: &mut Vec<_>| { // Using a vec and sending after to not end up with stuck threads in par iter bulk.par_iter_mut().for_each(|(_, block_state)| { - let raw_block = match block_state { - BlockState::Raw(vec) => vec, - _ => unreachable!(), - }; - - let mut cursor = Cursor::new(raw_block); - - let block = Block::consensus_decode(&mut cursor).unwrap(); - - *block_state = BlockState::Decoded(block); + BlockState::decode(block_state); }); bulk.drain(..).try_for_each(|(blk_metadata, block_state)| { @@ -170,7 +125,7 @@ pub fn new( _ => unreachable!(), }; - if send_block.send(BlkMetadataAndBlock::new(blk_metadata, block)).is_err() { + if send_block.send(BlkIndexAndBlock::new(blk_metadata, block)).is_err() { return ControlFlow::Break(()); } @@ -193,113 +148,83 @@ pub fn new( }); thread::spawn(move || { - let mut height = start_recap.map_or(Height::default(), |(_, recap)| recap.height()); + let mut current_height = start.unwrap_or_default(); let mut future_blocks = BTreeMap::default(); - let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default(); - let mut recent_hashes: BTreeSet = BTreeSet::default(); - let mut prev_hash = start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash()); + recv_block.iter().try_for_each(|tuple| -> ControlFlow<(), _> { + let blk_metadata = tuple.blk_metadata; + let block = tuple.block; + let hash = block.block_hash(); + let header = rpc.get_block_header_info(&hash); - let mut prepare_and_send = |(hash, tuple): (BlockHash, BlkMetadataAndBlock)| { - blk_index_to_blk_recap.update(&tuple, height); - - if start.map_or(true, |start| start <= height) { - send_height_block_hash.send((height, tuple.block, hash)).unwrap(); + if header.is_err() { + return ControlFlow::Continue(()); + } + let header = header.unwrap(); + if header.confirmations <= 0 { + return ControlFlow::Continue(()); } - if end == Some(height) { - return ControlFlow::Break(()); + let height = Height::from(header.height); + + let len = blk_index_to_blk_recap.tree.len(); + if blk_metadata.index == len as u16 || blk_metadata.index + 1 == len as u16 { + match (len as u16).cmp(&blk_metadata.index) { + Ordering::Equal => { + if len % 21 == 0 { + blk_index_to_blk_recap.export(); + } + } + Ordering::Less => panic!(), + Ordering::Greater => {} + } + + blk_index_to_blk_recap + .tree + .entry(blk_metadata.index) + .and_modify(|recap| { + if recap.max_height < height { + recap.max_height = height; + } + }) + .or_insert(BlkRecap { + max_height: height, + modified_time: blk_metadata.modified_time, + }); + } else { + dbg!(blk_metadata.index, len); + panic!() } - height += 1; + let mut opt = if current_height == height { + Some((block, hash)) + } else { + if start.map_or(true, |start| start <= height) && end.map_or(true, |end| end >= height) { + future_blocks.insert(height, (block, hash)); + } + None + }; - ControlFlow::Continue(()) - }; + while let Some((block, hash)) = opt.take().or_else(|| { + if !future_blocks.is_empty() { + future_blocks.remove(¤t_height) + } else { + None + } + }) { + send_height_block_hash.send((current_height, block, hash)).unwrap(); - let mut update_tip = |prev_hash: &mut BlockHash, - recent_hashes: &mut BTreeSet, - recent_chain: &mut VecDeque<(BlockHash, BlkMetadataAndBlock)>, - future_blocks: &mut BTreeMap, - tuple: BlkMetadataAndBlock| { - let mut tuple = Some(tuple); - - while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) { - let hash = tuple.block.block_hash(); - - *prev_hash = hash; - recent_hashes.insert(hash); - recent_chain.push_back((hash, tuple)); - } - - while recent_chain.len() > NUMBER_OF_UNSAFE_BLOCKS { - let (hash, tuple) = recent_chain.pop_front().unwrap(); - - recent_hashes.remove(&hash); - - if prepare_and_send((hash, tuple)).is_break() { + if end == Some(current_height) { return ControlFlow::Break(()); } - } - ControlFlow::Continue(()) - }; - - let flow = recv_block.iter().try_for_each(|tuple| { - // block isn't next after current tip - if prev_hash != tuple.block.header.prev_blockhash { - let is_block_active = |hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0; - - // block prev has already been processed - if recent_hashes.contains(&tuple.block.header.prev_blockhash) { - let hash = tuple.block.block_hash(); - - if is_block_active(&hash) { - let prev_index = recent_chain - .iter() - .position(|(hash, ..)| hash == &tuple.block.header.prev_blockhash) - .unwrap(); - - let bad_index_start = prev_index + 1; - - recent_chain.drain(bad_index_start..).for_each(|(hash, _)| { - recent_hashes.remove(&hash); - }); - - return update_tip( - &mut prev_hash, - &mut recent_hashes, - &mut recent_chain, - &mut future_blocks, - tuple, - ); - } - // Check if there was already a future block with the same prev hash - } else if let Some(prev_tuple) = future_blocks.insert(tuple.block.header.prev_blockhash, tuple) { - // If the previous was the active one - if is_block_active(&prev_tuple.block.block_hash()) { - // Rollback the insert - future_blocks.insert(prev_tuple.block.header.prev_blockhash, prev_tuple); - } - } - } else { - return update_tip( - &mut prev_hash, - &mut recent_hashes, - &mut recent_chain, - &mut future_blocks, - tuple, - ); + current_height.increment(); } ControlFlow::Continue(()) }); - if flow.is_continue() { - // Send the last (up to 100) blocks - recent_chain.into_iter().try_for_each(prepare_and_send); - } - blk_index_to_blk_recap.export(); }); @@ -310,3 +235,18 @@ enum BlockState { Raw(Vec), Decoded(Block), } + +impl BlockState { + pub fn decode(&mut self) { + let bytes = match self { + BlockState::Raw(bytes) => bytes, + _ => unreachable!(), + }; + + let mut cursor = Cursor::new(bytes); + + let block = Block::consensus_decode(&mut cursor).unwrap(); + + *self = BlockState::Decoded(block); + } +} diff --git a/iterator/src/main.rs b/iterator/src/main.rs index 4341a714d..d055ff53b 100644 --- a/iterator/src/main.rs +++ b/iterator/src/main.rs @@ -15,7 +15,7 @@ fn main() { )); let start = None; - let end = None; + let end = None; //Some(200_000_u32.into()); biterator::new(data_dir, start, end, rpc) .iter() diff --git a/server/Cargo.toml b/server/Cargo.toml index e01994f06..6b15f027a 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -13,7 +13,7 @@ derive_deref = { workspace = true } indexer = { workspace = true } jiff = { workspace = true } logger = { workspace = true } -oxc = { version = "0.51.0", features = ["codegen", "minifier"] } +oxc = { version = "0.52.0", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } storable_vec = { workspace = true }