From 677aca7a0376df2f7a4d20a4d70d0a68e2c2187c Mon Sep 17 00:00:00 2001 From: nym21 Date: Thu, 27 Feb 2025 00:50:35 +0100 Subject: [PATCH] indexer: rollback fixed via fjall v2.6.6 (conv on discord) --- Cargo.lock | 93 +++++++++---------- Cargo.toml | 4 +- README.md | 28 +++--- crates/brk_fetcher/Cargo.toml | 1 - crates/brk_fetcher/src/fetchers/binance.rs | 5 +- crates/brk_fetcher/src/fetchers/kraken.rs | 3 +- crates/brk_fetcher/src/lib.rs | 34 +------ crates/brk_fetcher/src/main.rs | 3 +- crates/brk_indexer/src/lib.rs | 75 ++++++++------- crates/brk_indexer/src/main.rs | 8 +- crates/brk_indexer/src/storage/fjalls/base.rs | 52 +++++++---- crates/brk_indexer/src/storage/fjalls/meta.rs | 3 - crates/brk_indexer/src/storage/fjalls/mod.rs | 38 +++++--- .../src/storage/storable_vecs/base.rs | 8 -- .../src/storage/storable_vecs/mod.rs | 2 +- crates/brk_server/Cargo.toml | 2 +- 16 files changed, 172 insertions(+), 187 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbc010479..8ffa62669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,7 +377,6 @@ dependencies = [ "log", "minreq", "serde_json", - "storable_vec", ] [[package]] @@ -780,9 +779,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "fjall" -version = "2.6.5" +version = "2.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e0fd3f3a8cbaa2b179ccc690ea0d37282d24787c06eab0dfd9137e1c4d4699" +checksum = "0ad81b05d96e456433c704ae51210be48241c43214f97820c9254b80f9428cae" dependencies = [ "byteorder", "byteview", @@ -1151,9 +1150,9 @@ checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "lsm-tree" -version = "2.6.5" +version = "2.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614a43954a8414dcca688f9d4c91f727860b55ffd9a029438b18f234ed33330f" +checksum = "49c29f6322847a38368942f42eb86c3875fb2e946276f9f3df805cc6ada80b81" dependencies = [ "byteorder", "crossbeam-skiplist", @@ -1331,9 +1330,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6be05b94a99c886e3c8f79c0330e746df6fe27f8440b498ea7a5eb2fee2f67" +checksum = "691141bf0b572018b65456ca183507ff06cc50d9d90a4d48132b764d32c95527" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1374,9 +1373,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf460405a383f3f7ac134a0e06afdb3525c8fa8f119453167f179ea6b12faaf6" +checksum = "eaf9c94c4b7eca262bed12a600ae43fdc7546081755af321de79821acd3c5e45" dependencies = [ "allocator-api2", "bumpalo", @@ -1387,9 +1386,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b273094a3e96e84d3d8ce82190fd7113f70bb2a8dcb652c78b46df79fdd156cc" +checksum = "e9e5af333869e3c080f8c9b15aa78d27071c8ca7eee8a86ad0a32eefd9a1ba1e" dependencies = [ "bitflags", "cow-utils", @@ -1404,9 +1403,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab1dff20655433e64452ee12a9c87dba5d307a47fc8e155b4e37d746d66b37dd" +checksum = "6eca883d41a92aa68620385a28bbc6d016221982e7d22c4120fd1a7ffcb9be8b" dependencies = [ "proc-macro2", "quote", @@ -1415,9 +1414,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f289d7b40c97d1b7d2a8cf1528cd5af0f3f4e832dffdec20b6485b9d6240de0b" +checksum = "768c0e8e0cd1ff280c39b5d291fab9937419a70bc9964cc1c4fe7e57541fbdd9" dependencies = [ "bitflags", "itertools", @@ -1430,11 +1429,10 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa20c10288eacf1a8ec93981198c1428c7c9d0436ae7e0f0037a02e87535c94" +checksum = "15bf00ecbc5cc550a9fe34aff598ea826e7c8fbb4f988f6fe08bd5521bdd4b6c" dependencies = [ - "assert-unchecked", "bitflags", "cow-utils", "nonmax", @@ -1452,9 +1450,9 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e902c119eca30d0016d3d2aed31052d6e3ef1f594d71ca3fe193286771a8088d" +checksum = "cd9268d8b0aefbc7d7c175e673c34f8b1be450f5e0af838ec3a2d5224fd116bb" dependencies = [ "assert-unchecked", "ropey", @@ -1462,9 +1460,9 @@ dependencies = [ [[package]] name = "oxc_diagnostics" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640a5f40eb0e09725a5e9c90e88727272c999d9805403747e2140cab2a742b6e" +checksum = "fd9549fb1a8f37c4e5fa61fadcaf439402e9c1541d3442987594e9d173389443" dependencies = [ "cow-utils", "oxc-miette", @@ -1472,9 +1470,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87543a59fd0d2d27102ccd8eee9f954564f7e82124992cc21975bbbe75851ce4" +checksum = "833da37fb0dcf12e58f2380dadade222c5b5caf2494f6ed0b4355f74af46d453" dependencies = [ "cow-utils", "num-bigint", @@ -1486,24 +1484,21 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1efeb3bb8ce6bced1e382e49d3978a2e5067cdc3881584f1cdef5aaf52d28b4" -dependencies = [ - "itoa", -] +checksum = "cb42f17f05b19f20edfef46916e62bdd03ecde89dc06e131b27adc715c19e1dc" [[package]] name = "oxc_index" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eca5d9726cd0a6e433debe003b7bc88b2ecad0bb6109f0cef7c55e692139a34" +checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74396fefe8eb5a62b2d79023b468f3012d0daa25b4a6fdff64816ade625a6dfc" +checksum = "ef1718e6788566d0d7d72596526c4e8c9b666939fa8b1948914a90a3122fe835" dependencies = [ "fixedbitset", "itertools", @@ -1517,9 +1512,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc9099e8cb43a9362ffde2cfcb5d69bbbf87f0303c36d6209379b4f88f3d866" +checksum = "358a1f68e7f76062ba3c53f140d52d50d34dfdeb53e5e7bbffdd8c19eb212b3f" dependencies = [ "cow-utils", "oxc_allocator", @@ -1538,9 +1533,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5195b60a980baf22e34c801c8a37901ee21149e523d2a2fc68111991dd0b6" +checksum = "416b26d59eb0c273239620de864c21f2bbfd098510dff92af554e2a0a464b70d" dependencies = [ "assert-unchecked", "bitflags", @@ -1561,9 +1556,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6b7199022f1e8a071a118737884cdf75bd3ba869471fd500f00dd4612a4faeb" +checksum = "691d451203f63304fd68cacac3bbbcea6578a545e2681d761eee53b47cc2a8ce" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -1577,9 +1572,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55da371ce5345676e602719ce048eb2343b3e7b1798c6275e06ea431917340af" +checksum = "c9d91a99031a2640eab43251d3108b129981c8dc049ded28d94378db929c140f" dependencies = [ "assert-unchecked", "itertools", @@ -1599,9 +1594,9 @@ dependencies = [ [[package]] name = "oxc_sourcemap" -version = "1.0.9" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb048fad7eee078b23dc3f3be7aba94eeba596c892c9255fc2646fba232a2ec" +checksum = "00b84540840cb31eaea9548f93a271da369d69792bd7149bff1a44b0af9eb7a1" dependencies = [ "base64-simd", "cfg-if", @@ -1613,9 +1608,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3cbaec227f5bb91e52a04f9028915973cbb7522b123b41af2d3189eb1ce41d8" +checksum = "bf5c92fba70b2fe8e1a34f249447915338b5165c6a0f563ff9b14a99204d0ba3" dependencies = [ "compact_str", "oxc-miette", @@ -1626,9 +1621,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91bd42fae0c0e6212ea704ed533df47748002abc0af5496911fdb09735028a77" +checksum = "1339878529f3a78b8b365d23e62a326dfc3dd21b4c128619105b83d2bc0cfbfb" dependencies = [ "assert-unchecked", "bitflags", @@ -1647,9 +1642,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.52.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3558bebe882232c4c6fd5f28f08ce58586970c56b486a694b29c59b8b976a8" +checksum = "88b9a49b46d61f4fea58bdba8430bfbe82814c0fbdf4a291d6c57d3d9ffc5ee0" dependencies = [ "compact_str", "itoa", diff --git a/Cargo.toml b/Cargo.toml index 557e7a09f..40b74541d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" members = ["crates/*"] -package.description = "The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node" +package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" package.version = "0.0.1" @@ -20,7 +20,7 @@ brk_server = { version = "0", path = "crates/brk_server" } byteview = "0.5.4" color-eyre = "0.6.3" derive_deref = "1.1.1" -fjall = "2.6.5" +fjall = "2.6.7" hodor = { version = "0", path = "crates/hodor" } jiff = "0.2.1" log = { version = "0.4.26" } diff --git a/README.md b/README.md index 6af995d39..b4c50cd50 100644 --- a/README.md +++ b/README.md @@ -5,71 +5,71 @@ ## Description -The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node. +The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node. ## Crates -`brk` +### `brk` Wrapper around all other `brk-*` crates. > Status: ⚠️ -`brk_cli` +### `brk_cli` A command line interface to interact with the Bitcoin Research Kit. > Status: ❌ -`brk_computer` +### `brk_computer` A Bitcoin dataset computer, built on top of brk_indexer. > Status: ⚠️ -`brk_core` +### `brk_core` The Core (Structs and Errors) of the Bitcoin Research Kit. > Status: ✅ -`brk_fetcher` +### `brk_fetcher` A Bitcoin price fetcher. -> Status: ⚠️ +> Status: ✅ -`brk_indexer` +### `brk_indexer` A Bitcoin Core indexer built on top of brk_parser. -> Status: ⚠️ +> Status: ✅ -`brk_logger` +### `brk_logger` A clean logger used in the Bitcoin Research Kit. > Status: ✅ -`brk_parser` +### `brk_parser` A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust. > Status: ✅ -`brk_server` +### `brk_server` A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer. > Status: ⚠️ -`hodor` +### `hodor` Hold the door, an exit blocker built on top of ctrlc. > Status: ✅ -`storable_vec` +### `storable_vec` A very small, fast, efficient and simple storable Vec. diff --git a/crates/brk_fetcher/Cargo.toml b/crates/brk_fetcher/Cargo.toml index 76d572085..67f937f22 100644 --- a/crates/brk_fetcher/Cargo.toml +++ b/crates/brk_fetcher/Cargo.toml @@ -14,4 +14,3 @@ jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } serde_json = { workspace = true } -storable_vec = { workspace = true } diff --git a/crates/brk_fetcher/src/fetchers/binance.rs b/crates/brk_fetcher/src/fetchers/binance.rs index 8dce5e336..dee92be38 100644 --- a/crates/brk_fetcher/src/fetchers/binance.rs +++ b/crates/brk_fetcher/src/fetchers/binance.rs @@ -10,7 +10,6 @@ use brk_core::{Cents, OHLCCents, Timestamp}; use color_eyre::eyre::{ContextCompat, eyre}; use log::info; use serde_json::Value; -use storable_vec::STATELESS; use crate::{Close, Date, Dollars, High, Low, Open, Pricer, fetchers::retry}; @@ -39,7 +38,7 @@ impl Binance { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { self._1mn.replace(Self::fetch_1mn()?); } - Pricer::::find_height_ohlc( + Pricer::find_height_ohlc( self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, @@ -90,7 +89,7 @@ impl Binance { if self.har.is_none() { self.har.replace(self.read_har().unwrap_or_default()); } - Pricer::::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har") + Pricer::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har") } fn read_har(&self) -> color_eyre::Result> { diff --git a/crates/brk_fetcher/src/fetchers/kraken.rs b/crates/brk_fetcher/src/fetchers/kraken.rs index 57d52842e..ed1f10ae9 100644 --- a/crates/brk_fetcher/src/fetchers/kraken.rs +++ b/crates/brk_fetcher/src/fetchers/kraken.rs @@ -4,7 +4,6 @@ use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestam use color_eyre::eyre::ContextCompat; use log::info; use serde_json::Value; -use storable_vec::STATELESS; use crate::{Pricer, fetchers::retry}; @@ -23,7 +22,7 @@ impl Kraken { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { self._1mn.replace(Self::fetch_1mn()?); } - Pricer::::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m") + Pricer::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m") } fn fetch_1mn() -> color_eyre::Result> { diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index df3e0c4a7..93de5eb62 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -1,25 +1,20 @@ -use std::{ - collections::BTreeMap, - fs, - path::{Path, PathBuf}, -}; +use std::{collections::BTreeMap, fs, path::Path}; -use brk_core::{Cents, Close, Date, Dateindex, Dollars, Height, High, Low, OHLCCents, Open, Timestamp}; +use brk_core::{Cents, Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp}; use color_eyre::eyre::Error; mod fetchers; // use brk_indexer::Indexer; pub use fetchers::*; -use storable_vec::{AnyJsonStorableVec, AnyStorableVec, SINGLE_THREAD, StorableVec, Version}; -pub struct Pricer { +pub struct Pricer { binance: Binance, kraken: Kraken, kibo: Kibo, } -impl Pricer { +impl Pricer { pub fn import(path: &Path) -> color_eyre::Result { fs::create_dir_all(path)?; @@ -30,27 +25,6 @@ impl Pricer { }) } - pub fn compute_if_needed(&mut self) { - // TODO: Remove all outdated - - // indexer - // .vecs - // .height_to_timestamp - // .iter_from(Height::default(), |v| Ok(())); - - // self.open - // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open); - - // self.high - // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high); - - // self.low - // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low); - - // self.close - // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close); - } - fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { todo!(); // if self.ohlc.date.is_key_safe(date) { diff --git a/crates/brk_fetcher/src/main.rs b/crates/brk_fetcher/src/main.rs index 2222ecc47..719c459cb 100644 --- a/crates/brk_fetcher/src/main.rs +++ b/crates/brk_fetcher/src/main.rs @@ -1,5 +1,4 @@ -use brk_fetcher::{Binance, Kibo, Kraken}; -use serde_json::Value; +use brk_fetcher::Binance; fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index a9c06529a..b89557dd0 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -14,7 +14,7 @@ pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; use color_eyre::eyre::{ContextCompat, eyre}; -use hodor::Exit; +use hodor::Hodor; use log::info; use rayon::prelude::*; use storable_vec::CACHED_GETS; @@ -35,10 +35,9 @@ pub struct Indexer { impl Indexer { pub fn import(indexes_dir: &Path) -> color_eyre::Result { - // info!("Increasing limit of opened files to 210_000..."); rlimit::setrlimit( rlimit::Resource::NOFILE, - 210_000, + rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().0.max(210_000), rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1, )?; @@ -51,7 +50,7 @@ impl Indexer { } impl Indexer { - pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> { + pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, hodor: &Hodor) -> color_eyre::Result<()> { let check_collisions = true; let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| { @@ -60,25 +59,29 @@ impl Indexer { indexes }); - // dbg!(starting_indexes); - // panic!(); + hodor.hold(); + self.stores.rollback_if_needed(&self.vecs, &starting_indexes)?; + self.vecs.rollback_if_needed(&starting_indexes)?; + hodor.release(); - exit.block(); - self.stores.rollback(&self.vecs, &starting_indexes)?; - self.vecs.rollback(&starting_indexes)?; - exit.unblock(); + let export_if_needed = |stores: &mut Fjalls, + vecs: &mut StorableVecs, + height: Height, + hodor: &Hodor| + -> color_eyre::Result<()> { + if height == 0 || height % SNAPSHOT_BLOCK_RANGE != 0 || hodor.triggered() { + return Ok(()); + } - let export = - |stores: &mut Fjalls, vecs: &mut StorableVecs, height: Height| -> color_eyre::Result<()> { - info!("Exporting..."); - exit.block(); - stores.commit(height)?; - info!("Exported stores"); - vecs.flush(height)?; - info!("Exported vecs"); - exit.unblock(); - Ok(()) - }; + info!("Exporting..."); + hodor.hold(); + stores.commit(height)?; + info!("Exported stores"); + vecs.flush(height)?; + info!("Exported vecs"); + hodor.release(); + Ok(()) + }; let vecs = &mut self.vecs; let stores = &mut self.stores; @@ -91,9 +94,8 @@ impl Indexer { info!("Started indexing..."); - parser.parse(Some(idxs.height), None) - .iter() - .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { + parser.parse(Some(idxs.height), None).iter().try_for_each( + |(height, block, blockhash)| -> color_eyre::Result<()> { info!("Indexing block {height}..."); idxs.height = height; @@ -115,8 +117,10 @@ impl Indexer { .insert_if_needed(blockhash_prefix, height, height); vecs.height_to_blockhash.push_if_needed(height, blockhash)?; - vecs.height_to_difficulty.push_if_needed(height, block.header.difficulty_float())?; - vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?; + vecs.height_to_difficulty + .push_if_needed(height, block.header.difficulty_float())?; + vecs.height_to_timestamp + .push_if_needed(height, Timestamp::from(block.header.time))?; vecs.height_to_size.push_if_needed(height, block.total_size())?; vecs.height_to_weight.push_if_needed(height, block.weight().into())?; @@ -583,10 +587,12 @@ impl Indexer { let only_known_dup_txids = [ bitcoin::Txid::from_str( "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599", - )?.into(), + )? + .into(), bitcoin::Txid::from_str( "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468", - )?.into(), + )? + .into(), ]; let is_dup = only_known_dup_txids.contains(prev_txid); @@ -623,18 +629,15 @@ impl Indexer { idxs.push_future_if_needed(vecs)?; - let should_snapshot = height != 0 && height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked(); - if should_snapshot { - export(stores, vecs, height)?; - } + export_if_needed(stores, vecs, height, hodor)?; Ok(()) - })?; + }, + )?; - if idxs.height % SNAPSHOT_BLOCK_RANGE != 0 { - export(stores, vecs, idxs.height)?; - } + export_if_needed(stores, vecs, idxs.height, hodor)?; + // To make sure that Fjall had the time to flush everything properly sleep(Duration::from_millis(100)); Ok(()) diff --git a/crates/brk_indexer/src/main.rs b/crates/brk_indexer/src/main.rs index 83ff7a7e6..a65bfc310 100644 --- a/crates/brk_indexer/src/main.rs +++ b/crates/brk_indexer/src/main.rs @@ -5,21 +5,21 @@ use brk_parser::{ Parser, rpc::{self}, }; -use hodor::Exit; +use hodor::Hodor; use log::info; use storable_vec::CACHED_GETS; fn main() -> color_eyre::Result<()> { color_eyre::install()?; - brk_logger::init(None); + brk_logger::init(Some(Path::new(".log"))); let data_dir = Path::new("../../../bitcoin"); let rpc = Box::leak(Box::new(rpc::Client::new( "http://localhost:8332", rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), )?)); - let exit = Exit::new(); + let hodor = Hodor::new(); let parser = Parser::new(data_dir, rpc); @@ -32,7 +32,7 @@ fn main() -> color_eyre::Result<()> { let mut indexer: Indexer = Indexer::import(Path::new("../../_outputs/indexes"))?; - indexer.index(&parser, rpc, &exit)?; + indexer.index(&parser, rpc, &hodor)?; info!("Took: {:?}", i.elapsed()); diff --git a/crates/brk_indexer/src/storage/fjalls/base.rs b/crates/brk_indexer/src/storage/fjalls/base.rs index 9a7b623c8..968ef0b03 100644 --- a/crates/brk_indexer/src/storage/fjalls/base.rs +++ b/crates/brk_indexer/src/storage/fjalls/base.rs @@ -1,6 +1,8 @@ use std::{ collections::{BTreeMap, BTreeSet}, - error, mem, + error, + fmt::Debug, + mem, path::Path, }; @@ -27,25 +29,32 @@ const CHECK_COLLISISONS: bool = true; impl Store where - K: Into + Ord + Immutable + IntoBytes, - V: Into + TryFrom, + K: Debug + Into + Ord + Immutable + IntoBytes, + V: Debug + Into + TryFrom, >::Error: error::Error + Send + Sync + 'static, { pub fn import(path: &Path, version: Version) -> color_eyre::Result { let meta = StoreMeta::checked_open(path, version)?; - let keyspace = if let Ok(keyspace) = Self::open_keyspace(path) { - keyspace - } else { - meta.reset()?; - return Self::import(path, version); + + let keyspace = match Self::open_keyspace(path) { + Ok(keyspace) => keyspace, + Err(e) => { + dbg!(e); + meta.reset()?; + return Self::import(path, version); + } }; - let part = if let Ok(part) = Self::open_partition_handle(&keyspace) { - part - } else { - drop(keyspace); - meta.reset()?; - return Self::import(path, version); + + let part = match Self::open_partition_handle(&keyspace) { + Ok(part) => part, + Err(e) => { + dbg!(e); + drop(keyspace); + meta.reset()?; + return Self::import(path, version); + } }; + let rtx = keyspace.read_tx(); Ok(Self { @@ -71,8 +80,8 @@ where pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) { if self.needs(height) { if !self.dels.is_empty() { - unreachable!("Shouldn't reach this"); // self.dels.remove(&key); + unreachable!("Shouldn't reach this"); } self.puts.insert(key, value); } @@ -83,7 +92,10 @@ where unreachable!("Shouldn't reach this"); // self.puts.remove(&key); } - self.dels.insert(key); + // dbg!(&key); + if !self.dels.insert(key) { + unreachable!(); + } } pub fn commit(&mut self, height: Height) -> Result<()> { @@ -101,8 +113,14 @@ where mem::take(&mut self.puts).into_iter().for_each(|(key, value)| { if CHECK_COLLISISONS { + #[allow(unused_must_use)] if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) { - dbg!(value, &self.meta); + dbg!( + &key, + V::try_from(value.into()).unwrap(), + &self.meta, + self.rtx.get(&self.part, key.as_bytes()) + ); unreachable!(); } } diff --git a/crates/brk_indexer/src/storage/fjalls/meta.rs b/crates/brk_indexer/src/storage/fjalls/meta.rs index 5c0bfdf56..85f45e2af 100644 --- a/crates/brk_indexer/src/storage/fjalls/meta.rs +++ b/crates/brk_indexer/src/storage/fjalls/meta.rs @@ -84,9 +84,6 @@ impl StoreMeta { path.join("height") } - // fn read_length(&self) -> color_eyre::Result { - // Self::read_length_(&self.pathbuf) - // } fn read_length_(path: &Path) -> color_eyre::Result { Ok(fs::read(Self::path_length(path)) .map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default()) diff --git a/crates/brk_indexer/src/storage/fjalls/mod.rs b/crates/brk_indexer/src/storage/fjalls/mod.rs index 64cc683bc..2c7b601cd 100644 --- a/crates/brk_indexer/src/storage/fjalls/mod.rs +++ b/crates/brk_indexer/src/storage/fjalls/mod.rs @@ -21,18 +21,27 @@ pub struct Fjalls { impl Fjalls { pub fn import(path: &Path) -> color_eyre::Result { - let addresshash_to_addressindex = Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))?; - let blockhash_prefix_to_height = Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?; - let txid_prefix_to_txindex = Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?; + thread::scope(|scope| { + let addresshash_to_addressindex = + scope.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))); + let blockhash_prefix_to_height = + scope.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))); + let txid_prefix_to_txindex = + scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))); - Ok(Self { - addresshash_to_addressindex, - blockhash_prefix_to_height, - txid_prefix_to_txindex, + Ok(Self { + addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?, + blockhash_prefix_to_height: blockhash_prefix_to_height.join().unwrap()?, + txid_prefix_to_txindex: txid_prefix_to_txindex.join().unwrap()?, + }) }) } - pub fn rollback(&mut self, vecs: &StorableVecs, starting_indexes: &Indexes) -> color_eyre::Result<()> { + pub fn rollback_if_needed( + &mut self, + vecs: &StorableVecs, + starting_indexes: &Indexes, + ) -> color_eyre::Result<()> { vecs.height_to_blockhash .iter_from(starting_indexes.height, |(_, blockhash)| { let blockhash = blockhash.as_ref(); @@ -41,12 +50,13 @@ impl Fjalls { Ok(()) })?; - vecs.txindex_to_txid.iter_from(starting_indexes.txindex, |(_, txid)| { - let txid = txid.as_ref(); - let txid_prefix = TxidPrefix::from(txid); - self.txid_prefix_to_txindex.remove(txid_prefix); - Ok(()) - })?; + vecs.txindex_to_txid + .iter_from(starting_indexes.txindex, |(_txindex, txid)| { + let txid = txid.as_ref(); + let txid_prefix = TxidPrefix::from(txid); + self.txid_prefix_to_txindex.remove(txid_prefix); + Ok(()) + })?; if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? { let mut index = index.into_inner(); diff --git a/crates/brk_indexer/src/storage/storable_vecs/base.rs b/crates/brk_indexer/src/storage/storable_vecs/base.rs index 571d063b2..05e1a53b8 100644 --- a/crates/brk_indexer/src/storage/storable_vecs/base.rs +++ b/crates/brk_indexer/src/storage/storable_vecs/base.rs @@ -48,14 +48,6 @@ where fn path_height_(path: &Path) -> PathBuf { path.join("height") } - - pub fn needs(&self, height: Height) -> bool { - self.height.is_none_or(|self_height| height > self_height) - } - #[allow(unused)] - pub fn has(&self, height: Height) -> bool { - !self.needs(height) - } } impl Deref for StorableVec { diff --git a/crates/brk_indexer/src/storage/storable_vecs/mod.rs b/crates/brk_indexer/src/storage/storable_vecs/mod.rs index df669e203..0ea9ebaf5 100644 --- a/crates/brk_indexer/src/storage/storable_vecs/mod.rs +++ b/crates/brk_indexer/src/storage/storable_vecs/mod.rs @@ -187,7 +187,7 @@ impl StorableVecs { }) } - pub fn rollback(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> { + pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> { let saved_height = starting_indexes.height.decremented(); // We don't want to override the starting indexes so we cut from n + 1 diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 53298095b..e7021f5c2 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -15,7 +15,7 @@ color-eyre = { workspace = true } derive_deref = { workspace = true } jiff = { workspace = true } log = { workspace = true } -oxc = { version = "0.52.0", features = ["codegen", "minifier"] } +oxc = { version = "0.53.0", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } storable_vec = { workspace = true }