mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-05-13 14:08:36 -07:00
indexer: rollback fixed via fjall v2.6.6 (conv on discord)
This commit is contained in:
nym21
93
Cargo.lock
generated
93
Cargo.lock
generated
@@ -377,7 +377,6 @@ dependencies = [
|
||||
"log",
|
||||
"minreq",
|
||||
"serde_json",
|
||||
"storable_vec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -780,9 +779,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
||||
|
||||
[[package]]
|
||||
name = "fjall"
|
||||
version = "2.6.5"
|
||||
version = "2.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33e0fd3f3a8cbaa2b179ccc690ea0d37282d24787c06eab0dfd9137e1c4d4699"
|
||||
checksum = "0ad81b05d96e456433c704ae51210be48241c43214f97820c9254b80f9428cae"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"byteview",
|
||||
@@ -1151,9 +1150,9 @@ checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
|
||||
|
||||
[[package]]
|
||||
name = "lsm-tree"
|
||||
version = "2.6.5"
|
||||
version = "2.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "614a43954a8414dcca688f9d4c91f727860b55ffd9a029438b18f234ed33330f"
|
||||
checksum = "49c29f6322847a38368942f42eb86c3875fb2e946276f9f3df805cc6ada80b81"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crossbeam-skiplist",
|
||||
@@ -1331,9 +1330,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
|
||||
|
||||
[[package]]
|
||||
name = "oxc"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd6be05b94a99c886e3c8f79c0330e746df6fe27f8440b498ea7a5eb2fee2f67"
|
||||
checksum = "691141bf0b572018b65456ca183507ff06cc50d9d90a4d48132b764d32c95527"
|
||||
dependencies = [
|
||||
"oxc_allocator",
|
||||
"oxc_ast",
|
||||
@@ -1374,9 +1373,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_allocator"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf460405a383f3f7ac134a0e06afdb3525c8fa8f119453167f179ea6b12faaf6"
|
||||
checksum = "eaf9c94c4b7eca262bed12a600ae43fdc7546081755af321de79821acd3c5e45"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"bumpalo",
|
||||
@@ -1387,9 +1386,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_ast"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b273094a3e96e84d3d8ce82190fd7113f70bb2a8dcb652c78b46df79fdd156cc"
|
||||
checksum = "e9e5af333869e3c080f8c9b15aa78d27071c8ca7eee8a86ad0a32eefd9a1ba1e"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cow-utils",
|
||||
@@ -1404,9 +1403,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_ast_macros"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab1dff20655433e64452ee12a9c87dba5d307a47fc8e155b4e37d746d66b37dd"
|
||||
checksum = "6eca883d41a92aa68620385a28bbc6d016221982e7d22c4120fd1a7ffcb9be8b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1415,9 +1414,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_cfg"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f289d7b40c97d1b7d2a8cf1528cd5af0f3f4e832dffdec20b6485b9d6240de0b"
|
||||
checksum = "768c0e8e0cd1ff280c39b5d291fab9937419a70bc9964cc1c4fe7e57541fbdd9"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"itertools",
|
||||
@@ -1430,11 +1429,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_codegen"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8aa20c10288eacf1a8ec93981198c1428c7c9d0436ae7e0f0037a02e87535c94"
|
||||
checksum = "15bf00ecbc5cc550a9fe34aff598ea826e7c8fbb4f988f6fe08bd5521bdd4b6c"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"bitflags",
|
||||
"cow-utils",
|
||||
"nonmax",
|
||||
@@ -1452,9 +1450,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_data_structures"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e902c119eca30d0016d3d2aed31052d6e3ef1f594d71ca3fe193286771a8088d"
|
||||
checksum = "cd9268d8b0aefbc7d7c175e673c34f8b1be450f5e0af838ec3a2d5224fd116bb"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"ropey",
|
||||
@@ -1462,9 +1460,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_diagnostics"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "640a5f40eb0e09725a5e9c90e88727272c999d9805403747e2140cab2a742b6e"
|
||||
checksum = "fd9549fb1a8f37c4e5fa61fadcaf439402e9c1541d3442987594e9d173389443"
|
||||
dependencies = [
|
||||
"cow-utils",
|
||||
"oxc-miette",
|
||||
@@ -1472,9 +1470,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_ecmascript"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87543a59fd0d2d27102ccd8eee9f954564f7e82124992cc21975bbbe75851ce4"
|
||||
checksum = "833da37fb0dcf12e58f2380dadade222c5b5caf2494f6ed0b4355f74af46d453"
|
||||
dependencies = [
|
||||
"cow-utils",
|
||||
"num-bigint",
|
||||
@@ -1486,24 +1484,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_estree"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1efeb3bb8ce6bced1e382e49d3978a2e5067cdc3881584f1cdef5aaf52d28b4"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
]
|
||||
checksum = "cb42f17f05b19f20edfef46916e62bdd03ecde89dc06e131b27adc715c19e1dc"
|
||||
|
||||
[[package]]
|
||||
name = "oxc_index"
|
||||
version = "2.0.0"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eca5d9726cd0a6e433debe003b7bc88b2ecad0bb6109f0cef7c55e692139a34"
|
||||
checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
|
||||
|
||||
[[package]]
|
||||
name = "oxc_mangler"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74396fefe8eb5a62b2d79023b468f3012d0daa25b4a6fdff64816ade625a6dfc"
|
||||
checksum = "ef1718e6788566d0d7d72596526c4e8c9b666939fa8b1948914a90a3122fe835"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
"itertools",
|
||||
@@ -1517,9 +1512,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_minifier"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fc9099e8cb43a9362ffde2cfcb5d69bbbf87f0303c36d6209379b4f88f3d866"
|
||||
checksum = "358a1f68e7f76062ba3c53f140d52d50d34dfdeb53e5e7bbffdd8c19eb212b3f"
|
||||
dependencies = [
|
||||
"cow-utils",
|
||||
"oxc_allocator",
|
||||
@@ -1538,9 +1533,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_parser"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0f5195b60a980baf22e34c801c8a37901ee21149e523d2a2fc68111991dd0b6"
|
||||
checksum = "416b26d59eb0c273239620de864c21f2bbfd098510dff92af554e2a0a464b70d"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"bitflags",
|
||||
@@ -1561,9 +1556,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_regular_expression"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6b7199022f1e8a071a118737884cdf75bd3ba869471fd500f00dd4612a4faeb"
|
||||
checksum = "691d451203f63304fd68cacac3bbbcea6578a545e2681d761eee53b47cc2a8ce"
|
||||
dependencies = [
|
||||
"oxc_allocator",
|
||||
"oxc_ast_macros",
|
||||
@@ -1577,9 +1572,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_semantic"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55da371ce5345676e602719ce048eb2343b3e7b1798c6275e06ea431917340af"
|
||||
checksum = "c9d91a99031a2640eab43251d3108b129981c8dc049ded28d94378db929c140f"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"itertools",
|
||||
@@ -1599,9 +1594,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_sourcemap"
|
||||
version = "1.0.9"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcb048fad7eee078b23dc3f3be7aba94eeba596c892c9255fc2646fba232a2ec"
|
||||
checksum = "00b84540840cb31eaea9548f93a271da369d69792bd7149bff1a44b0af9eb7a1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"cfg-if",
|
||||
@@ -1613,9 +1608,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_span"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3cbaec227f5bb91e52a04f9028915973cbb7522b123b41af2d3189eb1ce41d8"
|
||||
checksum = "bf5c92fba70b2fe8e1a34f249447915338b5165c6a0f563ff9b14a99204d0ba3"
|
||||
dependencies = [
|
||||
"compact_str",
|
||||
"oxc-miette",
|
||||
@@ -1626,9 +1621,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_syntax"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91bd42fae0c0e6212ea704ed533df47748002abc0af5496911fdb09735028a77"
|
||||
checksum = "1339878529f3a78b8b365d23e62a326dfc3dd21b4c128619105b83d2bc0cfbfb"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"bitflags",
|
||||
@@ -1647,9 +1642,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_traverse"
|
||||
version = "0.52.0"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c3558bebe882232c4c6fd5f28f08ce58586970c56b486a694b29c59b8b976a8"
|
||||
checksum = "88b9a49b46d61f4fea58bdba8430bfbe82814c0fbdf4a291d6c57d3d9ffc5ee0"
|
||||
dependencies = [
|
||||
"compact_str",
|
||||
"itoa",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node"
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.1"
|
||||
@@ -20,7 +20,7 @@ brk_server = { version = "0", path = "crates/brk_server" }
|
||||
byteview = "0.5.4"
|
||||
color-eyre = "0.6.3"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.6.5"
|
||||
fjall = "2.6.7"
|
||||
hodor = { version = "0", path = "crates/hodor" }
|
||||
jiff = "0.2.1"
|
||||
log = { version = "0.4.26" }
|
||||
|
||||
28
README.md
28
README.md
@@ -5,71 +5,71 @@
|
||||
|
||||
## Description
|
||||
|
||||
The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node.
|
||||
The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node.
|
||||
|
||||
## Crates
|
||||
|
||||
`brk`
|
||||
### `brk`
|
||||
|
||||
Wrapper around all other `brk-*` crates.
|
||||
|
||||
> Status: ⚠️
|
||||
|
||||
`brk_cli`
|
||||
### `brk_cli`
|
||||
|
||||
A command line interface to interact with the Bitcoin Research Kit.
|
||||
|
||||
> Status: ❌
|
||||
|
||||
`brk_computer`
|
||||
### `brk_computer`
|
||||
|
||||
A Bitcoin dataset computer, built on top of brk_indexer.
|
||||
|
||||
> Status: ⚠️
|
||||
|
||||
`brk_core`
|
||||
### `brk_core`
|
||||
|
||||
The Core (Structs and Errors) of the Bitcoin Research Kit.
|
||||
|
||||
> Status: ✅
|
||||
|
||||
`brk_fetcher`
|
||||
### `brk_fetcher`
|
||||
|
||||
A Bitcoin price fetcher.
|
||||
|
||||
> Status: ⚠️
|
||||
> Status: ✅
|
||||
|
||||
`brk_indexer`
|
||||
### `brk_indexer`
|
||||
|
||||
A Bitcoin Core indexer built on top of brk_parser.
|
||||
|
||||
> Status: ⚠️
|
||||
> Status: ✅
|
||||
|
||||
`brk_logger`
|
||||
### `brk_logger`
|
||||
|
||||
A clean logger used in the Bitcoin Research Kit.
|
||||
|
||||
> Status: ✅
|
||||
|
||||
`brk_parser`
|
||||
### `brk_parser`
|
||||
|
||||
A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust.
|
||||
|
||||
> Status: ✅
|
||||
|
||||
`brk_server`
|
||||
### `brk_server`
|
||||
|
||||
A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer.
|
||||
|
||||
> Status: ⚠️
|
||||
|
||||
`hodor`
|
||||
### `hodor`
|
||||
|
||||
Hold the door, an exit blocker built on top of ctrlc.
|
||||
|
||||
> Status: ✅
|
||||
|
||||
`storable_vec`
|
||||
### `storable_vec`
|
||||
|
||||
A very small, fast, efficient and simple storable Vec.
|
||||
|
||||
|
||||
@@ -14,4 +14,3 @@ jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
minreq = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
storable_vec = { workspace = true }
|
||||
|
||||
@@ -10,7 +10,6 @@ use brk_core::{Cents, OHLCCents, Timestamp};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
use storable_vec::STATELESS;
|
||||
|
||||
use crate::{Close, Date, Dollars, High, Low, Open, Pricer, fetchers::retry};
|
||||
|
||||
@@ -39,7 +38,7 @@ impl Binance {
|
||||
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp {
|
||||
self._1mn.replace(Self::fetch_1mn()?);
|
||||
}
|
||||
Pricer::<STATELESS>::find_height_ohlc(
|
||||
Pricer::find_height_ohlc(
|
||||
self._1mn.as_ref().unwrap(),
|
||||
timestamp,
|
||||
previous_timestamp,
|
||||
@@ -90,7 +89,7 @@ impl Binance {
|
||||
if self.har.is_none() {
|
||||
self.har.replace(self.read_har().unwrap_or_default());
|
||||
}
|
||||
Pricer::<STATELESS>::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har")
|
||||
Pricer::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har")
|
||||
}
|
||||
|
||||
fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
|
||||
|
||||
@@ -4,7 +4,6 @@ use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestam
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
use storable_vec::STATELESS;
|
||||
|
||||
use crate::{Pricer, fetchers::retry};
|
||||
|
||||
@@ -23,7 +22,7 @@ impl Kraken {
|
||||
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp {
|
||||
self._1mn.replace(Self::fetch_1mn()?);
|
||||
}
|
||||
Pricer::<STATELESS>::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
|
||||
Pricer::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
|
||||
}
|
||||
|
||||
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
|
||||
|
||||
@@ -1,25 +1,20 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
|
||||
use brk_core::{Cents, Close, Date, Dateindex, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
|
||||
use brk_core::{Cents, Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
|
||||
use color_eyre::eyre::Error;
|
||||
|
||||
mod fetchers;
|
||||
|
||||
// use brk_indexer::Indexer;
|
||||
pub use fetchers::*;
|
||||
use storable_vec::{AnyJsonStorableVec, AnyStorableVec, SINGLE_THREAD, StorableVec, Version};
|
||||
|
||||
pub struct Pricer<const MODE: u8> {
|
||||
pub struct Pricer {
|
||||
binance: Binance,
|
||||
kraken: Kraken,
|
||||
kibo: Kibo,
|
||||
}
|
||||
|
||||
impl<const MODE: u8> Pricer<MODE> {
|
||||
impl Pricer {
|
||||
pub fn import(path: &Path) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
@@ -30,27 +25,6 @@ impl<const MODE: u8> Pricer<MODE> {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_if_needed(&mut self) {
|
||||
// TODO: Remove all outdated
|
||||
|
||||
// indexer
|
||||
// .vecs
|
||||
// .height_to_timestamp
|
||||
// .iter_from(Height::default(), |v| Ok(()));
|
||||
|
||||
// self.open
|
||||
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open);
|
||||
|
||||
// self.high
|
||||
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high);
|
||||
|
||||
// self.low
|
||||
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low);
|
||||
|
||||
// self.close
|
||||
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close);
|
||||
}
|
||||
|
||||
fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
|
||||
todo!();
|
||||
// if self.ohlc.date.is_key_safe(date) {
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
use brk_fetcher::{Binance, Kibo, Kraken};
|
||||
use serde_json::Value;
|
||||
use brk_fetcher::Binance;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
@@ -14,7 +14,7 @@ pub use brk_parser::*;
|
||||
|
||||
use bitcoin::{Transaction, TxIn, TxOut};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use hodor::Exit;
|
||||
use hodor::Hodor;
|
||||
use log::info;
|
||||
use rayon::prelude::*;
|
||||
use storable_vec::CACHED_GETS;
|
||||
@@ -35,10 +35,9 @@ pub struct Indexer<const MODE: u8> {
|
||||
|
||||
impl<const MODE: u8> Indexer<MODE> {
|
||||
pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
|
||||
// info!("Increasing limit of opened files to 210_000...");
|
||||
rlimit::setrlimit(
|
||||
rlimit::Resource::NOFILE,
|
||||
210_000,
|
||||
rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().0.max(210_000),
|
||||
rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
|
||||
)?;
|
||||
|
||||
@@ -51,7 +50,7 @@ impl<const MODE: u8> Indexer<MODE> {
|
||||
}
|
||||
|
||||
impl Indexer<CACHED_GETS> {
|
||||
pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
|
||||
pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, hodor: &Hodor) -> color_eyre::Result<()> {
|
||||
let check_collisions = true;
|
||||
|
||||
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| {
|
||||
@@ -60,25 +59,29 @@ impl Indexer<CACHED_GETS> {
|
||||
indexes
|
||||
});
|
||||
|
||||
// dbg!(starting_indexes);
|
||||
// panic!();
|
||||
hodor.hold();
|
||||
self.stores.rollback_if_needed(&self.vecs, &starting_indexes)?;
|
||||
self.vecs.rollback_if_needed(&starting_indexes)?;
|
||||
hodor.release();
|
||||
|
||||
exit.block();
|
||||
self.stores.rollback(&self.vecs, &starting_indexes)?;
|
||||
self.vecs.rollback(&starting_indexes)?;
|
||||
exit.unblock();
|
||||
let export_if_needed = |stores: &mut Fjalls,
|
||||
vecs: &mut StorableVecs<CACHED_GETS>,
|
||||
height: Height,
|
||||
hodor: &Hodor|
|
||||
-> color_eyre::Result<()> {
|
||||
if height == 0 || height % SNAPSHOT_BLOCK_RANGE != 0 || hodor.triggered() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let export =
|
||||
|stores: &mut Fjalls, vecs: &mut StorableVecs<CACHED_GETS>, height: Height| -> color_eyre::Result<()> {
|
||||
info!("Exporting...");
|
||||
exit.block();
|
||||
stores.commit(height)?;
|
||||
info!("Exported stores");
|
||||
vecs.flush(height)?;
|
||||
info!("Exported vecs");
|
||||
exit.unblock();
|
||||
Ok(())
|
||||
};
|
||||
info!("Exporting...");
|
||||
hodor.hold();
|
||||
stores.commit(height)?;
|
||||
info!("Exported stores");
|
||||
vecs.flush(height)?;
|
||||
info!("Exported vecs");
|
||||
hodor.release();
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let vecs = &mut self.vecs;
|
||||
let stores = &mut self.stores;
|
||||
@@ -91,9 +94,8 @@ impl Indexer<CACHED_GETS> {
|
||||
|
||||
info!("Started indexing...");
|
||||
|
||||
parser.parse(Some(idxs.height), None)
|
||||
.iter()
|
||||
.try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> {
|
||||
parser.parse(Some(idxs.height), None).iter().try_for_each(
|
||||
|(height, block, blockhash)| -> color_eyre::Result<()> {
|
||||
info!("Indexing block {height}...");
|
||||
|
||||
idxs.height = height;
|
||||
@@ -115,8 +117,10 @@ impl Indexer<CACHED_GETS> {
|
||||
.insert_if_needed(blockhash_prefix, height, height);
|
||||
|
||||
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
|
||||
vecs.height_to_difficulty.push_if_needed(height, block.header.difficulty_float())?;
|
||||
vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?;
|
||||
vecs.height_to_difficulty
|
||||
.push_if_needed(height, block.header.difficulty_float())?;
|
||||
vecs.height_to_timestamp
|
||||
.push_if_needed(height, Timestamp::from(block.header.time))?;
|
||||
vecs.height_to_size.push_if_needed(height, block.total_size())?;
|
||||
vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
|
||||
|
||||
@@ -583,10 +587,12 @@ impl Indexer<CACHED_GETS> {
|
||||
let only_known_dup_txids = [
|
||||
bitcoin::Txid::from_str(
|
||||
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599",
|
||||
)?.into(),
|
||||
)?
|
||||
.into(),
|
||||
bitcoin::Txid::from_str(
|
||||
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468",
|
||||
)?.into(),
|
||||
)?
|
||||
.into(),
|
||||
];
|
||||
|
||||
let is_dup = only_known_dup_txids.contains(prev_txid);
|
||||
@@ -623,18 +629,15 @@ impl Indexer<CACHED_GETS> {
|
||||
|
||||
idxs.push_future_if_needed(vecs)?;
|
||||
|
||||
let should_snapshot = height != 0 && height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked();
|
||||
if should_snapshot {
|
||||
export(stores, vecs, height)?;
|
||||
}
|
||||
export_if_needed(stores, vecs, height, hodor)?;
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
},
|
||||
)?;
|
||||
|
||||
if idxs.height % SNAPSHOT_BLOCK_RANGE != 0 {
|
||||
export(stores, vecs, idxs.height)?;
|
||||
}
|
||||
export_if_needed(stores, vecs, idxs.height, hodor)?;
|
||||
|
||||
// To make sure that Fjall had the time to flush everything properly
|
||||
sleep(Duration::from_millis(100));
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -5,21 +5,21 @@ use brk_parser::{
|
||||
Parser,
|
||||
rpc::{self},
|
||||
};
|
||||
use hodor::Exit;
|
||||
use hodor::Hodor;
|
||||
use log::info;
|
||||
use storable_vec::CACHED_GETS;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
brk_logger::init(None);
|
||||
brk_logger::init(Some(Path::new(".log")));
|
||||
|
||||
let data_dir = Path::new("../../../bitcoin");
|
||||
let rpc = Box::leak(Box::new(rpc::Client::new(
|
||||
"http://localhost:8332",
|
||||
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
|
||||
)?));
|
||||
let exit = Exit::new();
|
||||
let hodor = Hodor::new();
|
||||
|
||||
let parser = Parser::new(data_dir, rpc);
|
||||
|
||||
@@ -32,7 +32,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../../_outputs/indexes"))?;
|
||||
|
||||
indexer.index(&parser, rpc, &exit)?;
|
||||
indexer.index(&parser, rpc, &hodor)?;
|
||||
|
||||
info!("Took: {:?}", i.elapsed());
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
error, mem,
|
||||
error,
|
||||
fmt::Debug,
|
||||
mem,
|
||||
path::Path,
|
||||
};
|
||||
|
||||
@@ -27,25 +29,32 @@ const CHECK_COLLISISONS: bool = true;
|
||||
|
||||
impl<K, V> Store<K, V>
|
||||
where
|
||||
K: Into<ByteView> + Ord + Immutable + IntoBytes,
|
||||
V: Into<ByteView> + TryFrom<ByteView>,
|
||||
K: Debug + Into<ByteView> + Ord + Immutable + IntoBytes,
|
||||
V: Debug + Into<ByteView> + TryFrom<ByteView>,
|
||||
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
|
||||
{
|
||||
pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
|
||||
let meta = StoreMeta::checked_open(path, version)?;
|
||||
let keyspace = if let Ok(keyspace) = Self::open_keyspace(path) {
|
||||
keyspace
|
||||
} else {
|
||||
meta.reset()?;
|
||||
return Self::import(path, version);
|
||||
|
||||
let keyspace = match Self::open_keyspace(path) {
|
||||
Ok(keyspace) => keyspace,
|
||||
Err(e) => {
|
||||
dbg!(e);
|
||||
meta.reset()?;
|
||||
return Self::import(path, version);
|
||||
}
|
||||
};
|
||||
let part = if let Ok(part) = Self::open_partition_handle(&keyspace) {
|
||||
part
|
||||
} else {
|
||||
drop(keyspace);
|
||||
meta.reset()?;
|
||||
return Self::import(path, version);
|
||||
|
||||
let part = match Self::open_partition_handle(&keyspace) {
|
||||
Ok(part) => part,
|
||||
Err(e) => {
|
||||
dbg!(e);
|
||||
drop(keyspace);
|
||||
meta.reset()?;
|
||||
return Self::import(path, version);
|
||||
}
|
||||
};
|
||||
|
||||
let rtx = keyspace.read_tx();
|
||||
|
||||
Ok(Self {
|
||||
@@ -71,8 +80,8 @@ where
|
||||
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
|
||||
if self.needs(height) {
|
||||
if !self.dels.is_empty() {
|
||||
unreachable!("Shouldn't reach this");
|
||||
// self.dels.remove(&key);
|
||||
unreachable!("Shouldn't reach this");
|
||||
}
|
||||
self.puts.insert(key, value);
|
||||
}
|
||||
@@ -83,7 +92,10 @@ where
|
||||
unreachable!("Shouldn't reach this");
|
||||
// self.puts.remove(&key);
|
||||
}
|
||||
self.dels.insert(key);
|
||||
// dbg!(&key);
|
||||
if !self.dels.insert(key) {
|
||||
unreachable!();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn commit(&mut self, height: Height) -> Result<()> {
|
||||
@@ -101,8 +113,14 @@ where
|
||||
|
||||
mem::take(&mut self.puts).into_iter().for_each(|(key, value)| {
|
||||
if CHECK_COLLISISONS {
|
||||
#[allow(unused_must_use)]
|
||||
if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) {
|
||||
dbg!(value, &self.meta);
|
||||
dbg!(
|
||||
&key,
|
||||
V::try_from(value.into()).unwrap(),
|
||||
&self.meta,
|
||||
self.rtx.get(&self.part, key.as_bytes())
|
||||
);
|
||||
unreachable!();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,9 +84,6 @@ impl StoreMeta {
|
||||
path.join("height")
|
||||
}
|
||||
|
||||
// fn read_length(&self) -> color_eyre::Result<usize> {
|
||||
// Self::read_length_(&self.pathbuf)
|
||||
// }
|
||||
fn read_length_(path: &Path) -> color_eyre::Result<usize> {
|
||||
Ok(fs::read(Self::path_length(path))
|
||||
.map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default())
|
||||
|
||||
@@ -21,18 +21,27 @@ pub struct Fjalls {
|
||||
|
||||
impl Fjalls {
|
||||
pub fn import(path: &Path) -> color_eyre::Result<Self> {
|
||||
let addresshash_to_addressindex = Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))?;
|
||||
let blockhash_prefix_to_height = Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?;
|
||||
let txid_prefix_to_txindex = Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?;
|
||||
thread::scope(|scope| {
|
||||
let addresshash_to_addressindex =
|
||||
scope.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::from(1)));
|
||||
let blockhash_prefix_to_height =
|
||||
scope.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1)));
|
||||
let txid_prefix_to_txindex =
|
||||
scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
|
||||
|
||||
Ok(Self {
|
||||
addresshash_to_addressindex,
|
||||
blockhash_prefix_to_height,
|
||||
txid_prefix_to_txindex,
|
||||
Ok(Self {
|
||||
addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?,
|
||||
blockhash_prefix_to_height: blockhash_prefix_to_height.join().unwrap()?,
|
||||
txid_prefix_to_txindex: txid_prefix_to_txindex.join().unwrap()?,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn rollback(&mut self, vecs: &StorableVecs<CACHED_GETS>, starting_indexes: &Indexes) -> color_eyre::Result<()> {
|
||||
pub fn rollback_if_needed(
|
||||
&mut self,
|
||||
vecs: &StorableVecs<CACHED_GETS>,
|
||||
starting_indexes: &Indexes,
|
||||
) -> color_eyre::Result<()> {
|
||||
vecs.height_to_blockhash
|
||||
.iter_from(starting_indexes.height, |(_, blockhash)| {
|
||||
let blockhash = blockhash.as_ref();
|
||||
@@ -41,12 +50,13 @@ impl Fjalls {
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
vecs.txindex_to_txid.iter_from(starting_indexes.txindex, |(_, txid)| {
|
||||
let txid = txid.as_ref();
|
||||
let txid_prefix = TxidPrefix::from(txid);
|
||||
self.txid_prefix_to_txindex.remove(txid_prefix);
|
||||
Ok(())
|
||||
})?;
|
||||
vecs.txindex_to_txid
|
||||
.iter_from(starting_indexes.txindex, |(_txindex, txid)| {
|
||||
let txid = txid.as_ref();
|
||||
let txid_prefix = TxidPrefix::from(txid);
|
||||
self.txid_prefix_to_txindex.remove(txid_prefix);
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? {
|
||||
let mut index = index.into_inner();
|
||||
|
||||
@@ -48,14 +48,6 @@ where
|
||||
fn path_height_(path: &Path) -> PathBuf {
|
||||
path.join("height")
|
||||
}
|
||||
|
||||
pub fn needs(&self, height: Height) -> bool {
|
||||
self.height.is_none_or(|self_height| height > self_height)
|
||||
}
|
||||
#[allow(unused)]
|
||||
pub fn has(&self, height: Height) -> bool {
|
||||
!self.needs(height)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T, const MODE: u8> Deref for StorableVec<I, T, MODE> {
|
||||
|
||||
@@ -187,7 +187,7 @@ impl<const MODE: u8> StorableVecs<MODE> {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn rollback(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> {
|
||||
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> {
|
||||
let saved_height = starting_indexes.height.decremented();
|
||||
|
||||
// We don't want to override the starting indexes so we cut from n + 1
|
||||
|
||||
@@ -15,7 +15,7 @@ color-eyre = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
oxc = { version = "0.52.0", features = ["codegen", "minifier"] }
|
||||
oxc = { version = "0.53.0", features = ["codegen", "minifier"] }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
storable_vec = { workspace = true }
|
||||
|
||||
Reference in New Issue
Block a user