Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-10 22:29:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: rollback fixed via fjall v2.6.6 (conv on discord)

Browse Source
This commit is contained in:
nym21
2025-02-27 00:50:35 +01:00
parent 66b31a62d0
commit 677aca7a03
16 changed files with 172 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
Cargo.lock generated
View File

@@ -377,7 +377,6 @@ dependencies = [
"log", "log",
"minreq", "minreq",
"serde_json", "serde_json",
"storable_vec",
] ]
[[package]] [[package]]
@@ -780,9 +779,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]] [[package]]
name = "fjall" name = "fjall"
version = "2.6.5" version = "2.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33e0fd3f3a8cbaa2b179ccc690ea0d37282d24787c06eab0dfd9137e1c4d4699" checksum = "0ad81b05d96e456433c704ae51210be48241c43214f97820c9254b80f9428cae"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"byteview", "byteview",
@@ -1151,9 +1150,9 @@ checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
[[package]] [[package]]
name = "lsm-tree" name = "lsm-tree"
version = "2.6.5" version = "2.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "614a43954a8414dcca688f9d4c91f727860b55ffd9a029438b18f234ed33330f" checksum = "49c29f6322847a38368942f42eb86c3875fb2e946276f9f3df805cc6ada80b81"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"crossbeam-skiplist", "crossbeam-skiplist",
@@ -1331,9 +1330,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
[[package]] [[package]]
name = "oxc" name = "oxc"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6be05b94a99c886e3c8f79c0330e746df6fe27f8440b498ea7a5eb2fee2f67" checksum = "691141bf0b572018b65456ca183507ff06cc50d9d90a4d48132b764d32c95527"
dependencies = [ dependencies = [
"oxc_allocator", "oxc_allocator",
"oxc_ast", "oxc_ast",
@@ -1374,9 +1373,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_allocator" name = "oxc_allocator"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf460405a383f3f7ac134a0e06afdb3525c8fa8f119453167f179ea6b12faaf6" checksum = "eaf9c94c4b7eca262bed12a600ae43fdc7546081755af321de79821acd3c5e45"
dependencies = [ dependencies = [
"allocator-api2", "allocator-api2",
"bumpalo", "bumpalo",
@@ -1387,9 +1386,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_ast" name = "oxc_ast"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b273094a3e96e84d3d8ce82190fd7113f70bb2a8dcb652c78b46df79fdd156cc" checksum = "e9e5af333869e3c080f8c9b15aa78d27071c8ca7eee8a86ad0a32eefd9a1ba1e"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"cow-utils", "cow-utils",
@@ -1404,9 +1403,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_ast_macros" name = "oxc_ast_macros"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab1dff20655433e64452ee12a9c87dba5d307a47fc8e155b4e37d746d66b37dd" checksum = "6eca883d41a92aa68620385a28bbc6d016221982e7d22c4120fd1a7ffcb9be8b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -1415,9 +1414,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_cfg" name = "oxc_cfg"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f289d7b40c97d1b7d2a8cf1528cd5af0f3f4e832dffdec20b6485b9d6240de0b" checksum = "768c0e8e0cd1ff280c39b5d291fab9937419a70bc9964cc1c4fe7e57541fbdd9"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"itertools", "itertools",
@@ -1430,11 +1429,10 @@ dependencies = [
[[package]] [[package]]
name = "oxc_codegen" name = "oxc_codegen"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aa20c10288eacf1a8ec93981198c1428c7c9d0436ae7e0f0037a02e87535c94" checksum = "15bf00ecbc5cc550a9fe34aff598ea826e7c8fbb4f988f6fe08bd5521bdd4b6c"
dependencies = [ dependencies = [
"assert-unchecked",
"bitflags", "bitflags",
"cow-utils", "cow-utils",
"nonmax", "nonmax",
@@ -1452,9 +1450,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_data_structures" name = "oxc_data_structures"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e902c119eca30d0016d3d2aed31052d6e3ef1f594d71ca3fe193286771a8088d" checksum = "cd9268d8b0aefbc7d7c175e673c34f8b1be450f5e0af838ec3a2d5224fd116bb"
dependencies = [ dependencies = [
"assert-unchecked", "assert-unchecked",
"ropey", "ropey",
@@ -1462,9 +1460,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_diagnostics" name = "oxc_diagnostics"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "640a5f40eb0e09725a5e9c90e88727272c999d9805403747e2140cab2a742b6e" checksum = "fd9549fb1a8f37c4e5fa61fadcaf439402e9c1541d3442987594e9d173389443"
dependencies = [ dependencies = [
"cow-utils", "cow-utils",
"oxc-miette", "oxc-miette",
@@ -1472,9 +1470,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_ecmascript" name = "oxc_ecmascript"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87543a59fd0d2d27102ccd8eee9f954564f7e82124992cc21975bbbe75851ce4" checksum = "833da37fb0dcf12e58f2380dadade222c5b5caf2494f6ed0b4355f74af46d453"
dependencies = [ dependencies = [
"cow-utils", "cow-utils",
"num-bigint", "num-bigint",
@@ -1486,24 +1484,21 @@ dependencies = [
[[package]] [[package]]
name = "oxc_estree" name = "oxc_estree"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1efeb3bb8ce6bced1e382e49d3978a2e5067cdc3881584f1cdef5aaf52d28b4" checksum = "cb42f17f05b19f20edfef46916e62bdd03ecde89dc06e131b27adc715c19e1dc"
dependencies = [
"itoa",
]
[[package]] [[package]]
name = "oxc_index" name = "oxc_index"
version = "2.0.0" version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eca5d9726cd0a6e433debe003b7bc88b2ecad0bb6109f0cef7c55e692139a34" checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
[[package]] [[package]]
name = "oxc_mangler" name = "oxc_mangler"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74396fefe8eb5a62b2d79023b468f3012d0daa25b4a6fdff64816ade625a6dfc" checksum = "ef1718e6788566d0d7d72596526c4e8c9b666939fa8b1948914a90a3122fe835"
dependencies = [ dependencies = [
"fixedbitset", "fixedbitset",
"itertools", "itertools",
@@ -1517,9 +1512,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_minifier" name = "oxc_minifier"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc9099e8cb43a9362ffde2cfcb5d69bbbf87f0303c36d6209379b4f88f3d866" checksum = "358a1f68e7f76062ba3c53f140d52d50d34dfdeb53e5e7bbffdd8c19eb212b3f"
dependencies = [ dependencies = [
"cow-utils", "cow-utils",
"oxc_allocator", "oxc_allocator",
@@ -1538,9 +1533,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_parser" name = "oxc_parser"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0f5195b60a980baf22e34c801c8a37901ee21149e523d2a2fc68111991dd0b6" checksum = "416b26d59eb0c273239620de864c21f2bbfd098510dff92af554e2a0a464b70d"
dependencies = [ dependencies = [
"assert-unchecked", "assert-unchecked",
"bitflags", "bitflags",
@@ -1561,9 +1556,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_regular_expression" name = "oxc_regular_expression"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6b7199022f1e8a071a118737884cdf75bd3ba869471fd500f00dd4612a4faeb" checksum = "691d451203f63304fd68cacac3bbbcea6578a545e2681d761eee53b47cc2a8ce"
dependencies = [ dependencies = [
"oxc_allocator", "oxc_allocator",
"oxc_ast_macros", "oxc_ast_macros",
@@ -1577,9 +1572,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_semantic" name = "oxc_semantic"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55da371ce5345676e602719ce048eb2343b3e7b1798c6275e06ea431917340af" checksum = "c9d91a99031a2640eab43251d3108b129981c8dc049ded28d94378db929c140f"
dependencies = [ dependencies = [
"assert-unchecked", "assert-unchecked",
"itertools", "itertools",
@@ -1599,9 +1594,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_sourcemap" name = "oxc_sourcemap"
version = "1.0.9" version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb048fad7eee078b23dc3f3be7aba94eeba596c892c9255fc2646fba232a2ec" checksum = "00b84540840cb31eaea9548f93a271da369d69792bd7149bff1a44b0af9eb7a1"
dependencies = [ dependencies = [
"base64-simd", "base64-simd",
"cfg-if", "cfg-if",
@@ -1613,9 +1608,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_span" name = "oxc_span"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3cbaec227f5bb91e52a04f9028915973cbb7522b123b41af2d3189eb1ce41d8" checksum = "bf5c92fba70b2fe8e1a34f249447915338b5165c6a0f563ff9b14a99204d0ba3"
dependencies = [ dependencies = [
"compact_str", "compact_str",
"oxc-miette", "oxc-miette",
@@ -1626,9 +1621,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_syntax" name = "oxc_syntax"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91bd42fae0c0e6212ea704ed533df47748002abc0af5496911fdb09735028a77" checksum = "1339878529f3a78b8b365d23e62a326dfc3dd21b4c128619105b83d2bc0cfbfb"
dependencies = [ dependencies = [
"assert-unchecked", "assert-unchecked",
"bitflags", "bitflags",
@@ -1647,9 +1642,9 @@ dependencies = [
[[package]] [[package]]
name = "oxc_traverse" name = "oxc_traverse"
version = "0.52.0" version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c3558bebe882232c4c6fd5f28f08ce58586970c56b486a694b29c59b8b976a8" checksum = "88b9a49b46d61f4fea58bdba8430bfbe82814c0fbdf4a291d6c57d3d9ffc5ee0"
dependencies = [ dependencies = [
"compact_str", "compact_str",
"itoa", "itoa",

4
Cargo.toml
View File

@@ -1,7 +1,7 @@
[workspace] [workspace]
resolver = "2" resolver = "2"
members = ["crates/*"] members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node" package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT" package.license = "MIT"
package.edition = "2024" package.edition = "2024"
package.version = "0.0.1" package.version = "0.0.1"
@@ -20,7 +20,7 @@ brk_server = { version = "0", path = "crates/brk_server" }
byteview = "0.5.4" byteview = "0.5.4"
color-eyre = "0.6.3" color-eyre = "0.6.3"
derive_deref = "1.1.1" derive_deref = "1.1.1"
fjall = "2.6.5" fjall = "2.6.7"
hodor = { version = "0", path = "crates/hodor" } hodor = { version = "0", path = "crates/hodor" }
jiff = "0.2.1" jiff = "0.2.1"
log = { version = "0.4.26" } log = { version = "0.4.26" }

28
README.md
View File

@@ -5,71 +5,71 @@
## Description ## Description
The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node. The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node.
## Crates ## Crates
`brk` ### `brk`
Wrapper around all other `brk-*` crates. Wrapper around all other `brk-*` crates.
> Status: ⚠️ > Status: ⚠️
`brk_cli` ### `brk_cli`
A command line interface to interact with the Bitcoin Research Kit. A command line interface to interact with the Bitcoin Research Kit.
> Status: ❌ > Status: ❌
`brk_computer` ### `brk_computer`
A Bitcoin dataset computer, built on top of brk_indexer. A Bitcoin dataset computer, built on top of brk_indexer.
> Status: ⚠️ > Status: ⚠️
`brk_core` ### `brk_core`
The Core (Structs and Errors) of the Bitcoin Research Kit. The Core (Structs and Errors) of the Bitcoin Research Kit.
> Status: ✅ > Status: ✅
`brk_fetcher` ### `brk_fetcher`
A Bitcoin price fetcher. A Bitcoin price fetcher.
> Status: ⚠️ > Status: ✅
`brk_indexer` ### `brk_indexer`
A Bitcoin Core indexer built on top of brk_parser. A Bitcoin Core indexer built on top of brk_parser.
> Status: ⚠️ > Status: ✅
`brk_logger` ### `brk_logger`
A clean logger used in the Bitcoin Research Kit. A clean logger used in the Bitcoin Research Kit.
> Status: ✅ > Status: ✅
`brk_parser` ### `brk_parser`
A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust. A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust.
> Status: ✅ > Status: ✅
`brk_server` ### `brk_server`
A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer. A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer.
> Status: ⚠️ > Status: ⚠️
`hodor` ### `hodor`
Hold the door, an exit blocker built on top of ctrlc. Hold the door, an exit blocker built on top of ctrlc.
> Status: ✅ > Status: ✅
`storable_vec` ### `storable_vec`
A very small, fast, efficient and simple storable Vec. A very small, fast, efficient and simple storable Vec.

1
crates/brk_fetcher/Cargo.toml
View File

@@ -14,4 +14,3 @@ jiff = { workspace = true }
log = { workspace = true } log = { workspace = true }
minreq = { workspace = true } minreq = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
storable_vec = { workspace = true }

5
crates/brk_fetcher/src/fetchers/binance.rs
View File

@@ -10,7 +10,6 @@ use brk_core::{Cents, OHLCCents, Timestamp};
use color_eyre::eyre::{ContextCompat, eyre}; use color_eyre::eyre::{ContextCompat, eyre};
use log::info; use log::info;
use serde_json::Value; use serde_json::Value;
use storable_vec::STATELESS;
use crate::{Close, Date, Dollars, High, Low, Open, Pricer, fetchers::retry}; use crate::{Close, Date, Dollars, High, Low, Open, Pricer, fetchers::retry};
@@ -39,7 +38,7 @@ impl Binance {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self._1mn.replace(Self::fetch_1mn()?); self._1mn.replace(Self::fetch_1mn()?);
} }
Pricer::<STATELESS>::find_height_ohlc( Pricer::find_height_ohlc(
self._1mn.as_ref().unwrap(), self._1mn.as_ref().unwrap(),
timestamp, timestamp,
previous_timestamp, previous_timestamp,
@@ -90,7 +89,7 @@ impl Binance {
if self.har.is_none() { if self.har.is_none() {
self.har.replace(self.read_har().unwrap_or_default()); self.har.replace(self.read_har().unwrap_or_default());
} }
Pricer::<STATELESS>::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har") Pricer::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har")
} }
fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> { fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {

3
crates/brk_fetcher/src/fetchers/kraken.rs
View File

@@ -4,7 +4,6 @@ use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestam
use color_eyre::eyre::ContextCompat; use color_eyre::eyre::ContextCompat;
use log::info; use log::info;
use serde_json::Value; use serde_json::Value;
use storable_vec::STATELESS;
use crate::{Pricer, fetchers::retry}; use crate::{Pricer, fetchers::retry};
@@ -23,7 +22,7 @@ impl Kraken {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self._1mn.replace(Self::fetch_1mn()?); self._1mn.replace(Self::fetch_1mn()?);
} }
Pricer::<STATELESS>::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m") Pricer::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
} }
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> { fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {

34
crates/brk_fetcher/src/lib.rs
View File

@@ -1,25 +1,20 @@
use std::{ use std::{collections::BTreeMap, fs, path::Path};
collections::BTreeMap,
fs,
path::{Path, PathBuf},
};
use brk_core::{Cents, Close, Date, Dateindex, Dollars, Height, High, Low, OHLCCents, Open, Timestamp}; use brk_core::{Cents, Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use color_eyre::eyre::Error; use color_eyre::eyre::Error;
mod fetchers; mod fetchers;
// use brk_indexer::Indexer; // use brk_indexer::Indexer;
pub use fetchers::*; pub use fetchers::*;
use storable_vec::{AnyJsonStorableVec, AnyStorableVec, SINGLE_THREAD, StorableVec, Version};
pub struct Pricer<const MODE: u8> { pub struct Pricer {
binance: Binance, binance: Binance,
kraken: Kraken, kraken: Kraken,
kibo: Kibo, kibo: Kibo,
} }
impl<const MODE: u8> Pricer<MODE> { impl Pricer {
pub fn import(path: &Path) -> color_eyre::Result<Self> { pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?; fs::create_dir_all(path)?;
@@ -30,27 +25,6 @@ impl<const MODE: u8> Pricer<MODE> {
}) })
} }
pub fn compute_if_needed(&mut self) {
// TODO: Remove all outdated
// indexer
// .vecs
// .height_to_timestamp
// .iter_from(Height::default(), |v| Ok(()));
// self.open
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open);
// self.high
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high);
// self.low
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low);
// self.close
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close);
}
fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLCCents> { fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
todo!(); todo!();
// if self.ohlc.date.is_key_safe(date) { // if self.ohlc.date.is_key_safe(date) {

3
crates/brk_fetcher/src/main.rs
View File

@@ -1,5 +1,4 @@
use brk_fetcher::{Binance, Kibo, Kraken}; use brk_fetcher::Binance;
use serde_json::Value;
fn main() -> color_eyre::Result<()> { fn main() -> color_eyre::Result<()> {
color_eyre::install()?; color_eyre::install()?;

75
crates/brk_indexer/src/lib.rs
View File

@@ -14,7 +14,7 @@ pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut}; use bitcoin::{Transaction, TxIn, TxOut};
use color_eyre::eyre::{ContextCompat, eyre}; use color_eyre::eyre::{ContextCompat, eyre};
use hodor::Exit; use hodor::Hodor;
use log::info; use log::info;
use rayon::prelude::*; use rayon::prelude::*;
use storable_vec::CACHED_GETS; use storable_vec::CACHED_GETS;
@@ -35,10 +35,9 @@ pub struct Indexer<const MODE: u8> {
impl<const MODE: u8> Indexer<MODE> { impl<const MODE: u8> Indexer<MODE> {
pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> { pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
// info!("Increasing limit of opened files to 210_000...");
rlimit::setrlimit( rlimit::setrlimit(
rlimit::Resource::NOFILE, rlimit::Resource::NOFILE,
210_000, rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().0.max(210_000),
rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1, rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
)?; )?;
@@ -51,7 +50,7 @@ impl<const MODE: u8> Indexer<MODE> {
} }
impl Indexer<CACHED_GETS> { impl Indexer<CACHED_GETS> {
pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> { pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, hodor: &Hodor) -> color_eyre::Result<()> {
let check_collisions = true; let check_collisions = true;
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| { let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| {
@@ -60,25 +59,29 @@ impl Indexer<CACHED_GETS> {
indexes indexes
}); });
// dbg!(starting_indexes); hodor.hold();
// panic!(); self.stores.rollback_if_needed(&self.vecs, &starting_indexes)?;
self.vecs.rollback_if_needed(&starting_indexes)?;
hodor.release();
exit.block(); let export_if_needed = |stores: &mut Fjalls,
self.stores.rollback(&self.vecs, &starting_indexes)?; vecs: &mut StorableVecs<CACHED_GETS>,
self.vecs.rollback(&starting_indexes)?; height: Height,
exit.unblock(); hodor: &Hodor|
-> color_eyre::Result<()> {
if height == 0 || height % SNAPSHOT_BLOCK_RANGE != 0 || hodor.triggered() {
return Ok(());
}
let export = info!("Exporting...");
|stores: &mut Fjalls, vecs: &mut StorableVecs<CACHED_GETS>, height: Height| -> color_eyre::Result<()> { hodor.hold();
info!("Exporting..."); stores.commit(height)?;
exit.block(); info!("Exported stores");
stores.commit(height)?; vecs.flush(height)?;
info!("Exported stores"); info!("Exported vecs");
vecs.flush(height)?; hodor.release();
info!("Exported vecs"); Ok(())
exit.unblock(); };
Ok(())
};
let vecs = &mut self.vecs; let vecs = &mut self.vecs;
let stores = &mut self.stores; let stores = &mut self.stores;
@@ -91,9 +94,8 @@ impl Indexer<CACHED_GETS> {
info!("Started indexing..."); info!("Started indexing...");
parser.parse(Some(idxs.height), None) parser.parse(Some(idxs.height), None).iter().try_for_each(
.iter() |(height, block, blockhash)| -> color_eyre::Result<()> {
.try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {height}..."); info!("Indexing block {height}...");
idxs.height = height; idxs.height = height;
@@ -115,8 +117,10 @@ impl Indexer<CACHED_GETS> {
.insert_if_needed(blockhash_prefix, height, height); .insert_if_needed(blockhash_prefix, height, height);
vecs.height_to_blockhash.push_if_needed(height, blockhash)?; vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
vecs.height_to_difficulty.push_if_needed(height, block.header.difficulty_float())?; vecs.height_to_difficulty
vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?; .push_if_needed(height, block.header.difficulty_float())?;
vecs.height_to_timestamp
.push_if_needed(height, Timestamp::from(block.header.time))?;
vecs.height_to_size.push_if_needed(height, block.total_size())?; vecs.height_to_size.push_if_needed(height, block.total_size())?;
vecs.height_to_weight.push_if_needed(height, block.weight().into())?; vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
@@ -583,10 +587,12 @@ impl Indexer<CACHED_GETS> {
let only_known_dup_txids = [ let only_known_dup_txids = [
bitcoin::Txid::from_str( bitcoin::Txid::from_str(
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599", "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599",
)?.into(), )?
.into(),
bitcoin::Txid::from_str( bitcoin::Txid::from_str(
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468", "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468",
)?.into(), )?
.into(),
]; ];
let is_dup = only_known_dup_txids.contains(prev_txid); let is_dup = only_known_dup_txids.contains(prev_txid);
@@ -623,18 +629,15 @@ impl Indexer<CACHED_GETS> {
idxs.push_future_if_needed(vecs)?; idxs.push_future_if_needed(vecs)?;
let should_snapshot = height != 0 && height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked(); export_if_needed(stores, vecs, height, hodor)?;
if should_snapshot {
export(stores, vecs, height)?;
}
Ok(()) Ok(())
})?; },
)?;
if idxs.height % SNAPSHOT_BLOCK_RANGE != 0 { export_if_needed(stores, vecs, idxs.height, hodor)?;
export(stores, vecs, idxs.height)?;
}
// To make sure that Fjall had the time to flush everything properly
sleep(Duration::from_millis(100)); sleep(Duration::from_millis(100));
Ok(()) Ok(())

8
crates/brk_indexer/src/main.rs
View File

@@ -5,21 +5,21 @@ use brk_parser::{
Parser, Parser,
rpc::{self}, rpc::{self},
}; };
use hodor::Exit; use hodor::Hodor;
use log::info; use log::info;
use storable_vec::CACHED_GETS; use storable_vec::CACHED_GETS;
fn main() -> color_eyre::Result<()> { fn main() -> color_eyre::Result<()> {
color_eyre::install()?; color_eyre::install()?;
brk_logger::init(None); brk_logger::init(Some(Path::new(".log")));
let data_dir = Path::new("../../../bitcoin"); let data_dir = Path::new("../../../bitcoin");
let rpc = Box::leak(Box::new(rpc::Client::new( let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332", "http://localhost:8332",
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
)?)); )?));
let exit = Exit::new(); let hodor = Hodor::new();
let parser = Parser::new(data_dir, rpc); let parser = Parser::new(data_dir, rpc);
@@ -32,7 +32,7 @@ fn main() -> color_eyre::Result<()> {
let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../../_outputs/indexes"))?; let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../../_outputs/indexes"))?;
indexer.index(&parser, rpc, &exit)?; indexer.index(&parser, rpc, &hodor)?;
info!("Took: {:?}", i.elapsed()); info!("Took: {:?}", i.elapsed());

52
crates/brk_indexer/src/storage/fjalls/base.rs
View File

@@ -1,6 +1,8 @@
use std::{ use std::{
collections::{BTreeMap, BTreeSet}, collections::{BTreeMap, BTreeSet},
error, mem, error,
fmt::Debug,
mem,
path::Path, path::Path,
}; };
@@ -27,25 +29,32 @@ const CHECK_COLLISISONS: bool = true;
impl<K, V> Store<K, V> impl<K, V> Store<K, V>
where where
K: Into<ByteView> + Ord + Immutable + IntoBytes, K: Debug + Into<ByteView> + Ord + Immutable + IntoBytes,
V: Into<ByteView> + TryFrom<ByteView>, V: Debug + Into<ByteView> + TryFrom<ByteView>,
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static, <V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
{ {
pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> { pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
let meta = StoreMeta::checked_open(path, version)?; let meta = StoreMeta::checked_open(path, version)?;
let keyspace = if let Ok(keyspace) = Self::open_keyspace(path) {
keyspace let keyspace = match Self::open_keyspace(path) {
} else { Ok(keyspace) => keyspace,
meta.reset()?; Err(e) => {
return Self::import(path, version); dbg!(e);
meta.reset()?;
return Self::import(path, version);
}
}; };
let part = if let Ok(part) = Self::open_partition_handle(&keyspace) {
part let part = match Self::open_partition_handle(&keyspace) {
} else { Ok(part) => part,
drop(keyspace); Err(e) => {
meta.reset()?; dbg!(e);
return Self::import(path, version); drop(keyspace);
meta.reset()?;
return Self::import(path, version);
}
}; };
let rtx = keyspace.read_tx(); let rtx = keyspace.read_tx();
Ok(Self { Ok(Self {
@@ -71,8 +80,8 @@ where
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) { pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) { if self.needs(height) {
if !self.dels.is_empty() { if !self.dels.is_empty() {
unreachable!("Shouldn't reach this");
// self.dels.remove(&key); // self.dels.remove(&key);
unreachable!("Shouldn't reach this");
} }
self.puts.insert(key, value); self.puts.insert(key, value);
} }
@@ -83,7 +92,10 @@ where
unreachable!("Shouldn't reach this"); unreachable!("Shouldn't reach this");
// self.puts.remove(&key); // self.puts.remove(&key);
} }
self.dels.insert(key); // dbg!(&key);
if !self.dels.insert(key) {
unreachable!();
}
} }
pub fn commit(&mut self, height: Height) -> Result<()> { pub fn commit(&mut self, height: Height) -> Result<()> {
@@ -101,8 +113,14 @@ where
mem::take(&mut self.puts).into_iter().for_each(|(key, value)| { mem::take(&mut self.puts).into_iter().for_each(|(key, value)| {
if CHECK_COLLISISONS { if CHECK_COLLISISONS {
#[allow(unused_must_use)]
if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) { if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) {
dbg!(value, &self.meta); dbg!(
&key,
V::try_from(value.into()).unwrap(),
&self.meta,
self.rtx.get(&self.part, key.as_bytes())
);
unreachable!(); unreachable!();
} }
} }

3
crates/brk_indexer/src/storage/fjalls/meta.rs
View File

@@ -84,9 +84,6 @@ impl StoreMeta {
path.join("height") path.join("height")
} }
// fn read_length(&self) -> color_eyre::Result<usize> {
// Self::read_length_(&self.pathbuf)
// }
fn read_length_(path: &Path) -> color_eyre::Result<usize> { fn read_length_(path: &Path) -> color_eyre::Result<usize> {
Ok(fs::read(Self::path_length(path)) Ok(fs::read(Self::path_length(path))
.map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default()) .map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default())

38
crates/brk_indexer/src/storage/fjalls/mod.rs
View File

@@ -21,18 +21,27 @@ pub struct Fjalls {
impl Fjalls { impl Fjalls {
pub fn import(path: &Path) -> color_eyre::Result<Self> { pub fn import(path: &Path) -> color_eyre::Result<Self> {
let addresshash_to_addressindex = Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))?; thread::scope(|scope| {
let blockhash_prefix_to_height = Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?; let addresshash_to_addressindex =
let txid_prefix_to_txindex = Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?; scope.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::from(1)));
let blockhash_prefix_to_height =
scope.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1)));
let txid_prefix_to_txindex =
scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
Ok(Self { Ok(Self {
addresshash_to_addressindex, addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?,
blockhash_prefix_to_height, blockhash_prefix_to_height: blockhash_prefix_to_height.join().unwrap()?,
txid_prefix_to_txindex, txid_prefix_to_txindex: txid_prefix_to_txindex.join().unwrap()?,
})
}) })
} }
pub fn rollback(&mut self, vecs: &StorableVecs<CACHED_GETS>, starting_indexes: &Indexes) -> color_eyre::Result<()> { pub fn rollback_if_needed(
&mut self,
vecs: &StorableVecs<CACHED_GETS>,
starting_indexes: &Indexes,
) -> color_eyre::Result<()> {
vecs.height_to_blockhash vecs.height_to_blockhash
.iter_from(starting_indexes.height, |(_, blockhash)| { .iter_from(starting_indexes.height, |(_, blockhash)| {
let blockhash = blockhash.as_ref(); let blockhash = blockhash.as_ref();
@@ -41,12 +50,13 @@ impl Fjalls {
Ok(()) Ok(())
})?; })?;
vecs.txindex_to_txid.iter_from(starting_indexes.txindex, |(_, txid)| { vecs.txindex_to_txid
let txid = txid.as_ref(); .iter_from(starting_indexes.txindex, |(_txindex, txid)| {
let txid_prefix = TxidPrefix::from(txid); let txid = txid.as_ref();
self.txid_prefix_to_txindex.remove(txid_prefix); let txid_prefix = TxidPrefix::from(txid);
Ok(()) self.txid_prefix_to_txindex.remove(txid_prefix);
})?; Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? { if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? {
let mut index = index.into_inner(); let mut index = index.into_inner();

8
crates/brk_indexer/src/storage/storable_vecs/base.rs
View File

@@ -48,14 +48,6 @@ where
fn path_height_(path: &Path) -> PathBuf { fn path_height_(path: &Path) -> PathBuf {
path.join("height") path.join("height")
} }
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
}
#[allow(unused)]
pub fn has(&self, height: Height) -> bool {
!self.needs(height)
}
} }
impl<I, T, const MODE: u8> Deref for StorableVec<I, T, MODE> { impl<I, T, const MODE: u8> Deref for StorableVec<I, T, MODE> {

2
crates/brk_indexer/src/storage/storable_vecs/mod.rs
View File

@@ -187,7 +187,7 @@ impl<const MODE: u8> StorableVecs<MODE> {
}) })
} }
pub fn rollback(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> { pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> {
let saved_height = starting_indexes.height.decremented(); let saved_height = starting_indexes.height.decremented();
// We don't want to override the starting indexes so we cut from n + 1 // We don't want to override the starting indexes so we cut from n + 1

2
crates/brk_server/Cargo.toml
View File

@@ -15,7 +15,7 @@ color-eyre = { workspace = true }
derive_deref = { workspace = true } derive_deref = { workspace = true }
jiff = { workspace = true } jiff = { workspace = true }
log = { workspace = true } log = { workspace = true }
oxc = { version = "0.52.0", features = ["codegen", "minifier"] } oxc = { version = "0.53.0", features = ["codegen", "minifier"] }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
storable_vec = { workspace = true } storable_vec = { workspace = true }