From 53175c9ed7b152d1e1debd47bce2df1cf02ceec9 Mon Sep 17 00:00:00 2001 From: nym21 Date: Mon, 24 Feb 2025 14:08:51 +0100 Subject: [PATCH] core: init --- Cargo.lock | 32 +++-- Cargo.toml | 3 +- LICENSE.md | 2 +- crates/brk_computer/Cargo.toml | 8 +- crates/brk_computer/src/lib.rs | 4 +- crates/brk_computer/src/main.rs | 2 - crates/brk_computer/src/storage/fjalls.rs | 3 +- .../brk_computer/src/storage/storable_vecs.rs | 5 +- crates/brk_computer/src/structs/mod.rs | 11 -- crates/brk_computer/src/structs/ohlc.rs | 36 ----- crates/brk_core/Cargo.toml | 20 +++ crates/brk_core/src/error.rs | 49 +++++++ crates/brk_core/src/lib.rs | 5 + .../src/structs/addressbytes.rs | 21 +-- .../src/structs/addressindex.rs | 4 +- .../src/structs/addressindextxoutindex.rs | 8 +- .../src/structs/addresstype.rs | 2 +- .../src/structs/addresstypeindex.rs | 0 .../src/structs/bitcoin.rs | 2 +- .../src/structs/blockhash.rs | 9 +- .../src/structs/cents.rs | 0 .../src/structs/compressed.rs | 8 +- .../src/structs/date.rs | 3 +- .../src/structs/dateindex.rs | 10 +- .../src/structs/dollars.rs | 0 .../src/structs/feerate.rs | 0 .../src => brk_core/src/structs}/height.rs | 15 +- .../src/structs/locktime.rs | 3 +- crates/brk_core/src/structs/mod.rs | 55 ++++++++ crates/brk_core/src/structs/ohlc.rs | 130 ++++++++++++++++++ .../src/structs/sats.rs | 4 +- .../src/structs/timestamp.rs | 0 .../src/structs/txid.rs | 0 .../src/structs/txindex.rs | 4 +- .../src/structs/txinindex.rs | 0 .../src/structs/txoutindex.rs | 0 .../src/structs/txversion.rs | 0 .../src/structs/unit.rs | 0 .../src/structs/vin.rs | 0 .../src/structs/vout.rs | 0 .../src/structs/weight.rs | 0 crates/brk_fetcher/Cargo.toml | 4 +- crates/brk_fetcher/src/fetchers/binance.rs | 36 +++-- crates/brk_fetcher/src/fetchers/kibo.rs | 22 ++- crates/brk_fetcher/src/fetchers/kraken.rs | 26 ++-- crates/brk_fetcher/src/lib.rs | 22 +-- crates/brk_fetcher/src/main.rs | 1 - crates/brk_fetcher/src/structs/close.rs | 27 ---- crates/brk_fetcher/src/structs/high.rs | 38 ----- crates/brk_fetcher/src/structs/low.rs | 38 ----- crates/brk_fetcher/src/structs/mod.rs | 19 --- crates/brk_fetcher/src/structs/open.rs | 38 ----- crates/brk_indexer/Cargo.toml | 7 +- crates/brk_indexer/src/lib.rs | 17 ++- crates/brk_indexer/src/main.rs | 2 +- crates/brk_indexer/src/storage/fjalls/base.rs | 2 +- crates/brk_indexer/src/storage/fjalls/mod.rs | 7 +- .../src/storage/storable_vecs/base.rs | 6 +- .../src/storage/storable_vecs/mod.rs | 18 +-- crates/brk_indexer/src/structs/indexes.rs | 33 +++-- crates/brk_indexer/src/structs/mod.rs | 34 ----- crates/brk_parser/Cargo.toml | 4 +- .../brk_parser/src/blk_index_to_blk_recap.rs | 3 +- crates/brk_parser/src/blk_recap.rs | 3 +- crates/brk_parser/src/lib.rs | 3 +- 65 files changed, 444 insertions(+), 424 deletions(-) delete mode 100644 crates/brk_computer/src/structs/mod.rs delete mode 100644 crates/brk_computer/src/structs/ohlc.rs create mode 100644 crates/brk_core/Cargo.toml create mode 100644 crates/brk_core/src/error.rs create mode 100644 crates/brk_core/src/lib.rs rename crates/{brk_indexer => brk_core}/src/structs/addressbytes.rs (91%) rename crates/{brk_indexer => brk_core}/src/structs/addressindex.rs (97%) rename crates/{brk_computer => brk_core}/src/structs/addressindextxoutindex.rs (86%) rename crates/{brk_indexer => brk_core}/src/structs/addresstype.rs (97%) rename crates/{brk_indexer => brk_core}/src/structs/addresstypeindex.rs (100%) rename crates/{brk_computer => brk_core}/src/structs/bitcoin.rs (91%) rename crates/{brk_indexer => brk_core}/src/structs/blockhash.rs (88%) rename crates/{brk_fetcher => brk_core}/src/structs/cents.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/compressed.rs (96%) rename crates/{brk_fetcher => brk_core}/src/structs/date.rs (96%) rename crates/{brk_fetcher => brk_core}/src/structs/dateindex.rs (87%) rename crates/{brk_fetcher => brk_core}/src/structs/dollars.rs (100%) rename crates/{brk_computer => brk_core}/src/structs/feerate.rs (100%) rename crates/{brk_parser/src => brk_core/src/structs}/height.rs (93%) rename crates/{brk_indexer => brk_core}/src/structs/locktime.rs (95%) create mode 100644 crates/brk_core/src/structs/mod.rs create mode 100644 crates/brk_core/src/structs/ohlc.rs rename crates/{brk_indexer => brk_core}/src/structs/sats.rs (97%) rename crates/{brk_indexer => brk_core}/src/structs/timestamp.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/txid.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/txindex.rs (97%) rename crates/{brk_indexer => brk_core}/src/structs/txinindex.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/txoutindex.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/txversion.rs (100%) rename crates/{brk_computer => brk_core}/src/structs/unit.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/vin.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/vout.rs (100%) rename crates/{brk_indexer => brk_core}/src/structs/weight.rs (100%) delete mode 100644 crates/brk_fetcher/src/structs/close.rs delete mode 100644 crates/brk_fetcher/src/structs/high.rs delete mode 100644 crates/brk_fetcher/src/structs/low.rs delete mode 100644 crates/brk_fetcher/src/structs/mod.rs delete mode 100644 crates/brk_fetcher/src/structs/open.rs diff --git a/Cargo.lock b/Cargo.lock index e6fd1bc01..27ea3f0a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,15 +342,26 @@ version = "0.0.0" name = "brk_computer" version = "0.0.0" dependencies = [ - "brk_fetcher", + "brk_core", "brk_indexer", "brk_parser", - "byteview", "color-eyre", - "derive_deref", - "fjall", "hodor", "storable_vec", +] + +[[package]] +name = "brk_core" +version = "0.0.0" +dependencies = [ + "bitcoin", + "bitcoincore-rpc", + "byteview", + "derive_deref", + "jiff", + "rapidhash", + "serde", + "serde_bytes", "zerocopy 0.8.20", ] @@ -358,17 +369,15 @@ dependencies = [ name = "brk_fetcher" version = "0.0.0" dependencies = [ + "brk_core", "brk_indexer", "brk_logger", "color-eyre", - "derive_deref", "jiff", "log", "minreq", - "serde", "serde_json", "storable_vec", - "zerocopy 0.8.20", ] [[package]] @@ -376,20 +385,17 @@ name = "brk_indexer" version = "0.0.0" dependencies = [ "bitcoin", + "bitcoincore-rpc", + "brk_core", "brk_logger", "brk_parser", "byteview", "color-eyre", - "derive_deref", "fjall", "hodor", - "jiff", "log", - "rapidhash", "rayon", "rlimit", - "serde", - "serde_bytes", "storable_vec", "zerocopy 0.8.20", ] @@ -409,7 +415,7 @@ version = "0.0.0" dependencies = [ "bitcoin", "bitcoincore-rpc", - "byteview", + "brk_core", "crossbeam", "derive_deref", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 48a9425e5..9036a25ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,9 @@ package.version = "0.0.0" [workspace.dependencies] bitcoin = { version = "0.32.5", features = ["serde"] } +bitcoincore-rpc = "0.19.0" brk_computer = { version = "0", path = "crates/brk_computer" } +brk_core = { version = "0", path = "crates/brk_core" } brk_fetcher = { version = "0", path = "crates/brk_fetcher" } brk_indexer = { version = "0", path = "crates/brk_indexer" } brk_parser = { version = "0", path = "crates/brk_parser" } @@ -23,7 +25,6 @@ log = { version = "0.4.26" } minreq = { version = "2.13.2", features = ["https", "serde_json"] } rayon = "1.10.0" serde = { version = "1.0.218", features = ["derive"] } -serde_bytes = "0.11.15" serde_json = { version = "1.0.139", features = ["float_roundtrip"] } storable_vec = { version = "0", path = "crates/storable_vec", features = [ "json", diff --git a/LICENSE.md b/LICENSE.md index 4827b5007..292c5cd8b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 kibō.money +Copyright (c) 2025 bitcoinresearchkit, kibō.money Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index add4956ff..a3d0de575 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -1,18 +1,14 @@ [package] name = "brk_computer" -description = "A Bitcoin dataset computer built on top of brk_indexer and brk_fetcher" +description = "A Bitcoin dataset computer, built on top of brk_indexer" version = { workspace = true } edition = { workspace = true } license = { workspace = true } [dependencies] -brk_fetcher = { workspace = true } +brk_core = { workspace = true } brk_indexer = { workspace = true } brk_parser = { workspace = true } -byteview = { workspace = true } color-eyre = { workspace = true } -derive_deref = { workspace = true } -fjall = { workspace = true } hodor = { workspace = true } storable_vec = { workspace = true } -zerocopy = { workspace = true } diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 5ae619db4..f7f2270a1 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -5,12 +5,10 @@ pub use brk_parser::rpc; use hodor::Exit; mod storage; -mod structs; -use brk_fetcher::Date; +use brk_core::Date; use storable_vec::SINGLE_THREAD; use storage::{Fjalls, StorableVecs}; -pub use structs::*; pub struct Computer { path: PathBuf, diff --git a/crates/brk_computer/src/main.rs b/crates/brk_computer/src/main.rs index acf96bc03..678cabe18 100644 --- a/crates/brk_computer/src/main.rs +++ b/crates/brk_computer/src/main.rs @@ -4,8 +4,6 @@ use brk_computer::Computer; use brk_indexer::Indexer; use hodor::Exit; -mod structs; - pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_computer/src/storage/fjalls.rs b/crates/brk_computer/src/storage/fjalls.rs index 38221d0be..f5af2c92a 100644 --- a/crates/brk_computer/src/storage/fjalls.rs +++ b/crates/brk_computer/src/storage/fjalls.rs @@ -1,10 +1,9 @@ use std::path::Path; +use brk_core::{AddressindexTxoutindex, Unit}; use brk_indexer::Store; use storable_vec::Version; -use crate::structs::{AddressindexTxoutindex, Unit}; - pub struct Fjalls { pub address_to_utxos_received: Store, pub address_to_utxos_spent: Store, diff --git a/crates/brk_computer/src/storage/storable_vecs.rs b/crates/brk_computer/src/storage/storable_vecs.rs index a39ec0de4..dd0a83116 100644 --- a/crates/brk_computer/src/storage/storable_vecs.rs +++ b/crates/brk_computer/src/storage/storable_vecs.rs @@ -1,11 +1,8 @@ use std::{fs, path::Path}; -use brk_fetcher::{Date, Dateindex}; -use brk_indexer::{Addressindex, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex}; +use brk_core::{Addressindex, Date, Dateindex, Feerate, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex}; use storable_vec::{StorableVec, Version}; -use crate::structs::Feerate; - // mod base; // use base::*; diff --git a/crates/brk_computer/src/structs/mod.rs b/crates/brk_computer/src/structs/mod.rs deleted file mode 100644 index 7edf1230f..000000000 --- a/crates/brk_computer/src/structs/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod addressindextxoutindex; -mod bitcoin; -mod feerate; -// mod ohlc; -mod unit; - -pub use addressindextxoutindex::*; -pub use bitcoin::*; -pub use feerate::*; -// pub use ohlc::*; -pub use unit::*; diff --git a/crates/brk_computer/src/structs/ohlc.rs b/crates/brk_computer/src/structs/ohlc.rs deleted file mode 100644 index a37e5ba2d..000000000 --- a/crates/brk_computer/src/structs/ohlc.rs +++ /dev/null @@ -1,36 +0,0 @@ -use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use super::{Cents, Close, High, Low, Open}; - -// #[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] -// #[repr(C)] -// pub struct OHLCCents(OHLC); - -#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] -#[repr(C)] -pub struct OHLCCents(Open, High, Low, Close); - -impl OHLCCents { - pub fn open(&self) -> Open { - self.0 - } - - pub fn high(&self) -> High { - self.1 - } - - pub fn low(&self) -> Low { - self.2 - } - - pub fn close(&self) -> Close { - self.3 - } -} - -impl From<(Open, High, Low, Close)> for OHLCCents { - fn from(value: (Open, High, Low, Close)) -> Self { - Self(value.0, value.1, value.2, value.3) - } -} diff --git a/crates/brk_core/Cargo.toml b/crates/brk_core/Cargo.toml new file mode 100644 index 000000000..56baddb32 --- /dev/null +++ b/crates/brk_core/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "brk_core" +description = "The Core (Structs and Errors) of the Bitcoin Research Kit" +license.workspace = true +edition.workspace = true +version.workspace = true + +[dependencies] +bitcoin = { workspace = true } +bitcoincore-rpc = { workspace = true } +byteview = { workspace = true } +derive_deref = { workspace = true } +jiff = { workspace = true } +rapidhash = "1.4.0" +serde = { workspace = true } +serde_bytes = "0.11.15" +zerocopy = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["serde_bytes"] diff --git a/crates/brk_core/src/error.rs b/crates/brk_core/src/error.rs new file mode 100644 index 000000000..2a00c66b3 --- /dev/null +++ b/crates/brk_core/src/error.rs @@ -0,0 +1,49 @@ +use std::{ + fmt::{self, Debug}, + io, +}; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum Error { + IO(io::Error), + Jiff(jiff::Error), + ZeroCopyError, + WrongLength, + WrongAddressType, + UnindexableDate, +} + +impl From for Error { + fn from(value: io::Error) -> Self { + Self::IO(value) + } +} + +impl From for Error { + fn from(value: jiff::Error) -> Self { + Self::Jiff(value) + } +} + +impl From> for Error { + fn from(_: zerocopy::error::SizeError) -> Self { + Self::ZeroCopyError + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::IO(error) => Debug::fmt(&error, f), + Error::Jiff(error) => Debug::fmt(&error, f), + Error::ZeroCopyError => write!(f, "Zero copy convert error"), + Error::WrongLength => write!(f, "Wrong length"), + Error::WrongAddressType => write!(f, "Wrong address type"), + Error::UnindexableDate => write!(f, "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"), + } + } +} + +impl std::error::Error for Error {} diff --git a/crates/brk_core/src/lib.rs b/crates/brk_core/src/lib.rs new file mode 100644 index 000000000..1bbcf827b --- /dev/null +++ b/crates/brk_core/src/lib.rs @@ -0,0 +1,5 @@ +mod error; +mod structs; + +pub use error::*; +pub use structs::*; diff --git a/crates/brk_indexer/src/structs/addressbytes.rs b/crates/brk_core/src/structs/addressbytes.rs similarity index 91% rename from crates/brk_indexer/src/structs/addressbytes.rs rename to crates/brk_core/src/structs/addressbytes.rs index 2d679a3cd..285f0d5f4 100644 --- a/crates/brk_indexer/src/structs/addressbytes.rs +++ b/crates/brk_core/src/structs/addressbytes.rs @@ -1,9 +1,10 @@ -use brk_parser::bitcoin::ScriptBuf; -use color_eyre::eyre::eyre; +use bitcoin::ScriptBuf; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::Error; + use super::Addresstype; #[derive(Debug, PartialEq, Eq)] @@ -32,7 +33,7 @@ impl Addressbytes { } impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { - type Error = color_eyre::Report; + type Error = Error; fn try_from(tuple: (&ScriptBuf, Addresstype)) -> Result { let (script, addresstype) = tuple; @@ -43,7 +44,7 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { 67 => &bytes[1..66], _ => { dbg!(bytes); - return Err(eyre!("Wrong len")); + return Err(Error::WrongLength); } }; Ok(Self::P2PK65(P2PK65AddressBytes(U8x65::from(bytes)))) @@ -54,7 +55,7 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { 35 => &bytes[1..34], _ => { dbg!(bytes); - return Err(eyre!("Wrong len")); + return Err(Error::WrongLength); } }; Ok(Self::P2PK33(P2PK33AddressBytes(U8x33::from(bytes)))) @@ -79,11 +80,11 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes { let bytes = &script.as_bytes()[2..]; Ok(Self::P2TR(P2TRAddressBytes(U8x32::from(bytes)))) } - Addresstype::Multisig => Err(eyre!("multisig address type")), - Addresstype::PushOnly => Err(eyre!("push_only address type")), - Addresstype::Unknown => Err(eyre!("unknown address type")), - Addresstype::Empty => Err(eyre!("empty address type")), - Addresstype::OpReturn => Err(eyre!("op_return address type")), + Addresstype::Multisig => Err(Error::WrongAddressType), + Addresstype::PushOnly => Err(Error::WrongAddressType), + Addresstype::Unknown => Err(Error::WrongAddressType), + Addresstype::Empty => Err(Error::WrongAddressType), + Addresstype::OpReturn => Err(Error::WrongAddressType), } } } diff --git a/crates/brk_indexer/src/structs/addressindex.rs b/crates/brk_core/src/structs/addressindex.rs similarity index 97% rename from crates/brk_indexer/src/structs/addressindex.rs rename to crates/brk_core/src/structs/addressindex.rs index c30d2ac4e..01977413f 100644 --- a/crates/brk_indexer/src/structs/addressindex.rs +++ b/crates/brk_core/src/structs/addressindex.rs @@ -5,6 +5,8 @@ use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::Error; + #[derive( Debug, PartialEq, @@ -69,7 +71,7 @@ impl From for usize { } impl TryFrom for Addressindex { - type Error = storable_vec::Error; + type Error = Error; fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } diff --git a/crates/brk_computer/src/structs/addressindextxoutindex.rs b/crates/brk_core/src/structs/addressindextxoutindex.rs similarity index 86% rename from crates/brk_computer/src/structs/addressindextxoutindex.rs rename to crates/brk_core/src/structs/addressindextxoutindex.rs index 11a743b50..d895ca47b 100644 --- a/crates/brk_computer/src/structs/addressindextxoutindex.rs +++ b/crates/brk_core/src/structs/addressindextxoutindex.rs @@ -1,8 +1,12 @@ -use brk_indexer::{Addressindex, Txoutindex}; use byteview::ByteView; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::Error; + +use super::{Addressindex, Txoutindex}; + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)] +#[repr(C)] pub struct AddressindexTxoutindex { addressindex: Addressindex, _padding: u32, @@ -10,7 +14,7 @@ pub struct AddressindexTxoutindex { } impl TryFrom for AddressindexTxoutindex { - type Error = storable_vec::Error; + type Error = Error; fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } diff --git a/crates/brk_indexer/src/structs/addresstype.rs b/crates/brk_core/src/structs/addresstype.rs similarity index 97% rename from crates/brk_indexer/src/structs/addresstype.rs rename to crates/brk_core/src/structs/addresstype.rs index 94a3e06a8..ddbeab956 100644 --- a/crates/brk_indexer/src/structs/addresstype.rs +++ b/crates/brk_core/src/structs/addresstype.rs @@ -1,4 +1,4 @@ -use brk_parser::bitcoin::ScriptBuf; +use bitcoin::ScriptBuf; use serde::Serialize; use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; diff --git a/crates/brk_indexer/src/structs/addresstypeindex.rs b/crates/brk_core/src/structs/addresstypeindex.rs similarity index 100% rename from crates/brk_indexer/src/structs/addresstypeindex.rs rename to crates/brk_core/src/structs/addresstypeindex.rs diff --git a/crates/brk_computer/src/structs/bitcoin.rs b/crates/brk_core/src/structs/bitcoin.rs similarity index 91% rename from crates/brk_computer/src/structs/bitcoin.rs rename to crates/brk_core/src/structs/bitcoin.rs index 36deb7800..e7f9ef5fe 100644 --- a/crates/brk_computer/src/structs/bitcoin.rs +++ b/crates/brk_core/src/structs/bitcoin.rs @@ -1,4 +1,4 @@ -use brk_indexer::Sats; +use super::Sats; #[derive(Debug, Default, Clone, Copy)] pub struct Bitcoin(f64); diff --git a/crates/brk_indexer/src/structs/blockhash.rs b/crates/brk_core/src/structs/blockhash.rs similarity index 88% rename from crates/brk_indexer/src/structs/blockhash.rs rename to crates/brk_core/src/structs/blockhash.rs index 4fd59e391..fd06c9fdb 100644 --- a/crates/brk_indexer/src/structs/blockhash.rs +++ b/crates/brk_core/src/structs/blockhash.rs @@ -1,13 +1,12 @@ use std::mem; -use brk_parser::{ - Height, - rpc::{Client, RpcApi}, -}; +use bitcoincore_rpc::{Client, RpcApi}; use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use super::Height; + #[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)] pub struct BlockHash([u8; 32]); @@ -24,7 +23,7 @@ impl From for bitcoin::BlockHash { } impl TryFrom<(&Client, Height)> for BlockHash { - type Error = brk_parser::rpc::Error; + type Error = bitcoincore_rpc::Error; fn try_from((rpc, height): (&Client, Height)) -> Result { Ok(Self::from(rpc.get_block_hash(u64::from(height))?)) } diff --git a/crates/brk_fetcher/src/structs/cents.rs b/crates/brk_core/src/structs/cents.rs similarity index 100% rename from crates/brk_fetcher/src/structs/cents.rs rename to crates/brk_core/src/structs/cents.rs diff --git a/crates/brk_indexer/src/structs/compressed.rs b/crates/brk_core/src/structs/compressed.rs similarity index 96% rename from crates/brk_indexer/src/structs/compressed.rs rename to crates/brk_core/src/structs/compressed.rs index 51a00f54d..4f78919bd 100644 --- a/crates/brk_indexer/src/structs/compressed.rs +++ b/crates/brk_core/src/structs/compressed.rs @@ -4,6 +4,8 @@ use byteview::ByteView; use derive_deref::Deref; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::Error; + use super::{Addressbytes, Addresstype, BlockHash, Txid}; #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] @@ -23,7 +25,7 @@ impl From<[u8; 8]> for AddressHash { } } impl TryFrom for AddressHash { - type Error = storable_vec::Error; + type Error = Error; fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } @@ -47,7 +49,7 @@ impl From<&BlockHash> for BlockHashPrefix { } } impl TryFrom for BlockHashPrefix { - type Error = storable_vec::Error; + type Error = Error; fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } @@ -71,7 +73,7 @@ impl From<&Txid> for TxidPrefix { } } impl TryFrom for TxidPrefix { - type Error = storable_vec::Error; + type Error = Error; fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } diff --git a/crates/brk_fetcher/src/structs/date.rs b/crates/brk_core/src/structs/date.rs similarity index 96% rename from crates/brk_fetcher/src/structs/date.rs rename to crates/brk_core/src/structs/date.rs index 21e7f112d..3a18de02d 100644 --- a/crates/brk_fetcher/src/structs/date.rs +++ b/crates/brk_core/src/structs/date.rs @@ -1,8 +1,7 @@ -use brk_indexer::Timestamp; use jiff::{Span, civil::Date as Date_, tz::TimeZone}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::Dateindex; +use super::{Dateindex, Timestamp}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct Date(u32); diff --git a/crates/brk_fetcher/src/structs/dateindex.rs b/crates/brk_core/src/structs/dateindex.rs similarity index 87% rename from crates/brk_fetcher/src/structs/dateindex.rs rename to crates/brk_core/src/structs/dateindex.rs index fc314e42e..45a243987 100644 --- a/crates/brk_fetcher/src/structs/dateindex.rs +++ b/crates/brk_core/src/structs/dateindex.rs @@ -1,8 +1,10 @@ use std::ops::Add; -use color_eyre::eyre::eyre; +// use color_eyre::eyre::eyre; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::Error; + use super::Date; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] @@ -34,15 +36,15 @@ impl Add for Dateindex { } impl TryFrom for Dateindex { - type Error = color_eyre::Report; + type Error = Error; fn try_from(value: Date) -> Result { let value_ = jiff::civil::Date::from(value); if value_ < Date::INDEX_ZERO_ { - Err(eyre!("Date is too early")) + Err(Error::UnindexableDate) } else if value == Date::INDEX_ZERO { Ok(Self(0)) } else if value_ < Date::INDEX_ONE_ { - Err(eyre!("Date is between first and second")) + Err(Error::UnindexableDate) } else if value == Date::INDEX_ONE { Ok(Self(1)) } else { diff --git a/crates/brk_fetcher/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs similarity index 100% rename from crates/brk_fetcher/src/structs/dollars.rs rename to crates/brk_core/src/structs/dollars.rs diff --git a/crates/brk_computer/src/structs/feerate.rs b/crates/brk_core/src/structs/feerate.rs similarity index 100% rename from crates/brk_computer/src/structs/feerate.rs rename to crates/brk_core/src/structs/feerate.rs diff --git a/crates/brk_parser/src/height.rs b/crates/brk_core/src/structs/height.rs similarity index 93% rename from crates/brk_parser/src/height.rs rename to crates/brk_core/src/structs/height.rs index c81c0bc3b..9d21ad5bc 100644 --- a/crates/brk_parser/src/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -3,12 +3,11 @@ use std::{ ops::{Add, AddAssign, Rem, Sub}, }; +use bitcoincore_rpc::{Client, RpcApi}; use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::rpc::{self, RpcApi}; - #[derive( Debug, Clone, @@ -148,6 +147,12 @@ impl From for Height { } } +impl From for Height { + fn from(value: u64) -> Self { + Self(value as u32) + } +} + impl From for Height { fn from(value: usize) -> Self { Self(value as u32) @@ -165,9 +170,9 @@ impl From for u64 { } } -impl TryFrom<&rpc::Client> for Height { - type Error = rpc::Error; - fn try_from(value: &rpc::Client) -> Result { +impl TryFrom<&Client> for Height { + type Error = bitcoincore_rpc::Error; + fn try_from(value: &Client) -> Result { Ok((value.get_blockchain_info()?.blocks as usize - 1).into()) } } diff --git a/crates/brk_indexer/src/structs/locktime.rs b/crates/brk_core/src/structs/locktime.rs similarity index 95% rename from crates/brk_indexer/src/structs/locktime.rs rename to crates/brk_core/src/structs/locktime.rs index ceaac5b46..58a2ea9f2 100644 --- a/crates/brk_indexer/src/structs/locktime.rs +++ b/crates/brk_core/src/structs/locktime.rs @@ -1,8 +1,7 @@ -use brk_parser::Height; use serde::Serialize; use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; -use super::Timestamp; +use super::{Height, Timestamp}; #[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)] #[repr(C)] diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_core/src/structs/mod.rs new file mode 100644 index 000000000..bbe35badf --- /dev/null +++ b/crates/brk_core/src/structs/mod.rs @@ -0,0 +1,55 @@ +mod addressbytes; +mod addressindex; +mod addressindextxoutindex; +mod addresstype; +mod addresstypeindex; +mod bitcoin; +mod blockhash; +mod cents; +mod compressed; +mod date; +mod dateindex; +mod dollars; +mod feerate; +mod height; +mod locktime; +mod ohlc; +mod sats; +mod timestamp; +mod txid; +mod txindex; +mod txinindex; +mod txoutindex; +mod txversion; +mod unit; +mod vin; +mod vout; +mod weight; + +pub use addressbytes::*; +pub use addressindex::*; +pub use addressindextxoutindex::*; +pub use addresstype::*; +pub use addresstypeindex::*; +pub use bitcoin::*; +pub use blockhash::*; +pub use cents::*; +pub use compressed::*; +pub use date::*; +pub use dateindex::*; +pub use dollars::*; +pub use feerate::*; +pub use height::*; +pub use locktime::*; +pub use ohlc::*; +pub use sats::*; +pub use timestamp::*; +pub use txid::*; +pub use txindex::*; +pub use txinindex::*; +pub use txoutindex::*; +pub use txversion::*; +pub use unit::*; +pub use vin::*; +pub use vout::*; +pub use weight::*; diff --git a/crates/brk_core/src/structs/ohlc.rs b/crates/brk_core/src/structs/ohlc.rs new file mode 100644 index 000000000..3f78f1d04 --- /dev/null +++ b/crates/brk_core/src/structs/ohlc.rs @@ -0,0 +1,130 @@ +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use super::Cents; + +pub type OHLCCents = (Open, High, Low, Close); + +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] +#[repr(C)] +pub struct Open(T); +impl From for Open { + fn from(value: T) -> Self { + Self(value) + } +} + +impl From> for Open +where + T: Copy, +{ + fn from(value: Close) -> Self { + Self(*value) + } +} + +#[derive( + Debug, + Default, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] +#[repr(C)] +pub struct High(T); +impl From for High { + fn from(value: T) -> Self { + Self(value) + } +} + +impl From> for High +where + T: Copy, +{ + fn from(value: Close) -> Self { + Self(*value) + } +} + +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] +#[repr(C)] +pub struct Low(T); +impl From for Low { + fn from(value: T) -> Self { + Self(value) + } +} + +impl From> for Low +where + T: Copy, +{ + fn from(value: Close) -> Self { + Self(*value) + } +} + +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] +#[repr(C)] +pub struct Close(T); +impl From for Close { + fn from(value: T) -> Self { + Self(value) + } +} diff --git a/crates/brk_indexer/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs similarity index 97% rename from crates/brk_indexer/src/structs/sats.rs rename to crates/brk_core/src/structs/sats.rs index 9d5881440..6c9c0d405 100644 --- a/crates/brk_indexer/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -3,11 +3,13 @@ use std::{ ops::{Add, AddAssign, Mul, Sub, SubAssign}, }; -use brk_parser::{Height, bitcoin::Amount}; +use bitcoin::Amount; use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use super::Height; + #[derive( Debug, PartialEq, diff --git a/crates/brk_indexer/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs similarity index 100% rename from crates/brk_indexer/src/structs/timestamp.rs rename to crates/brk_core/src/structs/timestamp.rs diff --git a/crates/brk_indexer/src/structs/txid.rs b/crates/brk_core/src/structs/txid.rs similarity index 100% rename from crates/brk_indexer/src/structs/txid.rs rename to crates/brk_core/src/structs/txid.rs diff --git a/crates/brk_indexer/src/structs/txindex.rs b/crates/brk_core/src/structs/txindex.rs similarity index 97% rename from crates/brk_indexer/src/structs/txindex.rs rename to crates/brk_core/src/structs/txindex.rs index 8ca6fba34..bb080a3eb 100644 --- a/crates/brk_indexer/src/structs/txindex.rs +++ b/crates/brk_core/src/structs/txindex.rs @@ -5,6 +5,8 @@ use derive_deref::{Deref, DerefMut}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::Error; + #[derive( Debug, PartialEq, @@ -90,7 +92,7 @@ impl From for usize { } impl TryFrom for Txindex { - type Error = storable_vec::Error; + type Error = Error; fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } diff --git a/crates/brk_indexer/src/structs/txinindex.rs b/crates/brk_core/src/structs/txinindex.rs similarity index 100% rename from crates/brk_indexer/src/structs/txinindex.rs rename to crates/brk_core/src/structs/txinindex.rs diff --git a/crates/brk_indexer/src/structs/txoutindex.rs b/crates/brk_core/src/structs/txoutindex.rs similarity index 100% rename from crates/brk_indexer/src/structs/txoutindex.rs rename to crates/brk_core/src/structs/txoutindex.rs diff --git a/crates/brk_indexer/src/structs/txversion.rs b/crates/brk_core/src/structs/txversion.rs similarity index 100% rename from crates/brk_indexer/src/structs/txversion.rs rename to crates/brk_core/src/structs/txversion.rs diff --git a/crates/brk_computer/src/structs/unit.rs b/crates/brk_core/src/structs/unit.rs similarity index 100% rename from crates/brk_computer/src/structs/unit.rs rename to crates/brk_core/src/structs/unit.rs diff --git a/crates/brk_indexer/src/structs/vin.rs b/crates/brk_core/src/structs/vin.rs similarity index 100% rename from crates/brk_indexer/src/structs/vin.rs rename to crates/brk_core/src/structs/vin.rs diff --git a/crates/brk_indexer/src/structs/vout.rs b/crates/brk_core/src/structs/vout.rs similarity index 100% rename from crates/brk_indexer/src/structs/vout.rs rename to crates/brk_core/src/structs/vout.rs diff --git a/crates/brk_indexer/src/structs/weight.rs b/crates/brk_core/src/structs/weight.rs similarity index 100% rename from crates/brk_indexer/src/structs/weight.rs rename to crates/brk_core/src/structs/weight.rs diff --git a/crates/brk_fetcher/Cargo.toml b/crates/brk_fetcher/Cargo.toml index 78349ea11..3b7ee8841 100644 --- a/crates/brk_fetcher/Cargo.toml +++ b/crates/brk_fetcher/Cargo.toml @@ -6,14 +6,12 @@ edition = { workspace = true } license = { workspace = true } [dependencies] +brk_core = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } color-eyre = { workspace = true } -derive_deref = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } storable_vec = { workspace = true } -zerocopy = { workspace = true } diff --git a/crates/brk_fetcher/src/fetchers/binance.rs b/crates/brk_fetcher/src/fetchers/binance.rs index 6bd444b60..8dce5e336 100644 --- a/crates/brk_fetcher/src/fetchers/binance.rs +++ b/crates/brk_fetcher/src/fetchers/binance.rs @@ -6,23 +6,19 @@ use std::{ str::FromStr, }; -use brk_indexer::Timestamp; +use brk_core::{Cents, OHLCCents, Timestamp}; use color_eyre::eyre::{ContextCompat, eyre}; use log::info; use serde_json::Value; use storable_vec::STATELESS; -use crate::{ - Close, Date, Dollars, High, Low, Open, Pricer, - fetchers::retry, - structs::{Cents, OHLC}, -}; +use crate::{Close, Date, Dollars, High, Low, Open, Pricer, fetchers::retry}; pub struct Binance { path: PathBuf, - _1mn: Option>, - _1d: Option>, - har: Option>, + _1mn: Option>, + _1d: Option>, + har: Option>, } impl Binance { @@ -39,7 +35,7 @@ impl Binance { &mut self, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> color_eyre::Result { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { self._1mn.replace(Self::fetch_1mn()?); } @@ -51,7 +47,7 @@ impl Binance { ) } - pub fn fetch_1mn() -> color_eyre::Result> { + pub fn fetch_1mn() -> color_eyre::Result> { info!("Fetching 1mn prices from Binance..."); retry( @@ -61,7 +57,7 @@ impl Binance { ) } - pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { + pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date { self._1d.replace(Self::fetch_1d()?); } @@ -74,7 +70,7 @@ impl Binance { .ok_or(color_eyre::eyre::Error::msg("Couldn't find date")) } - pub fn fetch_1d() -> color_eyre::Result> { + pub fn fetch_1d() -> color_eyre::Result> { info!("Fetching daily prices from Kraken..."); dbg!(&Self::url("interval=1d")); @@ -90,14 +86,14 @@ impl Binance { &mut self, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> color_eyre::Result { if self.har.is_none() { self.har.replace(self.read_har().unwrap_or_default()); } Pricer::::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har") } - fn read_har(&self) -> color_eyre::Result> { + fn read_har(&self) -> color_eyre::Result> { info!("Reading Binance har file..."); let path = &self.path; @@ -164,11 +160,11 @@ impl Binance { }) } - fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result> { + fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result> { Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc) } - fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { + fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { Self::json_to_btree(json, Self::array_to_date_and_ohlc) } @@ -184,7 +180,7 @@ impl Binance { .collect::, _>>() } - fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> { + fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> { let array = array.as_array().context("Expect to be array")?; let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32); @@ -197,7 +193,7 @@ impl Binance { Ok(( timestamp, - OHLC::from(( + OHLCCents::from(( Open::from(get_cents(1)), High::from(get_cents(2)), Low::from(get_cents(3)), @@ -206,7 +202,7 @@ impl Binance { )) } - fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> { + fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> { Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc)) } diff --git a/crates/brk_fetcher/src/fetchers/kibo.rs b/crates/brk_fetcher/src/fetchers/kibo.rs index b5327375b..242987de8 100644 --- a/crates/brk_fetcher/src/fetchers/kibo.rs +++ b/crates/brk_fetcher/src/fetchers/kibo.rs @@ -1,20 +1,16 @@ use std::{collections::BTreeMap, str::FromStr}; -use brk_indexer::Height; +use brk_core::{Date, Height, OHLCCents}; use color_eyre::eyre::ContextCompat; use log::info; use serde_json::Value; -use crate::{ - Cents, Close, Dollars, High, Low, Open, - fetchers::retry, - structs::{Date, OHLC}, -}; +use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry}; #[derive(Default)] pub struct Kibo { - height_to_ohlc_vec: BTreeMap>, - year_to_date_to_ohlc: BTreeMap>, + height_to_ohlc_vec: BTreeMap>, + year_to_date_to_ohlc: BTreeMap>, } const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api"; @@ -31,7 +27,7 @@ impl Kibo { } } - pub fn get_from_height_kibo(&mut self, height: Height) -> color_eyre::Result { + pub fn get_from_height_kibo(&mut self, height: Height) -> color_eyre::Result { #[allow(clippy::map_entry)] if !self.height_to_ohlc_vec.contains_key(&height) || ((usize::from(height) + self.height_to_ohlc_vec.get(&height).unwrap().len()) <= usize::from(height)) @@ -48,7 +44,7 @@ impl Kibo { .ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo")) } - fn fetch_height_prices(height: Height) -> color_eyre::Result> { + fn fetch_height_prices(height: Height) -> color_eyre::Result> { info!("Fetching Kibo height prices..."); retry( @@ -81,7 +77,7 @@ impl Kibo { ) } - pub fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result { + pub fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result { let year = date.year(); #[allow(clippy::map_entry)] @@ -106,7 +102,7 @@ impl Kibo { .ok_or(color_eyre::eyre::Error::msg("Couldn't find date in kibo")) } - fn fetch_date_prices(year: u16) -> color_eyre::Result> { + fn fetch_date_prices(year: u16) -> color_eyre::Result> { info!("Fetching Kibo date prices..."); retry( @@ -139,7 +135,7 @@ impl Kibo { ) } - fn value_to_ohlc(value: &Value) -> color_eyre::Result { + fn value_to_ohlc(value: &Value) -> color_eyre::Result { let ohlc = value.as_object().context("Expect as_object to work")?; let get_value = |key: &str| -> color_eyre::Result<_> { diff --git a/crates/brk_fetcher/src/fetchers/kraken.rs b/crates/brk_fetcher/src/fetchers/kraken.rs index 49618fcee..57d52842e 100644 --- a/crates/brk_fetcher/src/fetchers/kraken.rs +++ b/crates/brk_fetcher/src/fetchers/kraken.rs @@ -1,17 +1,17 @@ use std::collections::BTreeMap; -use brk_indexer::Timestamp; +use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp}; use color_eyre::eyre::ContextCompat; use log::info; use serde_json::Value; use storable_vec::STATELESS; -use crate::{Cents, Close, Dollars, High, Low, OHLC, Open, Pricer, fetchers::retry, structs::Date}; +use crate::{Pricer, fetchers::retry}; #[derive(Default)] pub struct Kraken { - _1mn: Option>, - _1d: Option>, + _1mn: Option>, + _1d: Option>, } impl Kraken { @@ -19,14 +19,14 @@ impl Kraken { &mut self, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> color_eyre::Result { if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { self._1mn.replace(Self::fetch_1mn()?); } Pricer::::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m") } - fn fetch_1mn() -> color_eyre::Result> { + fn fetch_1mn() -> color_eyre::Result> { info!("Fetching 1mn prices from Kraken..."); retry( @@ -36,7 +36,7 @@ impl Kraken { ) } - pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { + pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date { self._1d.replace(Kraken::fetch_1d()?); } @@ -48,7 +48,7 @@ impl Kraken { .ok_or(color_eyre::eyre::Error::msg("Couldn't find date")) } - fn fetch_1d() -> color_eyre::Result> { + fn fetch_1d() -> color_eyre::Result> { info!("Fetching daily prices from Kraken..."); retry( @@ -58,11 +58,11 @@ impl Kraken { ) } - fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result> { + fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result> { Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc) } - fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { + fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { Self::json_to_btree(json, Self::array_to_date_and_ohlc) } @@ -86,7 +86,7 @@ impl Kraken { .collect::, _>>() } - fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> { + fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> { let array = array.as_array().context("Expect to be array")?; let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32); @@ -99,7 +99,7 @@ impl Kraken { Ok(( timestamp, - OHLC::from(( + OHLCCents::from(( Open::from(get_cents(1)), High::from(get_cents(2)), Low::from(get_cents(3)), @@ -108,7 +108,7 @@ impl Kraken { )) } - fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> { + fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> { Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc)) } diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index 7de87c79c..4be2da90e 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -4,15 +4,14 @@ use std::{ path::{Path, PathBuf}, }; +use brk_core::{Cents, Close, Date, Dateindex, Dollars, Height, High, Low, OHLCCents, Open, Timestamp}; use color_eyre::eyre::Error; mod fetchers; -mod structs; -use brk_indexer::{Height, Indexer, Timestamp}; +use brk_indexer::Indexer; pub use fetchers::*; use storable_vec::{AnyJsonStorableVec, AnyStorableVec, SINGLE_THREAD, StorableVec, Version}; -pub use structs::*; pub struct Pricer { path: PathBuf, @@ -48,13 +47,6 @@ impl Pricer { kraken: Kraken::default(), kibo: Kibo::default(), - // binance_1mn: None, - // binance_daily: None, - // binance_har: None, - // kraken_1mn: None, - // kraken_daily: None, - // kibo_by_height: BTreeMap::default(), - // kibo_by_date: BTreeMap::default(), dateindex_to_close_in_cents: StorableVec::import( &path.join("dateindex_to_close_in_cents"), Version::from(1), @@ -119,7 +111,7 @@ impl Pricer { // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close); } - fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { + fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { todo!(); // if self.ohlc.date.is_key_safe(date) { // Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned()) @@ -140,7 +132,7 @@ impl Pricer { height: Height, timestamp: Timestamp, previous_timestamp: Option, - ) -> color_eyre::Result { + ) -> color_eyre::Result { todo!(); // if let Some(ohlc) = self.ohlc.height.get_or_import(&height) { @@ -191,12 +183,12 @@ impl Pricer { } fn find_height_ohlc( - tree: &BTreeMap, + tree: &BTreeMap, timestamp: Timestamp, previous_timestamp: Option, name: &str, - ) -> color_eyre::Result { - let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| { + ) -> color_eyre::Result { + let previous_ohlc = previous_timestamp.map_or(Some(OHLCCents::default()), |previous_timestamp| { tree.get(&previous_timestamp).cloned() }); diff --git a/crates/brk_fetcher/src/main.rs b/crates/brk_fetcher/src/main.rs index fd589f72a..2222ecc47 100644 --- a/crates/brk_fetcher/src/main.rs +++ b/crates/brk_fetcher/src/main.rs @@ -1,5 +1,4 @@ use brk_fetcher::{Binance, Kibo, Kraken}; -use brk_indexer::Height; use serde_json::Value; fn main() -> color_eyre::Result<()> { diff --git a/crates/brk_fetcher/src/structs/close.rs b/crates/brk_fetcher/src/structs/close.rs deleted file mode 100644 index a87318ea2..000000000 --- a/crates/brk_fetcher/src/structs/close.rs +++ /dev/null @@ -1,27 +0,0 @@ -use derive_deref::Deref; -use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -#[derive( - Debug, - Default, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Deref, - Serialize, -)] -#[repr(C)] -pub struct Close(T); -impl From for Close { - fn from(value: T) -> Self { - Self(value) - } -} diff --git a/crates/brk_fetcher/src/structs/high.rs b/crates/brk_fetcher/src/structs/high.rs deleted file mode 100644 index 53fa506cd..000000000 --- a/crates/brk_fetcher/src/structs/high.rs +++ /dev/null @@ -1,38 +0,0 @@ -use derive_deref::Deref; -use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use super::Close; - -#[derive( - Debug, - Default, - PartialEq, - Eq, - PartialOrd, - Ord, - Clone, - Copy, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Deref, - Serialize, -)] -#[repr(C)] -pub struct High(T); -impl From for High { - fn from(value: T) -> Self { - Self(value) - } -} - -impl From> for High -where - T: Copy, -{ - fn from(value: Close) -> Self { - Self(*value) - } -} diff --git a/crates/brk_fetcher/src/structs/low.rs b/crates/brk_fetcher/src/structs/low.rs deleted file mode 100644 index 39f8c52ea..000000000 --- a/crates/brk_fetcher/src/structs/low.rs +++ /dev/null @@ -1,38 +0,0 @@ -use derive_deref::Deref; -use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use super::Close; - -#[derive( - Debug, - Default, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Deref, - Serialize, -)] -#[repr(C)] -pub struct Low(T); -impl From for Low { - fn from(value: T) -> Self { - Self(value) - } -} - -impl From> for Low -where - T: Copy, -{ - fn from(value: Close) -> Self { - Self(*value) - } -} diff --git a/crates/brk_fetcher/src/structs/mod.rs b/crates/brk_fetcher/src/structs/mod.rs deleted file mode 100644 index 5d1dfcf2b..000000000 --- a/crates/brk_fetcher/src/structs/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -mod cents; -mod close; -mod date; -mod dateindex; -mod dollars; -mod high; -mod low; -mod open; - -pub use cents::*; -pub use close::*; -pub use date::*; -pub use dateindex::*; -pub use dollars::*; -pub use high::*; -pub use low::*; -pub use open::*; - -pub type OHLC = (Open, High, Low, Close); diff --git a/crates/brk_fetcher/src/structs/open.rs b/crates/brk_fetcher/src/structs/open.rs deleted file mode 100644 index fe2843268..000000000 --- a/crates/brk_fetcher/src/structs/open.rs +++ /dev/null @@ -1,38 +0,0 @@ -use derive_deref::Deref; -use serde::Serialize; -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use super::Close; - -#[derive( - Debug, - Default, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Deref, - Serialize, -)] -#[repr(C)] -pub struct Open(T); -impl From for Open { - fn from(value: T) -> Self { - Self(value) - } -} - -impl From> for Open -where - T: Copy, -{ - fn from(value: Close) -> Self { - Self(*value) - } -} diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 39fb037a5..00cdc2a3a 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -7,19 +7,16 @@ license = { workspace = true } [dependencies] bitcoin = { workspace = true } +bitcoincore-rpc = { workspace = true } +brk_core = { workspace = true } brk_parser = { workspace = true } brk_logger = { workspace = true } byteview = { workspace = true } color-eyre = { workspace = true } -derive_deref = { workspace = true } fjall = { workspace = true } hodor = { workspace = true } -jiff = { workspace = true } log = { workspace = true } -rapidhash = "1.4.0" rayon = { workspace = true } rlimit = { version = "0.10.2" } -serde = { workspace = true } -serde_bytes = { workspace = true } storable_vec = { workspace = true } zerocopy = { workspace = true } diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index ac7fab13a..fb932ec00 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -6,6 +6,10 @@ use std::{ time::Duration, }; +use brk_core::{ + AddressHash, Addressbytes, Addressindex, Addresstype, BlockHash, BlockHashPrefix, Height, Sats, Timestamp, Txid, + TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout, +}; pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; @@ -48,8 +52,6 @@ impl Indexer { impl Indexer { pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> { - info!("Started indexing..."); - let check_collisions = true; let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| { @@ -80,7 +82,13 @@ impl Indexer { let mut idxs = starting_indexes; - parser.parse(Some(idxs.height), None) + if idxs.height >= Height::try_from(rpc)? { + return Ok(()); + } + + info!("Started indexing..."); + + parser.parse(Some(idxs.height), Some(400_000_u32.into())) .iter() .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { info!("Indexing block {height}..."); @@ -256,7 +264,7 @@ impl Indexer { Txindex, Vout, Addresstype, - color_eyre::Result, + brk_core::Result, Option, &Transaction, ), @@ -621,7 +629,6 @@ impl Indexer { })?; export(stores, vecs, idxs.height)?; - sleep(Duration::from_millis(100)); Ok(()) diff --git a/crates/brk_indexer/src/main.rs b/crates/brk_indexer/src/main.rs index eb3d9d5a5..83ff7a7e6 100644 --- a/crates/brk_indexer/src/main.rs +++ b/crates/brk_indexer/src/main.rs @@ -34,7 +34,7 @@ fn main() -> color_eyre::Result<()> { indexer.index(&parser, rpc, &exit)?; - dbg!(i.elapsed()); + info!("Took: {:?}", i.elapsed()); info!("Waiting for a new block..."); diff --git a/crates/brk_indexer/src/storage/fjalls/base.rs b/crates/brk_indexer/src/storage/fjalls/base.rs index 6f9e1cafd..9a7b623c8 100644 --- a/crates/brk_indexer/src/storage/fjalls/base.rs +++ b/crates/brk_indexer/src/storage/fjalls/base.rs @@ -4,7 +4,7 @@ use std::{ path::Path, }; -use brk_parser::Height; +use brk_core::Height; use byteview::ByteView; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace, TransactionalPartitionHandle, diff --git a/crates/brk_indexer/src/storage/fjalls/mod.rs b/crates/brk_indexer/src/storage/fjalls/mod.rs index 7b2a0c029..058a84e0a 100644 --- a/crates/brk_indexer/src/storage/fjalls/mod.rs +++ b/crates/brk_indexer/src/storage/fjalls/mod.rs @@ -1,10 +1,9 @@ use std::{path::Path, thread}; -use storable_vec::{Value, Version, CACHED_GETS}; +use brk_core::{AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, Txindex}; +use storable_vec::{CACHED_GETS, Value, Version}; -use crate::{ - AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, Indexes, TxidPrefix, Txindex, -}; +use crate::Indexes; mod base; mod meta; diff --git a/crates/brk_indexer/src/storage/storable_vecs/base.rs b/crates/brk_indexer/src/storage/storable_vecs/base.rs index f762bb2b5..571d063b2 100644 --- a/crates/brk_indexer/src/storage/storable_vecs/base.rs +++ b/crates/brk_indexer/src/storage/storable_vecs/base.rs @@ -39,7 +39,7 @@ where self.vec.truncate_if_needed(index) } - pub fn height(&self) -> brk_parser::Result { + pub fn height(&self) -> brk_core::Result { Height::try_from(self.path_height().as_path()) } fn path_height(&self) -> PathBuf { @@ -71,7 +71,7 @@ impl DerefMut for StorableVec { } pub trait AnyStorableVec: Send + Sync { - fn height(&self) -> brk_parser::Result; + fn height(&self) -> brk_core::Result; fn flush(&mut self, height: Height) -> io::Result<()>; } @@ -80,7 +80,7 @@ where I: StoredIndex, T: StoredType, { - fn height(&self) -> brk_parser::Result { + fn height(&self) -> brk_core::Result { self.height() } diff --git a/crates/brk_indexer/src/storage/storable_vecs/mod.rs b/crates/brk_indexer/src/storage/storable_vecs/mod.rs index 481f92af2..df669e203 100644 --- a/crates/brk_indexer/src/storage/storable_vecs/mod.rs +++ b/crates/brk_indexer/src/storage/storable_vecs/mod.rs @@ -1,19 +1,15 @@ use std::{fs, io, path::Path}; -use brk_parser::Height; +use brk_core::{ + Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex, + Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, + P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, + P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, +}; use rayon::prelude::*; use storable_vec::{AnyJsonStorableVec, CACHED_GETS, Version}; -use crate::{ - Indexes, - structs::{ - Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, LockTime, Multisigindex, - Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, - P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, - P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, - Weight, - }, -}; +use crate::Indexes; mod base; diff --git a/crates/brk_indexer/src/structs/indexes.rs b/crates/brk_indexer/src/structs/indexes.rs index 5339b559d..91e485bfe 100644 --- a/crates/brk_indexer/src/structs/indexes.rs +++ b/crates/brk_indexer/src/structs/indexes.rs @@ -1,15 +1,14 @@ +use bitcoincore_rpc::Client; +use brk_core::{ + Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex, + P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, +}; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; -use brk_parser::{Height, rpc::Client}; use color_eyre::eyre::ContextCompat; use storable_vec::CACHED_GETS; use crate::storage::{Fjalls, StorableVecs}; -use super::{ - Addressindex, BlockHash, Emptyindex, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, - P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, -}; - #[derive(Debug, Default)] pub struct Indexes { pub addressindex: Addressindex, @@ -79,18 +78,28 @@ impl TryFrom<(&mut StorableVecs, &Fjalls, &Client)> for Indexes { // Height at which we wanna start: min last saved + 1 or 0 let starting_height = vecs.starting_height().min(trees.starting_height()); - // But we also need to check the chain and start earlier in case of a reorg - let height = (starting_height + let range = starting_height .checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32) - .unwrap_or_default()..*starting_height) // ..= because of last saved + 1 + .unwrap_or_default()..*starting_height; + + // But we also need to check the chain and start earlier in case of a reorg + let height = range // ..= because of last saved + 1 .map(Height::from) .find(|height| { - let rpc_blockhash = BlockHash::try_from((rpc, *height)).unwrap(); - let saved_blockhash = vecs.height_to_blockhash.get(*height).unwrap().unwrap(); - &rpc_blockhash != saved_blockhash.as_ref() + let rpc_blockhash = BlockHash::try_from((rpc, *height)) + .inspect_err(|e| { + dbg!(e, height); + }) + .unwrap(); + + vecs.height_to_blockhash.get(*height).map_or(true, |opt| { + opt.is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref()) + }) }) .unwrap_or(starting_height); + // let height = 885000_u32.into(); + Ok(Self { addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?, emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?, diff --git a/crates/brk_indexer/src/structs/mod.rs b/crates/brk_indexer/src/structs/mod.rs index 9c64bd3f8..db6175227 100644 --- a/crates/brk_indexer/src/structs/mod.rs +++ b/crates/brk_indexer/src/structs/mod.rs @@ -1,37 +1,3 @@ -mod addressbytes; -mod addressindex; -mod addresstype; -mod addresstypeindex; -mod blockhash; -mod compressed; mod indexes; -mod locktime; -mod sats; -mod timestamp; -mod txid; -mod txindex; -mod txinindex; -mod txoutindex; -mod txversion; -mod vin; -mod vout; -mod weight; -pub use addressbytes::*; -pub use addressindex::*; -pub use addresstype::*; -pub use addresstypeindex::*; -pub use blockhash::*; -pub use compressed::*; pub use indexes::*; -pub use locktime::*; -pub use sats::*; -pub use timestamp::*; -pub use txid::*; -pub use txindex::*; -pub use txinindex::*; -pub use txoutindex::*; -pub use txversion::*; -pub use vin::*; -pub use vout::*; -pub use weight::*; diff --git a/crates/brk_parser/Cargo.toml b/crates/brk_parser/Cargo.toml index 772740f35..9d2b52c05 100644 --- a/crates/brk_parser/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -10,8 +10,8 @@ license = { workspace = true } [dependencies] bitcoin = { workspace = true } -bitcoincore-rpc = "0.19.0" -byteview = { workspace = true } +bitcoincore-rpc = { workspace = true } +brk_core = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_deref = { workspace = true } rayon = { workspace = true } diff --git a/crates/brk_parser/src/blk_index_to_blk_recap.rs b/crates/brk_parser/src/blk_index_to_blk_recap.rs index f95ef13ae..57f1e5476 100644 --- a/crates/brk_parser/src/blk_index_to_blk_recap.rs +++ b/crates/brk_parser/src/blk_index_to_blk_recap.rs @@ -81,7 +81,8 @@ impl BlkIndexToBlkRecap { } } - start.unwrap() + // Should only be none if asking for a too high start + start.unwrap_or_else(|| self.tree.last_key_value().map_or(0, |(i, _)| *i)) } pub fn export(&self) { diff --git a/crates/brk_parser/src/blk_recap.rs b/crates/brk_parser/src/blk_recap.rs index d7c8e8ca6..0185084ca 100644 --- a/crates/brk_parser/src/blk_recap.rs +++ b/crates/brk_parser/src/blk_recap.rs @@ -1,8 +1,9 @@ use std::path::Path; +use brk_core::Height; use serde::{Deserialize, Serialize}; -use crate::{Height, path_to_modified_time}; +use crate::path_to_modified_time; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[repr(C)] diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index 1ad841f91..96224397c 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -11,6 +11,7 @@ use bitcoin::{Block, BlockHash}; use bitcoincore_rpc::RpcApi; use blk_index_to_blk_path::*; use blk_recap::BlkRecap; +use brk_core::Height; use crossbeam::channel::{Receiver, bounded}; use rayon::prelude::*; @@ -23,7 +24,6 @@ mod blk_metadata; mod blk_recap; mod block_state; mod error; -mod height; mod utils; mod xor_bytes; mod xor_index; @@ -32,7 +32,6 @@ use blk_index_to_blk_recap::*; use blk_metadata::*; use block_state::*; pub use error::*; -pub use height::*; use utils::*; use xor_bytes::*; use xor_index::*;