Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-07 20:59:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-03-10 23:08:07 +01:00
parent 9428beeae5
commit db70b05088
30 changed files with 326 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/brk_cli/Cargo.toml
View File

@@ -10,6 +10,7 @@ repository.workspace = true
brk_computer = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }

6
crates/brk_cli/src/lib.rs
View File

@@ -1,6 +1,6 @@
use std::fs;
use brk_core::{path_dot_brk, path_dot_brk_log};
use brk_core::{dot_brk_log_path, dot_brk_path};
use brk_query::Params as QueryArgs;
use clap::{Parser, Subcommand};
use query::query;
@@ -28,9 +28,9 @@ enum Commands {
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
fs::create_dir_all(path_dot_brk())?;
fs::create_dir_all(dot_brk_path())?;
brk_logger::init(Some(&path_dot_brk_log()));
brk_logger::init(Some(&dot_brk_log_path()));
let cli = Cli::parse();

2
crates/brk_cli/src/query.rs
View File

@@ -11,7 +11,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let mut indexer = Indexer::new(config.indexeddir())?;
indexer.import_vecs()?;
let mut computer = Computer::new(config.computeddir());
let mut computer = Computer::new(config.computeddir(), None);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

93
crates/brk_cli/src/run.rs
View File

@@ -6,8 +6,9 @@ use std::{
};
use brk_computer::Computer;
use brk_core::path_dot_brk;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::rpc::{self, Auth, Client, RpcApi};
use brk_server::{Frontend, Server, tokio};
@@ -23,13 +24,17 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let exit = Exit::new();
let parser = brk_parser::Parser::new(config.bitcoindir(), rpc);
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
let mut indexer = Indexer::new(config.indexeddir())?;
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::new(config.computeddir());
let fetcher = config
.fetch()
.then(|| Fetcher::import(Some(config.harsdir().as_path())).unwrap());
let mut computer = Computer::new(config.computeddir(), fetcher);
computer.import_stores()?;
computer.import_vecs()?;
@@ -82,11 +87,15 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
pub struct RunConfig {
/// Bitcoin data directory path, saved
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
#[arg(long, value_name = "PATH")]
bitcoindir: Option<String>,
/// Bitcoin Research Kit outputs directory path, saved
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
#[arg(long, value_name = "PATH")]
blocksdir: Option<String>,
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
@@ -94,8 +103,12 @@ pub struct RunConfig {
#[arg(short, long)]
mode: Option<Mode>,
/// Frontend served by the server (if active), default: kibo.money, saved
#[arg(short, long)]
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: false, saved
#[arg(short, long, value_name = "BOOL")]
fetch: Option<bool>,
/// Frontend served by the server (if active), default: none, saved
#[arg(short = 'F', long)]
frontend: Option<Frontend>,
/// Bitcoin RPC ip, default: localhost, saved
@@ -125,7 +138,7 @@ pub struct RunConfig {
impl RunConfig {
pub fn import(config_args: Option<RunConfig>) -> color_eyre::Result<Self> {
let path = path_dot_brk();
let path = dot_brk_path();
let _ = fs::create_dir_all(&path);
@@ -138,6 +151,10 @@ impl RunConfig {
config_saved.bitcoindir = Some(bitcoindir);
}
if let Some(blocksdir) = config_args.blocksdir.take() {
config_saved.blocksdir = Some(blocksdir);
}
if let Some(brkdir) = config_args.brkdir.take() {
config_saved.brkdir = Some(brkdir);
}
@@ -146,6 +163,10 @@ impl RunConfig {
config_saved.mode = Some(mode);
}
if let Some(fetch) = config_args.fetch.take() {
config_saved.fetch = Some(fetch);
}
if let Some(frontend) = config_args.frontend.take() {
config_saved.frontend = Some(frontend);
}
@@ -203,33 +224,24 @@ impl RunConfig {
}
fn check(&self) {
if self.bitcoindir.is_none() {
println!(
"You need to set the --bitcoindir parameter at least once to run the parser.\nRun the program with '-h' for help."
);
std::process::exit(1);
} else if !self.bitcoindir().is_dir() {
println!(
"Given --bitcoindir parameter doesn't seem to be a valid directory path.\nRun the program with '-h' for help."
);
if !self.bitcoindir().is_dir() {
println!("{:?} isn't a valid directory", self.bitcoindir());
println!("Please use the --bitcoindir parameter to set a valid path.");
println!("Run the program with '-h' for help.");
std::process::exit(1);
}
if self.brkdir.is_none() {
println!(
"You need to set the --brkdir parameter at least once to run the parser.\nRun the program with '-h' for help."
);
std::process::exit(1);
} else if !self.brkdir().is_dir() {
println!(
"Given --brkdir parameter doesn't seem to be a valid directory path.\nRun the program with '-h' for help."
);
if !self.blocksdir().is_dir() {
println!("{:?} isn't a valid directory", self.blocksdir());
println!("Please use the --blocksdir parameter to set a valid path.");
println!("Run the program with '-h' for help.");
std::process::exit(1);
}
let path = self.bitcoindir();
if !path.is_dir() {
println!("Expect path '{:#?}' to be a directory.", path);
if !self.brkdir().is_dir() {
println!("{:?} isn't a valid directory", self.brkdir());
println!("Please use the --brkdir parameter to set a valid path.");
println!("Run the program with '-h' for help.");
std::process::exit(1);
}
@@ -290,11 +302,22 @@ impl RunConfig {
}
pub fn bitcoindir(&self) -> PathBuf {
Self::fix_user_path(self.bitcoindir.as_ref().unwrap().as_ref())
self.bitcoindir
.as_ref()
.map_or_else(default_bitcoin_path, |s| Self::fix_user_path(s.as_ref()))
}
pub fn blocksdir(&self) -> PathBuf {
self.blocksdir.as_ref().map_or_else(
|| self.bitcoindir().join("blocks"),
|blocksdir| Self::fix_user_path(blocksdir.as_str()),
)
}
pub fn brkdir(&self) -> PathBuf {
Self::fix_user_path(self.brkdir.as_ref().unwrap().as_ref())
self.brkdir
.as_ref()
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
}
fn outputsdir(&self) -> PathBuf {
@@ -309,6 +332,10 @@ impl RunConfig {
self.outputsdir().join("computed")
}
pub fn harsdir(&self) -> PathBuf {
self.outputsdir().join("hars")
}
pub fn process(&self) -> bool {
self.mode
.is_none_or(|m| m == Mode::All || m == Mode::Processor)
@@ -347,6 +374,10 @@ impl RunConfig {
pub fn frontend(&self) -> Frontend {
self.frontend.unwrap_or_default()
}
pub fn fetch(&self) -> bool {
self.fetch.is_some_and(|b| b)
}
}
#[derive(

7
crates/brk_computer/examples/main.rs
View File

@@ -3,6 +3,7 @@ use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
@@ -23,7 +24,7 @@ pub fn main() -> color_eyre::Result<()> {
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let outputs_dir = Path::new("../../_outputs");
@@ -31,7 +32,9 @@ pub fn main() -> color_eyre::Result<()> {
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"));
let fetcher = Fetcher::import(None)?;
let mut computer = Computer::new(outputs_dir.join("computed"), Some(fetcher));
computer.import_stores()?;
computer.import_vecs()?;

19
crates/brk_computer/src/lib.rs
View File

@@ -6,6 +6,7 @@
use std::path::{Path, PathBuf};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::{Indexer, Indexes};
pub use brk_parser::rpc;
@@ -17,25 +18,30 @@ use storage::{Stores, Vecs};
#[derive(Clone)]
pub struct Computer {
path: PathBuf,
fetcher: Option<Fetcher>,
vecs: Option<Vecs>,
stores: Option<Stores>,
}
impl Computer {
pub fn new(computed_dir: PathBuf) -> Self {
pub fn new(computed_dir: PathBuf, fetcher: Option<Fetcher>) -> Self {
Self {
path: computed_dir,
fetcher,
vecs: None,
stores: None,
}
}
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(&self.path.join("vecs"))?);
self.vecs = Some(Vecs::import(
&self.path.join("vecs"),
self.fetcher.is_some(),
)?);
Ok(())
}
/// Do NOT import multiple times are things will break !!!
/// Do NOT import multiple times or things will break !!!
/// Clone struct instead
pub fn import_stores(&mut self) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(&self.path.join("stores"))?);
@@ -52,7 +58,12 @@ impl Computer {
) -> color_eyre::Result<()> {
info!("Computing...");
self.mut_vecs().compute(indexer, starting_indexes, exit)?;
self.vecs.as_mut().unwrap().compute(
indexer,
starting_indexes,
self.fetcher.as_mut(),
exit,
)?;
Ok(())
}

33
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -1,6 +1,8 @@
use std::{fs, path::Path};
use brk_core::{Cents, Close, Dateindex, Dollars, Height, High, Low, OHLCCents, OHLCDollars, Open};
use brk_core::{
Cents, Close, Dateindex, Dollars, Height, High, Low, OHLCCents, OHLCDollars, Open, Sats,
};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
@@ -20,6 +22,7 @@ pub struct Vecs {
pub dateindex_to_low: StorableVec<Dateindex, Low<Dollars>>,
pub dateindex_to_open_in_cents: StorableVec<Dateindex, Open<Cents>>,
pub dateindex_to_open: StorableVec<Dateindex, Open<Dollars>>,
pub dateindex_to_sats_per_dollar: StorableVec<Dateindex, Close<Sats>>,
pub height_to_ohlc_in_cents: StorableVec<Height, OHLCCents>,
pub height_to_ohlc: StorableVec<Height, OHLCDollars>,
pub height_to_close_in_cents: StorableVec<Height, Close<Cents>>,
@@ -30,6 +33,7 @@ pub struct Vecs {
pub height_to_low: StorableVec<Height, Low<Dollars>>,
pub height_to_open_in_cents: StorableVec<Height, Open<Cents>>,
pub height_to_open: StorableVec<Height, Open<Dollars>>,
pub height_to_sats_per_dollar: StorableVec<Height, Close<Sats>>,
}
impl Vecs {
@@ -77,6 +81,10 @@ impl Vecs {
&path.join("dateindex_to_open"),
Version::from(1),
)?,
dateindex_to_sats_per_dollar: StorableVec::import(
&path.join("dateindex_to_sats_per_dollar"),
Version::from(1),
)?,
height_to_ohlc_in_cents: StorableVec::import(
&path.join("height_to_ohlc_in_cents"),
Version::from(1),
@@ -102,6 +110,10 @@ impl Vecs {
Version::from(1),
)?,
height_to_open: StorableVec::import(&path.join("height_to_open"), Version::from(1))?,
height_to_sats_per_dollar: StorableVec::import(
&path.join("height_to_sats_per_dollar"),
Version::from(1),
)?,
})
}
@@ -110,10 +122,9 @@ impl Vecs {
indexer: &mut Indexer,
indexes: &mut indexes::Vecs,
starting_indexes: Indexes,
fetcher: &mut Fetcher,
exit: &Exit,
) -> color_eyre::Result<()> {
let mut fetcher = Fetcher::import(None)?;
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
@@ -199,6 +210,13 @@ impl Vecs {
exit,
)?;
self.height_to_sats_per_dollar.compute_transform(
starting_indexes.height,
&mut self.height_to_close,
|(di, close, ..)| (di, Close::from(Sats::ONE_BTC / **close)),
exit,
)?;
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
&mut indexes.dateindex_to_date,
@@ -272,6 +290,13 @@ impl Vecs {
exit,
)?;
self.dateindex_to_sats_per_dollar.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_close,
|(di, close, ..)| (di, Close::from(Sats::ONE_BTC / **close)),
exit,
)?;
Ok(())
}
@@ -287,6 +312,7 @@ impl Vecs {
&self.dateindex_to_ohlc_in_cents,
&self.dateindex_to_open,
&self.dateindex_to_open_in_cents,
&self.dateindex_to_sats_per_dollar,
&self.height_to_close,
&self.height_to_close_in_cents,
&self.height_to_high,
@@ -297,6 +323,7 @@ impl Vecs {
&self.height_to_ohlc_in_cents,
&self.height_to_open,
&self.height_to_open_in_cents,
&self.height_to_sats_per_dollar,
]
}
}

27
crates/brk_computer/src/storage/vecs/mod.rs
View File

@@ -2,6 +2,7 @@ use std::{fs, path::Path};
use brk_core::{Height, Sats, Txindex, Txinindex, Txoutindex};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, StorableVec, Version};
@@ -11,7 +12,7 @@ mod marketprice;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub marketprice: marketprice::Vecs,
pub marketprice: Option<marketprice::Vecs>,
// pub height_to_block_interval: StorableVec<Height, Timestamp>,
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Height, u32>,
@@ -38,13 +39,13 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
pub fn import(path: &Path, fetch: bool) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
indexes: indexes::Vecs::import(path)?,
marketprice: marketprice::Vecs::import(path)?,
marketprice: fetch.then(|| marketprice::Vecs::import(path).unwrap()),
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::forced_import(
@@ -107,12 +108,20 @@ impl Vecs {
&mut self,
indexer: &mut Indexer,
starting_indexes: brk_indexer::Indexes,
fetcher: Option<&mut Fetcher>,
exit: &Exit,
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
self.marketprice
.compute(indexer, &mut self.indexes, starting_indexes, exit)?;
if let Some(marketprice) = self.marketprice.as_mut() {
marketprice.compute(
indexer,
&mut self.indexes,
starting_indexes,
fetcher.unwrap(),
exit,
)?;
}
// self.mut_vecs().height_to_ohlc
@@ -197,6 +206,12 @@ impl Vecs {
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
[self.indexes.as_any_vecs(), self.marketprice.as_any_vecs()].concat()
[
self.indexes.as_any_vecs(),
self.marketprice
.as_ref()
.map_or(vec![], |v| v.as_any_vecs()),
]
.concat()
}
}

2
crates/brk_core/Cargo.toml
View File

@@ -16,7 +16,7 @@ log = { workspace = true }
rapidhash = "1.4.0"
rlimit = "0.10.2"
serde = { workspace = true }
serde_bytes = "0.11.16"
serde_bytes = "0.11.17"
zerocopy = { workspace = true }
[package.metadata.cargo-machete]

23
crates/brk_core/src/structs/bitcoin.rs
View File

@@ -1,14 +1,31 @@
use std::ops::Mul;
use super::Sats;
#[derive(Debug, Default, Clone, Copy)]
pub struct Bitcoin(f64);
impl Bitcoin {
const ONE: Self = Self(100_000_000.0);
impl Mul for Bitcoin {
type Output = Self;
fn mul(self, rhs: Self) -> Self::Output {
Self(self.0 * rhs.0)
}
}
impl From<Sats> for Bitcoin {
fn from(value: Sats) -> Self {
Self(f64::from(value) / Self::ONE.0)
Self(u64::from(value) as f64 / (u64::from(Sats::ONE_BTC) as f64))
}
}
impl From<f64> for Bitcoin {
fn from(value: f64) -> Self {
Self(value)
}
}
impl From<Bitcoin> for f64 {
fn from(value: Bitcoin) -> Self {
value.0
}
}

24
crates/brk_core/src/structs/sats.rs
View File

@@ -1,13 +1,13 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Mul, Sub, SubAssign},
ops::{Add, AddAssign, Div, Mul, Sub, SubAssign},
};
use bitcoin::Amount;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
use super::{Bitcoin, Dollars, Height};
#[derive(
Debug,
@@ -28,6 +28,7 @@ pub struct Sats(u64);
impl Sats {
pub const ZERO: Self = Self(0);
pub const ONE_BTC: Self = Self(100_000_000);
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
@@ -88,6 +89,13 @@ impl Sum for Sats {
}
}
impl Div<Dollars> for Sats {
type Output = Self;
fn div(self, rhs: Dollars) -> Self::Output {
Self((self.0 as f64 / f64::from(rhs)) as u64)
}
}
impl From<u64> for Sats {
fn from(value: u64) -> Self {
Self(value)
@@ -105,8 +113,14 @@ impl From<Sats> for Amount {
}
}
impl From<Sats> for f64 {
fn from(value: Sats) -> Self {
value.0 as f64
impl From<Bitcoin> for Sats {
fn from(value: Bitcoin) -> Self {
Self((f64::from(value) * (u64::from(Sats::ONE_BTC) as f64)) as u64)
}
}
impl From<Sats> for u64 {
fn from(value: Sats) -> Self {
value.0
}
}

14
crates/brk_core/src/utils/paths.rs
View File

@@ -3,17 +3,17 @@ use std::{
path::{Path, PathBuf},
};
pub fn path_dot_brk() -> PathBuf {
pub fn dot_brk_path() -> PathBuf {
let home = std::env::var("HOME").unwrap();
Path::new(&home).join(".brk")
}
pub fn path_dot_brk_log() -> PathBuf {
path_dot_brk().join("log")
pub fn dot_brk_log_path() -> PathBuf {
dot_brk_path().join("log")
}
pub fn default_brk() -> PathBuf {
path_dot_brk()
pub fn default_brk_path() -> PathBuf {
dot_brk_path()
}
pub fn default_bitcoin_path() -> PathBuf {
@@ -24,11 +24,11 @@ pub fn default_bitcoin_path() -> PathBuf {
}
}
fn default_linux_bitcoin_path() -> PathBuf {
pub fn default_linux_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap()).join(".bitcoin")
}
fn default_mac_bitcoin_path() -> PathBuf {
pub fn default_mac_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")

40
crates/brk_fetcher/src/fetchers/binance.rs
View File

@@ -13,6 +13,7 @@ use serde_json::Value;
use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, fetchers::retry};
#[derive(Clone)]
pub struct Binance {
path: Option<PathBuf>,
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
@@ -35,7 +36,9 @@ impl Binance {
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
self._1mn.replace(Self::fetch_1mn()?);
}
@@ -54,14 +57,25 @@ impl Binance {
self.har.replace(self.read_har().unwrap_or_default());
}
Fetcher::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har")
Fetcher::find_height_ohlc(
self.har.as_ref().unwrap(),
timestamp,
previous_timestamp,
"binance har",
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Binance...");
retry(
|_| Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url("interval=1m&limit=1000")).send()?.json()?),
|_| {
Self::json_to_timestamp_to_ohlc(
&minreq::get(Self::url("interval=1m&limit=1000"))
.send()?
.json()?,
)
},
30,
10,
)
@@ -141,7 +155,13 @@ impl Binance {
.contains("/uiKlines")
})
.map(|entry| {
let response = entry.as_object().unwrap().get("response").unwrap().as_object().unwrap();
let response = entry
.as_object()
.unwrap()
.get("response")
.unwrap()
.as_object()
.unwrap();
let content = response.get("content").unwrap().as_object().unwrap();
@@ -161,7 +181,9 @@ impl Binance {
})
}
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
@@ -188,7 +210,13 @@ impl Binance {
let get_cents = |index: usize| {
Cents::from(Dollars::from(
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
array
.get(index)
.unwrap()
.as_str()
.unwrap()
.parse::<f64>()
.unwrap(),
))
};

7
crates/brk_fetcher/src/fetchers/kibo.rs
View File

@@ -7,7 +7,7 @@ use serde_json::Value;
use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry};
#[derive(Default)]
#[derive(Default, Clone)]
pub struct Kibo {
height_to_ohlc_vec: BTreeMap<Height, Vec<OHLCCents>>,
year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLCCents>>,
@@ -92,10 +92,7 @@ impl Kibo {
.unwrap()
.get(date)
.cloned()
.ok_or({
dbg!(date);
eyre!("Couldn't find date in kibo")
})
.ok_or(eyre!("Couldn't find date in kibo"))
}
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {

25
crates/brk_fetcher/src/fetchers/kraken.rs
View File

@@ -7,7 +7,7 @@ use serde_json::Value;
use crate::{Fetcher, fetchers::retry};
#[derive(Default)]
#[derive(Default, Clone)]
pub struct Kraken {
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
_1d: Option<BTreeMap<Date, OHLCCents>>,
@@ -19,10 +19,17 @@ impl Kraken {
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
self._1mn.replace(Self::fetch_1mn()?);
}
Fetcher::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
Fetcher::find_height_ohlc(
self._1mn.as_ref().unwrap(),
timestamp,
previous_timestamp,
"kraken 1m",
)
}
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
@@ -57,7 +64,9 @@ impl Kraken {
)
}
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
@@ -92,7 +101,13 @@ impl Kraken {
let get_cents = |index: usize| {
Cents::from(Dollars::from(
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
array
.get(index)
.unwrap()
.as_str()
.unwrap()
.parse::<f64>()
.unwrap(),
))
};

2
crates/brk_fetcher/src/lib.rs
View File

@@ -10,9 +10,9 @@ use color_eyre::eyre::Error;
mod fetchers;
// use brk_indexer::Indexer;
use fetchers::*;
#[derive(Clone)]
pub struct Fetcher {
binance: Binance,
kraken: Kraken,

2
crates/brk_indexer/examples/main.rs
View File

@@ -22,7 +22,7 @@ fn main() -> color_eyre::Result<()> {
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let mut indexer = Indexer::new(Path::new("../../_outputs/indexed").to_owned())?;
indexer.import_stores()?;

2
crates/brk_indexer/src/stores/mod.rs
View File

@@ -181,7 +181,7 @@ impl Stores {
}
}
self.commit(starting_indexes.height.decremented().unwrap())?;
self.commit(starting_indexes.height.decremented().unwrap_or_default())?;
Ok(())
}

2
crates/brk_logger/Cargo.toml
View File

@@ -8,5 +8,5 @@ repository.workspace = true
[dependencies]
color-eyre = { workspace = true }
env_logger = "0.11.6"
env_logger = "0.11.7"
jiff = { workspace = true }

2
crates/brk_parser/examples/main.rs
View File

@@ -18,7 +18,7 @@ fn main() {
// let start = None;
// let end = None;
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
// parser
// .parse(start, end)

4
crates/brk_parser/src/blk_index_to_blk_path.rs
View File

@@ -13,9 +13,7 @@ const DAT: &str = ".dat";
pub struct BlkIndexToBlkPath(BTreeMap<u16, PathBuf>);
impl BlkIndexToBlkPath {
pub fn scan(bitcoin_dir: &Path) -> Self {
let blocks_dir = bitcoin_dir.join("blocks");
pub fn scan(blocks_dir: &Path) -> Self {
Self(
fs::read_dir(blocks_dir)
.unwrap()

14
crates/brk_parser/src/lib.rs
View File

@@ -47,13 +47,13 @@ const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 50;
pub struct Parser {
bitcoin_dir: PathBuf,
blocks_dir: PathBuf,
rpc: &'static bitcoincore_rpc::Client,
}
impl Parser {
pub fn new(bitcoin_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { bitcoin_dir, rpc }
pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { blocks_dir, rpc }
}
pub fn get(&self, height: Height) -> Block {
@@ -74,19 +74,19 @@ impl Parser {
start: Option<Height>,
end: Option<Height>,
) -> Receiver<(Height, Block, BlockHash)> {
let bitcoin_dir = self.bitcoin_dir.as_path();
let blocks_dir = self.blocks_dir.as_path();
let rpc = self.rpc;
let (send_bytes, recv_bytes) = bounded(BOUND_CAP);
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(bitcoin_dir);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);
let (mut blk_index_to_blk_recap, blk_index) =
BlkIndexToBlkRecap::import(bitcoin_dir, &blk_index_to_blk_path, start);
BlkIndexToBlkRecap::import(blocks_dir, &blk_index_to_blk_path, start);
let xor_bytes = XORBytes::from(bitcoin_dir);
let xor_bytes = XORBytes::from(blocks_dir);
thread::spawn(move || {
let xor_bytes = xor_bytes;

2
crates/brk_parser/src/xor_bytes.rs
View File

@@ -10,7 +10,7 @@ pub struct XORBytes([u8; XOR_LEN]);
impl From<&Path> for XORBytes {
fn from(value: &Path) -> Self {
Self(
fs::read(value.join("blocks/xor.dat"))
fs::read(value.join("xor.dat"))
.unwrap_or(vec![0; 8])
.try_into()
.unwrap(),

2
crates/brk_query/examples/main.rs
View File

@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
let mut indexer = Indexer::new(outputs_dir.join("indexed"))?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"));
let mut computer = Computer::new(outputs_dir.join("computed"), None);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

1
crates/brk_server/Cargo.toml
View File

@@ -10,6 +10,7 @@ repository.workspace = true
axum = "0.8.1"
brk_computer = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_core = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }

7
crates/brk_server/examples/main.rs
View File

@@ -3,6 +3,7 @@ use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
@@ -26,7 +27,7 @@ pub fn main() -> color_eyre::Result<()> {
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let outputs_dir = Path::new("../../_outputs");
@@ -34,7 +35,9 @@ pub fn main() -> color_eyre::Result<()> {
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"));
let fetcher = Some(Fetcher::import(None)?);
let mut computer = Computer::new(outputs_dir.join("computed"), fetcher);
computer.import_stores()?;
computer.import_vecs()?;

3
crates/brk_server/src/files/frontend.rs
View File

@@ -5,8 +5,9 @@ use serde::{Deserialize, Serialize};
Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, ValueEnum,
)]
pub enum Frontend {
None,
#[default]
None,
#[value(name = "kibo.money")]
KiboMoney,
Custom,
}

4
crates/brk_server/src/lib.rs
View File

@@ -18,7 +18,7 @@ use axum::{
serve,
};
use brk_computer::Computer;
use brk_core::path_dot_brk;
use brk_core::dot_brk_path;
use brk_indexer::Indexer;
use brk_query::Query;
use color_eyre::owo_colors::OwoColorize;
@@ -66,7 +66,7 @@ impl Server {
let websites_path = if fs::exists(&websites_dev_path)? {
websites_dev_path
} else {
let downloads_path = path_dot_brk().join(DOWNLOADS);
let downloads_path = dot_brk_path().join(DOWNLOADS);
let downloaded_websites_path = downloads_path.join("brk-main").join(WEBSITES);