global: snapshot

This commit is contained in:
nym21
2025-03-10 23:08:07 +01:00
parent 9428beeae5
commit db70b05088
30 changed files with 326 additions and 189 deletions

141
Cargo.lock generated
View File

@@ -396,6 +396,7 @@ dependencies = [
"brk_computer",
"brk_core",
"brk_exit",
"brk_fetcher",
"brk_indexer",
"brk_logger",
"brk_parser",
@@ -438,7 +439,7 @@ dependencies = [
"rlimit",
"serde",
"serde_bytes",
"zerocopy 0.8.23",
"zerocopy",
]
[[package]]
@@ -478,7 +479,7 @@ dependencies = [
"fjall",
"log",
"rayon",
"zerocopy 0.8.23",
"zerocopy",
]
[[package]]
@@ -502,7 +503,7 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"zerocopy 0.8.23",
"zerocopy",
]
[[package]]
@@ -529,6 +530,7 @@ dependencies = [
"brk_computer",
"brk_core",
"brk_exit",
"brk_fetcher",
"brk_indexer",
"brk_logger",
"brk_parser",
@@ -557,7 +559,7 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"zerocopy 0.8.23",
"zerocopy",
]
[[package]]
@@ -690,9 +692,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.31"
version = "4.5.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767"
checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83"
dependencies = [
"clap_builder",
"clap_derive",
@@ -700,9 +702,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.31"
version = "4.5.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863"
checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8"
dependencies = [
"anstream",
"anstyle",
@@ -712,14 +714,14 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.28"
version = "4.5.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed"
checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -939,7 +941,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -950,7 +952,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -991,7 +993,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -1024,7 +1026,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -1048,7 +1050,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -1063,14 +1065,14 @@ dependencies = [
[[package]]
name = "env_logger"
version = "0.11.6"
version = "0.11.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0"
checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697"
dependencies = [
"anstream",
"anstyle",
"env_filter",
"humantime",
"jiff",
"log",
]
@@ -1341,12 +1343,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "1.6.0"
@@ -1430,9 +1426,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.7.1"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
@@ -1501,7 +1497,7 @@ checksum = "dbc3e0019b0f5f43038cf46471b1312136f29e36f54436c6042c8f155fec8789"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -1764,9 +1760,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.20.3"
version = "1.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad"
[[package]]
name = "outref"
@@ -1826,7 +1822,7 @@ checksum = "e21f680e8c5f1900297d394627d495351b9e37761f7bbf90116bd5eeb6e80967"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -1867,7 +1863,7 @@ checksum = "8f1505d8622b2ea6ed0274f355bd5e4ee3f09df5d9b39c8a3a673f344d87b82a"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2207,7 +2203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
dependencies = [
"fixedbitset",
"indexmap 2.7.1",
"indexmap 2.8.0",
]
[[package]]
@@ -2240,7 +2236,7 @@ dependencies = [
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2293,11 +2289,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.20"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy 0.7.35",
"zerocopy",
]
[[package]]
@@ -2319,7 +2315,7 @@ dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2491,9 +2487,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustix"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657"
checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
dependencies = [
"bitflags",
"errno",
@@ -2593,31 +2589,31 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "serde"
version = "1.0.218"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_bytes"
version = "0.11.16"
version = "0.11.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "364fec0df39c49a083c9a8a18a23a6bcfd9af130fe9fe321d18520a0d113e09e"
checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96"
dependencies = [
"serde",
]
[[package]]
name = "serde_derive"
version = "1.0.218"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2673,7 +2669,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.7.1",
"indexmap 2.8.0",
"serde",
"serde_derive",
"serde_json",
@@ -2690,7 +2686,7 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2811,9 +2807,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.99"
version = "2.0.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2"
checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
dependencies = [
"proc-macro2",
"quote",
@@ -2846,7 +2842,7 @@ dependencies = [
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2900,7 +2896,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2911,7 +2907,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -2981,7 +2977,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -3024,7 +3020,7 @@ version = "0.22.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
dependencies = [
"indexmap 2.7.1",
"indexmap 2.8.0",
"serde",
"serde_spanned",
"toml_datetime",
@@ -3240,7 +3236,7 @@ dependencies = [
"log",
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
"wasm-bindgen-shared",
]
@@ -3262,7 +3258,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -3403,34 +3399,13 @@ version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
"zerocopy-derive 0.7.35",
]
[[package]]
name = "zerocopy"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6"
dependencies = [
"zerocopy-derive 0.8.23",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"zerocopy-derive",
]
[[package]]
@@ -3441,7 +3416,7 @@ checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -3461,7 +3436,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
"syn 2.0.100",
]
[[package]]
@@ -3480,7 +3455,7 @@ dependencies = [
"displaydoc",
"flate2",
"hmac",
"indexmap 2.7.1",
"indexmap 2.8.0",
"lzma-rs",
"memchr",
"pbkdf2",

View File

@@ -22,7 +22,7 @@ brk_query = { version = "0", path = "crates/brk_query" }
brk_server = { version = "0", path = "crates/brk_server" }
brk_vec = { version = "0", path = "crates/brk_vec" }
byteview = "0.5.4"
clap = { version = "4.5.31", features = ["derive", "string"] }
clap = { version = "4.5.32", features = ["derive", "string"] }
color-eyre = "0.6.3"
derive_deref = "1.1.1"
fjall = "2.6.7"
@@ -30,7 +30,7 @@ jiff = "0.2.3"
log = { version = "0.4.26" }
minreq = { version = "2.13.2", features = ["https", "serde_json"] }
rayon = "1.10.0"
serde = { version = "1.0.218", features = ["derive"] }
serde = { version = "1.0.219", features = ["derive"] }
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
tabled = "0.18.0"
zerocopy = { version = "0.8.23", features = ["derive"] }

View File

@@ -10,6 +10,7 @@ repository.workspace = true
brk_computer = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }

View File

@@ -1,6 +1,6 @@
use std::fs;
use brk_core::{path_dot_brk, path_dot_brk_log};
use brk_core::{dot_brk_log_path, dot_brk_path};
use brk_query::Params as QueryArgs;
use clap::{Parser, Subcommand};
use query::query;
@@ -28,9 +28,9 @@ enum Commands {
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
fs::create_dir_all(path_dot_brk())?;
fs::create_dir_all(dot_brk_path())?;
brk_logger::init(Some(&path_dot_brk_log()));
brk_logger::init(Some(&dot_brk_log_path()));
let cli = Cli::parse();

View File

@@ -11,7 +11,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let mut indexer = Indexer::new(config.indexeddir())?;
indexer.import_vecs()?;
let mut computer = Computer::new(config.computeddir());
let mut computer = Computer::new(config.computeddir(), None);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

View File

@@ -6,8 +6,9 @@ use std::{
};
use brk_computer::Computer;
use brk_core::path_dot_brk;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::rpc::{self, Auth, Client, RpcApi};
use brk_server::{Frontend, Server, tokio};
@@ -23,13 +24,17 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let exit = Exit::new();
let parser = brk_parser::Parser::new(config.bitcoindir(), rpc);
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
let mut indexer = Indexer::new(config.indexeddir())?;
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::new(config.computeddir());
let fetcher = config
.fetch()
.then(|| Fetcher::import(Some(config.harsdir().as_path())).unwrap());
let mut computer = Computer::new(config.computeddir(), fetcher);
computer.import_stores()?;
computer.import_vecs()?;
@@ -82,11 +87,15 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
pub struct RunConfig {
/// Bitcoin data directory path, saved
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
#[arg(long, value_name = "PATH")]
bitcoindir: Option<String>,
/// Bitcoin Research Kit outputs directory path, saved
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
#[arg(long, value_name = "PATH")]
blocksdir: Option<String>,
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
@@ -94,8 +103,12 @@ pub struct RunConfig {
#[arg(short, long)]
mode: Option<Mode>,
/// Frontend served by the server (if active), default: kibo.money, saved
#[arg(short, long)]
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: false, saved
#[arg(short, long, value_name = "BOOL")]
fetch: Option<bool>,
/// Frontend served by the server (if active), default: none, saved
#[arg(short = 'F', long)]
frontend: Option<Frontend>,
/// Bitcoin RPC ip, default: localhost, saved
@@ -125,7 +138,7 @@ pub struct RunConfig {
impl RunConfig {
pub fn import(config_args: Option<RunConfig>) -> color_eyre::Result<Self> {
let path = path_dot_brk();
let path = dot_brk_path();
let _ = fs::create_dir_all(&path);
@@ -138,6 +151,10 @@ impl RunConfig {
config_saved.bitcoindir = Some(bitcoindir);
}
if let Some(blocksdir) = config_args.blocksdir.take() {
config_saved.blocksdir = Some(blocksdir);
}
if let Some(brkdir) = config_args.brkdir.take() {
config_saved.brkdir = Some(brkdir);
}
@@ -146,6 +163,10 @@ impl RunConfig {
config_saved.mode = Some(mode);
}
if let Some(fetch) = config_args.fetch.take() {
config_saved.fetch = Some(fetch);
}
if let Some(frontend) = config_args.frontend.take() {
config_saved.frontend = Some(frontend);
}
@@ -203,33 +224,24 @@ impl RunConfig {
}
fn check(&self) {
if self.bitcoindir.is_none() {
println!(
"You need to set the --bitcoindir parameter at least once to run the parser.\nRun the program with '-h' for help."
);
std::process::exit(1);
} else if !self.bitcoindir().is_dir() {
println!(
"Given --bitcoindir parameter doesn't seem to be a valid directory path.\nRun the program with '-h' for help."
);
if !self.bitcoindir().is_dir() {
println!("{:?} isn't a valid directory", self.bitcoindir());
println!("Please use the --bitcoindir parameter to set a valid path.");
println!("Run the program with '-h' for help.");
std::process::exit(1);
}
if self.brkdir.is_none() {
println!(
"You need to set the --brkdir parameter at least once to run the parser.\nRun the program with '-h' for help."
);
std::process::exit(1);
} else if !self.brkdir().is_dir() {
println!(
"Given --brkdir parameter doesn't seem to be a valid directory path.\nRun the program with '-h' for help."
);
if !self.blocksdir().is_dir() {
println!("{:?} isn't a valid directory", self.blocksdir());
println!("Please use the --blocksdir parameter to set a valid path.");
println!("Run the program with '-h' for help.");
std::process::exit(1);
}
let path = self.bitcoindir();
if !path.is_dir() {
println!("Expect path '{:#?}' to be a directory.", path);
if !self.brkdir().is_dir() {
println!("{:?} isn't a valid directory", self.brkdir());
println!("Please use the --brkdir parameter to set a valid path.");
println!("Run the program with '-h' for help.");
std::process::exit(1);
}
@@ -290,11 +302,22 @@ impl RunConfig {
}
pub fn bitcoindir(&self) -> PathBuf {
Self::fix_user_path(self.bitcoindir.as_ref().unwrap().as_ref())
self.bitcoindir
.as_ref()
.map_or_else(default_bitcoin_path, |s| Self::fix_user_path(s.as_ref()))
}
pub fn blocksdir(&self) -> PathBuf {
self.blocksdir.as_ref().map_or_else(
|| self.bitcoindir().join("blocks"),
|blocksdir| Self::fix_user_path(blocksdir.as_str()),
)
}
pub fn brkdir(&self) -> PathBuf {
Self::fix_user_path(self.brkdir.as_ref().unwrap().as_ref())
self.brkdir
.as_ref()
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
}
fn outputsdir(&self) -> PathBuf {
@@ -309,6 +332,10 @@ impl RunConfig {
self.outputsdir().join("computed")
}
pub fn harsdir(&self) -> PathBuf {
self.outputsdir().join("hars")
}
pub fn process(&self) -> bool {
self.mode
.is_none_or(|m| m == Mode::All || m == Mode::Processor)
@@ -347,6 +374,10 @@ impl RunConfig {
pub fn frontend(&self) -> Frontend {
self.frontend.unwrap_or_default()
}
pub fn fetch(&self) -> bool {
self.fetch.is_some_and(|b| b)
}
}
#[derive(

View File

@@ -3,6 +3,7 @@ use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
@@ -23,7 +24,7 @@ pub fn main() -> color_eyre::Result<()> {
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let outputs_dir = Path::new("../../_outputs");
@@ -31,7 +32,9 @@ pub fn main() -> color_eyre::Result<()> {
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"));
let fetcher = Fetcher::import(None)?;
let mut computer = Computer::new(outputs_dir.join("computed"), Some(fetcher));
computer.import_stores()?;
computer.import_vecs()?;

View File

@@ -6,6 +6,7 @@
use std::path::{Path, PathBuf};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::{Indexer, Indexes};
pub use brk_parser::rpc;
@@ -17,25 +18,30 @@ use storage::{Stores, Vecs};
#[derive(Clone)]
pub struct Computer {
path: PathBuf,
fetcher: Option<Fetcher>,
vecs: Option<Vecs>,
stores: Option<Stores>,
}
impl Computer {
pub fn new(computed_dir: PathBuf) -> Self {
pub fn new(computed_dir: PathBuf, fetcher: Option<Fetcher>) -> Self {
Self {
path: computed_dir,
fetcher,
vecs: None,
stores: None,
}
}
pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(&self.path.join("vecs"))?);
self.vecs = Some(Vecs::import(
&self.path.join("vecs"),
self.fetcher.is_some(),
)?);
Ok(())
}
/// Do NOT import multiple times are things will break !!!
/// Do NOT import multiple times or things will break !!!
/// Clone struct instead
pub fn import_stores(&mut self) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(&self.path.join("stores"))?);
@@ -52,7 +58,12 @@ impl Computer {
) -> color_eyre::Result<()> {
info!("Computing...");
self.mut_vecs().compute(indexer, starting_indexes, exit)?;
self.vecs.as_mut().unwrap().compute(
indexer,
starting_indexes,
self.fetcher.as_mut(),
exit,
)?;
Ok(())
}

View File

@@ -1,6 +1,8 @@
use std::{fs, path::Path};
use brk_core::{Cents, Close, Dateindex, Dollars, Height, High, Low, OHLCCents, OHLCDollars, Open};
use brk_core::{
Cents, Close, Dateindex, Dollars, Height, High, Low, OHLCCents, OHLCDollars, Open, Sats,
};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
@@ -20,6 +22,7 @@ pub struct Vecs {
pub dateindex_to_low: StorableVec<Dateindex, Low<Dollars>>,
pub dateindex_to_open_in_cents: StorableVec<Dateindex, Open<Cents>>,
pub dateindex_to_open: StorableVec<Dateindex, Open<Dollars>>,
pub dateindex_to_sats_per_dollar: StorableVec<Dateindex, Close<Sats>>,
pub height_to_ohlc_in_cents: StorableVec<Height, OHLCCents>,
pub height_to_ohlc: StorableVec<Height, OHLCDollars>,
pub height_to_close_in_cents: StorableVec<Height, Close<Cents>>,
@@ -30,6 +33,7 @@ pub struct Vecs {
pub height_to_low: StorableVec<Height, Low<Dollars>>,
pub height_to_open_in_cents: StorableVec<Height, Open<Cents>>,
pub height_to_open: StorableVec<Height, Open<Dollars>>,
pub height_to_sats_per_dollar: StorableVec<Height, Close<Sats>>,
}
impl Vecs {
@@ -77,6 +81,10 @@ impl Vecs {
&path.join("dateindex_to_open"),
Version::from(1),
)?,
dateindex_to_sats_per_dollar: StorableVec::import(
&path.join("dateindex_to_sats_per_dollar"),
Version::from(1),
)?,
height_to_ohlc_in_cents: StorableVec::import(
&path.join("height_to_ohlc_in_cents"),
Version::from(1),
@@ -102,6 +110,10 @@ impl Vecs {
Version::from(1),
)?,
height_to_open: StorableVec::import(&path.join("height_to_open"), Version::from(1))?,
height_to_sats_per_dollar: StorableVec::import(
&path.join("height_to_sats_per_dollar"),
Version::from(1),
)?,
})
}
@@ -110,10 +122,9 @@ impl Vecs {
indexer: &mut Indexer,
indexes: &mut indexes::Vecs,
starting_indexes: Indexes,
fetcher: &mut Fetcher,
exit: &Exit,
) -> color_eyre::Result<()> {
let mut fetcher = Fetcher::import(None)?;
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
@@ -199,6 +210,13 @@ impl Vecs {
exit,
)?;
self.height_to_sats_per_dollar.compute_transform(
starting_indexes.height,
&mut self.height_to_close,
|(di, close, ..)| (di, Close::from(Sats::ONE_BTC / **close)),
exit,
)?;
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
&mut indexes.dateindex_to_date,
@@ -272,6 +290,13 @@ impl Vecs {
exit,
)?;
self.dateindex_to_sats_per_dollar.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_close,
|(di, close, ..)| (di, Close::from(Sats::ONE_BTC / **close)),
exit,
)?;
Ok(())
}
@@ -287,6 +312,7 @@ impl Vecs {
&self.dateindex_to_ohlc_in_cents,
&self.dateindex_to_open,
&self.dateindex_to_open_in_cents,
&self.dateindex_to_sats_per_dollar,
&self.height_to_close,
&self.height_to_close_in_cents,
&self.height_to_high,
@@ -297,6 +323,7 @@ impl Vecs {
&self.height_to_ohlc_in_cents,
&self.height_to_open,
&self.height_to_open_in_cents,
&self.height_to_sats_per_dollar,
]
}
}

View File

@@ -2,6 +2,7 @@ use std::{fs, path::Path};
use brk_core::{Height, Sats, Txindex, Txinindex, Txoutindex};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, StorableVec, Version};
@@ -11,7 +12,7 @@ mod marketprice;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub marketprice: marketprice::Vecs,
pub marketprice: Option<marketprice::Vecs>,
// pub height_to_block_interval: StorableVec<Height, Timestamp>,
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Height, u32>,
@@ -38,13 +39,13 @@ pub struct Vecs {
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
pub fn import(path: &Path, fetch: bool) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
indexes: indexes::Vecs::import(path)?,
marketprice: marketprice::Vecs::import(path)?,
marketprice: fetch.then(|| marketprice::Vecs::import(path).unwrap()),
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::forced_import(
@@ -107,12 +108,20 @@ impl Vecs {
&mut self,
indexer: &mut Indexer,
starting_indexes: brk_indexer::Indexes,
fetcher: Option<&mut Fetcher>,
exit: &Exit,
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
self.marketprice
.compute(indexer, &mut self.indexes, starting_indexes, exit)?;
if let Some(marketprice) = self.marketprice.as_mut() {
marketprice.compute(
indexer,
&mut self.indexes,
starting_indexes,
fetcher.unwrap(),
exit,
)?;
}
// self.mut_vecs().height_to_ohlc
@@ -197,6 +206,12 @@ impl Vecs {
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
[self.indexes.as_any_vecs(), self.marketprice.as_any_vecs()].concat()
[
self.indexes.as_any_vecs(),
self.marketprice
.as_ref()
.map_or(vec![], |v| v.as_any_vecs()),
]
.concat()
}
}

View File

@@ -16,7 +16,7 @@ log = { workspace = true }
rapidhash = "1.4.0"
rlimit = "0.10.2"
serde = { workspace = true }
serde_bytes = "0.11.16"
serde_bytes = "0.11.17"
zerocopy = { workspace = true }
[package.metadata.cargo-machete]

View File

@@ -1,14 +1,31 @@
use std::ops::Mul;
use super::Sats;
#[derive(Debug, Default, Clone, Copy)]
pub struct Bitcoin(f64);
impl Bitcoin {
const ONE: Self = Self(100_000_000.0);
impl Mul for Bitcoin {
type Output = Self;
fn mul(self, rhs: Self) -> Self::Output {
Self(self.0 * rhs.0)
}
}
impl From<Sats> for Bitcoin {
fn from(value: Sats) -> Self {
Self(f64::from(value) / Self::ONE.0)
Self(u64::from(value) as f64 / (u64::from(Sats::ONE_BTC) as f64))
}
}
impl From<f64> for Bitcoin {
fn from(value: f64) -> Self {
Self(value)
}
}
impl From<Bitcoin> for f64 {
fn from(value: Bitcoin) -> Self {
value.0
}
}

View File

@@ -1,13 +1,13 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Mul, Sub, SubAssign},
ops::{Add, AddAssign, Div, Mul, Sub, SubAssign},
};
use bitcoin::Amount;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
use super::{Bitcoin, Dollars, Height};
#[derive(
Debug,
@@ -28,6 +28,7 @@ pub struct Sats(u64);
impl Sats {
pub const ZERO: Self = Self(0);
pub const ONE_BTC: Self = Self(100_000_000);
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
@@ -88,6 +89,13 @@ impl Sum for Sats {
}
}
impl Div<Dollars> for Sats {
type Output = Self;
fn div(self, rhs: Dollars) -> Self::Output {
Self((self.0 as f64 / f64::from(rhs)) as u64)
}
}
impl From<u64> for Sats {
fn from(value: u64) -> Self {
Self(value)
@@ -105,8 +113,14 @@ impl From<Sats> for Amount {
}
}
impl From<Sats> for f64 {
fn from(value: Sats) -> Self {
value.0 as f64
impl From<Bitcoin> for Sats {
fn from(value: Bitcoin) -> Self {
Self((f64::from(value) * (u64::from(Sats::ONE_BTC) as f64)) as u64)
}
}
impl From<Sats> for u64 {
fn from(value: Sats) -> Self {
value.0
}
}

View File

@@ -3,17 +3,17 @@ use std::{
path::{Path, PathBuf},
};
pub fn path_dot_brk() -> PathBuf {
pub fn dot_brk_path() -> PathBuf {
let home = std::env::var("HOME").unwrap();
Path::new(&home).join(".brk")
}
pub fn path_dot_brk_log() -> PathBuf {
path_dot_brk().join("log")
pub fn dot_brk_log_path() -> PathBuf {
dot_brk_path().join("log")
}
pub fn default_brk() -> PathBuf {
path_dot_brk()
pub fn default_brk_path() -> PathBuf {
dot_brk_path()
}
pub fn default_bitcoin_path() -> PathBuf {
@@ -24,11 +24,11 @@ pub fn default_bitcoin_path() -> PathBuf {
}
}
fn default_linux_bitcoin_path() -> PathBuf {
pub fn default_linux_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap()).join(".bitcoin")
}
fn default_mac_bitcoin_path() -> PathBuf {
pub fn default_mac_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")

View File

@@ -13,6 +13,7 @@ use serde_json::Value;
use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, fetchers::retry};
#[derive(Clone)]
pub struct Binance {
path: Option<PathBuf>,
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
@@ -35,7 +36,9 @@ impl Binance {
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
self._1mn.replace(Self::fetch_1mn()?);
}
@@ -54,14 +57,25 @@ impl Binance {
self.har.replace(self.read_har().unwrap_or_default());
}
Fetcher::find_height_ohlc(self.har.as_ref().unwrap(), timestamp, previous_timestamp, "binance har")
Fetcher::find_height_ohlc(
self.har.as_ref().unwrap(),
timestamp,
previous_timestamp,
"binance har",
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Binance...");
retry(
|_| Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url("interval=1m&limit=1000")).send()?.json()?),
|_| {
Self::json_to_timestamp_to_ohlc(
&minreq::get(Self::url("interval=1m&limit=1000"))
.send()?
.json()?,
)
},
30,
10,
)
@@ -141,7 +155,13 @@ impl Binance {
.contains("/uiKlines")
})
.map(|entry| {
let response = entry.as_object().unwrap().get("response").unwrap().as_object().unwrap();
let response = entry
.as_object()
.unwrap()
.get("response")
.unwrap()
.as_object()
.unwrap();
let content = response.get("content").unwrap().as_object().unwrap();
@@ -161,7 +181,9 @@ impl Binance {
})
}
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
@@ -188,7 +210,13 @@ impl Binance {
let get_cents = |index: usize| {
Cents::from(Dollars::from(
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
array
.get(index)
.unwrap()
.as_str()
.unwrap()
.parse::<f64>()
.unwrap(),
))
};

View File

@@ -7,7 +7,7 @@ use serde_json::Value;
use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry};
#[derive(Default)]
#[derive(Default, Clone)]
pub struct Kibo {
height_to_ohlc_vec: BTreeMap<Height, Vec<OHLCCents>>,
year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLCCents>>,
@@ -92,10 +92,7 @@ impl Kibo {
.unwrap()
.get(date)
.cloned()
.ok_or({
dbg!(date);
eyre!("Couldn't find date in kibo")
})
.ok_or(eyre!("Couldn't find date in kibo"))
}
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {

View File

@@ -7,7 +7,7 @@ use serde_json::Value;
use crate::{Fetcher, fetchers::retry};
#[derive(Default)]
#[derive(Default, Clone)]
pub struct Kraken {
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
_1d: Option<BTreeMap<Date, OHLCCents>>,
@@ -19,10 +19,17 @@ impl Kraken {
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
self._1mn.replace(Self::fetch_1mn()?);
}
Fetcher::find_height_ohlc(self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
Fetcher::find_height_ohlc(
self._1mn.as_ref().unwrap(),
timestamp,
previous_timestamp,
"kraken 1m",
)
}
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
@@ -57,7 +64,9 @@ impl Kraken {
)
}
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
@@ -92,7 +101,13 @@ impl Kraken {
let get_cents = |index: usize| {
Cents::from(Dollars::from(
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
array
.get(index)
.unwrap()
.as_str()
.unwrap()
.parse::<f64>()
.unwrap(),
))
};

View File

@@ -10,9 +10,9 @@ use color_eyre::eyre::Error;
mod fetchers;
// use brk_indexer::Indexer;
use fetchers::*;
#[derive(Clone)]
pub struct Fetcher {
binance: Binance,
kraken: Kraken,

View File

@@ -22,7 +22,7 @@ fn main() -> color_eyre::Result<()> {
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let mut indexer = Indexer::new(Path::new("../../_outputs/indexed").to_owned())?;
indexer.import_stores()?;

View File

@@ -181,7 +181,7 @@ impl Stores {
}
}
self.commit(starting_indexes.height.decremented().unwrap())?;
self.commit(starting_indexes.height.decremented().unwrap_or_default())?;
Ok(())
}

View File

@@ -8,5 +8,5 @@ repository.workspace = true
[dependencies]
color-eyre = { workspace = true }
env_logger = "0.11.6"
env_logger = "0.11.7"
jiff = { workspace = true }

View File

@@ -18,7 +18,7 @@ fn main() {
// let start = None;
// let end = None;
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
// parser
// .parse(start, end)

View File

@@ -13,9 +13,7 @@ const DAT: &str = ".dat";
pub struct BlkIndexToBlkPath(BTreeMap<u16, PathBuf>);
impl BlkIndexToBlkPath {
pub fn scan(bitcoin_dir: &Path) -> Self {
let blocks_dir = bitcoin_dir.join("blocks");
pub fn scan(blocks_dir: &Path) -> Self {
Self(
fs::read_dir(blocks_dir)
.unwrap()

View File

@@ -47,13 +47,13 @@ const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 50;
pub struct Parser {
bitcoin_dir: PathBuf,
blocks_dir: PathBuf,
rpc: &'static bitcoincore_rpc::Client,
}
impl Parser {
pub fn new(bitcoin_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { bitcoin_dir, rpc }
pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { blocks_dir, rpc }
}
pub fn get(&self, height: Height) -> Block {
@@ -74,19 +74,19 @@ impl Parser {
start: Option<Height>,
end: Option<Height>,
) -> Receiver<(Height, Block, BlockHash)> {
let bitcoin_dir = self.bitcoin_dir.as_path();
let blocks_dir = self.blocks_dir.as_path();
let rpc = self.rpc;
let (send_bytes, recv_bytes) = bounded(BOUND_CAP);
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(bitcoin_dir);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);
let (mut blk_index_to_blk_recap, blk_index) =
BlkIndexToBlkRecap::import(bitcoin_dir, &blk_index_to_blk_path, start);
BlkIndexToBlkRecap::import(blocks_dir, &blk_index_to_blk_path, start);
let xor_bytes = XORBytes::from(bitcoin_dir);
let xor_bytes = XORBytes::from(blocks_dir);
thread::spawn(move || {
let xor_bytes = xor_bytes;

View File

@@ -10,7 +10,7 @@ pub struct XORBytes([u8; XOR_LEN]);
impl From<&Path> for XORBytes {
fn from(value: &Path) -> Self {
Self(
fs::read(value.join("blocks/xor.dat"))
fs::read(value.join("xor.dat"))
.unwrap_or(vec![0; 8])
.try_into()
.unwrap(),

View File

@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
let mut indexer = Indexer::new(outputs_dir.join("indexed"))?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"));
let mut computer = Computer::new(outputs_dir.join("computed"), None);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

View File

@@ -10,6 +10,7 @@ repository.workspace = true
axum = "0.8.1"
brk_computer = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_core = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }

View File

@@ -3,6 +3,7 @@ use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
@@ -26,7 +27,7 @@ pub fn main() -> color_eyre::Result<()> {
)?));
let exit = Exit::new();
let parser = Parser::new(bitcoin_dir.to_owned(), rpc);
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let outputs_dir = Path::new("../../_outputs");
@@ -34,7 +35,9 @@ pub fn main() -> color_eyre::Result<()> {
indexer.import_stores()?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir.join("computed"));
let fetcher = Some(Fetcher::import(None)?);
let mut computer = Computer::new(outputs_dir.join("computed"), fetcher);
computer.import_stores()?;
computer.import_vecs()?;

View File

@@ -5,8 +5,9 @@ use serde::{Deserialize, Serialize};
Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, ValueEnum,
)]
pub enum Frontend {
None,
#[default]
None,
#[value(name = "kibo.money")]
KiboMoney,
Custom,
}

View File

@@ -18,7 +18,7 @@ use axum::{
serve,
};
use brk_computer::Computer;
use brk_core::path_dot_brk;
use brk_core::dot_brk_path;
use brk_indexer::Indexer;
use brk_query::Query;
use color_eyre::owo_colors::OwoColorize;
@@ -66,7 +66,7 @@ impl Server {
let websites_path = if fs::exists(&websites_dev_path)? {
websites_dev_path
} else {
let downloads_path = path_dot_brk().join(DOWNLOADS);
let downloads_path = dot_brk_path().join(DOWNLOADS);
let downloaded_websites_path = downloads_path.join("brk-main").join(WEBSITES);