Compare commits

...

44 Commits

Author SHA1 Message Date
nym21 c87b1c133c release: v0.0.65 2025-06-17 22:30:32 +02:00
nym21 9b275ecdae web: fix hang when candles fetched are slightly diff than before 2025-06-17 22:30:11 +02:00
nym21 49d66a133e release: v0.0.64 2025-06-17 18:49:17 +02:00
nym21 c559f26d0e global: add a bunch of realized datasets + charts 2025-06-17 18:47:04 +02:00
nym21 bbe9f1bad2 global: 4y zscore + 200d sma + mayer's multiple 2025-06-17 10:01:49 +02:00
nym21 7e1fb6472d release: v0.0.63 2025-06-16 23:46:57 +02:00
nym21 0ff8d20573 web: fix the fix for the stutter + pwa assets 2025-06-16 23:46:39 +02:00
nym21 9c1f9448dc release: v0.0.62 2025-06-16 18:56:59 +02:00
nym21 43a6081dd6 web: fix stutter on update and save default chart settings to url params 2025-06-16 18:56:38 +02:00
nym21 985e961876 web: fix error in lockdown safari + charts: update instead of setData when possible 2025-06-16 18:20:56 +02:00
nym21 098f6de047 release: v0.0.61 2025-06-15 17:30:49 +02:00
nym21 1b0f90fd68 release: v0.0.60 2025-06-15 17:27:41 +02:00
nym21 12252f407b computer: fix open of ohlc if fetched from different API than prev ohlc 2025-06-15 17:27:16 +02:00
nym21 3b6e3f47ab release: v0.0.59 2025-06-15 12:40:46 +02:00
nym21 6a9ac9b025 brk: fix bundler use + bundler: remove minify html crate 2025-06-15 12:39:50 +02:00
nym21 ae6aa4088b release: v0.0.58 2025-06-15 01:50:22 +02:00
nym21 c08f431180 bundler: deploy brk_rolldown + fix edge case 2025-06-15 01:50:01 +02:00
nym21 123c1f56e9 release: v0.0.57 2025-06-14 22:47:57 +02:00
nym21 35ac65a864 server: update cache control for bundled websites 2025-06-14 22:47:26 +02:00
nym21 e9f362cc87 bundler: init working version 2025-06-14 20:17:49 +02:00
nym21 65685c23e1 release: v0.0.56 2025-06-13 18:03:28 +02:00
nym21 2f74748cea computer: stateful: reset when reorg detected 2025-06-13 18:03:09 +02:00
nym21 f477bd66f3 release: v0.0.55 2025-06-13 10:23:38 +02:00
nym21 d7d77ae8f0 global: multiple fixes 2025-06-13 10:22:03 +02:00
nym21 31110a740d release: v0.0.54 2025-06-12 22:18:36 +02:00
nym21 b64d8b1d7f release: v0.0.53 2025-06-12 22:16:33 +02:00
nym21 c46006aacc web: filter possible index choices in charts 2025-06-12 22:09:33 +02:00
nym21 92f81b1493 web: fix css 2025-06-12 20:23:23 +02:00
nym21 70213cfc8f websites: default: add auto price series type 2025-06-12 18:41:56 +02:00
nym21 8a82bf5c50 websites: default: add live price 2025-06-12 18:10:24 +02:00
nym21 37405384a2 vec: fixed compressed, still slow par read, cli: made raw the default 2025-06-12 16:31:54 +02:00
nym21 54ea6cc53b indexer: only raw format + global: fixes 2025-06-12 12:33:43 +02:00
nym21 339c00d815 release: v0.0.52 2025-06-11 21:19:41 +02:00
nym21 ea6b4dcde2 websites: default: remove scrollToSelected 2025-06-11 21:19:22 +02:00
nym21 2b84623d1e release: v0.0.51 2025-06-11 21:09:07 +02:00
nym21 c8b3afa56b websites: default: fix sw adn co 2025-06-11 21:08:42 +02:00
nym21 1348f3c24c release: v0.0.50 2025-06-11 18:11:22 +02:00
nym21 62208ce3e1 websites: default: fix minBarSpacing 2025-06-11 18:11:11 +02:00
nym21 813b2481de release: v0.0.49 2025-06-11 17:51:31 +02:00
nym21 27b924ba61 cargo: set full version of crates 2025-06-11 17:51:11 +02:00
nym21 b40170b8ce websites: default: snapshot 2025-06-11 17:45:17 +02:00
nym21 8bfa9d2734 websites: default: snapshot 2025-06-11 11:25:25 +02:00
nym21 c7cf76d4a8 websites: default: snapshot 2025-06-10 18:54:18 +02:00
nym21 dfd2969b3e websites: default: snapshot 2025-06-09 17:58:26 +02:00
187 changed files with 8854 additions and 4464 deletions
+1
View File
@@ -3,6 +3,7 @@
# Builds
target
dist
# Copies
*\ copy*
Generated
+1275 -150
View File
File diff suppressed because it is too large Load Diff
+18 -17
View File
@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.0.48"
package.version = "0.0.65"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
@@ -22,26 +22,27 @@ axum = "0.8.4"
bincode = { version = "2.0.1", features = ["serde"] }
bitcoin = { version = "0.32.6", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_cli = { version = "0", path = "crates/brk_cli" }
brk_computer = { version = "0", path = "crates/brk_computer" }
brk_core = { version = "0", path = "crates/brk_core" }
brk_exit = { version = "0", path = "crates/brk_exit" }
brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
brk_indexer = { version = "0", path = "crates/brk_indexer" }
brk_logger = { version = "0", path = "crates/brk_logger" }
brk_parser = { version = "0", path = "crates/brk_parser" }
brk_query = { version = "0", path = "crates/brk_query" }
brk_server = { version = "0", path = "crates/brk_server" }
brk_state = { version = "0", path = "crates/brk_state" }
brk_store = { version = "0", path = "crates/brk_store" }
brk_vec = { version = "0", path = "crates/brk_vec" }
brk_bundler = { version = "0.0.65", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.65", path = "crates/brk_cli" }
brk_computer = { version = "0.0.65", path = "crates/brk_computer" }
brk_core = { version = "0.0.65", path = "crates/brk_core" }
brk_exit = { version = "0.0.65", path = "crates/brk_exit" }
brk_fetcher = { version = "0.0.65", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.0.65", path = "crates/brk_indexer" }
brk_logger = { version = "0.0.65", path = "crates/brk_logger" }
brk_parser = { version = "0.0.65", path = "crates/brk_parser" }
brk_query = { version = "0.0.65", path = "crates/brk_query" }
brk_server = { version = "0.0.65", path = "crates/brk_server" }
brk_state = { version = "0.0.65", path = "crates/brk_state" }
brk_store = { version = "0.0.65", path = "crates/brk_store" }
brk_vec = { version = "0.0.65", path = "crates/brk_vec" }
byteview = "=0.6.1"
clap = { version = "4.5.39", features = ["string"] }
clap_derive = "4.5.32"
clap = { version = "4.5.40", features = ["string"] }
clap_derive = "4.5.40"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall = "2.11.0"
jiff = "0.2.14"
jiff = "0.2.15"
log = { version = "0.4.27" }
minreq = { version = "2.13.4", features = ["https", "serde_json"] }
rayon = "1.10.0"
+1
View File
@@ -71,6 +71,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
## Hosting as a service
+3
View File
@@ -10,6 +10,7 @@ version.workspace = true
[features]
full = [
"bundler",
"core",
"computer",
"exit",
@@ -23,6 +24,7 @@ full = [
"store",
"vec",
]
bundler = ["brk_bundler"]
core = ["brk_core"]
computer = ["brk_computer"]
exit = ["brk_exit"]
@@ -37,6 +39,7 @@ store = ["brk_store"]
vec = ["brk_vec"]
[dependencies]
brk_bundler = { workspace = true, optional = true }
brk_cli = { workspace = true }
brk_core = { workspace = true, optional = true }
brk_computer = { workspace = true, optional = true }
+1
View File
@@ -0,0 +1 @@
fn main() {}
+7
View File
@@ -1,5 +1,12 @@
#![doc = include_str!(concat!("../", env!("CARGO_PKG_README")))]
#[cfg(feature = "bundler")]
#[doc(inline)]
pub use brk_bundler as bundler;
#[doc(inline)]
pub use brk_cli as cli;
#[cfg(feature = "core")]
#[doc(inline)]
pub use brk_core as core;
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "brk_bundler"
description = "A thin wrapper around rolldown"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
log = { workspace = true }
notify = "8.0.0"
brk_rolldown = "0.0.1"
sugar_path = "1.2.0"
tokio = { workspace = true }
+145
View File
@@ -0,0 +1,145 @@
use std::{fs, io, path::Path, sync::Arc};
use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
use log::error;
use notify::{EventKind, RecursiveMode, Watcher};
use sugar_path::SugarPath;
use tokio::sync::Mutex;
const VERSION: &str = env!("CARGO_PKG_VERSION");
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
let source_path = websites_path.join(source_folder);
let dist_path = websites_path.join("dist");
let _ = fs::remove_dir_all(&dist_path);
copy_dir_all(&source_path, &dist_path)?;
let source_scripts = format!("./{source_folder}/scripts");
let source_entry = format!("{source_scripts}/entry.js");
let absolute_websites_path = websites_path.absolutize();
let mut bundler = Bundler::new(BundlerOptions {
input: Some(vec![source_entry.into()]),
dir: Some("./dist/scripts".to_string()),
cwd: Some(absolute_websites_path),
minify: Some(RawMinifyOptions::Bool(true)),
sourcemap: Some(SourceMapType::File),
..Default::default()
});
bundler.write().await.unwrap();
let absolute_source_index_path = source_path.join("index.html").absolutize();
let absolute_source_index_path_clone = absolute_source_index_path.clone();
let absolute_source_path = source_path.absolutize();
let absolute_source_path_clone = absolute_source_path.clone();
let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
let absolute_dist_index_path = dist_path.join("index.html").absolutize();
let absolute_dist_path = dist_path.absolutize();
let absolute_dist_path_clone = absolute_dist_path.clone();
let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
let write_index = move || {
let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
if let Some(start) = entry.find("main") {
if let Some(end) = entry.find(".js") {
let main_hashed = &entry[start..end];
contents =
contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
}
}
}
let _ = fs::write(&absolute_dist_index_path, contents);
};
let write_sw = move || {
let contents = fs::read_to_string(&absolute_source_sw_path)
.unwrap()
.replace("__VERSION__", &format!("v{VERSION}"));
let _ = fs::write(&absolute_dist_sw_path, contents);
};
write_index();
write_sw();
if !watch {
return Ok(());
}
tokio::spawn(async move {
let write_index_clone = write_index.clone();
let mut entry_watcher = notify::recommended_watcher(
move |res: Result<notify::Event, notify::Error>| match res {
Ok(_) => write_index_clone(),
Err(e) => error!("watch error: {:?}", e),
},
)
.unwrap();
entry_watcher
.watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
.unwrap();
let mut source_watcher = notify::recommended_watcher(
move |res: Result<notify::Event, notify::Error>| match res {
Ok(event) => match event.kind {
EventKind::Create(_) => event.paths,
EventKind::Modify(_) => event.paths,
_ => vec![],
}
.into_iter()
.filter(|path| path.starts_with(&absolute_source_path))
.filter(|path| !path.starts_with(&absolute_source_scripts_path))
.for_each(|source_path| {
let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
let dist_path = absolute_dist_path.join(suffix);
if source_path == absolute_source_index_path_clone {
write_index();
} else if source_path == absolute_source_sw_path_clone {
write_sw();
} else {
let _ = fs::copy(&source_path, &dist_path);
}
}),
Err(e) => error!("watch error: {:?}", e),
},
)
.unwrap();
source_watcher
.watch(&absolute_source_path_clone, RecursiveMode::Recursive)
.unwrap();
let watcher =
brk_rolldown::Watcher::new(vec![Arc::new(Mutex::new(bundler))], None).unwrap();
watcher.start().await;
});
Ok(())
}
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(&dst)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_dir() {
copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
} else {
fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
}
}
Ok(())
}
+2 -2
View File
@@ -20,9 +20,9 @@ struct Cli {
#[derive(Subcommand, Debug)]
enum Commands {
/// Run the indexer, computer and server
/// Run the indexer, computer and server, use `run -h` for more information
Run(RunConfig),
/// Query generated datasets via the `run` command in a similar fashion as the server's API
/// Query generated datasets via the `run` command in a similar fashion as the server's API, use `query -h` for more information
Query(QueryArgs),
}
+1 -1
View File
@@ -10,7 +10,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let format = config.format();
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
indexer.import_vecs()?;
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
+44 -28
View File
@@ -7,7 +7,7 @@ use std::{
use bitcoincore_rpc::{self, Auth, Client, RpcApi};
use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
@@ -29,7 +29,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let format = config.format();
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
indexer.import_stores()?;
indexer.import_vecs()?;
@@ -63,8 +63,9 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let server = Server::new(served_indexer, served_computer, config.website())?;
let watch = config.watch();
let opt = Some(tokio::spawn(async move {
server.serve().await.unwrap();
server.serve(watch).await.unwrap();
}));
sleep(Duration::from_secs(1));
@@ -109,62 +110,82 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
pub struct RunConfig {
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
bitcoindir: Option<String>,
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
blocksdir: Option<String>,
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
/// Executed by the runner, default: all, saved
/// Activated services, default: all, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
mode: Option<Mode>,
services: Option<Services>,
/// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
computation: Option<Computation>,
/// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved
#[arg(short, long, value_name = "FORMAT")]
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
format: Option<Format>,
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short = 'F', long, value_name = "BOOL")]
fetch: Option<bool>,
/// Website served by the server (if active), default: default, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
website: Option<Website>,
/// Bitcoin RPC ip, default: localhost, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "IP")]
rpcconnect: Option<String>,
/// Bitcoin RPC port, default: 8332, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PORT")]
rpcport: Option<u16>,
/// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
rpccookiefile: Option<String>,
/// Bitcoin RPC username, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "USERNAME")]
rpcuser: Option<String>,
/// Bitcoin RPC password, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PASSWORD")]
rpcpassword: Option<String>,
/// Delay between runs, default: 0, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "SECONDS")]
delay: Option<u64>,
/// DEV: Activate to watch the selected website's folder for changes, default: false, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "BOOL")]
watch: Option<bool>,
/// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "BOOL")]
check_collisions: Option<bool>,
}
@@ -192,8 +213,8 @@ impl RunConfig {
config_saved.brkdir = Some(brkdir);
}
if let Some(mode) = config_args.mode.take() {
config_saved.mode = Some(mode);
if let Some(services) = config_args.services.take() {
config_saved.services = Some(services);
}
if let Some(computation) = config_args.computation.take() {
@@ -240,6 +261,10 @@ impl RunConfig {
config_saved.check_collisions = Some(check_collisions);
}
if let Some(watch) = config_args.watch.take() {
config_saved.watch = Some(watch);
}
if config_args != RunConfig::default() {
dbg!(config_args);
panic!("Didn't consume the full config")
@@ -252,19 +277,6 @@ impl RunConfig {
config.write(&path)?;
// info!("Configuration {{");
// info!(" bitcoindir: {:?}", config.bitcoindir);
// info!(" brkdir: {:?}", config.brkdir);
// info!(" mode: {:?}", config.mode);
// info!(" website: {:?}", config.website);
// info!(" rpcconnect: {:?}", config.rpcconnect);
// info!(" rpcport: {:?}", config.rpcport);
// info!(" rpccookiefile: {:?}", config.rpccookiefile);
// info!(" rpcuser: {:?}", config.rpcuser);
// info!(" rpcpassword: {:?}", config.rpcpassword);
// info!(" delay: {:?}", config.delay);
// info!("}}");
Ok(config)
}
@@ -375,13 +387,13 @@ impl RunConfig {
}
pub fn process(&self) -> bool {
self.mode
.is_none_or(|m| m == Mode::All || m == Mode::Processor)
self.services
.is_none_or(|m| m == Services::All || m == Services::Processor)
}
pub fn serve(&self) -> bool {
self.mode
.is_none_or(|m| m == Mode::All || m == Mode::Server)
self.services
.is_none_or(|m| m == Services::All || m == Services::Server)
}
fn path_cookiefile(&self) -> PathBuf {
@@ -433,6 +445,10 @@ impl RunConfig {
pub fn check_collisions(&self) -> bool {
self.check_collisions.is_some_and(|b| b)
}
pub fn watch(&self) -> bool {
self.watch.is_some_and(|b| b)
}
}
#[derive(
@@ -449,7 +465,7 @@ impl RunConfig {
PartialOrd,
Ord,
)]
pub enum Mode {
pub enum Services {
#[default]
All,
Processor,
+1 -1
View File
@@ -33,7 +33,7 @@ pub fn main() -> color_eyre::Result<()> {
let format = Format::Raw;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
let mut indexer = Indexer::new(outputs_dir, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
+13 -3
View File
@@ -7,7 +7,7 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format, StoredIndex};
use super::{
Indexes,
@@ -429,8 +429,18 @@ impl Vecs {
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
&indexes.dateindex_to_date,
|(di, d, ..)| {
let ohlc = fetcher.get_date(d).unwrap();
|(di, d, this)| {
let mut ohlc = fetcher.get_date(d).unwrap();
if let Some(prev) = di.decremented() {
let prev_open = *this
.get_or_read(prev, &this.mmap().load())
.unwrap()
.unwrap()
.close;
*ohlc.open = prev_open;
*ohlc.high = (*ohlc.high).max(prev_open);
*ohlc.low = (*ohlc.low).min(prev_open);
}
(di, ohlc)
},
exit,
@@ -130,7 +130,11 @@ where
Box::new(
EagerVec::forced_import(
path,
&maybe_suffix("sum"),
&(if !options.last {
name.to_string()
} else {
maybe_suffix("sum")
}),
version + VERSION + Version::ZERO,
format,
)
@@ -24,8 +24,10 @@ pub struct ComputedRatioVecsFromDateIndex {
pub ratio_1w_sma: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_1m_sma: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_1y_sma: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_4y_sma: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_standard_deviation: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_sd: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_4y_sd: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_p99_9: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_p99_5: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_p99: ComputedVecsFromDateIndex<StoredF32>,
@@ -51,6 +53,7 @@ pub struct ComputedRatioVecsFromDateIndex {
pub ratio_m2sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_m3sd_as_price: ComputedVecsFromDateIndex<Dollars>,
pub ratio_zscore: ComputedVecsFromDateIndex<StoredF32>,
pub ratio_4y_zscore: ComputedVecsFromDateIndex<StoredF32>,
}
const VERSION: Version = Version::ZERO;
@@ -116,6 +119,14 @@ impl ComputedRatioVecsFromDateIndex {
format,
options,
)?,
ratio_4y_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_4y_sma"),
true,
version + VERSION + Version::ZERO,
format,
options,
)?,
ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
@@ -124,9 +135,17 @@ impl ComputedRatioVecsFromDateIndex {
format,
options,
)?,
ratio_standard_deviation: ComputedVecsFromDateIndex::forced_import(
ratio_sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_standard_deviation"),
&format!("{name}_ratio_sd"),
true,
version + VERSION + Version::ZERO,
format,
options,
)?,
ratio_4y_sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_4y_sd"),
true,
version + VERSION + Version::ZERO,
format,
@@ -332,6 +351,14 @@ impl ComputedRatioVecsFromDateIndex {
format,
options,
)?,
ratio_4y_zscore: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_4y_zscore"),
true,
version + VERSION + Version::ZERO,
format,
options,
)?,
})
}
@@ -476,6 +503,22 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_4y_sma.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
4 * 365,
exit,
Some(min_ratio_date),
)
},
)?;
self.ratio_1y_sma_momentum_oscillator.compute_all(
indexer,
indexes,
@@ -528,6 +571,8 @@ impl ComputedRatioVecsFromDateIndex {
let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
let mut _4y_sma_iter = self.ratio_4y_sma.dateindex.as_ref().unwrap().into_iter();
let nan = StoredF32::from(f32::NAN);
self.ratio
.dateindex
@@ -566,7 +611,12 @@ impl ComputedRatioVecsFromDateIndex {
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_standard_deviation
self.ratio_sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_4y_sd
.dateindex
.as_mut()
.unwrap()
@@ -645,12 +695,26 @@ impl ComputedRatioVecsFromDateIndex {
.sqrt(),
);
self.ratio_standard_deviation
self.ratio_sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, sd, exit)?;
let _4y_avg = _4y_sma_iter.unwrap_get_inner(index);
let _4y_sd = StoredF32::from(
(sorted.iter().map(|v| (**v - *_4y_avg).powi(2)).sum::<f32>()
/ (index.unwrap_to_usize() + 1) as f32)
.sqrt(),
);
self.ratio_4y_sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, _4y_sd, exit)?;
self.ratio_p1sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + sd,
@@ -726,7 +790,13 @@ impl ComputedRatioVecsFromDateIndex {
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_standard_deviation.compute_rest(
self.ratio_sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_4y_sd.compute_rest(
indexes,
starting_indexes,
exit,
@@ -1007,21 +1077,27 @@ impl ComputedRatioVecsFromDateIndex {
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
let mut sd_iter = self
.ratio_standard_deviation
.dateindex
.as_ref()
.unwrap()
.into_iter();
vec.compute_transform(
vec.compute_zscore(
starting_indexes.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
|(i, ratio, ..)| {
let sma = sma_iter.unwrap_get_inner(i);
let sd = sd_iter.unwrap_get_inner(i);
(i, (ratio - sma) / sd)
},
self.ratio_sma.dateindex.as_ref().unwrap(),
self.ratio_sd.dateindex.as_ref().unwrap(),
exit,
)
},
)?;
self.ratio_4y_zscore.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_zscore(
starting_indexes.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
self.ratio_4y_sma.dateindex.as_ref().unwrap(),
self.ratio_4y_sd.dateindex.as_ref().unwrap(),
exit,
)
},
@@ -1032,7 +1108,8 @@ impl ComputedRatioVecsFromDateIndex {
fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec<DateIndex, StoredF32>> {
vec![
self.ratio_standard_deviation.dateindex.as_mut().unwrap(),
self.ratio_sd.dateindex.as_mut().unwrap(),
self.ratio_4y_sd.dateindex.as_mut().unwrap(),
self.ratio_p99_9.dateindex.as_mut().unwrap(),
self.ratio_p99_5.dateindex.as_mut().unwrap(),
self.ratio_p99.dateindex.as_mut().unwrap(),
@@ -1056,8 +1133,10 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_1w_sma.vecs(),
self.ratio_1m_sma.vecs(),
self.ratio_1y_sma.vecs(),
self.ratio_4y_sma.vecs(),
self.ratio_1y_sma_momentum_oscillator.vecs(),
self.ratio_standard_deviation.vecs(),
self.ratio_sd.vecs(),
self.ratio_4y_sd.vecs(),
self.ratio_p99_9.vecs(),
self.ratio_p99_5.vecs(),
self.ratio_p99.vecs(),
@@ -1083,6 +1162,7 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_m2sd_as_price.vecs(),
self.ratio_m3sd_as_price.vecs(),
self.ratio_zscore.vecs(),
self.ratio_4y_zscore.vecs(),
]
.into_iter()
.flatten()
@@ -226,7 +226,11 @@ impl ComputedValueVecsFromTxindex {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.sats.vecs(),
vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
self.bitcoin.vecs(),
self.dollars_txindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
+78 -1
View File
@@ -36,11 +36,15 @@ pub struct Vecs {
pub indexes_to_55d_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_89d_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_144d_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_200d_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_1y_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_2y_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_200w_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_4y_sma: ComputedRatioVecsFromDateIndex,
pub indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex<Dollars>,
pub indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex<Dollars>,
pub price_1d_ago: ComputedVecsFromDateIndex<Dollars>,
pub price_1w_ago: ComputedVecsFromDateIndex<Dollars>,
pub price_1m_ago: ComputedVecsFromDateIndex<Dollars>,
@@ -306,6 +310,14 @@ impl Vecs {
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_200d_sma: ComputedRatioVecsFromDateIndex::forced_import(
path,
"200d_sma",
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_1y_sma: ComputedRatioVecsFromDateIndex::forced_import(
path,
"1y_sma",
@@ -1215,6 +1227,23 @@ impl Vecs {
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex::forced_import(
path,
"200d_sma_x2_4",
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex::forced_import(
path,
"200d_sma_x0_8",
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)?,
})
}
@@ -1771,6 +1800,7 @@ impl Vecs {
(&mut self.indexes_to_55d_sma, 55),
(&mut self.indexes_to_89d_sma, 89),
(&mut self.indexes_to_144d_sma, 144),
(&mut self.indexes_to_200d_sma, 200),
(&mut self.indexes_to_1y_sma, 365),
(&mut self.indexes_to_2y_sma, 2 * 365),
(&mut self.indexes_to_200w_sma, 200 * 7),
@@ -1797,7 +1827,51 @@ impl Vecs {
});
});
Ok(())
})
})?;
self.indexes_to_200d_sma_x0_8.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.indexes_to_200d_sma
.price
.as_ref()
.unwrap()
.dateindex
.as_ref()
.unwrap(),
|(i, v, ..)| (i, v * 0.8),
exit,
)
},
)?;
self.indexes_to_200d_sma_x2_4.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
self.indexes_to_200d_sma
.price
.as_ref()
.unwrap()
.dateindex
.as_ref()
.unwrap(),
|(i, v, ..)| (i, v * 2.4),
exit,
)
},
)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -1817,10 +1891,13 @@ impl Vecs {
self.indexes_to_55d_sma.vecs(),
self.indexes_to_89d_sma.vecs(),
self.indexes_to_144d_sma.vecs(),
self.indexes_to_200d_sma.vecs(),
self.indexes_to_1y_sma.vecs(),
self.indexes_to_2y_sma.vecs(),
self.indexes_to_200w_sma.vecs(),
self.indexes_to_4y_sma.vecs(),
self.indexes_to_200d_sma_x0_8.vecs(),
self.indexes_to_200d_sma_x2_4.vecs(),
self.price_1d_ago.vecs(),
self.price_1w_ago.vecs(),
self.price_1m_ago.vecs(),
+1
View File
@@ -124,6 +124,7 @@ impl Vecs {
fetcher: Option<&mut Fetcher>,
exit: &Exit,
) -> color_eyre::Result<()> {
info!("Computing indexes...");
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
+208 -20
View File
@@ -57,7 +57,7 @@ pub struct Vecs {
pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
pub dateindex_to_adjusted_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
pub dateindex_to_realized_cap_30d_change: Option<EagerVec<DateIndex, Dollars>>,
pub indexes_to_realized_cap_30d_change: Option<ComputedVecsFromDateIndex<Dollars>>,
pub dateindex_to_sell_side_risk_ratio: Option<EagerVec<DateIndex, StoredF32>>,
pub dateindex_to_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
@@ -89,6 +89,10 @@ pub struct Vecs {
Option<EagerVec<Height, StoredF32>>,
pub indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_realized_profit_relative_to_realized_cap:
Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_realized_loss_relative_to_realized_cap:
Option<ComputedVecsFromHeight<StoredF32>>,
pub indexes_to_net_realized_profit_and_loss_relative_to_realized_cap:
Option<ComputedVecsFromHeight<StoredF32>>,
pub height_to_supply_even_value: Option<ComputedHeightValueVecs>,
@@ -117,6 +121,12 @@ pub struct Vecs {
Option<ComputedVecsFromDateIndex<StoredF64>>,
pub indexes_to_supply_in_profit_relative_to_circulating_supply:
Option<ComputedVecsFromDateIndex<StoredF64>>,
pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change:
Option<ComputedVecsFromDateIndex<Dollars>>,
pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap:
Option<ComputedVecsFromDateIndex<StoredF32>>,
}
impl Vecs {
@@ -432,7 +442,9 @@ impl Vecs {
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_sum(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)
.unwrap()
}),
@@ -452,7 +464,9 @@ impl Vecs {
false,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_sum(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)
.unwrap()
}),
@@ -463,7 +477,7 @@ impl Vecs {
true,
version + VERSION + Version::ONE,
format,
StorableVecGeneatorOptions::default().add_sum(),
StorableVecGeneatorOptions::default().add_sum().add_cumulative(),
)
.unwrap()
}),
@@ -558,12 +572,14 @@ impl Vecs {
)
.unwrap()
}),
dateindex_to_realized_cap_30d_change: compute_dollars.then(|| {
EagerVec::forced_import(
indexes_to_realized_cap_30d_change: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("realized_cap_30d_change"),
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
@@ -574,7 +590,9 @@ impl Vecs {
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_sum(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)
.unwrap()
}),
@@ -686,15 +704,37 @@ impl Vecs {
.unwrap()
},
),
indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_profit_relative_to_realized_cap"),
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
}),
indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_loss_relative_to_realized_cap"),
true,
version + VERSION + Version::ZERO,
format,
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
}),
indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then(
|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("net_realized_profit_and_loss_relative_to_realized_cap"),
true,
version + VERSION + Version::ZERO,
version + VERSION + Version::ONE,
format,
StorableVecGeneatorOptions::default().add_last(),
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
},
@@ -903,6 +943,39 @@ impl Vecs {
format,
StorableVecGeneatorOptions::default().add_sum(),
)?,
indexes_to_cumulative_net_realized_profit_and_loss_30d_change: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change")),
true,
version + VERSION + Version::new(3),
format,
StorableVecGeneatorOptions::default().add_last()
)
.unwrap()
}),
indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change_relative_to_realized_cap")),
true,
version + VERSION + Version::new(3),
format,
StorableVecGeneatorOptions::default().add_last()
)
.unwrap()
}),
indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change_relative_to_market_cap")),
true,
version + VERSION + Version::new(3),
format,
StorableVecGeneatorOptions::default().add_last()
)
.unwrap()
}),
})
}
@@ -1877,6 +1950,7 @@ impl Vecs {
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
dateindex_to_supply: &impl AnyIterableVec<DateIndex, Bitcoin>,
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> color_eyre::Result<()> {
if let Some(v) = self
@@ -2021,18 +2095,26 @@ impl Vecs {
Some(self.height_to_adjusted_value_destroyed.as_ref().unwrap()),
)?;
self.dateindex_to_realized_cap_30d_change
self.indexes_to_realized_cap_30d_change
.as_mut()
.unwrap()
.compute_change(
starting_indexes.dateindex,
self.indexes_to_realized_cap
.as_ref()
.unwrap()
.dateindex
.unwrap_last(),
30,
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_change(
starting_indexes.dateindex,
self.indexes_to_realized_cap
.as_ref()
.unwrap()
.dateindex
.unwrap_last(),
30,
exit,
)
},
)?;
self.indexes_to_net_realized_profit_and_loss
@@ -2280,6 +2362,42 @@ impl Vecs {
},
)?;
self.indexes_to_realized_profit_relative_to_realized_cap
.as_mut()
.unwrap()
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.height,
self.height_to_realized_profit.as_ref().unwrap(),
*height_to_realized_cap.as_ref().unwrap(),
exit,
)
},
)?;
self.indexes_to_realized_loss_relative_to_realized_cap
.as_mut()
.unwrap()
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.height,
self.height_to_realized_loss.as_ref().unwrap(),
*height_to_realized_cap.as_ref().unwrap(),
exit,
)
},
)?;
self.indexes_to_net_realized_profit_and_loss_relative_to_realized_cap
.as_mut()
.unwrap()
@@ -2435,6 +2553,64 @@ impl Vecs {
},
)?;
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change
.as_mut()
.unwrap()
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_change(
starting_indexes.dateindex,
self.indexes_to_net_realized_profit_and_loss
.as_ref()
.unwrap()
.dateindex
.unwrap_cumulative(),
30,
exit,
)
},
)?;
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap.
as_mut()
.unwrap()
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(),
*dateindex_to_realized_cap.as_ref().unwrap(),
exit,
)
},
)?;
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap.
as_mut()
.unwrap()
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(),
market.indexes_to_marketcap.dateindex.as_ref().unwrap(),
exit,
)
},
)?;
if let Some(height_to_supply_even_relative_to_circulating_supply) = self
.height_to_supply_even_relative_to_circulating_supply
.as_mut()
@@ -2618,9 +2794,9 @@ impl Vecs {
self.indexes_to_adjusted_value_destroyed
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.dateindex_to_realized_cap_30d_change
self.indexes_to_realized_cap_30d_change
.as_ref()
.map_or(vec![], |v| vec![v]),
.map_or(vec![], |v| v.vecs()),
self.indexes_to_net_realized_profit_and_loss
.as_ref()
.map_or(vec![], |v| v.vecs()),
@@ -2703,6 +2879,12 @@ impl Vecs {
self.indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_realized_profit_relative_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_realized_loss_relative_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_net_realized_profit_and_loss_relative_to_realized_cap
.as_ref()
.map_or(vec![], |v| v.vecs()),
@@ -2756,6 +2938,12 @@ impl Vecs {
.map_or(vec![], |v| v.vecs()),
self.indexes_to_coinblocks_destroyed.vecs(),
self.indexes_to_coindays_destroyed.vecs(),
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
+267 -260
View File
@@ -4,8 +4,8 @@ use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Result, S
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Computation, EagerVec, Format,
GenericStoredVec, StoredIndex, StoredVec, UnsafeSlice, VecIterator,
AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, GenericStoredVec,
StoredIndex, StoredVec, UnsafeSlice, VecIterator,
};
use log::info;
use outputs::OutputCohorts;
@@ -1289,7 +1289,7 @@ impl Vecs {
base_version + self.height_to_opreturn_supply.inner_version(),
)?;
let mut chain_state: Vec<BlockState>;
let mut chain_state: Vec<BlockState> = vec![];
let mut chain_state_starting_height = Height::from(self.chain_state.len());
let stateful_starting_height = match separate_utxo_vecs
@@ -1322,287 +1322,286 @@ impl Vecs {
.collect::<Vec<_>>();
chain_state_starting_height
}
Ordering::Less => {
// todo!("rollback instead");
chain_state = vec![];
chain_state_starting_height = Height::ZERO;
Height::ZERO
}
Ordering::Less => Height::ZERO,
};
if stateful_starting_height.is_zero() {
info!("Starting processing utxos from the start");
separate_utxo_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
}
let starting_height = starting_indexes
.height
.min(stateful_starting_height)
.min(Height::from(self.height_to_unspendable_supply.len()))
.min(Height::from(self.height_to_opreturn_supply.len()));
if starting_height == Height::from(height_to_date_fixed.len()) {
return Ok(());
if starting_height.is_zero() {
info!("Starting processing utxos from the start");
// todo!("rollback instead");
chain_state = vec![];
chain_state_starting_height = Height::ZERO;
separate_utxo_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
}
// ---
// INIT
// ---
if starting_height < Height::from(height_to_date_fixed.len()) {
starting_indexes.update_from_height(starting_height, indexes);
separate_utxo_vecs
.par_iter_mut()
.for_each(|(_, v)| v.init(starting_height));
separate_utxo_vecs
.par_iter_mut()
.for_each(|(_, v)| v.init(starting_height));
let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_unspendable_supply
.into_iter()
.unwrap_get_inner(prev_height)
} else {
Sats::ZERO
};
let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_opreturn_supply
.into_iter()
.unwrap_get_inner(prev_height)
} else {
Sats::ZERO
};
let mut height = starting_height;
starting_indexes.update_from_height(height, indexes);
(height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
.map(Height::from)
.try_for_each(|_height| -> color_eyre::Result<()> {
height = _height;
self.utxos_vecs
.as_mut_separate_vecs()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
info!("Processing chain at {height}...");
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
let price = height_to_close_iter
.as_mut()
.map(|i| *i.unwrap_get_inner(height));
let first_outputindex = height_to_first_outputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
let first_inputindex = height_to_first_inputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let (mut height_to_sent, mut received) = thread::scope(|s| {
if chain_state_starting_height <= height {
s.spawn(|| {
self.utxos_vecs
.tick_tock_next_block(&chain_state, timestamp);
});
}
let sent_handle = s.spawn(|| {
// Skip coinbase
(first_inputindex + 1..first_inputindex + *input_count)
.into_par_iter()
.map(InputIndex::from)
.map(|inputindex| {
let outputindex = inputindex_to_outputindex
.get_or_read(inputindex, &inputindex_to_outputindex_mmap)
.unwrap()
.unwrap()
.into_inner();
let value = outputindex_to_value
.get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap()
.unwrap()
.into_inner();
let input_type = outputindex_to_outputtype
.get_or_read(outputindex, &outputindex_to_outputtype_mmap)
.unwrap()
.unwrap()
.into_inner();
// dbg!(input_type);
if input_type.is_unspendable() {
unreachable!()
}
let input_txindex = outputindex_to_txindex
.get_or_read(outputindex, &outputindex_to_txindex_mmap)
.unwrap()
.unwrap()
.into_inner();
let height = txindex_to_height
.get_or_read(input_txindex, &txindex_to_height_mmap)
.unwrap()
.unwrap()
.into_inner();
(height, value, input_type)
})
.fold(
BTreeMap::<Height, Transacted>::default,
|mut tree, (height, value, input_type)| {
tree.entry(height).or_default().iterate(value, input_type);
tree
},
)
.reduce(BTreeMap::<Height, Transacted>::default, |first, second| {
let (mut source, to_consume) = if first.len() > second.len() {
(first, second)
} else {
(second, first)
};
to_consume.into_iter().for_each(|(k, v)| {
*source.entry(k).or_default() += v;
});
source
})
});
let received_handle = s.spawn(|| {
(first_outputindex..first_outputindex + *output_count)
.into_par_iter()
.map(OutputIndex::from)
.map(|outputindex| {
let value = outputindex_to_value
.get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap()
.unwrap()
.into_inner();
let output_type = outputindex_to_outputtype
.get_or_read(outputindex, &outputindex_to_outputtype_mmap)
.unwrap()
.unwrap()
.into_inner();
(value, output_type)
})
.fold(
Transacted::default,
|mut transacted, (value, output_type)| {
transacted.iterate(value, output_type);
transacted
},
)
.reduce(Transacted::default, |acc, transacted| acc + transacted)
});
(sent_handle.join().unwrap(), received_handle.join().unwrap())
});
unspendable_supply += received
.by_type
.unspendable
.as_vec()
let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_unspendable_supply
.into_iter()
.map(|state| state.value)
.sum::<Sats>()
+ height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
opreturn_supply += received.by_type.unspendable.opreturn.value;
if height == Height::new(0) {
received = Transacted::default();
unspendable_supply += Sats::FIFTY_BTC;
} else if height == Height::new(91_842) || height == Height::new(91_880) {
// Need to destroy invalid coinbases due to duplicate txids
if height == Height::new(91_842) {
height_to_sent.entry(Height::new(91_812)).or_default()
} else {
height_to_sent.entry(Height::new(91_722)).or_default()
}
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
};
if chain_state_starting_height <= height {
// Push current block state before processing sends and receives
chain_state.push(BlockState {
supply: received.spendable_supply.clone(),
price,
timestamp,
});
self.utxos_vecs.receive(received, height, price);
let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
height_to_sent.par_iter().for_each(|(height, sent)| unsafe {
(*unsafe_chain_state.get(height.unwrap_to_usize())).supply -=
&sent.spendable_supply;
});
self.utxos_vecs.send(height_to_sent, chain_state.as_slice());
} else {
dbg!(chain_state_starting_height, height);
panic!("temp, just making sure")
}
let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs();
separate_utxo_vecs
.iter_mut()
.try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
self.height_to_unspendable_supply.forced_push_at(
height,
unspendable_supply,
exit,
)?;
.unwrap_get_inner(prev_height)
} else {
Sats::ZERO
};
let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_opreturn_supply
.forced_push_at(height, opreturn_supply, exit)?;
.into_iter()
.unwrap_get_inner(prev_height)
} else {
Sats::ZERO
};
let date = height_to_date_fixed_iter.unwrap_get_inner(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
let is_date_last_height = date_first_height
+ Height::from(date_height_count).decremented().unwrap()
== height;
let date_price = dateindex_to_close_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
let mut height = starting_height;
separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
(height.unwrap_to_usize()..height_to_date_fixed.len())
.map(Height::from)
.try_for_each(|_height| -> color_eyre::Result<()> {
height = _height;
self.utxos_vecs
.as_mut_separate_vecs()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
info!("Processing chain at {height}...");
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
let price = height_to_close_iter
.as_mut()
.map(|i| *i.unwrap_get_inner(height));
let first_outputindex = height_to_first_outputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
let first_inputindex = height_to_first_inputindex_iter
.unwrap_get_inner(height)
.unwrap_to_usize();
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let (mut height_to_sent, mut received) = thread::scope(|s| {
if chain_state_starting_height <= height {
s.spawn(|| {
self.utxos_vecs
.tick_tock_next_block(&chain_state, timestamp);
});
}
let sent_handle = s.spawn(|| {
// Skip coinbase
(first_inputindex + 1..first_inputindex + *input_count)
.into_par_iter()
.map(InputIndex::from)
.map(|inputindex| {
let outputindex = inputindex_to_outputindex
.get_or_read(inputindex, &inputindex_to_outputindex_mmap)
.unwrap()
.unwrap()
.into_inner();
let value = outputindex_to_value
.get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap()
.unwrap()
.into_inner();
let input_type = outputindex_to_outputtype
.get_or_read(outputindex, &outputindex_to_outputtype_mmap)
.unwrap()
.unwrap()
.into_inner();
// dbg!(input_type);
if input_type.is_unspendable() {
unreachable!()
}
let input_txindex = outputindex_to_txindex
.get_or_read(outputindex, &outputindex_to_txindex_mmap)
.unwrap()
.unwrap()
.into_inner();
let height = txindex_to_height
.get_or_read(input_txindex, &txindex_to_height_mmap)
.unwrap()
.unwrap()
.into_inner();
(height, value, input_type)
})
.fold(
BTreeMap::<Height, Transacted>::default,
|mut tree, (height, value, input_type)| {
tree.entry(height).or_default().iterate(value, input_type);
tree
},
)
.reduce(BTreeMap::<Height, Transacted>::default, |first, second| {
let (mut source, to_consume) = if first.len() > second.len() {
(first, second)
} else {
(second, first)
};
to_consume.into_iter().for_each(|(k, v)| {
*source.entry(k).or_default() += v;
});
source
})
});
let received_handle = s.spawn(|| {
(first_outputindex..first_outputindex + *output_count)
.into_par_iter()
.map(OutputIndex::from)
.map(|outputindex| {
let value = outputindex_to_value
.get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap()
.unwrap()
.into_inner();
let output_type = outputindex_to_outputtype
.get_or_read(outputindex, &outputindex_to_outputtype_mmap)
.unwrap()
.unwrap()
.into_inner();
(value, output_type)
})
.fold(
Transacted::default,
|mut transacted, (value, output_type)| {
transacted.iterate(value, output_type);
transacted
},
)
.reduce(Transacted::default, |acc, transacted| acc + transacted)
});
(sent_handle.join().unwrap(), received_handle.join().unwrap())
});
unspendable_supply += received
.by_type
.unspendable
.as_vec()
.into_iter()
.map(|state| state.value)
.sum::<Sats>()
+ height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
opreturn_supply += received.by_type.unspendable.opreturn.value;
if height == Height::new(0) {
received = Transacted::default();
unspendable_supply += Sats::FIFTY_BTC;
} else if height == Height::new(91_842) || height == Height::new(91_880) {
// Need to destroy invalid coinbases due to duplicate txids
if height == Height::new(91_842) {
height_to_sent.entry(Height::new(91_812)).or_default()
} else {
height_to_sent.entry(Height::new(91_722)).or_default()
}
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
};
if chain_state_starting_height <= height {
// Push current block state before processing sends and receives
chain_state.push(BlockState {
supply: received.spendable_supply.clone(),
price,
timestamp,
});
self.utxos_vecs.receive(received, height, price);
let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
height_to_sent.par_iter().for_each(|(height, sent)| unsafe {
(*unsafe_chain_state.get(height.unwrap_to_usize())).supply -=
&sent.spendable_supply;
});
self.utxos_vecs.send(height_to_sent, chain_state.as_slice());
} else {
dbg!(chain_state_starting_height, height);
panic!("temp, just making sure")
}
let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs();
separate_utxo_vecs
.iter_mut()
.try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
self.height_to_unspendable_supply.forced_push_at(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
unspendable_supply,
exit,
)
)?;
self.height_to_opreturn_supply
.forced_push_at(height, opreturn_supply, exit)?;
let date = height_to_date_fixed_iter.unwrap_get_inner(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height =
dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
let date_height_count =
dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
let is_date_last_height = date_first_height
+ Height::from(date_height_count).decremented().unwrap()
== height;
let date_price = dateindex_to_close_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})?;
if height != Height::ZERO && height.unwrap_to_usize() % 20_000 == 0 {
info!("Flushing...");
exit.block();
self.flush_states(height, &chain_state, exit)?;
exit.release();
}
Ok(())
})?;
if height != Height::ZERO && height.unwrap_to_usize() % 20_000 == 0 {
info!("Flushing...");
exit.block();
self.flush_states(height, &chain_state, exit)?;
exit.release();
}
exit.block();
Ok(())
})?;
info!("Flushing...");
exit.block();
self.flush_states(height, &chain_state, exit)?;
}
info!("Flushing...");
self.flush_states(height, &chain_state, exit)?;
info!("Computing overlaping...");
info!("Computing overlapping...");
self.utxos_vecs
.compute_overlaping_vecs(&starting_indexes, exit)?;
.compute_overlapping_vecs(&starting_indexes, exit)?;
info!("Computing rest part 1...");
@@ -1625,6 +1624,13 @@ impl Vecs {
.dateindex
.clone();
let height_to_realized_cap = self.utxos_vecs.all.1.height_to_realized_cap.clone();
let dateindex_to_realized_cap = self
.utxos_vecs
.all
.1
.indexes_to_realized_cap
.as_ref()
.map(|v| v.dateindex.unwrap_last().clone());
self.utxos_vecs
.as_mut_vecs()
@@ -1639,6 +1645,7 @@ impl Vecs {
&height_to_supply,
dateindex_to_supply.as_ref().unwrap(),
height_to_realized_cap.as_ref(),
dateindex_to_realized_cap.as_ref(),
exit,
)
})?;
@@ -14,7 +14,7 @@ pub trait OutputCohorts {
fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp);
fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]);
fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>);
fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
fn compute_overlapping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
}
impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
@@ -172,7 +172,7 @@ impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
});
}
fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
fn compute_overlapping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
let by_date_range = self.by_date_range.as_vec();
let by_size_range = self.by_size_range.as_vec();
+19 -5
View File
@@ -178,6 +178,17 @@ impl Mul<usize> for Close<Dollars> {
}
}
impl Mul<f64> for Dollars {
type Output = Dollars;
fn mul(self, rhs: f64) -> Self::Output {
if rhs.fract() != 0.0 {
Self::from(self.0 * rhs)
} else {
self * rhs as i64
}
}
}
impl Mul<Bitcoin> for Dollars {
type Output = Self;
fn mul(self, rhs: Bitcoin) -> Self::Output {
@@ -208,11 +219,14 @@ impl Mul<Sats> for Dollars {
impl Mul<StoredF32> for Dollars {
type Output = Self;
fn mul(self, rhs: StoredF32) -> Self::Output {
if rhs.fract() != 0.0 {
Self::from(self.0 * *rhs as f64)
} else {
self * *rhs as i64
}
self * *rhs as f64
}
}
impl Mul<StoredF64> for Dollars {
type Output = Self;
fn mul(self, rhs: StoredF64) -> Self::Output {
self * *rhs
}
}
+5 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, Div};
use derive_deref::Deref;
use derive_deref::{Deref, DerefMut};
use serde::{Serialize, Serializer, ser::SerializeTuple};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -172,6 +172,7 @@ impl From<Close<Sats>> for OHLCSats {
IntoBytes,
KnownLayout,
Deref,
DerefMut,
Serialize,
)]
#[repr(C)]
@@ -259,6 +260,7 @@ where
IntoBytes,
KnownLayout,
Deref,
DerefMut,
Serialize,
)]
#[repr(C)]
@@ -346,6 +348,7 @@ where
IntoBytes,
KnownLayout,
Deref,
DerefMut,
Serialize,
)]
#[repr(C)]
@@ -433,6 +436,7 @@ where
IntoBytes,
KnownLayout,
Deref,
DerefMut,
Serialize,
)]
#[repr(C)]
+2
View File
@@ -3,9 +3,11 @@ mod checked_sub;
mod paths;
mod pause;
mod rlimit;
mod serde;
pub use bytes::*;
pub use checked_sub::*;
pub use paths::*;
pub use pause::*;
pub use rlimit::*;
pub use serde::*;
+3 -1
View File
@@ -3,5 +3,7 @@ use log::info;
pub fn pause() {
info!("Press enter to continue...");
let mut buffer = String::new();
std::io::stdin().read_line(&mut buffer).expect("Failed to read line");
std::io::stdin()
.read_line(&mut buffer)
.expect("Failed to read line");
}
+12
View File
@@ -0,0 +1,12 @@
use serde::{Deserialize, Deserializer};
pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de> + Default,
{
match T::deserialize(deserializer) {
Ok(v) => Ok(v),
Err(_) => Ok(T::default()),
}
}
+14 -1
View File
@@ -1,5 +1,5 @@
use brk_core::{Date, Height};
use brk_fetcher::{BRK, Fetcher};
use brk_fetcher::{BRK, Binance, Fetcher, Kraken};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -12,6 +12,19 @@ fn main() -> color_eyre::Result<()> {
let mut fetcher = Fetcher::import(None)?;
Binance::fetch_1d().map(|b| {
dbg!(b.last_key_value());
})?;
Kraken::fetch_1d().map(|b| {
dbg!(b.last_key_value());
})?;
Binance::fetch_1mn().map(|b| {
dbg!(b.last_key_value());
})?;
Kraken::fetch_1mn().map(|b| {
dbg!(b.last_key_value());
})?;
dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);
dbg!(fetcher.get_height(
899911_u32.into(),
+2 -2
View File
@@ -32,7 +32,7 @@ impl Kraken {
)
}
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Kraken...");
retry(
@@ -54,7 +54,7 @@ impl Kraken {
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
}
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Kraken...");
retry(
+5 -5
View File
@@ -40,11 +40,11 @@ impl Fetcher {
}
fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
self.binance
self.kraken
.get_from_1d(&date)
.or_else(|_| {
// eprintln!("{e}");
self.kraken.get_from_1d(&date)
self.binance.get_from_1d(&date)
})
.or_else(|_| {
// eprintln!("{e}");
@@ -90,11 +90,11 @@ impl Fetcher {
let previous_timestamp = previous_timestamp.map(|t| t.floor_seconds());
let ohlc = self
.binance
.kraken
.get_from_1mn(timestamp, previous_timestamp)
.unwrap_or_else(|_report| {
// eprintln!("{_report}");
self.kraken
self.binance
.get_from_1mn(timestamp, previous_timestamp)
.unwrap_or_else(|_report| {
// // eprintln!("{_report}");
@@ -185,8 +185,8 @@ How to fix this:
}
pub fn clear(&mut self) {
self.kraken.clear();
self.binance.clear();
self.brk.clear();
self.kraken.clear();
}
}
+1 -2
View File
@@ -4,7 +4,6 @@ use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vec::Format;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -25,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
let mut indexer = Indexer::new(outputs, false)?;
indexer.import_stores()?;
indexer.import_vecs()?;
+1 -7
View File
@@ -110,13 +110,7 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
vecs.height_to_blockhash
.iter()
.get(*height)
.is_none_or(|saved_blockhash| {
let b = &rpc_blockhash != saved_blockhash.as_ref();
if b {
dbg!(rpc_blockhash, saved_blockhash.as_ref());
}
b
})
.is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref())
})
.unwrap_or(starting_height);
+2 -9
View File
@@ -19,7 +19,7 @@ use brk_core::{
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_parser::Parser;
use brk_vec::{AnyVec, Format, VecIterator};
use brk_vec::{AnyVec, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -42,21 +42,15 @@ pub struct Indexer {
vecs: Option<Vecs>,
stores: Option<Stores>,
check_collisions: bool,
format: Format,
}
impl Indexer {
pub fn new(
outputs_dir: &Path,
format: Format,
check_collisions: bool,
) -> color_eyre::Result<Self> {
pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result<Self> {
setrlimit()?;
Ok(Self {
path: outputs_dir.to_owned(),
vecs: None,
stores: None,
format,
check_collisions,
})
}
@@ -65,7 +59,6 @@ impl Indexer {
self.vecs = Some(Vecs::forced_import(
&self.path.join("vecs/indexed"),
VERSION + Version::ZERO,
self.format,
)?);
Ok(())
}
+34 -38
View File
@@ -66,11 +66,7 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
format: Format,
) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -78,7 +74,7 @@ impl Vecs {
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_blockhash: IndexedVec::forced_import(
path,
@@ -90,145 +86,145 @@ impl Vecs {
path,
"difficulty",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_emptyoutputindex: IndexedVec::forced_import(
path,
"first_emptyoutputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_inputindex: IndexedVec::forced_import(
path,
"first_inputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_opreturnindex: IndexedVec::forced_import(
path,
"first_opreturnindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_outputindex: IndexedVec::forced_import(
path,
"first_outputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2aindex: IndexedVec::forced_import(
path,
"first_p2aindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2msindex: IndexedVec::forced_import(
path,
"first_p2msindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2pk33index: IndexedVec::forced_import(
path,
"first_p2pk33index",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2pk65index: IndexedVec::forced_import(
path,
"first_p2pk65index",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2pkhindex: IndexedVec::forced_import(
path,
"first_p2pkhindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2shindex: IndexedVec::forced_import(
path,
"first_p2shindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2trindex: IndexedVec::forced_import(
path,
"first_p2trindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2wpkhindex: IndexedVec::forced_import(
path,
"first_p2wpkhindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2wshindex: IndexedVec::forced_import(
path,
"first_p2wshindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_txindex: IndexedVec::forced_import(
path,
"first_txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_unknownoutputindex: IndexedVec::forced_import(
path,
"first_unknownoutputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_timestamp: IndexedVec::forced_import(
path,
"timestamp",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_total_size: IndexedVec::forced_import(
path,
"total_size",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_weight: IndexedVec::forced_import(
path,
"weight",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
inputindex_to_outputindex: IndexedVec::forced_import(
path,
"outputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
opreturnindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
outputindex_to_outputtype: IndexedVec::forced_import(
path,
"outputtype",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
outputindex_to_outputtypeindex: IndexedVec::forced_import(
path,
"outputtypeindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
outputindex_to_value: IndexedVec::forced_import(
path,
"value",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
p2aindex_to_p2abytes: IndexedVec::forced_import(
path,
@@ -240,7 +236,7 @@ impl Vecs {
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
path,
@@ -288,13 +284,13 @@ impl Vecs {
path,
"base_size",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_first_inputindex: IndexedVec::forced_import(
path,
"first_inputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_first_outputindex: IndexedVec::forced_import(
path,
@@ -306,19 +302,19 @@ impl Vecs {
path,
"is_explicitly_rbf",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_rawlocktime: IndexedVec::forced_import(
path,
"rawlocktime",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_total_size: IndexedVec::forced_import(
path,
"total_size",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_txid: IndexedVec::forced_import(
path,
@@ -330,13 +326,13 @@ impl Vecs {
path,
"txversion",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
unknownoutputindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
})
}
+49 -42
View File
@@ -25,50 +25,53 @@ pub fn init(path: Option<&Path>) {
.unwrap()
});
Builder::from_env(Env::default().default_filter_or("info,fjall=off,lsm_tree=off"))
.format(move |buf, record| {
let date_time = Timestamp::now()
.to_zoned(tz::TimeZone::system())
.strftime("%Y-%m-%d %H:%M:%S")
.to_string();
let level = record.level().as_str().to_lowercase();
let level = format!("{:5}", level);
let target = record.target();
let dash = "-";
let args = record.args();
Builder::from_env(
Env::default()
.default_filter_or("info,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off"),
)
.format(move |buf, record| {
let date_time = Timestamp::now()
.to_zoned(tz::TimeZone::system())
.strftime("%Y-%m-%d %H:%M:%S")
.to_string();
let level = record.level().as_str().to_lowercase();
let level = format!("{:5}", level);
let target = record.target();
let dash = "-";
let args = record.args();
if let Some(file) = file.as_ref() {
let _ = write(
file.try_clone().unwrap(),
&date_time,
target,
&level,
dash,
args,
);
}
let colored_date_time = date_time.bright_black();
let colored_level = match level.chars().next().unwrap() {
'e' => level.red().to_string(),
'w' => level.yellow().to_string(),
'i' => level.green().to_string(),
'd' => level.blue().to_string(),
't' => level.cyan().to_string(),
_ => panic!(),
};
let colored_dash = dash.bright_black();
write(
buf,
colored_date_time,
if let Some(file) = file.as_ref() {
let _ = write(
file.try_clone().unwrap(),
&date_time,
target,
colored_level,
colored_dash,
&level,
dash,
args,
)
})
.init();
);
}
let colored_date_time = date_time.bright_black();
let colored_level = match level.chars().next().unwrap() {
'e' => level.red().to_string(),
'w' => level.yellow().to_string(),
'i' => level.green().to_string(),
'd' => level.blue().to_string(),
't' => level.cyan().to_string(),
_ => panic!(),
};
let colored_dash = dash.bright_black();
write(
buf,
colored_date_time,
target,
colored_level,
colored_dash,
args,
)
})
.init();
}
fn write(
@@ -80,5 +83,9 @@ fn write(
args: impl Display,
) -> Result<(), std::io::Error> {
writeln!(buf, "{} {} {} {}", date_time, dash, level, args)
// writeln!(buf, "{} {} {} {} {}", date_time, _target, level, dash, args)
// writeln!(
// buf,
// "{} {} {} {} {}",
// date_time, _target, level, dash, args
// )
}
@@ -96,9 +96,10 @@ impl BlkIndexToBlkRecap {
}
pub fn export(&self) {
let file = File::create(&self.path).unwrap_or_else(|_| {
let file = File::create(&self.path).unwrap_or_else(|e| {
dbg!(e);
dbg!(&self.path);
panic!("No such file or directory")
panic!("Cannot write file");
});
serde_json::to_writer(&mut BufWriter::new(file), &self.tree).unwrap();
+1 -1
View File
@@ -18,5 +18,5 @@ color-eyre = { workspace = true }
derive_deref = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_with = "3.12.0"
serde_with = "3.13.0"
tabled = { workspace = true }
+1 -1
View File
@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
let format = Format::Compressed;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
let mut indexer = Indexer::new(outputs_dir, true)?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir, None, format);
+8 -3
View File
@@ -28,9 +28,14 @@ impl<'a> VecTrees<'a> {
|| s.starts_with("cumulative_from")
}))
&& !(split.len() == 4
&& split
.get(1)
.is_some_and(|s| s == &"up" || s == &"start" || s.starts_with("from"))
&& split.get(1).is_some_and(|s| {
s == &"up"
|| s == &"start"
|| s.starts_with("from")
|| s == &"cumulative_up"
|| s == &"cumulative_start"
|| s.starts_with("cumulative_from")
})
&& split.get(2).is_some_and(|s| s.ends_with("relative")))
{
dbg!(&name, &split);
+3 -2
View File
@@ -10,6 +10,7 @@ repository.workspace = true
[dependencies]
axum = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_bundler = { workspace = true }
brk_computer = { workspace = true }
brk_exit = { workspace = true }
brk_core = { workspace = true }
@@ -25,11 +26,11 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.72.3", features = ["codegen", "minifier"] }
oxc = { version = "0.73.0", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { workspace = true }
tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }
zip = "4.0.0"
zip = "4.1.0"
tracing = "0.1.41"
[package.metadata.cargo-machete]
+1 -1
View File
@@ -73,7 +73,7 @@ A list of all possible vec ids and their supported vec indexes
A list of all possible vec indexes and their supported vec ids
#### `GET /api/{INDEX}-to-{VALUE}`
#### `GET /api/{INDEX}-to-{ID}`
This endpoint retrieves data based on the specified vector index and id.
+2 -2
View File
@@ -31,7 +31,7 @@ pub fn main() -> color_eyre::Result<()> {
let format = Format::Compressed;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
let mut indexer = Indexer::new(outputs_dir, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
@@ -51,7 +51,7 @@ pub fn main() -> color_eyre::Result<()> {
let server = Server::new(served_indexer, served_computer, Website::Default)?;
let server = tokio::spawn(async move {
server.serve().await.unwrap();
server.serve(true).await.unwrap();
});
if process {
@@ -0,0 +1 @@
+1
View File
@@ -124,6 +124,7 @@ pub async fn variant_handler(
Query(params_opt): Query<ParamsOpt>,
state: State<AppState>,
) -> Response {
let variant = variant.replace("_", "-");
let mut split = variant.split(TO_SEPARATOR);
let params = Params::from((
(
+6 -7
View File
@@ -61,9 +61,12 @@ export const VERSION = \"v{}\";
.join(" | ")
);
contents += "\n\nexport function createVecIdToIndexes() {\n";
contents += "\n\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";
contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n";
contents += " return /** @type {const} */ ({\n";
contents += "\nexport function createVecIdToIndexes() {\n";
contents += " return {\n";
self.vec_trees
.id_to_index_to_vec
@@ -79,11 +82,7 @@ export const VERSION = \"v{}\";
contents += &format!(" \"{id}\": [{indexes}],\n");
});
contents += " });\n";
contents.push('}');
contents += "\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";
contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n";
contents += " };\n}\n";
fs::write(path, contents)
}
+26 -46
View File
@@ -13,8 +13,6 @@ use crate::{
traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
};
use super::minify::minify_js;
pub async fn file_handler(
headers: HeaderMap,
State(app_state): State<AppState>,
@@ -32,16 +30,12 @@ fn any_handler(
app_state: AppState,
path: Option<extract::Path<String>>,
) -> Response {
let website_path = app_state
.websites_path
.as_ref()
.expect("Should never reach here is websites_path is None")
.join(app_state.website.to_folder_name());
let dist_path = app_state.dist_path();
if let Some(path) = path.as_ref() {
let path = path.0.replace("..", "").replace("\\", "");
let mut path = website_path.join(&path);
let mut path = dist_path.join(&path);
if !path.exists() || path.is_dir() {
if path.extension().is_some() {
@@ -55,13 +49,13 @@ fn any_handler(
return response;
} else {
path = website_path.join("index.html");
path = dist_path.join("index.html");
}
}
path_to_response(&headers, &path)
} else {
path_to_response(&headers, &website_path.join("index.html"))
path_to_response(&headers, &dist_path.join("index.html"))
}
}
@@ -85,49 +79,35 @@ fn path_to_response_(headers: &HeaderMap, path: &Path) -> color_eyre::Result<Res
return Ok(Response::new_not_modified());
}
let mut response;
let content = fs::read(path).unwrap_or_else(|error| {
error!("{error}");
let path = path.to_str().unwrap();
info!("Can't read file {path}");
panic!("")
});
let is_localhost = headers.check_if_host_is_localhost();
if !is_localhost
&& path.extension().unwrap_or_else(|| {
dbg!(path);
panic!();
}) == "js"
{
let content = minify_js(path);
response = Response::new(content.into());
} else {
let content = fs::read(path).unwrap_or_else(|error| {
error!("{error}");
let path = path.to_str().unwrap();
info!("Can't read file {path}");
panic!("")
});
response = Response::new(content.into());
}
let mut response = Response::new(content.into());
let headers = response.headers_mut();
headers.insert_cors();
headers.insert_content_type(path);
if !is_localhost {
let serialized_path = path.to_str().unwrap();
let serialized_path = path.to_str().unwrap();
if serialized_path.contains("fonts/")
|| serialized_path.contains("assets/")
|| serialized_path.contains("packages/")
|| path.extension().is_some_and(|extension| {
extension == "pdf"
|| extension == "jpg"
|| extension == "png"
|| extension == "woff2"
})
{
headers.insert_cache_control_immutable();
}
if path
.extension()
.is_some_and(|extension| extension == "html")
|| serialized_path.ends_with("service-worker.js")
{
headers.insert_cache_control_must_revalidate();
} else if path.extension().is_some_and(|extension| {
extension == "jpg"
|| extension == "png"
|| extension == "woff2"
|| extension == "js"
|| extension == "map"
}) {
headers.insert_cache_control_immutable();
}
headers.insert_last_modified(date);
-41
View File
@@ -1,41 +0,0 @@
// Source: https://github.com/oxc-project/oxc/blob/main/crates/oxc_minifier/examples/minifier.rs
use std::{fs, path::Path};
use oxc::{
allocator::Allocator,
codegen::{Codegen, CodegenOptions, LegalComment},
minifier::{CompressOptions, MangleOptions, Minifier, MinifierOptions},
parser::Parser,
span::SourceType,
};
pub fn minify_js(path: &Path) -> String {
let source_text = fs::read_to_string(path).unwrap();
let source_type = SourceType::from_path(path).unwrap();
let allocator = Allocator::default();
let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
let mut program = parser_return.program;
let minifier_return = Minifier::new(MinifierOptions {
mangle: Some(MangleOptions::default()),
compress: Some(CompressOptions::default()),
})
.build(&allocator, &mut program);
Codegen::new()
.with_options(CodegenOptions {
minify: true,
single_quote: false,
comments: false,
annotation_comments: false,
source_map_path: None,
legal_comments: LegalComment::None,
})
.with_scoping(minifier_return.scoping)
.build(&program)
.code
}
-1
View File
@@ -3,7 +3,6 @@ use axum::{Router, routing::get};
use super::AppState;
mod file;
mod minify;
mod website;
use file::{file_handler, index_handler};
+15 -1
View File
@@ -19,6 +19,7 @@ use axum::{
routing::get,
serve,
};
use brk_bundler::bundle;
use brk_computer::Computer;
use brk_core::dot_brk_path;
use brk_indexer::Indexer;
@@ -45,6 +46,15 @@ pub struct AppState {
websites_path: Option<PathBuf>,
}
impl AppState {
pub fn dist_path(&self) -> PathBuf {
self.websites_path
.as_ref()
.expect("Should never reach here is websites_path is None")
.join("dist")
}
}
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
const DEV_PATH: &str = "../..";
@@ -103,9 +113,13 @@ impl Server {
}))
}
pub async fn serve(self) -> color_eyre::Result<()> {
pub async fn serve(self, watch: bool) -> color_eyre::Result<()> {
let state = self.0;
if let Some(websites_path) = state.websites_path.clone() {
bundle(&websites_path, state.website.to_folder_name(), watch).await?;
}
let compression_layer = CompressionLayer::new()
.br(true)
.deflate(true)
+11 -47
View File
@@ -5,12 +5,11 @@ use std::{
use axum::http::{
HeaderMap,
header::{self, HOST, IF_MODIFIED_SINCE},
header::{self, IF_MODIFIED_SINCE},
};
use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone};
use log::info;
const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish
const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT";
#[derive(PartialEq, Eq)]
@@ -20,12 +19,6 @@ pub enum ModifiedState {
}
pub trait HeaderMapExtended {
fn _get_scheme(&self) -> &str;
fn get_host(&self) -> &str;
fn check_if_host_is_local(&self) -> bool;
fn check_if_host_is_0000(&self) -> bool;
fn check_if_host_is_localhost(&self) -> bool;
fn insert_cors(&mut self);
fn get_if_modified_since(&self) -> Option<DateTime>;
@@ -36,8 +29,8 @@ pub trait HeaderMapExtended {
duration: Duration,
) -> color_eyre::Result<(ModifiedState, DateTime)>;
fn insert_cache_control_must_revalidate(&mut self);
fn insert_cache_control_immutable(&mut self);
fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
fn insert_last_modified(&mut self, date: DateTime);
fn insert_content_disposition_attachment(&mut self);
@@ -59,41 +52,22 @@ pub trait HeaderMapExtended {
}
impl HeaderMapExtended for HeaderMap {
fn _get_scheme(&self) -> &str {
if self.check_if_host_is_local() {
"http"
} else {
"https"
}
}
fn get_host(&self) -> &str {
self[HOST].to_str().unwrap()
}
fn check_if_host_is_local(&self) -> bool {
self.check_if_host_is_localhost() || self.check_if_host_is_0000()
}
fn check_if_host_is_0000(&self) -> bool {
self.get_host().contains("0.0.0.0")
}
fn check_if_host_is_localhost(&self) -> bool {
self.get_host().contains("localhost")
}
fn insert_cors(&mut self) {
self.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
self.insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap());
}
fn insert_cache_control_must_revalidate(&mut self) {
self.insert(
header::CACHE_CONTROL,
"public, max-age=0, must-revalidate".parse().unwrap(),
);
}
fn insert_cache_control_immutable(&mut self) {
self.insert(
header::CACHE_CONTROL,
format!("public, max-age=604800, immutable, stale-if-error={STALE_IF_ERROR}")
.parse()
.unwrap(),
"public, max-age=31536000, immutable".parse().unwrap(),
);
}
@@ -101,16 +75,6 @@ impl HeaderMapExtended for HeaderMap {
self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap());
}
fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64) {
self.insert(
header::CACHE_CONTROL,
format!(
"public, max-age={max_age}, stale-while-revalidate={stale_while_revalidate}, stale-if-error={STALE_IF_ERROR}")
.parse()
.unwrap(),
);
}
fn insert_last_modified(&mut self, date: DateTime) {
let formatted = date
.to_zoned(TimeZone::system())
@@ -167,7 +131,7 @@ impl HeaderMapExtended for HeaderMap {
fn insert_content_type(&mut self, path: &Path) {
match path.extension().unwrap().to_str().unwrap() {
"js" => self.insert_content_type_application_javascript(),
"json" => self.insert_content_type_application_json(),
"json" | "map" => self.insert_content_type_application_json(),
"html" => self.insert_content_type_text_html(),
"css" => self.insert_content_type_text_css(),
"toml" | "txt" => self.insert_content_type_text_plain(),
+1 -1
View File
@@ -70,7 +70,7 @@ impl<T> Outputs<T> {
.collect::<Vec<_>>()
}
pub fn as_mut_overlaping_vecs(&mut self) -> Vec<&mut T> {
pub fn as_mut_overlapping_vecs(&mut self) -> Vec<&mut T> {
[&mut self.all]
.into_iter()
.chain(self.by_term.as_mut_vec())
+1 -1
View File
@@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum,
)]
pub enum Format {
#[default]
Compressed,
#[default]
Raw,
}
+2 -1
View File
@@ -31,7 +31,7 @@ where
}
#[inline]
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<T>>> {
let stored_len = mmap.len() / Self::SIZE_OF_T;
let stored_len = self.stored_len_(mmap);
if index >= stored_len {
let pushed = self.pushed();
@@ -53,6 +53,7 @@ where
fn mmap(&self) -> &ArcSwap<Mmap>;
fn stored_len(&self) -> usize;
fn stored_len_(&self, mmap: &Mmap) -> usize;
fn pushed(&self) -> &[T];
#[inline]
+13 -5
View File
@@ -21,7 +21,9 @@ use crate::{
const ONE_KIB: usize = 1024;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB;
const VERSION: Version = Version::ONE;
#[derive(Debug)]
pub struct CompressedVec<I, T> {
@@ -39,7 +41,9 @@ where
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
/// Same as import but will reset the folder under certain errors, so be careful !
pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
pub fn forced_import(path: &Path, mut version: Version) -> Result<Self> {
version = version + VERSION;
let res = Self::import(path, version);
match res {
Err(Error::WrongEndian)
@@ -129,7 +133,7 @@ where
page_index * Self::PER_PAGE
}
fn stored_len_(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
fn stored_len__(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
if let Some(last) = pages_meta.last() {
(pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
} else {
@@ -178,7 +182,11 @@ where
#[inline]
fn stored_len(&self) -> usize {
Self::stored_len_(&self.pages_meta.load())
Self::stored_len__(&self.pages_meta.load())
}
#[inline]
fn stored_len_(&self, _: &Mmap) -> usize {
self.stored_len()
}
#[inline]
@@ -477,7 +485,7 @@ where
fn into_iter(self) -> Self::IntoIter {
let pages_meta = self.pages_meta.load();
let stored_len = CompressedVec::<I, T>::stored_len_(&pages_meta);
let stored_len = CompressedVec::<I, T>::stored_len__(&pages_meta);
CompressedVecIterator {
vec: self,
guard: self.mmap().load(),
+28 -2
View File
@@ -1015,6 +1015,32 @@ where
self.safe_flush(exit)
}
pub fn compute_zscore(
&mut self,
max_from: I,
ratio: &impl AnyIterableVec<I, StoredF32>,
sma: &impl AnyIterableVec<I, StoredF32>,
sd: &impl AnyIterableVec<I, StoredF32>,
exit: &Exit,
) -> Result<()>
where
T: From<StoredF32>,
{
let mut sma_iter = sma.iter();
let mut sd_iter = sd.iter();
self.compute_transform(
max_from,
ratio,
|(i, ratio, ..)| {
let sma = sma_iter.unwrap_get_inner(i);
let sd = sd_iter.unwrap_get_inner(i);
(i, T::from((ratio - sma) / sd))
},
exit,
)
}
}
impl EagerVec<DateIndex, Sats> {
@@ -1116,12 +1142,12 @@ impl EagerVec<DateIndex, Dollars> {
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
Version::ZERO + self.inner.version() + stacks.version(),
Version::ONE + self.inner.version() + stacks.version(),
)?;
let index = max_from.min(DateIndex::from(self.len()));
let first_price_date = DateIndex::try_from(Date::new(2010, 8, 16)).unwrap();
let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap();
stacks.iter_at(index).try_for_each(|(i, stack)| {
let stack = stack.into_inner();
+5 -1
View File
@@ -104,7 +104,11 @@ where
#[inline]
fn stored_len(&self) -> usize {
self.mmap.load().len() / Self::SIZE_OF_T
self.stored_len_(&self.mmap.load())
}
#[inline]
fn stored_len_(&self, mmap: &Mmap) -> usize {
mmap.len() / Self::SIZE_OF_T
}
#[inline]
+7
View File
@@ -73,6 +73,13 @@ where
StoredVec::Compressed(v) => v.stored_len(),
}
}
#[inline]
fn stored_len_(&self, mmap: &Mmap) -> usize {
match self {
StoredVec::Raw(v) => v.stored_len_(mmap),
StoredVec::Compressed(v) => v.stored_len_(mmap),
}
}
#[inline]
fn pushed(&self) -> &[T] {
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.7 KiB

@@ -1,187 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<link
rel="icon"
type="image/png"
sizes="196x196"
href="/assets/pwa/2025-03-22_10-00-00/favicon-196.png"
/>
<link
rel="apple-touch-icon"
href="/assets/pwa/2025-03-22_10-00-00/apple-icon-180.png"
/>
<meta name="apple-mobile-web-app-capable" content="yes" />
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2048-2732.jpg"
media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2732-2048.jpg"
media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1668-2388.jpg"
media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2388-1668.jpg"
media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1536-2048.jpg"
media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2048-1536.jpg"
media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1488-2266.jpg"
media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2266-1488.jpg"
media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1640-2360.jpg"
media="(device-width: 820px) and (device-height: 1180px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2360-1640.jpg"
media="(device-width: 820px) and (device-height: 1180px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1668-2224.jpg"
media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2224-1668.jpg"
media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1620-2160.jpg"
media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2160-1620.jpg"
media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1290-2796.jpg"
media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2796-1290.jpg"
media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1179-2556.jpg"
media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2556-1179.jpg"
media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1284-2778.jpg"
media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2778-1284.jpg"
media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1170-2532.jpg"
media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2532-1170.jpg"
media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1125-2436.jpg"
media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2436-1125.jpg"
media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1242-2688.jpg"
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2688-1242.jpg"
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-828-1792.jpg"
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1792-828.jpg"
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1242-2208.jpg"
media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2208-1242.jpg"
media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-750-1334.jpg"
media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1334-750.jpg"
media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-640-1136.jpg"
media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
/>
<link
rel="apple-touch-startup-image"
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1136-640.jpg"
media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
/>
</head>
<body></body>
</html>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.2 KiB

Some files were not shown because too many files have changed in this diff Show More