Compare commits
42 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7e1fb6472d | |||
| 0ff8d20573 | |||
| 9c1f9448dc | |||
| 43a6081dd6 | |||
| 985e961876 | |||
| 098f6de047 | |||
| 1b0f90fd68 | |||
| 12252f407b | |||
| 3b6e3f47ab | |||
| 6a9ac9b025 | |||
| ae6aa4088b | |||
| c08f431180 | |||
| 123c1f56e9 | |||
| 35ac65a864 | |||
| e9f362cc87 | |||
| 65685c23e1 | |||
| 2f74748cea | |||
| f477bd66f3 | |||
| d7d77ae8f0 | |||
| 31110a740d | |||
| b64d8b1d7f | |||
| c46006aacc | |||
| 92f81b1493 | |||
| 70213cfc8f | |||
| 8a82bf5c50 | |||
| 37405384a2 | |||
| 54ea6cc53b | |||
| 339c00d815 | |||
| ea6b4dcde2 | |||
| 2b84623d1e | |||
| c8b3afa56b | |||
| 1348f3c24c | |||
| 62208ce3e1 | |||
| 813b2481de | |||
| 27b924ba61 | |||
| b40170b8ce | |||
| 8bfa9d2734 | |||
| c7cf76d4a8 | |||
| dfd2969b3e | |||
| 0e1866fe1d | |||
| b9ae46b913 | |||
| 06e7284055 |
@@ -3,6 +3,7 @@
|
||||
|
||||
# Builds
|
||||
target
|
||||
dist
|
||||
|
||||
# Copies
|
||||
*\ copy*
|
||||
|
||||
@@ -4,7 +4,7 @@ members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.47"
|
||||
package.version = "0.0.63"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
|
||||
@@ -22,26 +22,27 @@ axum = "0.8.4"
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.6", features = ["serde"] }
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
brk_cli = { version = "0", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0", path = "crates/brk_indexer" }
|
||||
brk_logger = { version = "0", path = "crates/brk_logger" }
|
||||
brk_parser = { version = "0", path = "crates/brk_parser" }
|
||||
brk_query = { version = "0", path = "crates/brk_query" }
|
||||
brk_server = { version = "0", path = "crates/brk_server" }
|
||||
brk_state = { version = "0", path = "crates/brk_state" }
|
||||
brk_store = { version = "0", path = "crates/brk_store" }
|
||||
brk_vec = { version = "0", path = "crates/brk_vec" }
|
||||
brk_bundler = { version = "0.0.63", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.0.63", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0.0.63", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0.0.63", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0.0.63", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0.0.63", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.0.63", path = "crates/brk_indexer" }
|
||||
brk_logger = { version = "0.0.63", path = "crates/brk_logger" }
|
||||
brk_parser = { version = "0.0.63", path = "crates/brk_parser" }
|
||||
brk_query = { version = "0.0.63", path = "crates/brk_query" }
|
||||
brk_server = { version = "0.0.63", path = "crates/brk_server" }
|
||||
brk_state = { version = "0.0.63", path = "crates/brk_state" }
|
||||
brk_store = { version = "0.0.63", path = "crates/brk_store" }
|
||||
brk_vec = { version = "0.0.63", path = "crates/brk_vec" }
|
||||
byteview = "=0.6.1"
|
||||
clap = { version = "4.5.39", features = ["string"] }
|
||||
clap_derive = "4.5.32"
|
||||
clap = { version = "4.5.40", features = ["string"] }
|
||||
clap_derive = "4.5.40"
|
||||
color-eyre = "0.6.5"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.11.0"
|
||||
jiff = "0.2.14"
|
||||
jiff = "0.2.15"
|
||||
log = { version = "0.4.27" }
|
||||
minreq = { version = "2.13.4", features = ["https", "serde_json"] }
|
||||
rayon = "1.10.0"
|
||||
|
||||
@@ -31,20 +31,22 @@
|
||||
</a>
|
||||
</p>
|
||||
|
||||
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin Core node, enabling users to gain deeper insights into the Bitcoin network.
|
||||
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
|
||||
|
||||
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and the self-hosting experience.
|
||||
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and ease of use.
|
||||
|
||||
The toolkit can be used in various ways to accommodate as many needs as possible:
|
||||
|
||||
- **[Website](https://kibo.money)** \
|
||||
Everyone is welcome to visit [kibo.money](https://kibo.money) which is the official showcase of the suite's capabilities and served by default when running BRK. \
|
||||
Researchers and developers are free to use the API which endpoints documentation can be found [here](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#endpoints). \
|
||||
As a token of gratitude to the community and to stimulate curiosity, both the website and the API are entirely free, allowing anyone to use them.
|
||||
- **[Website](https://bitcoinresearchkit.org)** \
|
||||
Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
|
||||
It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
|
||||
Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
|
||||
- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#endpoints)** \
|
||||
Researchers and developers are free to use BRK's public API with  dataset variants at your disposal. \
|
||||
Just like the website, it's entirely free, with no authentication or rate-limiting.
|
||||
- **[CLI](https://crates.io/crates/brk_cli)** \
|
||||
Node runners are strongly encouraged to try out and self-host their own instance. \
|
||||
A lot of effort has gone into making this as easy as possible. \
|
||||
For more information visit: [`brk_cli`](https://crates.io/crates/brk_cli)
|
||||
Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
|
||||
The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
|
||||
- **[Crates](https://crates.io/crates/brk)** \
|
||||
Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility.
|
||||
PRs are welcome, especially if their goal is to introduce additional datasets.
|
||||
@@ -69,22 +71,15 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
|
||||
- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer
|
||||
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
|
||||
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
|
||||
|
||||
Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [kibo.money](https://kibo.money) public instance.
|
||||
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
|
||||
|
||||
## Hosting as a service
|
||||
|
||||
*Soon™*
|
||||
|
||||
If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
|
||||
|
||||
- 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
|
||||
- 99.99% SLA
|
||||
- Configurated for speed (`raw + eager`)
|
||||
- Configured for speed
|
||||
- Updates delivered at your convenience
|
||||
- Direct communication for feature requests and support
|
||||
- Bitcoin Core or Knots with desired version
|
||||
@@ -93,6 +88,12 @@ If you'd like to have your own instance hosted for you please contact [hosting@b
|
||||
|
||||
Pricing: `0.01 BTC / month` *or* `0.1 BTC / year`
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
|
||||
|
||||
Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [kibo.money](https://kibo.money) public instance.
|
||||
|
||||
## Donate
|
||||
|
||||
[`bc1q09 8zsm89 m7kgyz e338vf ejhpdt 92ua9p 3peuve`](bitcoin:bc1q098zsm89m7kgyze338vfejhpdt92ua9p3peuve)
|
||||
|
||||
@@ -10,6 +10,7 @@ version.workspace = true
|
||||
|
||||
[features]
|
||||
full = [
|
||||
"bundler",
|
||||
"core",
|
||||
"computer",
|
||||
"exit",
|
||||
@@ -23,6 +24,7 @@ full = [
|
||||
"store",
|
||||
"vec",
|
||||
]
|
||||
bundler = ["brk_bundler"]
|
||||
core = ["brk_core"]
|
||||
computer = ["brk_computer"]
|
||||
exit = ["brk_exit"]
|
||||
@@ -37,6 +39,7 @@ store = ["brk_store"]
|
||||
vec = ["brk_vec"]
|
||||
|
||||
[dependencies]
|
||||
brk_bundler = { workspace = true, optional = true }
|
||||
brk_cli = { workspace = true }
|
||||
brk_core = { workspace = true, optional = true }
|
||||
brk_computer = { workspace = true, optional = true }
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
fn main() {}
|
||||
@@ -1,5 +1,12 @@
|
||||
#![doc = include_str!(concat!("../", env!("CARGO_PKG_README")))]
|
||||
|
||||
#[cfg(feature = "bundler")]
|
||||
#[doc(inline)]
|
||||
pub use brk_bundler as bundler;
|
||||
|
||||
#[doc(inline)]
|
||||
pub use brk_cli as cli;
|
||||
|
||||
#[cfg(feature = "core")]
|
||||
#[doc(inline)]
|
||||
pub use brk_core as core;
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
[package]
|
||||
name = "brk_bundler"
|
||||
description = "A thin wrapper around rolldown"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
log = { workspace = true }
|
||||
notify = "8.0.0"
|
||||
brk_rolldown = "0.0.1"
|
||||
sugar_path = "1.2.0"
|
||||
tokio = { workspace = true }
|
||||
@@ -0,0 +1,144 @@
|
||||
use std::{fs, io, path::Path, sync::Arc};
|
||||
|
||||
use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
|
||||
use log::error;
|
||||
use notify::{EventKind, RecursiveMode, Watcher};
|
||||
use sugar_path::SugarPath;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
|
||||
let source_path = websites_path.join(source_folder);
|
||||
let dist_path = websites_path.join("dist");
|
||||
|
||||
let _ = fs::remove_dir_all(&dist_path);
|
||||
copy_dir_all(&source_path, &dist_path)?;
|
||||
|
||||
let source_scripts = format!("./{source_folder}/scripts");
|
||||
let source_entry = format!("{source_scripts}/entry.js");
|
||||
|
||||
let absolute_websites_path = websites_path.absolutize();
|
||||
|
||||
let mut bundler = Bundler::new(BundlerOptions {
|
||||
input: Some(vec![source_entry.into()]),
|
||||
dir: Some("./dist/scripts".to_string()),
|
||||
cwd: Some(absolute_websites_path),
|
||||
minify: Some(RawMinifyOptions::Bool(true)),
|
||||
sourcemap: Some(SourceMapType::File),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
bundler.write().await.unwrap();
|
||||
|
||||
let absolute_source_index_path = source_path.join("index.html").absolutize();
|
||||
let absolute_source_index_path_clone = absolute_source_index_path.clone();
|
||||
let absolute_source_path = source_path.absolutize();
|
||||
let absolute_source_path_clone = absolute_source_path.clone();
|
||||
let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
|
||||
let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
|
||||
let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
|
||||
|
||||
let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
|
||||
let absolute_dist_index_path = dist_path.join("index.html").absolutize();
|
||||
let absolute_dist_path = dist_path.absolutize();
|
||||
let absolute_dist_path_clone = absolute_dist_path.clone();
|
||||
let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
|
||||
|
||||
let write_index = move || {
|
||||
let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
|
||||
|
||||
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
|
||||
if let Some(start) = entry.find("main") {
|
||||
if let Some(end) = entry.find(".js") {
|
||||
let main_hashed = &entry[start..end];
|
||||
contents = contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = fs::write(&absolute_dist_index_path, contents);
|
||||
};
|
||||
|
||||
let write_sw = move || {
|
||||
let contents = fs::read_to_string(&absolute_source_sw_path)
|
||||
.unwrap()
|
||||
.replace("__VERSION__", &format!("v{VERSION}"));
|
||||
let _ = fs::write(&absolute_dist_sw_path, contents);
|
||||
};
|
||||
|
||||
write_index();
|
||||
write_sw();
|
||||
|
||||
if !watch {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let write_index_clone = write_index.clone();
|
||||
|
||||
let mut entry_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(_) => write_index_clone(),
|
||||
Err(e) => error!("watch error: {:?}", e),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
entry_watcher
|
||||
.watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let mut source_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(event) => match event.kind {
|
||||
EventKind::Create(_) => event.paths,
|
||||
EventKind::Modify(_) => event.paths,
|
||||
_ => vec![],
|
||||
}
|
||||
.into_iter()
|
||||
.filter(|path| path.starts_with(&absolute_source_path))
|
||||
.filter(|path| !path.starts_with(&absolute_source_scripts_path))
|
||||
.for_each(|source_path| {
|
||||
let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
|
||||
let dist_path = absolute_dist_path.join(suffix);
|
||||
|
||||
if source_path == absolute_source_index_path_clone {
|
||||
write_index();
|
||||
} else if source_path == absolute_source_sw_path_clone {
|
||||
write_sw();
|
||||
} else {
|
||||
let _ = fs::copy(&source_path, &dist_path);
|
||||
}
|
||||
}),
|
||||
Err(e) => error!("watch error: {:?}", e),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
source_watcher
|
||||
.watch(&absolute_source_path_clone, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let watcher =
|
||||
brk_rolldown::Watcher::new(vec![Arc::new(Mutex::new(bundler))], None).unwrap();
|
||||
|
||||
watcher.start().await;
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(&dst)?;
|
||||
for entry in fs::read_dir(src)? {
|
||||
let entry = entry?;
|
||||
let ty = entry.file_type()?;
|
||||
if ty.is_dir() {
|
||||
copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
|
||||
} else {
|
||||
fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -20,9 +20,9 @@ struct Cli {
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Commands {
|
||||
/// Run the indexer, computer and server
|
||||
/// Run the indexer, computer and server, use `run -h` for more information
|
||||
Run(RunConfig),
|
||||
/// Query generated datasets via the `run` command in a similar fashion as the server's API
|
||||
/// Query generated datasets via the `run` command in a similar fashion as the server's API, use `query -h` for more information
|
||||
Query(QueryArgs),
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
|
||||
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::{
|
||||
|
||||
use bitcoincore_rpc::{self, Auth, Client, RpcApi};
|
||||
use brk_computer::Computer;
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
@@ -29,7 +29,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
|
||||
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
|
||||
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
@@ -63,8 +63,9 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
|
||||
|
||||
let server = Server::new(served_indexer, served_computer, config.website())?;
|
||||
|
||||
let watch = config.watch();
|
||||
let opt = Some(tokio::spawn(async move {
|
||||
server.serve().await.unwrap();
|
||||
server.serve(watch).await.unwrap();
|
||||
}));
|
||||
|
||||
sleep(Duration::from_secs(1));
|
||||
@@ -109,62 +110,82 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
|
||||
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
|
||||
pub struct RunConfig {
|
||||
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
bitcoindir: Option<String>,
|
||||
|
||||
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
blocksdir: Option<String>,
|
||||
|
||||
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
brkdir: Option<String>,
|
||||
|
||||
/// Executed by the runner, default: all, saved
|
||||
/// Activated services, default: all, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
mode: Option<Mode>,
|
||||
services: Option<Services>,
|
||||
|
||||
/// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
|
||||
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
computation: Option<Computation>,
|
||||
|
||||
/// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved
|
||||
#[arg(short, long, value_name = "FORMAT")]
|
||||
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
format: Option<Format>,
|
||||
|
||||
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short = 'F', long, value_name = "BOOL")]
|
||||
fetch: Option<bool>,
|
||||
|
||||
/// Website served by the server (if active), default: default, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
website: Option<Website>,
|
||||
|
||||
/// Bitcoin RPC ip, default: localhost, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "IP")]
|
||||
rpcconnect: Option<String>,
|
||||
|
||||
/// Bitcoin RPC port, default: 8332, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PORT")]
|
||||
rpcport: Option<u16>,
|
||||
|
||||
/// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PATH")]
|
||||
rpccookiefile: Option<String>,
|
||||
|
||||
/// Bitcoin RPC username, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "USERNAME")]
|
||||
rpcuser: Option<String>,
|
||||
|
||||
/// Bitcoin RPC password, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "PASSWORD")]
|
||||
rpcpassword: Option<String>,
|
||||
|
||||
/// Delay between runs, default: 0, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "SECONDS")]
|
||||
delay: Option<u64>,
|
||||
|
||||
/// DEV: Activate to watch the selected website's folder for changes, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
watch: Option<bool>,
|
||||
|
||||
/// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
check_collisions: Option<bool>,
|
||||
}
|
||||
@@ -192,8 +213,8 @@ impl RunConfig {
|
||||
config_saved.brkdir = Some(brkdir);
|
||||
}
|
||||
|
||||
if let Some(mode) = config_args.mode.take() {
|
||||
config_saved.mode = Some(mode);
|
||||
if let Some(services) = config_args.services.take() {
|
||||
config_saved.services = Some(services);
|
||||
}
|
||||
|
||||
if let Some(computation) = config_args.computation.take() {
|
||||
@@ -240,6 +261,10 @@ impl RunConfig {
|
||||
config_saved.check_collisions = Some(check_collisions);
|
||||
}
|
||||
|
||||
if let Some(watch) = config_args.watch.take() {
|
||||
config_saved.watch = Some(watch);
|
||||
}
|
||||
|
||||
if config_args != RunConfig::default() {
|
||||
dbg!(config_args);
|
||||
panic!("Didn't consume the full config")
|
||||
@@ -252,19 +277,6 @@ impl RunConfig {
|
||||
|
||||
config.write(&path)?;
|
||||
|
||||
// info!("Configuration {{");
|
||||
// info!(" bitcoindir: {:?}", config.bitcoindir);
|
||||
// info!(" brkdir: {:?}", config.brkdir);
|
||||
// info!(" mode: {:?}", config.mode);
|
||||
// info!(" website: {:?}", config.website);
|
||||
// info!(" rpcconnect: {:?}", config.rpcconnect);
|
||||
// info!(" rpcport: {:?}", config.rpcport);
|
||||
// info!(" rpccookiefile: {:?}", config.rpccookiefile);
|
||||
// info!(" rpcuser: {:?}", config.rpcuser);
|
||||
// info!(" rpcpassword: {:?}", config.rpcpassword);
|
||||
// info!(" delay: {:?}", config.delay);
|
||||
// info!("}}");
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
@@ -375,13 +387,13 @@ impl RunConfig {
|
||||
}
|
||||
|
||||
pub fn process(&self) -> bool {
|
||||
self.mode
|
||||
.is_none_or(|m| m == Mode::All || m == Mode::Processor)
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Processor)
|
||||
}
|
||||
|
||||
pub fn serve(&self) -> bool {
|
||||
self.mode
|
||||
.is_none_or(|m| m == Mode::All || m == Mode::Server)
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Server)
|
||||
}
|
||||
|
||||
fn path_cookiefile(&self) -> PathBuf {
|
||||
@@ -433,6 +445,10 @@ impl RunConfig {
|
||||
pub fn check_collisions(&self) -> bool {
|
||||
self.check_collisions.is_some_and(|b| b)
|
||||
}
|
||||
|
||||
pub fn watch(&self) -> bool {
|
||||
self.watch.is_some_and(|b| b)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(
|
||||
@@ -449,7 +465,7 @@ impl RunConfig {
|
||||
PartialOrd,
|
||||
Ord,
|
||||
)]
|
||||
pub enum Mode {
|
||||
pub enum Services {
|
||||
#[default]
|
||||
All,
|
||||
Processor,
|
||||
|
||||
@@ -33,7 +33,7 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let format = Format::Raw;
|
||||
|
||||
let mut indexer = Indexer::new(outputs_dir, format, true)?;
|
||||
let mut indexer = Indexer::new(outputs_dir, true)?;
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ use brk_core::{
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format, StoredIndex};
|
||||
|
||||
use super::{
|
||||
Indexes,
|
||||
@@ -429,8 +429,18 @@ impl Vecs {
|
||||
self.dateindex_to_ohlc_in_cents.compute_transform(
|
||||
starting_indexes.dateindex,
|
||||
&indexes.dateindex_to_date,
|
||||
|(di, d, ..)| {
|
||||
let ohlc = fetcher.get_date(d).unwrap();
|
||||
|(di, d, this)| {
|
||||
let mut ohlc = fetcher.get_date(d).unwrap();
|
||||
if let Some(prev) = di.decremented() {
|
||||
let prev_open = *this
|
||||
.get_or_read(prev, &this.mmap().load())
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.close;
|
||||
*ohlc.open = prev_open;
|
||||
*ohlc.high = (*ohlc.high).max(prev_open);
|
||||
*ohlc.low = (*ohlc.low).min(prev_open);
|
||||
}
|
||||
(di, ohlc)
|
||||
},
|
||||
exit,
|
||||
|
||||
@@ -226,7 +226,11 @@ impl ComputedValueVecsFromTxindex {
|
||||
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
|
||||
[
|
||||
self.sats.vecs(),
|
||||
vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
|
||||
self.bitcoin.vecs(),
|
||||
self.dollars_txindex
|
||||
.as_ref()
|
||||
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
|
||||
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
|
||||
]
|
||||
.into_iter()
|
||||
|
||||
@@ -124,6 +124,7 @@ impl Vecs {
|
||||
fetcher: Option<&mut Fetcher>,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
info!("Computing indexes...");
|
||||
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
|
||||
|
||||
info!("Computing constants...");
|
||||
|
||||
@@ -1289,7 +1289,7 @@ impl Vecs {
|
||||
base_version + self.height_to_opreturn_supply.inner_version(),
|
||||
)?;
|
||||
|
||||
let mut chain_state: Vec<BlockState>;
|
||||
let mut chain_state: Vec<BlockState> = vec![];
|
||||
let mut chain_state_starting_height = Height::from(self.chain_state.len());
|
||||
|
||||
let stateful_starting_height = match separate_utxo_vecs
|
||||
@@ -1322,25 +1322,27 @@ impl Vecs {
|
||||
.collect::<Vec<_>>();
|
||||
chain_state_starting_height
|
||||
}
|
||||
Ordering::Less => {
|
||||
// todo!("rollback instead");
|
||||
chain_state = vec![];
|
||||
chain_state_starting_height = Height::ZERO;
|
||||
Height::ZERO
|
||||
}
|
||||
Ordering::Less => Height::ZERO,
|
||||
};
|
||||
if stateful_starting_height.is_zero() {
|
||||
info!("Starting processing utxos from the start");
|
||||
separate_utxo_vecs
|
||||
.par_iter_mut()
|
||||
.try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
|
||||
}
|
||||
|
||||
let starting_height = starting_indexes
|
||||
.height
|
||||
.min(stateful_starting_height)
|
||||
.min(Height::from(self.height_to_unspendable_supply.len()))
|
||||
.min(Height::from(self.height_to_opreturn_supply.len()));
|
||||
|
||||
if starting_height.is_zero() {
|
||||
info!("Starting processing utxos from the start");
|
||||
|
||||
// todo!("rollback instead");
|
||||
chain_state = vec![];
|
||||
chain_state_starting_height = Height::ZERO;
|
||||
|
||||
separate_utxo_vecs
|
||||
.par_iter_mut()
|
||||
.try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
|
||||
}
|
||||
|
||||
if starting_height == Height::from(height_to_date_fixed.len()) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::ops::{Add, Div};
|
||||
|
||||
use derive_deref::Deref;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::{Serialize, Serializer, ser::SerializeTuple};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
@@ -172,6 +172,7 @@ impl From<Close<Sats>> for OHLCSats {
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
@@ -259,6 +260,7 @@ where
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
@@ -346,6 +348,7 @@ where
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
@@ -433,6 +436,7 @@ where
|
||||
IntoBytes,
|
||||
KnownLayout,
|
||||
Deref,
|
||||
DerefMut,
|
||||
Serialize,
|
||||
)]
|
||||
#[repr(C)]
|
||||
|
||||
@@ -3,9 +3,11 @@ mod checked_sub;
|
||||
mod paths;
|
||||
mod pause;
|
||||
mod rlimit;
|
||||
mod serde;
|
||||
|
||||
pub use bytes::*;
|
||||
pub use checked_sub::*;
|
||||
pub use paths::*;
|
||||
pub use pause::*;
|
||||
pub use rlimit::*;
|
||||
pub use serde::*;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
T: Deserialize<'de> + Default,
|
||||
{
|
||||
match T::deserialize(deserializer) {
|
||||
Ok(v) => Ok(v),
|
||||
Err(_) => Ok(T::default()),
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use brk_core::{Date, Height};
|
||||
use brk_fetcher::{BRK, Fetcher};
|
||||
use brk_fetcher::{BRK, Binance, Fetcher, Kraken};
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
@@ -12,6 +12,19 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let mut fetcher = Fetcher::import(None)?;
|
||||
|
||||
Binance::fetch_1d().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
Kraken::fetch_1d().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
Binance::fetch_1mn().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
Kraken::fetch_1mn().map(|b| {
|
||||
dbg!(b.last_key_value());
|
||||
})?;
|
||||
|
||||
dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);
|
||||
dbg!(fetcher.get_height(
|
||||
899911_u32.into(),
|
||||
|
||||
@@ -32,7 +32,7 @@ impl Kraken {
|
||||
)
|
||||
}
|
||||
|
||||
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
|
||||
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
|
||||
info!("Fetching 1mn prices from Kraken...");
|
||||
|
||||
retry(
|
||||
@@ -54,7 +54,7 @@ impl Kraken {
|
||||
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
|
||||
}
|
||||
|
||||
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
|
||||
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
|
||||
info!("Fetching daily prices from Kraken...");
|
||||
|
||||
retry(
|
||||
|
||||
@@ -40,11 +40,11 @@ impl Fetcher {
|
||||
}
|
||||
|
||||
fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
|
||||
self.binance
|
||||
self.kraken
|
||||
.get_from_1d(&date)
|
||||
.or_else(|_| {
|
||||
// eprintln!("{e}");
|
||||
self.kraken.get_from_1d(&date)
|
||||
self.binance.get_from_1d(&date)
|
||||
})
|
||||
.or_else(|_| {
|
||||
// eprintln!("{e}");
|
||||
@@ -90,11 +90,11 @@ impl Fetcher {
|
||||
let previous_timestamp = previous_timestamp.map(|t| t.floor_seconds());
|
||||
|
||||
let ohlc = self
|
||||
.binance
|
||||
.kraken
|
||||
.get_from_1mn(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|_report| {
|
||||
// eprintln!("{_report}");
|
||||
self.kraken
|
||||
self.binance
|
||||
.get_from_1mn(timestamp, previous_timestamp)
|
||||
.unwrap_or_else(|_report| {
|
||||
// // eprintln!("{_report}");
|
||||
@@ -185,8 +185,8 @@ How to fix this:
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.kraken.clear();
|
||||
self.binance.clear();
|
||||
self.brk.clear();
|
||||
self.kraken.clear();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ use brk_core::default_bitcoin_path;
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::Parser;
|
||||
use brk_vec::Format;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
@@ -25,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let outputs = Path::new("../../_outputs");
|
||||
|
||||
let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
|
||||
let mut indexer = Indexer::new(outputs, false)?;
|
||||
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
@@ -110,13 +110,7 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
|
||||
vecs.height_to_blockhash
|
||||
.iter()
|
||||
.get(*height)
|
||||
.is_none_or(|saved_blockhash| {
|
||||
let b = &rpc_blockhash != saved_blockhash.as_ref();
|
||||
if b {
|
||||
dbg!(rpc_blockhash, saved_blockhash.as_ref());
|
||||
}
|
||||
b
|
||||
})
|
||||
.is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref())
|
||||
})
|
||||
.unwrap_or(starting_height);
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ use brk_core::{
|
||||
use bitcoin::{Transaction, TxIn, TxOut};
|
||||
use brk_exit::Exit;
|
||||
use brk_parser::Parser;
|
||||
use brk_vec::{AnyVec, Format, VecIterator};
|
||||
use brk_vec::{AnyVec, VecIterator};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use fjall::TransactionalKeyspace;
|
||||
use log::{error, info};
|
||||
@@ -42,21 +42,15 @@ pub struct Indexer {
|
||||
vecs: Option<Vecs>,
|
||||
stores: Option<Stores>,
|
||||
check_collisions: bool,
|
||||
format: Format,
|
||||
}
|
||||
|
||||
impl Indexer {
|
||||
pub fn new(
|
||||
outputs_dir: &Path,
|
||||
format: Format,
|
||||
check_collisions: bool,
|
||||
) -> color_eyre::Result<Self> {
|
||||
pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result<Self> {
|
||||
setrlimit()?;
|
||||
Ok(Self {
|
||||
path: outputs_dir.to_owned(),
|
||||
vecs: None,
|
||||
stores: None,
|
||||
format,
|
||||
check_collisions,
|
||||
})
|
||||
}
|
||||
@@ -65,7 +59,6 @@ impl Indexer {
|
||||
self.vecs = Some(Vecs::forced_import(
|
||||
&self.path.join("vecs/indexed"),
|
||||
VERSION + Version::ZERO,
|
||||
self.format,
|
||||
)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -66,11 +66,7 @@ pub struct Vecs {
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self {
|
||||
@@ -78,7 +74,7 @@ impl Vecs {
|
||||
path,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_blockhash: IndexedVec::forced_import(
|
||||
path,
|
||||
@@ -90,145 +86,145 @@ impl Vecs {
|
||||
path,
|
||||
"difficulty",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_emptyoutputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_emptyoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_inputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_inputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_opreturnindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_opreturnindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_outputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_outputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2aindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2aindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2msindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2msindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2pk33index: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2pk33index",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2pk65index: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2pk65index",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2pkhindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2pkhindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2shindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2shindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2trindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2trindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2wpkhindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2wpkhindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2wshindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_p2wshindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_unknownoutputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_unknownoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_timestamp: IndexedVec::forced_import(
|
||||
path,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_total_size: IndexedVec::forced_import(
|
||||
path,
|
||||
"total_size",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_weight: IndexedVec::forced_import(
|
||||
path,
|
||||
"weight",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
inputindex_to_outputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"outputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
opreturnindex_to_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
outputindex_to_outputtype: IndexedVec::forced_import(
|
||||
path,
|
||||
"outputtype",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
outputindex_to_outputtypeindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"outputtypeindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
outputindex_to_value: IndexedVec::forced_import(
|
||||
path,
|
||||
"value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2aindex_to_p2abytes: IndexedVec::forced_import(
|
||||
path,
|
||||
@@ -240,7 +236,7 @@ impl Vecs {
|
||||
path,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
|
||||
path,
|
||||
@@ -288,13 +284,13 @@ impl Vecs {
|
||||
path,
|
||||
"base_size",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_first_inputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"first_inputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_first_outputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
@@ -306,19 +302,19 @@ impl Vecs {
|
||||
path,
|
||||
"is_explicitly_rbf",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_rawlocktime: IndexedVec::forced_import(
|
||||
path,
|
||||
"rawlocktime",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_total_size: IndexedVec::forced_import(
|
||||
path,
|
||||
"total_size",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_txid: IndexedVec::forced_import(
|
||||
path,
|
||||
@@ -330,13 +326,13 @@ impl Vecs {
|
||||
path,
|
||||
"txversion",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
unknownoutputindex_to_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
Format::Raw,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -25,50 +25,53 @@ pub fn init(path: Option<&Path>) {
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
Builder::from_env(Env::default().default_filter_or("info,fjall=off,lsm_tree=off"))
|
||||
.format(move |buf, record| {
|
||||
let date_time = Timestamp::now()
|
||||
.to_zoned(tz::TimeZone::system())
|
||||
.strftime("%Y-%m-%d %H:%M:%S")
|
||||
.to_string();
|
||||
let level = record.level().as_str().to_lowercase();
|
||||
let level = format!("{:5}", level);
|
||||
let target = record.target();
|
||||
let dash = "-";
|
||||
let args = record.args();
|
||||
Builder::from_env(
|
||||
Env::default()
|
||||
.default_filter_or("info,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off"),
|
||||
)
|
||||
.format(move |buf, record| {
|
||||
let date_time = Timestamp::now()
|
||||
.to_zoned(tz::TimeZone::system())
|
||||
.strftime("%Y-%m-%d %H:%M:%S")
|
||||
.to_string();
|
||||
let level = record.level().as_str().to_lowercase();
|
||||
let level = format!("{:5}", level);
|
||||
let target = record.target();
|
||||
let dash = "-";
|
||||
let args = record.args();
|
||||
|
||||
if let Some(file) = file.as_ref() {
|
||||
let _ = write(
|
||||
file.try_clone().unwrap(),
|
||||
&date_time,
|
||||
target,
|
||||
&level,
|
||||
dash,
|
||||
args,
|
||||
);
|
||||
}
|
||||
|
||||
let colored_date_time = date_time.bright_black();
|
||||
let colored_level = match level.chars().next().unwrap() {
|
||||
'e' => level.red().to_string(),
|
||||
'w' => level.yellow().to_string(),
|
||||
'i' => level.green().to_string(),
|
||||
'd' => level.blue().to_string(),
|
||||
't' => level.cyan().to_string(),
|
||||
_ => panic!(),
|
||||
};
|
||||
let colored_dash = dash.bright_black();
|
||||
|
||||
write(
|
||||
buf,
|
||||
colored_date_time,
|
||||
if let Some(file) = file.as_ref() {
|
||||
let _ = write(
|
||||
file.try_clone().unwrap(),
|
||||
&date_time,
|
||||
target,
|
||||
colored_level,
|
||||
colored_dash,
|
||||
&level,
|
||||
dash,
|
||||
args,
|
||||
)
|
||||
})
|
||||
.init();
|
||||
);
|
||||
}
|
||||
|
||||
let colored_date_time = date_time.bright_black();
|
||||
let colored_level = match level.chars().next().unwrap() {
|
||||
'e' => level.red().to_string(),
|
||||
'w' => level.yellow().to_string(),
|
||||
'i' => level.green().to_string(),
|
||||
'd' => level.blue().to_string(),
|
||||
't' => level.cyan().to_string(),
|
||||
_ => panic!(),
|
||||
};
|
||||
let colored_dash = dash.bright_black();
|
||||
|
||||
write(
|
||||
buf,
|
||||
colored_date_time,
|
||||
target,
|
||||
colored_level,
|
||||
colored_dash,
|
||||
args,
|
||||
)
|
||||
})
|
||||
.init();
|
||||
}
|
||||
|
||||
fn write(
|
||||
@@ -80,5 +83,9 @@ fn write(
|
||||
args: impl Display,
|
||||
) -> Result<(), std::io::Error> {
|
||||
writeln!(buf, "{} {} {} {}", date_time, dash, level, args)
|
||||
// writeln!(buf, "{} {} {} {} {}", date_time, _target, level, dash, args)
|
||||
// writeln!(
|
||||
// buf,
|
||||
// "{} {} {} {} {}",
|
||||
// date_time, _target, level, dash, args
|
||||
// )
|
||||
}
|
||||
|
||||
@@ -96,9 +96,10 @@ impl BlkIndexToBlkRecap {
|
||||
}
|
||||
|
||||
pub fn export(&self) {
|
||||
let file = File::create(&self.path).unwrap_or_else(|_| {
|
||||
let file = File::create(&self.path).unwrap_or_else(|e| {
|
||||
dbg!(e);
|
||||
dbg!(&self.path);
|
||||
panic!("No such file or directory")
|
||||
panic!("Cannot write file");
|
||||
});
|
||||
|
||||
serde_json::to_writer(&mut BufWriter::new(file), &self.tree).unwrap();
|
||||
|
||||
@@ -18,5 +18,5 @@ color-eyre = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde_with = "3.12.0"
|
||||
serde_with = "3.13.0"
|
||||
tabled = { workspace = true }
|
||||
|
||||
@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let format = Format::Compressed;
|
||||
|
||||
let mut indexer = Indexer::new(outputs_dir, format, true)?;
|
||||
let mut indexer = Indexer::new(outputs_dir, true)?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
let mut computer = Computer::new(outputs_dir, None, format);
|
||||
|
||||
@@ -10,6 +10,7 @@ repository.workspace = true
|
||||
[dependencies]
|
||||
axum = { workspace = true }
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_bundler = { workspace = true }
|
||||
brk_computer = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
@@ -25,11 +26,11 @@ color-eyre = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
minreq = { workspace = true }
|
||||
oxc = { version = "0.72.3", features = ["codegen", "minifier"] }
|
||||
oxc = { version = "0.73.0", features = ["codegen", "minifier"] }
|
||||
serde = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }
|
||||
zip = "4.0.0"
|
||||
zip = "4.1.0"
|
||||
tracing = "0.1.41"
|
||||
|
||||
[package.metadata.cargo-machete]
|
||||
|
||||
@@ -73,7 +73,7 @@ A list of all possible vec ids and their supported vec indexes
|
||||
|
||||
A list of all possible vec indexes and their supported vec ids
|
||||
|
||||
#### `GET /api/{INDEX}-to-{VALUE}`
|
||||
#### `GET /api/{INDEX}-to-{ID}`
|
||||
|
||||
This endpoint retrieves data based on the specified vector index and id.
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let format = Format::Compressed;
|
||||
|
||||
let mut indexer = Indexer::new(outputs_dir, format, true)?;
|
||||
let mut indexer = Indexer::new(outputs_dir, true)?;
|
||||
indexer.import_stores()?;
|
||||
indexer.import_vecs()?;
|
||||
|
||||
@@ -51,7 +51,7 @@ pub fn main() -> color_eyre::Result<()> {
|
||||
let server = Server::new(served_indexer, served_computer, Website::Default)?;
|
||||
|
||||
let server = tokio::spawn(async move {
|
||||
server.serve().await.unwrap();
|
||||
server.serve(true).await.unwrap();
|
||||
});
|
||||
|
||||
if process {
|
||||
|
||||
@@ -124,6 +124,7 @@ pub async fn variant_handler(
|
||||
Query(params_opt): Query<ParamsOpt>,
|
||||
state: State<AppState>,
|
||||
) -> Response {
|
||||
let variant = variant.replace("_", "-");
|
||||
let mut split = variant.split(TO_SEPARATOR);
|
||||
let params = Params::from((
|
||||
(
|
||||
|
||||
@@ -61,9 +61,12 @@ export const VERSION = \"v{}\";
|
||||
.join(" | ")
|
||||
);
|
||||
|
||||
contents += "\n\nexport function createVecIdToIndexes() {\n";
|
||||
contents += "\n\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";
|
||||
contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n";
|
||||
|
||||
contents += " return /** @type {const} */ ({\n";
|
||||
contents += "\nexport function createVecIdToIndexes() {\n";
|
||||
|
||||
contents += " return {\n";
|
||||
|
||||
self.vec_trees
|
||||
.id_to_index_to_vec
|
||||
@@ -79,11 +82,7 @@ export const VERSION = \"v{}\";
|
||||
contents += &format!(" \"{id}\": [{indexes}],\n");
|
||||
});
|
||||
|
||||
contents += " });\n";
|
||||
contents.push('}');
|
||||
|
||||
contents += "\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";
|
||||
contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n";
|
||||
contents += " };\n}\n";
|
||||
|
||||
fs::write(path, contents)
|
||||
}
|
||||
|
||||
@@ -13,8 +13,6 @@ use crate::{
|
||||
traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
|
||||
};
|
||||
|
||||
use super::minify::minify_js;
|
||||
|
||||
pub async fn file_handler(
|
||||
headers: HeaderMap,
|
||||
State(app_state): State<AppState>,
|
||||
@@ -32,16 +30,12 @@ fn any_handler(
|
||||
app_state: AppState,
|
||||
path: Option<extract::Path<String>>,
|
||||
) -> Response {
|
||||
let website_path = app_state
|
||||
.websites_path
|
||||
.as_ref()
|
||||
.expect("Should never reach here is websites_path is None")
|
||||
.join(app_state.website.to_folder_name());
|
||||
let dist_path = app_state.dist_path();
|
||||
|
||||
if let Some(path) = path.as_ref() {
|
||||
let path = path.0.replace("..", "").replace("\\", "");
|
||||
|
||||
let mut path = website_path.join(&path);
|
||||
let mut path = dist_path.join(&path);
|
||||
|
||||
if !path.exists() || path.is_dir() {
|
||||
if path.extension().is_some() {
|
||||
@@ -55,13 +49,13 @@ fn any_handler(
|
||||
|
||||
return response;
|
||||
} else {
|
||||
path = website_path.join("index.html");
|
||||
path = dist_path.join("index.html");
|
||||
}
|
||||
}
|
||||
|
||||
path_to_response(&headers, &path)
|
||||
} else {
|
||||
path_to_response(&headers, &website_path.join("index.html"))
|
||||
path_to_response(&headers, &dist_path.join("index.html"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,49 +79,35 @@ fn path_to_response_(headers: &HeaderMap, path: &Path) -> color_eyre::Result<Res
|
||||
return Ok(Response::new_not_modified());
|
||||
}
|
||||
|
||||
let mut response;
|
||||
let content = fs::read(path).unwrap_or_else(|error| {
|
||||
error!("{error}");
|
||||
let path = path.to_str().unwrap();
|
||||
info!("Can't read file {path}");
|
||||
panic!("")
|
||||
});
|
||||
|
||||
let is_localhost = headers.check_if_host_is_localhost();
|
||||
|
||||
if !is_localhost
|
||||
&& path.extension().unwrap_or_else(|| {
|
||||
dbg!(path);
|
||||
panic!();
|
||||
}) == "js"
|
||||
{
|
||||
let content = minify_js(path);
|
||||
|
||||
response = Response::new(content.into());
|
||||
} else {
|
||||
let content = fs::read(path).unwrap_or_else(|error| {
|
||||
error!("{error}");
|
||||
let path = path.to_str().unwrap();
|
||||
info!("Can't read file {path}");
|
||||
panic!("")
|
||||
});
|
||||
|
||||
response = Response::new(content.into());
|
||||
}
|
||||
let mut response = Response::new(content.into());
|
||||
|
||||
let headers = response.headers_mut();
|
||||
headers.insert_cors();
|
||||
headers.insert_content_type(path);
|
||||
|
||||
if !is_localhost {
|
||||
let serialized_path = path.to_str().unwrap();
|
||||
let serialized_path = path.to_str().unwrap();
|
||||
|
||||
if serialized_path.contains("fonts/")
|
||||
|| serialized_path.contains("assets/")
|
||||
|| serialized_path.contains("packages/")
|
||||
|| path.extension().is_some_and(|extension| {
|
||||
extension == "pdf"
|
||||
|| extension == "jpg"
|
||||
|| extension == "png"
|
||||
|| extension == "woff2"
|
||||
})
|
||||
{
|
||||
headers.insert_cache_control_immutable();
|
||||
}
|
||||
if path
|
||||
.extension()
|
||||
.is_some_and(|extension| extension == "html")
|
||||
|| serialized_path.ends_with("service-worker.js")
|
||||
{
|
||||
headers.insert_cache_control_must_revalidate();
|
||||
} else if path.extension().is_some_and(|extension| {
|
||||
extension == "jpg"
|
||||
|| extension == "png"
|
||||
|| extension == "woff2"
|
||||
|| extension == "js"
|
||||
|| extension == "map"
|
||||
}) {
|
||||
headers.insert_cache_control_immutable();
|
||||
}
|
||||
|
||||
headers.insert_last_modified(date);
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
// Source: https://github.com/oxc-project/oxc/blob/main/crates/oxc_minifier/examples/minifier.rs
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use oxc::{
|
||||
allocator::Allocator,
|
||||
codegen::{Codegen, CodegenOptions, LegalComment},
|
||||
minifier::{CompressOptions, MangleOptions, Minifier, MinifierOptions},
|
||||
parser::Parser,
|
||||
span::SourceType,
|
||||
};
|
||||
|
||||
pub fn minify_js(path: &Path) -> String {
|
||||
let source_text = fs::read_to_string(path).unwrap();
|
||||
let source_type = SourceType::from_path(path).unwrap();
|
||||
|
||||
let allocator = Allocator::default();
|
||||
|
||||
let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
|
||||
|
||||
let mut program = parser_return.program;
|
||||
|
||||
let minifier_return = Minifier::new(MinifierOptions {
|
||||
mangle: Some(MangleOptions::default()),
|
||||
compress: Some(CompressOptions::default()),
|
||||
})
|
||||
.build(&allocator, &mut program);
|
||||
|
||||
Codegen::new()
|
||||
.with_options(CodegenOptions {
|
||||
minify: true,
|
||||
single_quote: false,
|
||||
comments: false,
|
||||
annotation_comments: false,
|
||||
source_map_path: None,
|
||||
legal_comments: LegalComment::None,
|
||||
})
|
||||
.with_scoping(minifier_return.scoping)
|
||||
.build(&program)
|
||||
.code
|
||||
}
|
||||
@@ -3,7 +3,6 @@ use axum::{Router, routing::get};
|
||||
use super::AppState;
|
||||
|
||||
mod file;
|
||||
mod minify;
|
||||
mod website;
|
||||
|
||||
use file::{file_handler, index_handler};
|
||||
|
||||
@@ -19,6 +19,7 @@ use axum::{
|
||||
routing::get,
|
||||
serve,
|
||||
};
|
||||
use brk_bundler::bundle;
|
||||
use brk_computer::Computer;
|
||||
use brk_core::dot_brk_path;
|
||||
use brk_indexer::Indexer;
|
||||
@@ -45,6 +46,15 @@ pub struct AppState {
|
||||
websites_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn dist_path(&self) -> PathBuf {
|
||||
self.websites_path
|
||||
.as_ref()
|
||||
.expect("Should never reach here is websites_path is None")
|
||||
.join("dist")
|
||||
}
|
||||
}
|
||||
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
const DEV_PATH: &str = "../..";
|
||||
@@ -103,9 +113,13 @@ impl Server {
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn serve(self) -> color_eyre::Result<()> {
|
||||
pub async fn serve(self, watch: bool) -> color_eyre::Result<()> {
|
||||
let state = self.0;
|
||||
|
||||
if let Some(websites_path) = state.websites_path.clone() {
|
||||
bundle(&websites_path, state.website.to_folder_name(), watch).await?;
|
||||
}
|
||||
|
||||
let compression_layer = CompressionLayer::new()
|
||||
.br(true)
|
||||
.deflate(true)
|
||||
|
||||
@@ -5,12 +5,11 @@ use std::{
|
||||
|
||||
use axum::http::{
|
||||
HeaderMap,
|
||||
header::{self, HOST, IF_MODIFIED_SINCE},
|
||||
header::{self, IF_MODIFIED_SINCE},
|
||||
};
|
||||
use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone};
|
||||
use log::info;
|
||||
|
||||
const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish
|
||||
const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT";
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
@@ -20,12 +19,6 @@ pub enum ModifiedState {
|
||||
}
|
||||
|
||||
pub trait HeaderMapExtended {
|
||||
fn _get_scheme(&self) -> &str;
|
||||
fn get_host(&self) -> &str;
|
||||
fn check_if_host_is_local(&self) -> bool;
|
||||
fn check_if_host_is_0000(&self) -> bool;
|
||||
fn check_if_host_is_localhost(&self) -> bool;
|
||||
|
||||
fn insert_cors(&mut self);
|
||||
|
||||
fn get_if_modified_since(&self) -> Option<DateTime>;
|
||||
@@ -36,8 +29,8 @@ pub trait HeaderMapExtended {
|
||||
duration: Duration,
|
||||
) -> color_eyre::Result<(ModifiedState, DateTime)>;
|
||||
|
||||
fn insert_cache_control_must_revalidate(&mut self);
|
||||
fn insert_cache_control_immutable(&mut self);
|
||||
fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
|
||||
fn insert_last_modified(&mut self, date: DateTime);
|
||||
|
||||
fn insert_content_disposition_attachment(&mut self);
|
||||
@@ -59,41 +52,22 @@ pub trait HeaderMapExtended {
|
||||
}
|
||||
|
||||
impl HeaderMapExtended for HeaderMap {
|
||||
fn _get_scheme(&self) -> &str {
|
||||
if self.check_if_host_is_local() {
|
||||
"http"
|
||||
} else {
|
||||
"https"
|
||||
}
|
||||
}
|
||||
|
||||
fn get_host(&self) -> &str {
|
||||
self[HOST].to_str().unwrap()
|
||||
}
|
||||
|
||||
fn check_if_host_is_local(&self) -> bool {
|
||||
self.check_if_host_is_localhost() || self.check_if_host_is_0000()
|
||||
}
|
||||
|
||||
fn check_if_host_is_0000(&self) -> bool {
|
||||
self.get_host().contains("0.0.0.0")
|
||||
}
|
||||
|
||||
fn check_if_host_is_localhost(&self) -> bool {
|
||||
self.get_host().contains("localhost")
|
||||
}
|
||||
|
||||
fn insert_cors(&mut self) {
|
||||
self.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
|
||||
self.insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_cache_control_must_revalidate(&mut self) {
|
||||
self.insert(
|
||||
header::CACHE_CONTROL,
|
||||
"public, max-age=0, must-revalidate".parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_cache_control_immutable(&mut self) {
|
||||
self.insert(
|
||||
header::CACHE_CONTROL,
|
||||
format!("public, max-age=604800, immutable, stale-if-error={STALE_IF_ERROR}")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
"public, max-age=31536000, immutable".parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -101,16 +75,6 @@ impl HeaderMapExtended for HeaderMap {
|
||||
self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap());
|
||||
}
|
||||
|
||||
fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64) {
|
||||
self.insert(
|
||||
header::CACHE_CONTROL,
|
||||
format!(
|
||||
"public, max-age={max_age}, stale-while-revalidate={stale_while_revalidate}, stale-if-error={STALE_IF_ERROR}")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_last_modified(&mut self, date: DateTime) {
|
||||
let formatted = date
|
||||
.to_zoned(TimeZone::system())
|
||||
@@ -167,7 +131,7 @@ impl HeaderMapExtended for HeaderMap {
|
||||
fn insert_content_type(&mut self, path: &Path) {
|
||||
match path.extension().unwrap().to_str().unwrap() {
|
||||
"js" => self.insert_content_type_application_javascript(),
|
||||
"json" => self.insert_content_type_application_json(),
|
||||
"json" | "map" => self.insert_content_type_application_json(),
|
||||
"html" => self.insert_content_type_text_html(),
|
||||
"css" => self.insert_content_type_text_css(),
|
||||
"toml" | "txt" => self.insert_content_type_text_plain(),
|
||||
|
||||
@@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
|
||||
Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum,
|
||||
)]
|
||||
pub enum Format {
|
||||
#[default]
|
||||
Compressed,
|
||||
#[default]
|
||||
Raw,
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ where
|
||||
}
|
||||
#[inline]
|
||||
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<T>>> {
|
||||
let stored_len = mmap.len() / Self::SIZE_OF_T;
|
||||
let stored_len = self.stored_len_(mmap);
|
||||
|
||||
if index >= stored_len {
|
||||
let pushed = self.pushed();
|
||||
@@ -53,6 +53,7 @@ where
|
||||
fn mmap(&self) -> &ArcSwap<Mmap>;
|
||||
|
||||
fn stored_len(&self) -> usize;
|
||||
fn stored_len_(&self, mmap: &Mmap) -> usize;
|
||||
|
||||
fn pushed(&self) -> &[T];
|
||||
#[inline]
|
||||
|
||||
@@ -21,7 +21,9 @@ use crate::{
|
||||
const ONE_KIB: usize = 1024;
|
||||
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
|
||||
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
|
||||
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
|
||||
pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB;
|
||||
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CompressedVec<I, T> {
|
||||
@@ -39,7 +41,9 @@ where
|
||||
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
|
||||
|
||||
/// Same as import but will reset the folder under certain errors, so be careful !
|
||||
pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
|
||||
pub fn forced_import(path: &Path, mut version: Version) -> Result<Self> {
|
||||
version = version + VERSION;
|
||||
|
||||
let res = Self::import(path, version);
|
||||
match res {
|
||||
Err(Error::WrongEndian)
|
||||
@@ -129,7 +133,7 @@ where
|
||||
page_index * Self::PER_PAGE
|
||||
}
|
||||
|
||||
fn stored_len_(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
|
||||
fn stored_len__(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
|
||||
if let Some(last) = pages_meta.last() {
|
||||
(pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
|
||||
} else {
|
||||
@@ -178,7 +182,11 @@ where
|
||||
|
||||
#[inline]
|
||||
fn stored_len(&self) -> usize {
|
||||
Self::stored_len_(&self.pages_meta.load())
|
||||
Self::stored_len__(&self.pages_meta.load())
|
||||
}
|
||||
#[inline]
|
||||
fn stored_len_(&self, _: &Mmap) -> usize {
|
||||
self.stored_len()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -477,7 +485,7 @@ where
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
let pages_meta = self.pages_meta.load();
|
||||
let stored_len = CompressedVec::<I, T>::stored_len_(&pages_meta);
|
||||
let stored_len = CompressedVec::<I, T>::stored_len__(&pages_meta);
|
||||
CompressedVecIterator {
|
||||
vec: self,
|
||||
guard: self.mmap().load(),
|
||||
|
||||
@@ -1116,12 +1116,12 @@ impl EagerVec<DateIndex, Dollars> {
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.validate_computed_version_or_reset_file(
|
||||
Version::ZERO + self.inner.version() + stacks.version(),
|
||||
Version::ONE + self.inner.version() + stacks.version(),
|
||||
)?;
|
||||
|
||||
let index = max_from.min(DateIndex::from(self.len()));
|
||||
|
||||
let first_price_date = DateIndex::try_from(Date::new(2010, 8, 16)).unwrap();
|
||||
let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap();
|
||||
|
||||
stacks.iter_at(index).try_for_each(|(i, stack)| {
|
||||
let stack = stack.into_inner();
|
||||
|
||||
@@ -104,7 +104,11 @@ where
|
||||
|
||||
#[inline]
|
||||
fn stored_len(&self) -> usize {
|
||||
self.mmap.load().len() / Self::SIZE_OF_T
|
||||
self.stored_len_(&self.mmap.load())
|
||||
}
|
||||
#[inline]
|
||||
fn stored_len_(&self, mmap: &Mmap) -> usize {
|
||||
mmap.len() / Self::SIZE_OF_T
|
||||
}
|
||||
|
||||
#[inline]
|
||||
|
||||
@@ -73,6 +73,13 @@ where
|
||||
StoredVec::Compressed(v) => v.stored_len(),
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
fn stored_len_(&self, mmap: &Mmap) -> usize {
|
||||
match self {
|
||||
StoredVec::Raw(v) => v.stored_len_(mmap),
|
||||
StoredVec::Compressed(v) => v.stored_len_(mmap),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn pushed(&self) -> &[T] {
|
||||
|
||||
|
After Width: | Height: | Size: 4.8 KiB |
|
After Width: | Height: | Size: 3.9 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 36 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 18 KiB |
|
Before Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 46 KiB |
|
Before Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 47 KiB |
|
Before Width: | Height: | Size: 49 KiB |
|
Before Width: | Height: | Size: 22 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 64 KiB |
|
Before Width: | Height: | Size: 46 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 49 KiB |
|
Before Width: | Height: | Size: 49 KiB |
|
Before Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 36 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 64 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 18 KiB |
|
Before Width: | Height: | Size: 22 KiB |
|
Before Width: | Height: | Size: 6.7 KiB |
@@ -1,187 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<link
|
||||
rel="icon"
|
||||
type="image/png"
|
||||
sizes="196x196"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/favicon-196.png"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-icon"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-icon-180.png"
|
||||
/>
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2048-2732.jpg"
|
||||
media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2732-2048.jpg"
|
||||
media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1668-2388.jpg"
|
||||
media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2388-1668.jpg"
|
||||
media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1536-2048.jpg"
|
||||
media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2048-1536.jpg"
|
||||
media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1488-2266.jpg"
|
||||
media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2266-1488.jpg"
|
||||
media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1640-2360.jpg"
|
||||
media="(device-width: 820px) and (device-height: 1180px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2360-1640.jpg"
|
||||
media="(device-width: 820px) and (device-height: 1180px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1668-2224.jpg"
|
||||
media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2224-1668.jpg"
|
||||
media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1620-2160.jpg"
|
||||
media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2160-1620.jpg"
|
||||
media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1290-2796.jpg"
|
||||
media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2796-1290.jpg"
|
||||
media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1179-2556.jpg"
|
||||
media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2556-1179.jpg"
|
||||
media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1284-2778.jpg"
|
||||
media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2778-1284.jpg"
|
||||
media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1170-2532.jpg"
|
||||
media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2532-1170.jpg"
|
||||
media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1125-2436.jpg"
|
||||
media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2436-1125.jpg"
|
||||
media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1242-2688.jpg"
|
||||
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2688-1242.jpg"
|
||||
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-828-1792.jpg"
|
||||
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1792-828.jpg"
|
||||
media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1242-2208.jpg"
|
||||
media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-2208-1242.jpg"
|
||||
media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-750-1334.jpg"
|
||||
media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1334-750.jpg"
|
||||
media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-640-1136.jpg"
|
||||
media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)"
|
||||
/>
|
||||
<link
|
||||
rel="apple-touch-startup-image"
|
||||
href="/assets/pwa/2025-03-22_10-00-00/apple-splash-1136-640.jpg"
|
||||
media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)"
|
||||
/>
|
||||
</head>
|
||||
<body></body>
|
||||
</html>
|
||||
|
Before Width: | Height: | Size: 3.2 KiB |
|
Before Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 511 B |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 4.9 KiB |
|
After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 20 KiB |
|
After Width: | Height: | Size: 21 KiB |