diff --git a/Cargo.lock b/Cargo.lock index 17b51f81e..5dc73c58c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1698,9 +1698,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" @@ -3352,9 +3352,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "d3bfe459f85da17560875b8bf1423d6f113b7a87a5d942e7da0ac71be7c61f8b" [[package]] name = "windows-result" diff --git a/crates/brk_cli/src/query.rs b/crates/brk_cli/src/query.rs index 57b3d2379..3f884b3e2 100644 --- a/crates/brk_cli/src/query.rs +++ b/crates/brk_cli/src/query.rs @@ -10,7 +10,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> { let format = config.format(); - let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?; + let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?; indexer.import_vecs()?; let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format); diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index fa3afff07..3c9707ae4 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -7,7 +7,7 @@ use std::{ use bitcoincore_rpc::{self, Auth, Client, RpcApi}; use brk_computer::Computer; -use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path}; +use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path}; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; @@ -29,7 +29,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { let format = config.format(); - let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?; + let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?; indexer.import_stores()?; indexer.import_vecs()?; @@ -109,62 +109,77 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] pub struct RunConfig { /// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "PATH")] bitcoindir: Option, /// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "PATH")] blocksdir: Option, /// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "PATH")] brkdir: Option, - /// Executed by the runner, default: all, saved + /// Activated services, default: all, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(short, long)] - mode: Option, + services: Option, - /// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved + /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(short, long)] computation: Option, - /// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: compressed, saved - #[arg(short, long, value_name = "FORMAT")] + /// Format of computed datasets, `compressed` to save disk space, `raw` to prioritize speed, default: `compressed`, saved + #[serde(default, deserialize_with = "default_on_error")] + #[arg(short, long)] format: Option, /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(short = 'F', long, value_name = "BOOL")] fetch: Option, /// Website served by the server (if active), default: default, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(short, long)] website: Option, /// Bitcoin RPC ip, default: localhost, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "IP")] rpcconnect: Option, /// Bitcoin RPC port, default: 8332, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "PORT")] rpcport: Option, /// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "PATH")] rpccookiefile: Option, /// Bitcoin RPC username, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "USERNAME")] rpcuser: Option, /// Bitcoin RPC password, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "PASSWORD")] rpcpassword: Option, /// Delay between runs, default: 0, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "SECONDS")] delay: Option, /// DEV: Activate checking address hashes for collisions when indexing, default: false, saved + #[serde(default, deserialize_with = "default_on_error")] #[arg(long, value_name = "BOOL")] check_collisions: Option, } @@ -192,8 +207,8 @@ impl RunConfig { config_saved.brkdir = Some(brkdir); } - if let Some(mode) = config_args.mode.take() { - config_saved.mode = Some(mode); + if let Some(services) = config_args.services.take() { + config_saved.services = Some(services); } if let Some(computation) = config_args.computation.take() { @@ -255,7 +270,7 @@ impl RunConfig { // info!("Configuration {{"); // info!(" bitcoindir: {:?}", config.bitcoindir); // info!(" brkdir: {:?}", config.brkdir); - // info!(" mode: {:?}", config.mode); + // info!(" services: {:?}", config.services); // info!(" website: {:?}", config.website); // info!(" rpcconnect: {:?}", config.rpcconnect); // info!(" rpcport: {:?}", config.rpcport); @@ -375,13 +390,13 @@ impl RunConfig { } pub fn process(&self) -> bool { - self.mode - .is_none_or(|m| m == Mode::All || m == Mode::Processor) + self.services + .is_none_or(|m| m == Services::All || m == Services::Processor) } pub fn serve(&self) -> bool { - self.mode - .is_none_or(|m| m == Mode::All || m == Mode::Server) + self.services + .is_none_or(|m| m == Services::All || m == Services::Server) } fn path_cookiefile(&self) -> PathBuf { @@ -449,7 +464,7 @@ impl RunConfig { PartialOrd, Ord, )] -pub enum Mode { +pub enum Services { #[default] All, Processor, diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index 559483d55..b065c4a39 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -33,7 +33,7 @@ pub fn main() -> color_eyre::Result<()> { let format = Format::Raw; - let mut indexer = Indexer::new(outputs_dir, format, true)?; + let mut indexer = Indexer::new(outputs_dir, true)?; indexer.import_stores()?; indexer.import_vecs()?; diff --git a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs index 5b7a7607e..0860c9b17 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs @@ -226,7 +226,11 @@ impl ComputedValueVecsFromTxindex { pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ self.sats.vecs(), + vec![&self.bitcoin_txindex as &dyn AnyCollectableVec], self.bitcoin.vecs(), + self.dollars_txindex + .as_ref() + .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]), self.dollars.as_ref().map_or(vec![], |v| v.vecs()), ] .into_iter() diff --git a/crates/brk_computer/src/vecs/mod.rs b/crates/brk_computer/src/vecs/mod.rs index 66ec810d1..ab60aee42 100644 --- a/crates/brk_computer/src/vecs/mod.rs +++ b/crates/brk_computer/src/vecs/mod.rs @@ -124,6 +124,7 @@ impl Vecs { fetcher: Option<&mut Fetcher>, exit: &Exit, ) -> color_eyre::Result<()> { + info!("Computing indexes..."); let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; info!("Computing constants..."); diff --git a/crates/brk_core/src/utils/mod.rs b/crates/brk_core/src/utils/mod.rs index d8fc983a2..b410029b1 100644 --- a/crates/brk_core/src/utils/mod.rs +++ b/crates/brk_core/src/utils/mod.rs @@ -3,9 +3,11 @@ mod checked_sub; mod paths; mod pause; mod rlimit; +mod serde; pub use bytes::*; pub use checked_sub::*; pub use paths::*; pub use pause::*; pub use rlimit::*; +pub use serde::*; diff --git a/crates/brk_core/src/utils/serde.rs b/crates/brk_core/src/utils/serde.rs new file mode 100644 index 000000000..b3ae164a6 --- /dev/null +++ b/crates/brk_core/src/utils/serde.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Deserializer}; + +pub fn default_on_error<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Deserialize<'de> + Default, +{ + match T::deserialize(deserializer) { + Ok(v) => Ok(v), + Err(_) => Ok(T::default()), + } +} diff --git a/crates/brk_indexer/examples/main.rs b/crates/brk_indexer/examples/main.rs index 05cacfde1..66adb0e0e 100644 --- a/crates/brk_indexer/examples/main.rs +++ b/crates/brk_indexer/examples/main.rs @@ -4,7 +4,6 @@ use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_vec::Format; fn main() -> color_eyre::Result<()> { color_eyre::install()?; @@ -25,7 +24,7 @@ fn main() -> color_eyre::Result<()> { let outputs = Path::new("../../_outputs"); - let mut indexer = Indexer::new(outputs, Format::Raw, false)?; + let mut indexer = Indexer::new(outputs, false)?; indexer.import_stores()?; indexer.import_vecs()?; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index e403ece87..f282692dc 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -19,7 +19,7 @@ use brk_core::{ use bitcoin::{Transaction, TxIn, TxOut}; use brk_exit::Exit; use brk_parser::Parser; -use brk_vec::{AnyVec, Format, VecIterator}; +use brk_vec::{AnyVec, VecIterator}; use color_eyre::eyre::{ContextCompat, eyre}; use fjall::TransactionalKeyspace; use log::{error, info}; @@ -42,21 +42,15 @@ pub struct Indexer { vecs: Option, stores: Option, check_collisions: bool, - format: Format, } impl Indexer { - pub fn new( - outputs_dir: &Path, - format: Format, - check_collisions: bool, - ) -> color_eyre::Result { + pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result { setrlimit()?; Ok(Self { path: outputs_dir.to_owned(), vecs: None, stores: None, - format, check_collisions, }) } @@ -65,7 +59,6 @@ impl Indexer { self.vecs = Some(Vecs::forced_import( &self.path.join("vecs/indexed"), VERSION + Version::ZERO, - self.format, )?); Ok(()) } diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index 8b4ab1071..2e329b0b7 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -66,11 +66,7 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import( - path: &Path, - version: Version, - format: Format, - ) -> color_eyre::Result { + pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { @@ -78,7 +74,7 @@ impl Vecs { path, "txindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_blockhash: IndexedVec::forced_import( path, @@ -90,145 +86,145 @@ impl Vecs { path, "difficulty", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_emptyoutputindex: IndexedVec::forced_import( path, "first_emptyoutputindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_inputindex: IndexedVec::forced_import( path, "first_inputindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_opreturnindex: IndexedVec::forced_import( path, "first_opreturnindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_outputindex: IndexedVec::forced_import( path, "first_outputindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2aindex: IndexedVec::forced_import( path, "first_p2aindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2msindex: IndexedVec::forced_import( path, "first_p2msindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2pk33index: IndexedVec::forced_import( path, "first_p2pk33index", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2pk65index: IndexedVec::forced_import( path, "first_p2pk65index", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2pkhindex: IndexedVec::forced_import( path, "first_p2pkhindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2shindex: IndexedVec::forced_import( path, "first_p2shindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2trindex: IndexedVec::forced_import( path, "first_p2trindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2wpkhindex: IndexedVec::forced_import( path, "first_p2wpkhindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_p2wshindex: IndexedVec::forced_import( path, "first_p2wshindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_txindex: IndexedVec::forced_import( path, "first_txindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_first_unknownoutputindex: IndexedVec::forced_import( path, "first_unknownoutputindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_timestamp: IndexedVec::forced_import( path, "timestamp", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_total_size: IndexedVec::forced_import( path, "total_size", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, height_to_weight: IndexedVec::forced_import( path, "weight", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, inputindex_to_outputindex: IndexedVec::forced_import( path, "outputindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, opreturnindex_to_txindex: IndexedVec::forced_import( path, "txindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, outputindex_to_outputtype: IndexedVec::forced_import( path, "outputtype", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, outputindex_to_outputtypeindex: IndexedVec::forced_import( path, "outputtypeindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, outputindex_to_value: IndexedVec::forced_import( path, "value", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, p2aindex_to_p2abytes: IndexedVec::forced_import( path, @@ -240,7 +236,7 @@ impl Vecs { path, "txindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, p2pk33index_to_p2pk33bytes: IndexedVec::forced_import( path, @@ -288,13 +284,13 @@ impl Vecs { path, "base_size", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, txindex_to_first_inputindex: IndexedVec::forced_import( path, "first_inputindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, txindex_to_first_outputindex: IndexedVec::forced_import( path, @@ -306,19 +302,19 @@ impl Vecs { path, "is_explicitly_rbf", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, txindex_to_rawlocktime: IndexedVec::forced_import( path, "rawlocktime", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, txindex_to_total_size: IndexedVec::forced_import( path, "total_size", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, txindex_to_txid: IndexedVec::forced_import( path, @@ -330,13 +326,13 @@ impl Vecs { path, "txversion", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, unknownoutputindex_to_txindex: IndexedVec::forced_import( path, "txindex", version + VERSION + Version::ZERO, - format, + Format::Raw, )?, }) } diff --git a/crates/brk_query/examples/main.rs b/crates/brk_query/examples/main.rs index cbf633828..b28d7f88e 100644 --- a/crates/brk_query/examples/main.rs +++ b/crates/brk_query/examples/main.rs @@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> { let format = Format::Compressed; - let mut indexer = Indexer::new(outputs_dir, format, true)?; + let mut indexer = Indexer::new(outputs_dir, true)?; indexer.import_vecs()?; let mut computer = Computer::new(outputs_dir, None, format); diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 078848259..bc917c751 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -31,7 +31,7 @@ pub fn main() -> color_eyre::Result<()> { let format = Format::Compressed; - let mut indexer = Indexer::new(outputs_dir, format, true)?; + let mut indexer = Indexer::new(outputs_dir, true)?; indexer.import_stores()?; indexer.import_vecs()?; diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index d1d2ed766..841216cbc 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -21,7 +21,9 @@ use crate::{ const ONE_KIB: usize = 1024; const ONE_MIB: usize = ONE_KIB * ONE_KIB; pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB; -pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB; +pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB; + +const VERSION: Version = Version::ONE; #[derive(Debug)] pub struct CompressedVec { @@ -39,7 +41,9 @@ where pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; /// Same as import but will reset the folder under certain errors, so be careful ! - pub fn forced_import(path: &Path, version: Version) -> Result { + pub fn forced_import(path: &Path, mut version: Version) -> Result { + version = version + VERSION; + let res = Self::import(path, version); match res { Err(Error::WrongEndian) diff --git a/websites/default/index.html b/websites/default/index.html index 0d12b7b37..ccecdb3fd 100644 --- a/websites/default/index.html +++ b/websites/default/index.html @@ -1140,22 +1140,11 @@ window.document.documentElement.dataset.display = "standalone"; } - window.addEventListener("load", () => { - navigator.serviceWorker - .register("/service-worker.js") - .then((registration) => { - console.log( - "Service Worker registered with scope:", - registration.scope, - ); - }) - .catch((error) => { - console.error("Service Worker registration failed:", error); - }); - }); - // if ("serviceWorker" in navigator) { - // navigator.serviceWorker.register("/service-worker.js"); - // } + if ("serviceWorker" in navigator) { + window.addEventListener("load", () => { + navigator.serviceWorker.register("/service-worker.js"); + }); + } diff --git a/websites/default/scripts/vecid-to-indexes.js b/websites/default/scripts/vecid-to-indexes.js index bf5493df2..95886652d 100644 --- a/websites/default/scripts/vecid-to-indexes.js +++ b/websites/default/scripts/vecid-to-indexes.js @@ -2,7 +2,7 @@ // File auto-generated, any modifications will be overwritten // -export const VERSION = "v0.0.48"; +export const VERSION = "v0.0.52"; /** @typedef {0} DateIndex */ /** @typedef {1} DecadeIndex */ @@ -1251,6 +1251,7 @@ export function createVecIdToIndexes() { "fee-75p": [5], "fee-90p": [5], "fee-average": [0, 1, 2, 5, 7, 19, 22, 23], + "fee-in-btc": [20], "fee-in-btc-10p": [5], "fee-in-btc-25p": [5], "fee-in-btc-75p": [5], @@ -1260,6 +1261,7 @@ export function createVecIdToIndexes() { "fee-in-btc-median": [5], "fee-in-btc-min": [0, 1, 2, 5, 7, 19, 22, 23], "fee-in-btc-sum": [0, 1, 2, 5, 7, 19, 22, 23], + "fee-in-usd": [20], "fee-in-usd-10p": [5], "fee-in-usd-25p": [5], "fee-in-usd-75p": [5], diff --git a/websites/default/tsconfig.json b/websites/default/tsconfig.json index b3f335a8c..141a88943 100644 --- a/websites/default/tsconfig.json +++ b/websites/default/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "allowJs": true, "checkJs": true, "strict": true, "target": "ESNext",