indexer: only raw format + global: fixes

Cargo.lock

@@ -1698,9 +1698,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
 
 [[package]]
 name = "memchr"
-version = "2.7.4"
+version = "2.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
 
 [[package]]
 name = "memmap2"
@@ -3352,9 +3352,9 @@ dependencies = [
 
 [[package]]
 name = "windows-link"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
+checksum = "d3bfe459f85da17560875b8bf1423d6f113b7a87a5d942e7da0ac71be7c61f8b"
 
 [[package]]
 name = "windows-result"

crates/brk_cli/src/query.rs

@@ -10,7 +10,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
 
     let format = config.format();
 
-    let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
+    let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
     indexer.import_vecs()?;
 
     let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);

crates/brk_cli/src/run.rs

@@ -7,7 +7,7 @@ use std::{
 
 use bitcoincore_rpc::{self, Auth, Client, RpcApi};
 use brk_computer::Computer;
-use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
+use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
@@ -29,7 +29,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
 
     let format = config.format();
 
-    let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
+    let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
     indexer.import_stores()?;
     indexer.import_vecs()?;
 
@@ -109,62 +109,77 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
 pub struct RunConfig {
     /// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     bitcoindir: Option<String>,
 
     /// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     blocksdir: Option<String>,
 
     /// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     brkdir: Option<String>,
 
-    /// Executed by the runner, default: all, saved
+    /// Activated services, default: all, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short, long)]
-    mode: Option<Mode>,
+    services: Option<Services>,
 
-    /// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
+    /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short, long)]
     computation: Option<Computation>,
 
-    /// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: compressed, saved
+    /// Format of computed datasets, `compressed` to save disk space, `raw` to prioritize speed, default: `compressed`, saved
-    #[arg(short, long, value_name = "FORMAT")]
+    #[serde(default, deserialize_with = "default_on_error")]
+    #[arg(short, long)]
     format: Option<Format>,
 
     /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short = 'F', long, value_name = "BOOL")]
     fetch: Option<bool>,
 
     /// Website served by the server (if active), default: default, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short, long)]
     website: Option<Website>,
 
     /// Bitcoin RPC ip, default: localhost, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "IP")]
     rpcconnect: Option<String>,
 
     /// Bitcoin RPC port, default: 8332, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PORT")]
     rpcport: Option<u16>,
 
     /// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     rpccookiefile: Option<String>,
 
     /// Bitcoin RPC username, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "USERNAME")]
     rpcuser: Option<String>,
 
     /// Bitcoin RPC password, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PASSWORD")]
     rpcpassword: Option<String>,
 
     /// Delay between runs, default: 0, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "SECONDS")]
     delay: Option<u64>,
 
     /// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "BOOL")]
     check_collisions: Option<bool>,
 }
@@ -192,8 +207,8 @@ impl RunConfig {
                 config_saved.brkdir = Some(brkdir);
             }
 
-            if let Some(mode) = config_args.mode.take() {
+            if let Some(services) = config_args.services.take() {
-                config_saved.mode = Some(mode);
+                config_saved.services = Some(services);
             }
 
             if let Some(computation) = config_args.computation.take() {
@@ -255,7 +270,7 @@ impl RunConfig {
         // info!("Configuration {{");
         // info!("  bitcoindir: {:?}", config.bitcoindir);
         // info!("  brkdir: {:?}", config.brkdir);
-        // info!("  mode: {:?}", config.mode);
+        // info!("  services: {:?}", config.services);
         // info!("  website: {:?}", config.website);
         // info!("  rpcconnect: {:?}", config.rpcconnect);
         // info!("  rpcport: {:?}", config.rpcport);
@@ -375,13 +390,13 @@ impl RunConfig {
     }
 
     pub fn process(&self) -> bool {
-        self.mode
+        self.services
-            .is_none_or(|m| m == Mode::All || m == Mode::Processor)
+            .is_none_or(|m| m == Services::All || m == Services::Processor)
     }
 
     pub fn serve(&self) -> bool {
-        self.mode
+        self.services
-            .is_none_or(|m| m == Mode::All || m == Mode::Server)
+            .is_none_or(|m| m == Services::All || m == Services::Server)
     }
 
     fn path_cookiefile(&self) -> PathBuf {
@@ -449,7 +464,7 @@ impl RunConfig {
     PartialOrd,
     Ord,
 )]
-pub enum Mode {
+pub enum Services {
     #[default]
     All,
     Processor,

crates/brk_computer/examples/main.rs

@@ -33,7 +33,7 @@ pub fn main() -> color_eyre::Result<()> {
 
             let format = Format::Raw;
 
-            let mut indexer = Indexer::new(outputs_dir, format, true)?;
+            let mut indexer = Indexer::new(outputs_dir, true)?;
             indexer.import_stores()?;
             indexer.import_vecs()?;
 

crates/brk_computer/src/vecs/grouped/value_from_txindex.rs

@@ -226,7 +226,11 @@ impl ComputedValueVecsFromTxindex {
     pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
         [
             self.sats.vecs(),
+            vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
             self.bitcoin.vecs(),
+            self.dollars_txindex
+                .as_ref()
+                .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
             self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
         ]
         .into_iter()

crates/brk_computer/src/vecs/mod.rs

@@ -124,6 +124,7 @@ impl Vecs {
         fetcher: Option<&mut Fetcher>,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
+        info!("Computing indexes...");
         let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
 
         info!("Computing constants...");

crates/brk_core/src/utils/mod.rs

@@ -3,9 +3,11 @@ mod checked_sub;
 mod paths;
 mod pause;
 mod rlimit;
+mod serde;
 
 pub use bytes::*;
 pub use checked_sub::*;
 pub use paths::*;
 pub use pause::*;
 pub use rlimit::*;
+pub use serde::*;

crates/brk_core/src/utils/serde.rs

@@ -0,0 +1,12 @@
+use serde::{Deserialize, Deserializer};
+
+pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: Deserializer<'de>,
+    T: Deserialize<'de> + Default,
+{
+    match T::deserialize(deserializer) {
+        Ok(v) => Ok(v),
+        Err(_) => Ok(T::default()),
+    }
+}

crates/brk_indexer/examples/main.rs

@@ -4,7 +4,6 @@ use brk_core::default_bitcoin_path;
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_parser::Parser;
-use brk_vec::Format;
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -25,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
 
     let outputs = Path::new("../../_outputs");
 
-    let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
+    let mut indexer = Indexer::new(outputs, false)?;
 
     indexer.import_stores()?;
     indexer.import_vecs()?;

crates/brk_indexer/src/lib.rs

@@ -19,7 +19,7 @@ use brk_core::{
 use bitcoin::{Transaction, TxIn, TxOut};
 use brk_exit::Exit;
 use brk_parser::Parser;
-use brk_vec::{AnyVec, Format, VecIterator};
+use brk_vec::{AnyVec, VecIterator};
 use color_eyre::eyre::{ContextCompat, eyre};
 use fjall::TransactionalKeyspace;
 use log::{error, info};
@@ -42,21 +42,15 @@ pub struct Indexer {
     vecs: Option<Vecs>,
     stores: Option<Stores>,
     check_collisions: bool,
-    format: Format,
 }
 
 impl Indexer {
-    pub fn new(
+    pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result<Self> {
-        outputs_dir: &Path,
-        format: Format,
-        check_collisions: bool,
-    ) -> color_eyre::Result<Self> {
         setrlimit()?;
         Ok(Self {
             path: outputs_dir.to_owned(),
             vecs: None,
             stores: None,
-            format,
             check_collisions,
         })
     }
@@ -65,7 +59,6 @@ impl Indexer {
         self.vecs = Some(Vecs::forced_import(
             &self.path.join("vecs/indexed"),
             VERSION + Version::ZERO,
-            self.format,
         )?);
         Ok(())
     }

crates/brk_indexer/src/vecs.rs

@@ -66,11 +66,7 @@ pub struct Vecs {
 }
 
 impl Vecs {
-    pub fn forced_import(
+    pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
-        path: &Path,
-        version: Version,
-        format: Format,
-    ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
         Ok(Self {
@@ -78,7 +74,7 @@ impl Vecs {
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_blockhash: IndexedVec::forced_import(
                 path,
@@ -90,145 +86,145 @@ impl Vecs {
                 path,
                 "difficulty",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_emptyoutputindex: IndexedVec::forced_import(
                 path,
                 "first_emptyoutputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_inputindex: IndexedVec::forced_import(
                 path,
                 "first_inputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_opreturnindex: IndexedVec::forced_import(
                 path,
                 "first_opreturnindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_outputindex: IndexedVec::forced_import(
                 path,
                 "first_outputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2aindex: IndexedVec::forced_import(
                 path,
                 "first_p2aindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2msindex: IndexedVec::forced_import(
                 path,
                 "first_p2msindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2pk33index: IndexedVec::forced_import(
                 path,
                 "first_p2pk33index",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2pk65index: IndexedVec::forced_import(
                 path,
                 "first_p2pk65index",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2pkhindex: IndexedVec::forced_import(
                 path,
                 "first_p2pkhindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2shindex: IndexedVec::forced_import(
                 path,
                 "first_p2shindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2trindex: IndexedVec::forced_import(
                 path,
                 "first_p2trindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2wpkhindex: IndexedVec::forced_import(
                 path,
                 "first_p2wpkhindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2wshindex: IndexedVec::forced_import(
                 path,
                 "first_p2wshindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_txindex: IndexedVec::forced_import(
                 path,
                 "first_txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_unknownoutputindex: IndexedVec::forced_import(
                 path,
                 "first_unknownoutputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_timestamp: IndexedVec::forced_import(
                 path,
                 "timestamp",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_total_size: IndexedVec::forced_import(
                 path,
                 "total_size",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_weight: IndexedVec::forced_import(
                 path,
                 "weight",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             inputindex_to_outputindex: IndexedVec::forced_import(
                 path,
                 "outputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             opreturnindex_to_txindex: IndexedVec::forced_import(
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             outputindex_to_outputtype: IndexedVec::forced_import(
                 path,
                 "outputtype",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             outputindex_to_outputtypeindex: IndexedVec::forced_import(
                 path,
                 "outputtypeindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             outputindex_to_value: IndexedVec::forced_import(
                 path,
                 "value",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             p2aindex_to_p2abytes: IndexedVec::forced_import(
                 path,
@@ -240,7 +236,7 @@ impl Vecs {
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
                 path,
@@ -288,13 +284,13 @@ impl Vecs {
                 path,
                 "base_size",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_first_inputindex: IndexedVec::forced_import(
                 path,
                 "first_inputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_first_outputindex: IndexedVec::forced_import(
                 path,
@@ -306,19 +302,19 @@ impl Vecs {
                 path,
                 "is_explicitly_rbf",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_rawlocktime: IndexedVec::forced_import(
                 path,
                 "rawlocktime",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_total_size: IndexedVec::forced_import(
                 path,
                 "total_size",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_txid: IndexedVec::forced_import(
                 path,
@@ -330,13 +326,13 @@ impl Vecs {
                 path,
                 "txversion",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             unknownoutputindex_to_txindex: IndexedVec::forced_import(
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
         })
     }

crates/brk_query/examples/main.rs

@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
 
     let format = Format::Compressed;
 
-    let mut indexer = Indexer::new(outputs_dir, format, true)?;
+    let mut indexer = Indexer::new(outputs_dir, true)?;
     indexer.import_vecs()?;
 
     let mut computer = Computer::new(outputs_dir, None, format);

crates/brk_server/examples/main.rs

@@ -31,7 +31,7 @@ pub fn main() -> color_eyre::Result<()> {
 
     let format = Format::Compressed;
 
-    let mut indexer = Indexer::new(outputs_dir, format, true)?;
+    let mut indexer = Indexer::new(outputs_dir, true)?;
     indexer.import_stores()?;
     indexer.import_vecs()?;
 

crates/brk_vec/src/variants/compressed.rs

@@ -21,7 +21,9 @@ use crate::{
 const ONE_KIB: usize = 1024;
 const ONE_MIB: usize = ONE_KIB * ONE_KIB;
 pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
-pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
+pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB;
+
+const VERSION: Version = Version::ONE;
 
 #[derive(Debug)]
 pub struct CompressedVec<I, T> {
@@ -39,7 +41,9 @@ where
     pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
 
     /// Same as import but will reset the folder under certain errors, so be careful !
-    pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
+    pub fn forced_import(path: &Path, mut version: Version) -> Result<Self> {
+        version = version + VERSION;
+
         let res = Self::import(path, version);
         match res {
             Err(Error::WrongEndian)

websites/default/index.html

@@ -1140,22 +1140,11 @@
         window.document.documentElement.dataset.display = "standalone";
       }
 
-      window.addEventListener("load", () => {
+      if ("serviceWorker" in navigator) {
-        navigator.serviceWorker
+        window.addEventListener("load", () => {
-          .register("/service-worker.js")
+          navigator.serviceWorker.register("/service-worker.js");
-          .then((registration) => {
+        });
-            console.log(
+      }
-              "Service Worker registered with scope:",
-              registration.scope,
-            );
-          })
-          .catch((error) => {
-            console.error("Service Worker registration failed:", error);
-          });
-      });
-      // if ("serviceWorker" in navigator) {
-      //   navigator.serviceWorker.register("/service-worker.js");
-      // }
     </script>
 
     <!-- --- -->

websites/default/scripts/vecid-to-indexes.js

@@ -2,7 +2,7 @@
 // File auto-generated, any modifications will be overwritten
 //
 
-export const VERSION = "v0.0.48";
+export const VERSION = "v0.0.52";
 
 /** @typedef {0} DateIndex */
 /** @typedef {1} DecadeIndex */
@@ -1251,6 +1251,7 @@ export function createVecIdToIndexes() {
     "fee-75p": [5],
     "fee-90p": [5],
     "fee-average": [0, 1, 2, 5, 7, 19, 22, 23],
+    "fee-in-btc": [20],
     "fee-in-btc-10p": [5],
     "fee-in-btc-25p": [5],
     "fee-in-btc-75p": [5],
@@ -1260,6 +1261,7 @@ export function createVecIdToIndexes() {
     "fee-in-btc-median": [5],
     "fee-in-btc-min": [0, 1, 2, 5, 7, 19, 22, 23],
     "fee-in-btc-sum": [0, 1, 2, 5, 7, 19, 22, 23],
+    "fee-in-usd": [20],
     "fee-in-usd-10p": [5],
     "fee-in-usd-25p": [5],
     "fee-in-usd-75p": [5],

websites/default/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "allowJs": true,
     "checkJs": true,
     "strict": true,
     "target": "ESNext",