release: v0.0.70

Cargo.lock

@@ -443,7 +443,7 @@ dependencies = [
 
 [[package]]
 name = "brk"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "brk_bundler",
  "brk_cli",
@@ -464,7 +464,7 @@ dependencies = [
 
 [[package]]
 name = "brk_bundler"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "brk_rolldown",
  "log",
@@ -475,7 +475,7 @@ dependencies = [
 
 [[package]]
 name = "brk_cli"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "bitcoincore-rpc",
  "brk_computer",
@@ -498,7 +498,7 @@ dependencies = [
 
 [[package]]
 name = "brk_computer"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -519,7 +519,7 @@ dependencies = [
 
 [[package]]
 name = "brk_core"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "bincode",
  "bitcoin",
@@ -540,7 +540,7 @@ dependencies = [
 
 [[package]]
 name = "brk_exit"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "brk_logger",
  "ctrlc",
@@ -549,7 +549,7 @@ dependencies = [
 
 [[package]]
 name = "brk_fetcher"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "brk_core",
  "brk_logger",
@@ -561,7 +561,7 @@ dependencies = [
 
 [[package]]
 name = "brk_indexer"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -579,7 +579,7 @@ dependencies = [
 
 [[package]]
 name = "brk_interface"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "brk_computer",
  "brk_core",
@@ -597,7 +597,7 @@ dependencies = [
 
 [[package]]
 name = "brk_logger"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "color-eyre",
  "env_logger",
@@ -607,7 +607,7 @@ dependencies = [
 
 [[package]]
 name = "brk_mcp"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "axum",
  "brk_interface",
@@ -618,7 +618,7 @@ dependencies = [
 
 [[package]]
 name = "brk_parser"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -980,7 +980,7 @@ dependencies = [
 
 [[package]]
 name = "brk_server"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "axum",
  "bitcoincore-rpc",
@@ -1010,7 +1010,7 @@ dependencies = [
 
 [[package]]
 name = "brk_state"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "bincode",
  "brk_core",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "brk_store"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "brk_core",
  "byteview",
@@ -1043,7 +1043,7 @@ dependencies = [
 
 [[package]]
 name = "brk_vec"
-version = "0.0.68"
+version = "0.0.70"
 dependencies = [
  "arc-swap",
  "brk_core",

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.68"
+package.version = "0.0.70"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 
@@ -22,22 +22,22 @@ axum = "0.8.4"
 bincode = { version = "2.0.1", features = ["serde"] }
 bitcoin = { version = "0.32.6", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
-brk_bundler = { version = "0.0.68", path = "crates/brk_bundler" }
+brk_bundler = { version = "0.0.70", path = "crates/brk_bundler" }
-brk_cli = { version = "0.0.68", path = "crates/brk_cli" }
+brk_cli = { version = "0.0.70", path = "crates/brk_cli" }
-brk_computer = { version = "0.0.68", path = "crates/brk_computer" }
+brk_computer = { version = "0.0.70", path = "crates/brk_computer" }
-brk_core = { version = "0.0.68", path = "crates/brk_core" }
+brk_core = { version = "0.0.70", path = "crates/brk_core" }
-brk_exit = { version = "0.0.68", path = "crates/brk_exit" }
+brk_exit = { version = "0.0.70", path = "crates/brk_exit" }
-brk_fetcher = { version = "0.0.68", path = "crates/brk_fetcher" }
+brk_fetcher = { version = "0.0.70", path = "crates/brk_fetcher" }
-brk_indexer = { version = "0.0.68", path = "crates/brk_indexer" }
+brk_indexer = { version = "0.0.70", path = "crates/brk_indexer" }
-brk_interface = { version = "0.0.68", path = "crates/brk_interface" }
+brk_interface = { version = "0.0.70", path = "crates/brk_interface" }
-brk_logger = { version = "0.0.68", path = "crates/brk_logger" }
+brk_logger = { version = "0.0.70", path = "crates/brk_logger" }
-brk_mcp = { version = "0.0.68", path = "crates/brk_mcp" }
+brk_mcp = { version = "0.0.70", path = "crates/brk_mcp" }
-brk_parser = { version = "0.0.68", path = "crates/brk_parser" }
+brk_parser = { version = "0.0.70", path = "crates/brk_parser" }
 brk_rmcp = { version = "0.1.7", features = ["transport-streamable-http-server", "transport-worker"]}
-brk_server = { version = "0.0.68", path = "crates/brk_server" }
+brk_server = { version = "0.0.70", path = "crates/brk_server" }
-brk_state = { version = "0.0.68", path = "crates/brk_state" }
+brk_state = { version = "0.0.70", path = "crates/brk_state" }
-brk_store = { version = "0.0.68", path = "crates/brk_store" }
+brk_store = { version = "0.0.70", path = "crates/brk_store" }
-brk_vec = { version = "0.0.68", path = "crates/brk_vec" }
+brk_vec = { version = "0.0.70", path = "crates/brk_vec" }
 byteview = "=0.6.1"
 clap = { version = "4.5.40", features = ["string"] }
 clap_derive = "4.5.40"

README.md

@@ -42,7 +42,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible
   It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
   Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
 - **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
-  Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvariant-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
+  Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
   Just like the website, it's entirely free, with no authentication or rate-limiting.
 - **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \
   LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \

crates/brk_cli/src/config.rs

@@ -88,7 +88,7 @@ pub struct Config {
     #[arg(long, value_name = "SECONDS")]
     delay: Option<u64>,
 
-    /// Activate the Model Context Protocol (MCP) endpoint to give LLMs access to BRK (experimental), default: false, saved
+    /// Activate the Model Context Protocol (MCP) endpoint to give LLMs access to BRK (experimental), default: true, saved
     #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "BOOL")]
     mcp: Option<bool>,
@@ -367,7 +367,7 @@ impl Config {
     }
 
     pub fn mcp(&self) -> bool {
-        self.mcp.is_some_and(|b| b)
+        self.mcp.is_none_or(|b| b)
     }
 
     pub fn watch(&self) -> bool {

crates/brk_computer/src/vecs/mod.rs

@@ -1,4 +1,4 @@
-use std::{path::Path, thread};
+use std::path::Path;
 
 use brk_core::Version;
 use brk_exit::Exit;
@@ -44,31 +44,22 @@ impl Vecs {
         computation: Computation,
         format: Format,
     ) -> color_eyre::Result<Self> {
-        let (indexes, fetched) = thread::scope(|s| {
+        let indexes = indexes::Vecs::forced_import(
-            let indexes_handle = s.spawn(|| {
+            path,
-                indexes::Vecs::forced_import(
+            version + VERSION + Version::ZERO,
-                    path,
+            indexer,
-                    version + VERSION + Version::ZERO,
+            computation,
-                    indexer,
+            format,
-                    computation,
+        )?;
-                    format,
-                )
-                .unwrap()
-            });
 
-            let fetch_handle = s.spawn(|| {
+        let fetched = fetch.then(|| {
-                fetch.then(|| {
+            fetched::Vecs::forced_import(
-                    fetched::Vecs::forced_import(
+                path,
-                        path,
+                version + VERSION + Version::ZERO,
-                        version + VERSION + Version::ZERO,
+                computation,
-                        computation,
+                format,
-                        format,
+            )
-                    )
+            .unwrap()
-                    .unwrap()
-                })
-            });
-
-            (indexes_handle.join().unwrap(), fetch_handle.join().unwrap())
         });
 
         Ok(Self {

crates/brk_core/src/utils/rlimit.rs

@@ -7,7 +7,7 @@ pub fn setrlimit() -> io::Result<()> {
 
     rlimit::setrlimit(
         Resource::NOFILE,
-        no_file_limit.0.max(420_000),
+        no_file_limit.0.max(210_000),
         no_file_limit.1,
     )?;
 

crates/brk_fetcher/src/binance.rs

@@ -11,7 +11,7 @@ use color_eyre::eyre::{ContextCompat, eyre};
 use log::info;
 use serde_json::Value;
 
-use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, fetchers::retry};
+use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, retry};
 
 #[derive(Clone)]
 pub struct Binance {

crates/brk_fetcher/src/brk.rs

@@ -5,9 +5,10 @@ use color_eyre::eyre::{ContextCompat, eyre};
 use log::info;
 use serde_json::Value;
 
-use crate::{Close, Dollars, High, Low, Open, fetchers::retry};
+use crate::{Close, Dollars, High, Low, Open, retry};
 
 #[derive(Default, Clone)]
+#[allow(clippy::upper_case_acronyms)]
 pub struct BRK {
     height_to_ohlc: BTreeMap<Height, Vec<OHLCCents>>,
     dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCents>>,
@@ -47,7 +48,7 @@ impl BRK {
         retry(
             |_| {
                 let url = format!(
-                    "{API_URL}/query?index=height&values=ohlc&from={}&to={}",
+                    "{API_URL}/height-to-ohlc?from={}&to={}",
                     height,
                     height + CHUNK_SIZE
                 );
@@ -96,7 +97,7 @@ impl BRK {
         retry(
             |_| {
                 let url = format!(
-                    "{API_URL}/query?index=dateindex&values=ohlc&from={}&to={}",
+                    "{API_URL}/dateindex-to-ohlc?from={}&to={}",
                     dateindex,
                     dateindex + CHUNK_SIZE
                 );

crates/brk_fetcher/src/fetchers/mod.rs

@@ -1,9 +0,0 @@
-mod binance;
-mod brk;
-mod kraken;
-mod retry;
-
-pub use binance::*;
-pub use brk::*;
-pub use kraken::*;
-use retry::*;

crates/brk_fetcher/src/kraken.rs

@@ -5,7 +5,7 @@ use color_eyre::eyre::ContextCompat;
 use log::info;
 use serde_json::Value;
 
-use crate::{Fetcher, fetchers::retry};
+use crate::{Fetcher, retry};
 
 #[derive(Default, Clone)]
 pub struct Kraken {

crates/brk_fetcher/src/lib.rs

@@ -7,12 +7,18 @@ use std::{collections::BTreeMap, path::Path, thread::sleep, time::Duration};
 
 use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
 use color_eyre::eyre::Error;
-
-mod fetchers;
-
-pub use fetchers::*;
 use log::info;
 
+mod binance;
+mod brk;
+mod kraken;
+mod retry;
+
+pub use binance::*;
+pub use brk::*;
+pub use kraken::*;
+use retry::*;
+
 const TRIES: usize = 12 * 60;
 
 #[derive(Clone)]

crates/brk_mcp/src/lib.rs

@@ -142,7 +142,7 @@ The response's format will depend on the given parameters, it will be:
     }
 
     #[tool(description = "
-Get the running version of the Bitcoin Research Kit
+Get the running version of the Bitcoin Research Kit.
 ")]
     async fn get_version(&self) -> Result<CallToolResult, McpError> {
         info!("mcp: get_version");

crates/brk_vec/src/structs/header.rs

@@ -57,6 +57,10 @@ impl Header {
         });
     }
 
+    pub fn modified(&self) -> bool {
+        self.modified
+    }
+
     pub fn vec_version(&self) -> Version {
         self.inner.load().vec_version
     }
@@ -69,11 +73,9 @@ impl Header {
         self.inner.load().height
     }
 
-    pub fn write_if_needed(&mut self, file: &mut File) -> io::Result<()> {
+    pub fn write(&mut self, file: &mut File) -> io::Result<()> {
-        if self.modified {
+        self.inner.load().write(file)?;
-            self.inner.load().write(file)?;
+        self.modified = false;
-            self.modified = false;
-        }
         Ok(())
     }
 }

crates/brk_vec/src/traits/generic.rs

@@ -95,6 +95,9 @@ where
 
     // ---
 
+    fn open_file(&self) -> io::Result<File> {
+        Self::open_file_(&self.path())
+    }
     fn open_file_(path: &Path) -> io::Result<File> {
         let mut file = OpenOptions::new()
             .read(true)
@@ -106,13 +109,9 @@ where
         Ok(file)
     }
 
-    fn file(&self) -> &File;
+    fn file_set_len(&mut self, file: &mut File, len: u64) -> Result<()> {
-    fn mut_file(&mut self) -> &mut File;
-
-    fn file_set_len(&mut self, len: u64) -> Result<()> {
-        let file = self.mut_file();
         Self::file_set_len_(file, len)?;
-        self.update_mmap()
+        self.update_mmap(file)
     }
     fn file_set_len_(file: &mut File, len: u64) -> Result<()> {
         file.set_len(len)?;
@@ -120,32 +119,31 @@ where
         Ok(())
     }
 
-    fn file_write_all(&mut self, buf: &[u8]) -> Result<()> {
+    fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> {
-        let file = self.mut_file();
         file.write_all(buf)?;
-        self.update_mmap()
+        self.update_mmap(file)
     }
 
-    fn file_truncate_and_write_all(&mut self, len: u64, buf: &[u8]) -> Result<()> {
+    fn file_truncate_and_write_all(&mut self, file: &mut File, len: u64, buf: &[u8]) -> Result<()> {
-        let file = self.mut_file();
         Self::file_set_len_(file, len)?;
         file.write_all(buf)?;
-        self.update_mmap()
+        self.update_mmap(file)
     }
 
     fn reset(&mut self) -> Result<()>;
 
     #[inline]
     fn reset_(&mut self) -> Result<()> {
-        self.file_truncate_and_write_all(HEADER_OFFSET as u64, &[])
+        let mut file = self.open_file()?;
+        self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[])
     }
 
     fn new_mmap(file: &File) -> Result<Arc<Mmap>> {
         Ok(Arc::new(unsafe { Mmap::map(file)? }))
     }
 
-    fn update_mmap(&mut self) -> Result<()> {
+    fn update_mmap(&mut self, file: &File) -> Result<()> {
-        let mmap = Self::new_mmap(self.file())?;
+        let mmap = Self::new_mmap(file)?;
         self.mmap().store(mmap);
         Ok(())
     }

crates/brk_vec/src/variants/compressed.rs

@@ -1,6 +1,5 @@
 use std::{
-    fs::{self, File},
+    fs, mem,
-    mem,
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -189,14 +188,6 @@ where
         self.inner.parent()
     }
 
-    fn file(&self) -> &File {
-        self.inner.file()
-    }
-
-    fn mut_file(&mut self) -> &mut File {
-        self.inner.mut_file()
-    }
-
     #[inline]
     fn stored_len(&self) -> usize {
         Self::stored_len__(&self.pages_meta.load())
@@ -221,7 +212,7 @@ where
     }
 
     fn flush(&mut self) -> Result<()> {
-        self.inner.write_header_if_needed()?;
+        let file_opt = self.inner.write_header_if_needed()?;
 
         let pushed_len = self.pushed_len();
 
@@ -295,11 +286,13 @@ where
 
         pages_meta.write()?;
 
+        let mut file = file_opt.unwrap_or(self.open_file()?);
+
         if let Some(truncate_at) = truncate_at {
-            self.file_set_len(truncate_at)?;
+            self.file_set_len(&mut file, truncate_at)?;
         }
 
-        self.file_write_all(&buf)?;
+        self.file_write_all(&mut file, &buf)?;
 
         self.pages_meta.store(Arc::new(pages_meta));
 
@@ -354,7 +347,9 @@ where
 
         self.pages_meta.store(Arc::new(pages_meta));
 
-        self.file_truncate_and_write_all(len, &buf)?;
+        let mut file = self.open_file()?;
+
+        self.file_truncate_and_write_all(&mut file, len, &buf)?;
 
         Ok(())
     }

crates/brk_vec/src/variants/raw.rs

@@ -24,7 +24,6 @@ pub struct RawVec<I, T> {
     header: Header,
     parent: PathBuf,
     name: String,
-    file: Option<File>,
     // Consider  Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
     mmap: Arc<ArcSwap<Mmap>>,
     pushed: Vec<T>,
@@ -55,18 +54,16 @@ where
 
     pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
         let path = Self::path_(parent, name);
-        let (file, mmap, header) = match Self::open_file_(&path) {
+        let (mmap, header) = match Self::open_file_(&path) {
             Ok(mut file) => {
                 if file.metadata()?.len() == 0 {
                     let header = Header::create_and_write(&mut file, version, Format::Raw)?;
                     let mmap = Self::new_mmap(&file)?;
-                    (file, mmap, header)
+                    (mmap, header)
                 } else {
                     let mmap = Self::new_mmap(&file)?;
-                    // dbg!(&mmap[..]);
                     let header = Header::import_and_verify(&mmap, version, Format::Raw)?;
-                    // dbg!((&header, name, I::to_string()));
+                    (mmap, header)
-                    (file, mmap, header)
                 }
             }
             Err(e) => match e.kind() {
@@ -75,7 +72,7 @@ where
                     let mut file = Self::open_file_(&path)?;
                     let header = Header::create_and_write(&mut file, version, Format::Raw)?;
                     let mmap = Self::new_mmap(&file)?;
-                    (file, mmap, header)
+                    (mmap, header)
                 }
                 _ => return Err(e.into()),
             },
@@ -86,7 +83,6 @@ where
         Ok(Self {
             mmap,
             header,
-            file: Some(file),
             name: name.to_string(),
             parent: parent.to_owned(),
             pushed: vec![],
@@ -111,8 +107,14 @@ where
         iter
     }
 
-    pub fn write_header_if_needed(&mut self) -> io::Result<()> {
+    pub fn write_header_if_needed(&mut self) -> io::Result<Option<File>> {
-        self.header.write_if_needed(self.file.as_mut().unwrap())
+        if self.header.modified() {
+            let mut file = self.open_file()?;
+            self.header.write(&mut file)?;
+            Ok(Some(file))
+        } else {
+            Ok(None)
+        }
     }
 }
 
@@ -143,16 +145,6 @@ where
         &self.mmap
     }
 
-    #[inline]
-    fn file(&self) -> &File {
-        self.file.as_ref().unwrap()
-    }
-
-    #[inline]
-    fn mut_file(&mut self) -> &mut File {
-        self.file.as_mut().unwrap()
-    }
-
     #[inline]
     fn stored_len(&self) -> usize {
         self.stored_len_(&self.mmap.load())
@@ -177,7 +169,7 @@ where
     }
 
     fn flush(&mut self) -> Result<()> {
-        self.write_header_if_needed()?;
+        let file_opt = self.write_header_if_needed()?;
 
         let pushed_len = self.pushed_len();
 
@@ -200,7 +192,9 @@ where
             bytes
         };
 
-        self.file_write_all(&bytes)?;
+        let mut file = file_opt.unwrap_or(self.open_file()?);
+
+        self.file_write_all(&mut file, &bytes)?;
 
         Ok(())
     }
@@ -219,7 +213,8 @@ where
 
         let len = index * Self::SIZE_OF_T + HEADER_OFFSET;
 
-        self.file_set_len(len as u64)?;
+        let mut file = self.open_file()?;
+        self.file_set_len(&mut file, len as u64)?;
 
         Ok(())
     }
@@ -266,7 +261,6 @@ impl<I, T> Clone for RawVec<I, T> {
             header: self.header.clone(),
             parent: self.parent.clone(),
             name: self.name.clone(),
-            file: None,
             mmap: self.mmap.clone(),
             pushed: vec![],
             phantom: PhantomData,

crates/brk_vec/src/variants/stored.rs

@@ -1,7 +1,4 @@
-use std::{
+use std::path::{Path, PathBuf};
-    fs::File,
-    path::{Path, PathBuf},
-};
 
 use arc_swap::ArcSwap;
 use brk_core::{Result, Value, Version};
@@ -91,22 +88,6 @@ where
         }
     }
 
-    #[inline]
-    fn file(&self) -> &File {
-        match self {
-            StoredVec::Raw(v) => v.file(),
-            StoredVec::Compressed(v) => v.file(),
-        }
-    }
-
-    #[inline]
-    fn mut_file(&mut self) -> &mut File {
-        match self {
-            StoredVec::Raw(v) => v.mut_file(),
-            StoredVec::Compressed(v) => v.mut_file(),
-        }
-    }
-
     #[inline]
     fn stored_len(&self) -> usize {
         match self {