biter: fix ?

.gitignore

@@ -4,9 +4,6 @@
 # Builds
 target
 
-# Sync
-.stfolder
-
 # Copies
 *\ copy*
 
@@ -32,6 +29,3 @@ docker/kibo
 
 # Types
 paths.d.ts
-
-# Git
-.git-*

Cargo.lock

@@ -481,7 +481,7 @@ dependencies = [
 
 [[package]]
 name = "biter"
-version = "0.2.1"
+version = "0.2.2"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -2623,7 +2623,7 @@ dependencies = [
 
 [[package]]
 name = "snkrj"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "sanakirja",
 ]

Cargo.toml

@@ -12,7 +12,7 @@ bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
     "serde",
 ] }
 bitcoin_hashes = { version = "0.15.0" }
-biter = { path = "./crates/biter" }
+biter = { path = "./src/crates/biter" }
 chrono = { version = "0.4.39", features = ["serde"] }
 clap = { version = "4.5.23", features = ["derive"] }
 color-eyre = "0.6.3"
@@ -27,10 +27,10 @@ rayon = "1.10.0"
 regex = "1.11.1"
 reqwest = { version = "0.12.9", features = ["blocking", "json"] }
 rlimit = "0.10.2"
-snkrj = { path = "./crates/snkrj" }
+snkrj = { path = "./src/crates/snkrj" }
 serde = { version = "1.0.216", features = ["derive"] }
 serde_json = "1.0.133"
-struct_iterable = { path = "./crates/iterable" }
+struct_iterable = { path = "./src/crates/iterable" }
 swc = "9.0.0"
 swc_common = "5.0.0"
 tokio = { version = "1.42.0", features = ["full"] }

README.md

@@ -63,17 +63,19 @@ Please open an issue if you want to add another instance
 ### Requirements
 
 - At least 16 GB of RAM
-- 1 TB of free space (will use 70% of that without defragmentation and 40% after)
+  - Recommended: 32 GB
+- A disk with 1 TB of free space (will use between 40% to 80% depending on several things)
+  - Recommended: Rated at 3 GB/s (Thunderbolt 4 speed)
 - A running instance of bitcoin-core with:
   - `-txindex=1`
   - `-blocksxor=0`
   - RPC credentials
   - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly -txindex=1 -blocksxor=0`
 - Git
+- Unix based operating system (Mac OS or Linux)
+  - Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config`
 
-### Manual
+### Build
-
-_Mac OS and Linux only, Windows is unsupported_
 
 First we need to install Rust (https://www.rust-lang.org/tools/install)
 
@@ -81,75 +83,33 @@ First we need to install Rust (https://www.rust-lang.org/tools/install)
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 ```
 
-If you already had Rust installed you could update it just in case
+If you already had Rust installed you could update it
 
 ```bash
 rustup update
 ```
 
-> If you're on Ubuntu you'll probably also need to install `open-ssl` with
+Then you need to choose a path where the project will reside and then clone it
->
-> ```bash
-> sudo apt install libssl-dev pkg-config
-> ```
-
-Optionally, you can also install `cargo-watch` for the server to automatically restart it on file change, which will be triggered by new code and new datasets from the parser (https://github.com/watchexec/cargo-watch?tab=readme-ov-file#install)
-
-```bash
-cargo install cargo-watch --locked
-```
-
-Then you need to choose a path where all files related to **kibō** will live
 
 ```bash
 cd ???
-```
-
-We can now clone the repository
-
-```bash
 git clone https://github.com/kibo-money/kibo.git
+cd kibo
 ```
 
-In a new terminal, go to the `parser`'s folder of the repository
+If it's your first time running kibo, it will need several information such as:
 
-```bash
+- `--bitcoindir PATH`: path to bitcoin core data directory, `???/bitcoin`
-cd ???/kibo/parser
+- `--kibodir PATH`: path to kibo outputs, if you have enough space on your main disk `~/.kibo` is fine
-```
 
-Now we can finally start by running the parser, you need to use the `./run.sh` script instead of `cargo run -r` as we need to set various system variables for the program to run smoothly
+Everything will be saved at `~/.kibo/config.toml`, which will allow you to simply run `cargo run -r` next time
 
-For the first launch, the parser will need several information such as:
+If you need more options please run `cargo run -r --help` to see what parameters are available.
-
-- `--datadir`: which is bitcoin data directory path, prefer `$HOME` to `~` as the latter might not work
-- `--outdir`: where all outputs will be saved, prefer `$HOME` to `~` as the latter might not work
-
-Optionally you can also specify:
-
-- `--rpccookiefile`: the path to the cookie file if not default
-- `--rpcuser`: the username of the RPC credentials to talk to the bitcoin server if set
-- `--rpcpassword`: the password of the RPC credentials if set
-- `--rpcconnect`: if the bitcoin core server's IP is different than `localhost`
-- `--rpcport`: if the port is different than `8332`
-
-Everything will be saved in a `config.toml` file, which will allow you to simply run `./run.sh` next time
 
 Here's an example
 
 ```bash
-./run.sh --datadir=$HOME/Developer/bitcoin --outdir=$HOME/.kibo/out
+cargo run -r -- --bitcoindir=~/Developer/bitcoin --kibodir=~/.kibo
-```
-
-In a **new** terminal, go to the `server`'s folder of the repository
-
-```bash
-cd ???/kibo/server
-```
-
-And start it also with the `run.sh` script instead of `cargo run -r`
-
-```bash
-./run.sh
 ```
 
 Then the easiest to let others access your server is to use `cloudflared` which will also cache requests. For more information go to: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/

crates/snkrj/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "snkrj"
-description = "A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap"
-version = "0.1.0"
-license = "MIT"
-repository = "https://github.com/kibo-money/kibo/tree/main/crates/snkrj"
-keywords = ["database", "sanakirja", "btreemap"]
-categories = ["database"]
-edition = "2021"
-
-[dependencies]
-sanakirja = "1.4.3"

src/crates/biter/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "biter"
 description = "A very fast Bitcoin block iterator"
-version = "0.2.1"
+version = "0.2.2"
 license = "MIT"
-repository = "https://github.com/kibo-money/kibo/tree/main/crates/biter"
+repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter"
 keywords = ["bitcoin", "block", "iterator"]
 categories = ["cryptography::cryptocurrencies", "encoding"]
 edition = "2021"

src/crates/biter/src/blk_index_to_blk_recap.rs

@@ -17,7 +17,7 @@ pub struct BlkIndexToBlkRecap {
     path: PathBuf,
     #[target]
     tree: BTreeMap<usize, BlkRecap>,
-    last_safe_recap: Option<BlkRecap>,
+    last_safe_height: Option<usize>,
 }
 
 impl BlkIndexToBlkRecap {
@@ -38,7 +38,7 @@ impl BlkIndexToBlkRecap {
         let mut this = Self {
             path,
             tree,
-            last_safe_recap: None,
+            last_safe_height: None,
         };
 
         this.clean_outdated(blocks_dir);
@@ -58,11 +58,11 @@ impl BlkIndexToBlkRecap {
             }
         });
 
-        unprocessed_keys.iter().for_each(|blk_index| {
+        unprocessed_keys.into_iter().for_each(|blk_index| {
-            self.remove(blk_index);
+            self.remove(&blk_index);
         });
 
-        self.last_safe_recap = self.last_entry().map(|e| e.get().clone());
+        self.last_safe_height = self.iter().map(|(_, recap)| recap.height()).max();
     }
 
     pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
@@ -108,8 +108,8 @@ impl BlkIndexToBlkRecap {
         }
 
         if self
-            .last_safe_recap
+            .last_safe_height
-            .map_or(true, |recap| recap.height() >= height)
+            .map_or(true, |safe_height| height >= safe_height)
             && (height % TARGET_BLOCKS_PER_MONTH) == 0
         {
             self.export();

src/crates/biter/src/main.rs

@@ -11,8 +11,8 @@ fn main() {
     let auth = Auth::CookieFile(cookie);
     let rpc = Client::new(url, auth).unwrap();
 
-    let start = Some(800_000);
+    let start = Some(810078);
-    let end = Some(855_000);
+    let end = None;
 
     biter::new(data_dir, start, end, rpc)
         .iter()

src/crates/snkrj/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "snkrj"
+description = "A simple wrapper around Sanakirja's database that acts as a very fast on disk BTreeMap"
+version = "0.1.1"
+license = "MIT"
+repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/snkrj"
+keywords = ["database", "sanakirja", "btreemap"]
+categories = ["database"]
+edition = "2021"
+
+[dependencies]
+sanakirja = "1.4.3"

src/parser/mod.rs

@@ -37,7 +37,7 @@ pub fn main(
             &mut datasets,
         )?;
 
-        if let Some(delay) = config.delay {
+        if let Some(delay) = config.delay() {
             sleep(Duration::from_secs(delay))
         }
 

src/structs/config.rs

@@ -15,36 +15,36 @@ use super::MapPath;
 #[command(version, about, long_about = None)]
 pub struct Config {
     /// Bitcoin data directory path, saved
-    #[arg(long, value_name = "DIR")]
+    #[arg(long, value_name = "PATH")]
     bitcoindir: Option<String>,
 
-    /// Kibo data directory path, saved
+    /// Kibo output directory path, saved
-    #[arg(long, value_name = "DIR")]
+    #[arg(long, value_name = "PATH")]
     kibodir: Option<String>,
 
     /// Bitcoin RPC ip, default: localhost, saved
     #[arg(long, value_name = "IP")]
-    pub rpcconnect: Option<String>,
+    rpcconnect: Option<String>,
 
     /// Bitcoin RPC port, default: 8332, saved
     #[arg(long, value_name = "PORT")]
-    pub rpcport: Option<u16>,
+    rpcport: Option<u16>,
 
     /// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
     #[arg(long, value_name = "PATH")]
-    pub rpccookiefile: Option<String>,
+    rpccookiefile: Option<String>,
 
     /// Bitcoin RPC username, saved
     #[arg(long, value_name = "USERNAME")]
-    pub rpcuser: Option<String>,
+    rpcuser: Option<String>,
 
     /// Bitcoin RPC password, saved
     #[arg(long, value_name = "PASSWORD")]
-    pub rpcpassword: Option<String>,
+    rpcpassword: Option<String>,
 
     /// Delay between runs, default: 0, saved
     #[arg(long, value_name = "SECONDS")]
-    pub delay: Option<u64>,
+    delay: Option<u64>,
 
     // Maximum ram you want the program to use in GB, default: 50% of total, not saved
     // #[arg(long, value_name = "GB")]
@@ -181,7 +181,7 @@ impl Config {
             std::process::exit(1);
         }
 
-        let path = Path::new(self.bitcoindir.as_ref().unwrap());
+        let path = self.path_bitcoindir();
         if !path.is_dir() {
             println!("Expect path '{:#?}' to be a directory.", path);
             std::process::exit(1);
@@ -206,18 +206,10 @@ impl Config {
     }
 
     pub fn to_rpc_auth(&self) -> color_eyre::Result<Auth> {
-        let cookie = Path::new(self.bitcoindir.as_ref().unwrap()).join(".cookie");
+        let cookie = self.path_cookiefile();
 
         if cookie.is_file() {
             Ok(Auth::CookieFile(cookie))
-        } else if self
-            .rpccookiefile
-            .as_ref()
-            .is_some_and(|cookie| Path::new(cookie).is_file())
-        {
-            Ok(Auth::CookieFile(PathBuf::from(
-                self.rpccookiefile.as_ref().unwrap(),
-            )))
         } else if self.rpcuser.is_some() && self.rpcpassword.is_some() {
             Ok(Auth::UserPass(
                 self.rpcuser.clone().unwrap(),
@@ -228,6 +220,18 @@ impl Config {
         }
     }
 
+    pub fn rpcconnect(&self) -> Option<&String> {
+        self.rpcconnect.as_ref()
+    }
+
+    pub fn rpcport(&self) -> Option<u16> {
+        self.rpcport
+    }
+
+    pub fn delay(&self) -> Option<u64> {
+        self.delay
+    }
+
     pub fn dry_run(&self) -> bool {
         self.dry_run.is_some_and(|b| b)
     }
@@ -241,11 +245,36 @@ impl Config {
     }
 
     pub fn path_bitcoindir(&self) -> PathBuf {
-        PathBuf::from(self.bitcoindir.as_ref().unwrap())
+        Self::fix_user_path(self.bitcoindir.as_ref().unwrap().as_ref())
     }
 
     fn path_kibodir(&self) -> PathBuf {
-        PathBuf::from(self.kibodir.as_ref().unwrap())
+        Self::fix_user_path(self.kibodir.as_ref().unwrap().as_ref())
+    }
+
+    fn path_cookiefile(&self) -> PathBuf {
+        self.rpccookiefile.as_ref().map_or_else(
+            || self.path_bitcoindir().join(".cookie"),
+            |p| Self::fix_user_path(p.as_str()),
+        )
+    }
+
+    fn fix_user_path(path: &str) -> PathBuf {
+        let fix = move |pattern: &str| {
+            if path.starts_with(pattern) {
+                let path = &path
+                    .replace(&format!("{pattern}/"), "")
+                    .replace(pattern, "");
+
+                let home = std::env::var("HOME").unwrap();
+
+                Some(Path::new(&home).join(path))
+            } else {
+                None
+            }
+        };
+
+        fix("~").unwrap_or_else(|| fix("$HOME").unwrap_or_else(|| PathBuf::from(&path)))
     }
 
     pub fn path_datasets(&self) -> MapPath {

src/utils/rpc.rs

@@ -12,11 +12,8 @@ fn create_rpc(config: &Config) -> color_eyre::Result<Client> {
     Ok(Client::new(
         &format!(
             "http://{}:{}",
-            config
+            config.rpcconnect().unwrap_or(&"localhost".to_owned()),
-                .rpcconnect
+            config.rpcport().unwrap_or(8332)
-                .as_ref()
-                .unwrap_or(&"localhost".to_owned()),
-            config.rpcport.unwrap_or(8332)
         ),
         config.to_rpc_auth().unwrap(),
     )?)