diff --git a/.gitignore b/.gitignore index 1c9f9abdc..7df94636e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,6 @@ # Builds target -# Sync -.stfolder - # Copies *\ copy* @@ -32,6 +29,3 @@ docker/kibo # Types paths.d.ts - -# Git -.git-* diff --git a/Cargo.lock b/Cargo.lock index e72dc3740..4122381e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "biter" -version = "0.2.1" +version = "0.2.2" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -2623,7 +2623,7 @@ dependencies = [ [[package]] name = "snkrj" -version = "0.1.0" +version = "0.1.1" dependencies = [ "sanakirja", ] diff --git a/Cargo.toml b/Cargo.toml index 19d50bc1f..b35e6c734 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ bincode = { git = "https://github.com/bincode-org/bincode.git", features = [ "serde", ] } bitcoin_hashes = { version = "0.15.0" } -biter = { path = "./crates/biter" } +biter = { path = "./src/crates/biter" } chrono = { version = "0.4.39", features = ["serde"] } clap = { version = "4.5.23", features = ["derive"] } color-eyre = "0.6.3" @@ -27,10 +27,10 @@ rayon = "1.10.0" regex = "1.11.1" reqwest = { version = "0.12.9", features = ["blocking", "json"] } rlimit = "0.10.2" -snkrj = { path = "./crates/snkrj" } +snkrj = { path = "./src/crates/snkrj" } serde = { version = "1.0.216", features = ["derive"] } serde_json = "1.0.133" -struct_iterable = { path = "./crates/iterable" } +struct_iterable = { path = "./src/crates/iterable" } swc = "9.0.0" swc_common = "5.0.0" tokio = { version = "1.42.0", features = ["full"] } diff --git a/README.md b/README.md index d602d55d8..d364690a2 100644 --- a/README.md +++ b/README.md @@ -63,17 +63,19 @@ Please open an issue if you want to add another instance ### Requirements - At least 16 GB of RAM -- 1 TB of free space (will use 70% of that without defragmentation and 40% after) + - Recommended: 32 GB +- A disk with 1 TB of free space (will use between 40% to 80% depending on several things) + - Recommended: Rated at 3 GB/s (Thunderbolt 4 speed) - A running instance of bitcoin-core with: - `-txindex=1` - `-blocksxor=0` - RPC credentials - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly -txindex=1 -blocksxor=0` - Git +- Unix based operating system (Mac OS or Linux) + - Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config` -### Manual - -_Mac OS and Linux only, Windows is unsupported_ +### Build First we need to install Rust (https://www.rust-lang.org/tools/install) @@ -81,75 +83,33 @@ First we need to install Rust (https://www.rust-lang.org/tools/install) curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` -If you already had Rust installed you could update it just in case +If you already had Rust installed you could update it ```bash rustup update ``` -> If you're on Ubuntu you'll probably also need to install `open-ssl` with -> -> ```bash -> sudo apt install libssl-dev pkg-config -> ``` - -Optionally, you can also install `cargo-watch` for the server to automatically restart it on file change, which will be triggered by new code and new datasets from the parser (https://github.com/watchexec/cargo-watch?tab=readme-ov-file#install) - -```bash -cargo install cargo-watch --locked -``` - -Then you need to choose a path where all files related to **kibō** will live +Then you need to choose a path where the project will reside and then clone it ```bash cd ??? -``` - -We can now clone the repository - -```bash git clone https://github.com/kibo-money/kibo.git +cd kibo ``` -In a new terminal, go to the `parser`'s folder of the repository +If it's your first time running kibo, it will need several information such as: -```bash -cd ???/kibo/parser -``` +- `--bitcoindir PATH`: path to bitcoin core data directory, `???/bitcoin` +- `--kibodir PATH`: path to kibo outputs, if you have enough space on your main disk `~/.kibo` is fine -Now we can finally start by running the parser, you need to use the `./run.sh` script instead of `cargo run -r` as we need to set various system variables for the program to run smoothly +Everything will be saved at `~/.kibo/config.toml`, which will allow you to simply run `cargo run -r` next time -For the first launch, the parser will need several information such as: - -- `--datadir`: which is bitcoin data directory path, prefer `$HOME` to `~` as the latter might not work -- `--outdir`: where all outputs will be saved, prefer `$HOME` to `~` as the latter might not work - -Optionally you can also specify: - -- `--rpccookiefile`: the path to the cookie file if not default -- `--rpcuser`: the username of the RPC credentials to talk to the bitcoin server if set -- `--rpcpassword`: the password of the RPC credentials if set -- `--rpcconnect`: if the bitcoin core server's IP is different than `localhost` -- `--rpcport`: if the port is different than `8332` - -Everything will be saved in a `config.toml` file, which will allow you to simply run `./run.sh` next time +If you need more options please run `cargo run -r --help` to see what parameters are available. Here's an example ```bash -./run.sh --datadir=$HOME/Developer/bitcoin --outdir=$HOME/.kibo/out -``` - -In a **new** terminal, go to the `server`'s folder of the repository - -```bash -cd ???/kibo/server -``` - -And start it also with the `run.sh` script instead of `cargo run -r` - -```bash -./run.sh +cargo run -r -- --bitcoindir=~/Developer/bitcoin --kibodir=~/.kibo ``` Then the easiest to let others access your server is to use `cloudflared` which will also cache requests. For more information go to: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/ diff --git a/crates/snkrj/Cargo.toml b/crates/snkrj/Cargo.toml deleted file mode 100644 index 864b234b6..000000000 --- a/crates/snkrj/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "snkrj" -description = "A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap" -version = "0.1.0" -license = "MIT" -repository = "https://github.com/kibo-money/kibo/tree/main/crates/snkrj" -keywords = ["database", "sanakirja", "btreemap"] -categories = ["database"] -edition = "2021" - -[dependencies] -sanakirja = "1.4.3" diff --git a/crates/biter/CHANGELOG.md b/src/crates/biter/CHANGELOG.md similarity index 100% rename from crates/biter/CHANGELOG.md rename to src/crates/biter/CHANGELOG.md diff --git a/crates/biter/Cargo.lock b/src/crates/biter/Cargo.lock similarity index 100% rename from crates/biter/Cargo.lock rename to src/crates/biter/Cargo.lock diff --git a/crates/biter/Cargo.toml b/src/crates/biter/Cargo.toml similarity index 85% rename from crates/biter/Cargo.toml rename to src/crates/biter/Cargo.toml index fb39c739d..e9ad3f63a 100644 --- a/crates/biter/Cargo.toml +++ b/src/crates/biter/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "biter" description = "A very fast Bitcoin block iterator" -version = "0.2.1" +version = "0.2.2" license = "MIT" -repository = "https://github.com/kibo-money/kibo/tree/main/crates/biter" +repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter" keywords = ["bitcoin", "block", "iterator"] categories = ["cryptography::cryptocurrencies", "encoding"] edition = "2021" diff --git a/crates/biter/LICENSE.md b/src/crates/biter/LICENSE.md similarity index 100% rename from crates/biter/LICENSE.md rename to src/crates/biter/LICENSE.md diff --git a/crates/biter/README.md b/src/crates/biter/README.md similarity index 100% rename from crates/biter/README.md rename to src/crates/biter/README.md diff --git a/crates/biter/src/blk_index_to_blk_recap.rs b/src/crates/biter/src/blk_index_to_blk_recap.rs similarity index 90% rename from crates/biter/src/blk_index_to_blk_recap.rs rename to src/crates/biter/src/blk_index_to_blk_recap.rs index c3823447c..b0353261f 100644 --- a/crates/biter/src/blk_index_to_blk_recap.rs +++ b/src/crates/biter/src/blk_index_to_blk_recap.rs @@ -17,7 +17,7 @@ pub struct BlkIndexToBlkRecap { path: PathBuf, #[target] tree: BTreeMap, - last_safe_recap: Option, + last_safe_height: Option, } impl BlkIndexToBlkRecap { @@ -38,7 +38,7 @@ impl BlkIndexToBlkRecap { let mut this = Self { path, tree, - last_safe_recap: None, + last_safe_height: None, }; this.clean_outdated(blocks_dir); @@ -58,11 +58,11 @@ impl BlkIndexToBlkRecap { } }); - unprocessed_keys.iter().for_each(|blk_index| { - self.remove(blk_index); + unprocessed_keys.into_iter().for_each(|blk_index| { + self.remove(&blk_index); }); - self.last_safe_recap = self.last_entry().map(|e| e.get().clone()); + self.last_safe_height = self.iter().map(|(_, recap)| recap.height()).max(); } pub fn get_start_recap(&self, start: Option) -> Option<(usize, BlkRecap)> { @@ -108,8 +108,8 @@ impl BlkIndexToBlkRecap { } if self - .last_safe_recap - .map_or(true, |recap| recap.height() >= height) + .last_safe_height + .map_or(true, |safe_height| height >= safe_height) && (height % TARGET_BLOCKS_PER_MONTH) == 0 { self.export(); diff --git a/crates/biter/src/blk_metadata.rs b/src/crates/biter/src/blk_metadata.rs similarity index 100% rename from crates/biter/src/blk_metadata.rs rename to src/crates/biter/src/blk_metadata.rs diff --git a/crates/biter/src/blk_metadata_and_block.rs b/src/crates/biter/src/blk_metadata_and_block.rs similarity index 100% rename from crates/biter/src/blk_metadata_and_block.rs rename to src/crates/biter/src/blk_metadata_and_block.rs diff --git a/crates/biter/src/blk_recap.rs b/src/crates/biter/src/blk_recap.rs similarity index 100% rename from crates/biter/src/blk_recap.rs rename to src/crates/biter/src/blk_recap.rs diff --git a/crates/biter/src/lib.rs b/src/crates/biter/src/lib.rs similarity index 100% rename from crates/biter/src/lib.rs rename to src/crates/biter/src/lib.rs diff --git a/crates/biter/src/main.rs b/src/crates/biter/src/main.rs similarity index 89% rename from crates/biter/src/main.rs rename to src/crates/biter/src/main.rs index ed1da58cb..a8bea0066 100644 --- a/crates/biter/src/main.rs +++ b/src/crates/biter/src/main.rs @@ -11,8 +11,8 @@ fn main() { let auth = Auth::CookieFile(cookie); let rpc = Client::new(url, auth).unwrap(); - let start = Some(800_000); - let end = Some(855_000); + let start = Some(810078); + let end = None; biter::new(data_dir, start, end, rpc) .iter() diff --git a/crates/biter/src/utils.rs b/src/crates/biter/src/utils.rs similarity index 100% rename from crates/biter/src/utils.rs rename to src/crates/biter/src/utils.rs diff --git a/crates/iterable/Cargo.lock b/src/crates/iterable/Cargo.lock similarity index 100% rename from crates/iterable/Cargo.lock rename to src/crates/iterable/Cargo.lock diff --git a/crates/iterable/Cargo.toml b/src/crates/iterable/Cargo.toml similarity index 100% rename from crates/iterable/Cargo.toml rename to src/crates/iterable/Cargo.toml diff --git a/crates/iterable/README.md b/src/crates/iterable/README.md similarity index 100% rename from crates/iterable/README.md rename to src/crates/iterable/README.md diff --git a/crates/iterable/src/lib.rs b/src/crates/iterable/src/lib.rs similarity index 100% rename from crates/iterable/src/lib.rs rename to src/crates/iterable/src/lib.rs diff --git a/crates/iterable/struct_iterable_derive/Cargo.toml b/src/crates/iterable/struct_iterable_derive/Cargo.toml similarity index 100% rename from crates/iterable/struct_iterable_derive/Cargo.toml rename to src/crates/iterable/struct_iterable_derive/Cargo.toml diff --git a/crates/iterable/struct_iterable_derive/README.md b/src/crates/iterable/struct_iterable_derive/README.md similarity index 100% rename from crates/iterable/struct_iterable_derive/README.md rename to src/crates/iterable/struct_iterable_derive/README.md diff --git a/crates/iterable/struct_iterable_derive/src/lib.rs b/src/crates/iterable/struct_iterable_derive/src/lib.rs similarity index 100% rename from crates/iterable/struct_iterable_derive/src/lib.rs rename to src/crates/iterable/struct_iterable_derive/src/lib.rs diff --git a/crates/iterable/struct_iterable_internal/Cargo.toml b/src/crates/iterable/struct_iterable_internal/Cargo.toml similarity index 100% rename from crates/iterable/struct_iterable_internal/Cargo.toml rename to src/crates/iterable/struct_iterable_internal/Cargo.toml diff --git a/crates/iterable/struct_iterable_internal/README.md b/src/crates/iterable/struct_iterable_internal/README.md similarity index 100% rename from crates/iterable/struct_iterable_internal/README.md rename to src/crates/iterable/struct_iterable_internal/README.md diff --git a/crates/iterable/struct_iterable_internal/src/lib.rs b/src/crates/iterable/struct_iterable_internal/src/lib.rs similarity index 100% rename from crates/iterable/struct_iterable_internal/src/lib.rs rename to src/crates/iterable/struct_iterable_internal/src/lib.rs diff --git a/crates/snkrj/Cargo.lock b/src/crates/snkrj/Cargo.lock similarity index 100% rename from crates/snkrj/Cargo.lock rename to src/crates/snkrj/Cargo.lock diff --git a/src/crates/snkrj/Cargo.toml b/src/crates/snkrj/Cargo.toml new file mode 100644 index 000000000..2341d6537 --- /dev/null +++ b/src/crates/snkrj/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "snkrj" +description = "A simple wrapper around Sanakirja's database that acts as a very fast on disk BTreeMap" +version = "0.1.1" +license = "MIT" +repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/snkrj" +keywords = ["database", "sanakirja", "btreemap"] +categories = ["database"] +edition = "2021" + +[dependencies] +sanakirja = "1.4.3" diff --git a/crates/snkrj/LICENSE.md b/src/crates/snkrj/LICENSE.md similarity index 100% rename from crates/snkrj/LICENSE.md rename to src/crates/snkrj/LICENSE.md diff --git a/crates/snkrj/README.md b/src/crates/snkrj/README.md similarity index 100% rename from crates/snkrj/README.md rename to src/crates/snkrj/README.md diff --git a/crates/snkrj/src/lib.rs b/src/crates/snkrj/src/lib.rs similarity index 100% rename from crates/snkrj/src/lib.rs rename to src/crates/snkrj/src/lib.rs diff --git a/crates/snkrj/src/main.rs b/src/crates/snkrj/src/main.rs similarity index 100% rename from crates/snkrj/src/main.rs rename to src/crates/snkrj/src/main.rs diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 21acaa3ce..b720f435c 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -37,7 +37,7 @@ pub fn main( &mut datasets, )?; - if let Some(delay) = config.delay { + if let Some(delay) = config.delay() { sleep(Duration::from_secs(delay)) } diff --git a/src/structs/config.rs b/src/structs/config.rs index 9ad22a1c0..ed5acbedc 100644 --- a/src/structs/config.rs +++ b/src/structs/config.rs @@ -15,36 +15,36 @@ use super::MapPath; #[command(version, about, long_about = None)] pub struct Config { /// Bitcoin data directory path, saved - #[arg(long, value_name = "DIR")] + #[arg(long, value_name = "PATH")] bitcoindir: Option, - /// Kibo data directory path, saved - #[arg(long, value_name = "DIR")] + /// Kibo output directory path, saved + #[arg(long, value_name = "PATH")] kibodir: Option, /// Bitcoin RPC ip, default: localhost, saved #[arg(long, value_name = "IP")] - pub rpcconnect: Option, + rpcconnect: Option, /// Bitcoin RPC port, default: 8332, saved #[arg(long, value_name = "PORT")] - pub rpcport: Option, + rpcport: Option, /// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved #[arg(long, value_name = "PATH")] - pub rpccookiefile: Option, + rpccookiefile: Option, /// Bitcoin RPC username, saved #[arg(long, value_name = "USERNAME")] - pub rpcuser: Option, + rpcuser: Option, /// Bitcoin RPC password, saved #[arg(long, value_name = "PASSWORD")] - pub rpcpassword: Option, + rpcpassword: Option, /// Delay between runs, default: 0, saved #[arg(long, value_name = "SECONDS")] - pub delay: Option, + delay: Option, // Maximum ram you want the program to use in GB, default: 50% of total, not saved // #[arg(long, value_name = "GB")] @@ -181,7 +181,7 @@ impl Config { std::process::exit(1); } - let path = Path::new(self.bitcoindir.as_ref().unwrap()); + let path = self.path_bitcoindir(); if !path.is_dir() { println!("Expect path '{:#?}' to be a directory.", path); std::process::exit(1); @@ -206,18 +206,10 @@ impl Config { } pub fn to_rpc_auth(&self) -> color_eyre::Result { - let cookie = Path::new(self.bitcoindir.as_ref().unwrap()).join(".cookie"); + let cookie = self.path_cookiefile(); if cookie.is_file() { Ok(Auth::CookieFile(cookie)) - } else if self - .rpccookiefile - .as_ref() - .is_some_and(|cookie| Path::new(cookie).is_file()) - { - Ok(Auth::CookieFile(PathBuf::from( - self.rpccookiefile.as_ref().unwrap(), - ))) } else if self.rpcuser.is_some() && self.rpcpassword.is_some() { Ok(Auth::UserPass( self.rpcuser.clone().unwrap(), @@ -228,6 +220,18 @@ impl Config { } } + pub fn rpcconnect(&self) -> Option<&String> { + self.rpcconnect.as_ref() + } + + pub fn rpcport(&self) -> Option { + self.rpcport + } + + pub fn delay(&self) -> Option { + self.delay + } + pub fn dry_run(&self) -> bool { self.dry_run.is_some_and(|b| b) } @@ -241,11 +245,36 @@ impl Config { } pub fn path_bitcoindir(&self) -> PathBuf { - PathBuf::from(self.bitcoindir.as_ref().unwrap()) + Self::fix_user_path(self.bitcoindir.as_ref().unwrap().as_ref()) } fn path_kibodir(&self) -> PathBuf { - PathBuf::from(self.kibodir.as_ref().unwrap()) + Self::fix_user_path(self.kibodir.as_ref().unwrap().as_ref()) + } + + fn path_cookiefile(&self) -> PathBuf { + self.rpccookiefile.as_ref().map_or_else( + || self.path_bitcoindir().join(".cookie"), + |p| Self::fix_user_path(p.as_str()), + ) + } + + fn fix_user_path(path: &str) -> PathBuf { + let fix = move |pattern: &str| { + if path.starts_with(pattern) { + let path = &path + .replace(&format!("{pattern}/"), "") + .replace(pattern, ""); + + let home = std::env::var("HOME").unwrap(); + + Some(Path::new(&home).join(path)) + } else { + None + } + }; + + fix("~").unwrap_or_else(|| fix("$HOME").unwrap_or_else(|| PathBuf::from(&path))) } pub fn path_datasets(&self) -> MapPath { diff --git a/src/utils/rpc.rs b/src/utils/rpc.rs index a25275d27..398989639 100644 --- a/src/utils/rpc.rs +++ b/src/utils/rpc.rs @@ -12,11 +12,8 @@ fn create_rpc(config: &Config) -> color_eyre::Result { Ok(Client::new( &format!( "http://{}:{}", - config - .rpcconnect - .as_ref() - .unwrap_or(&"localhost".to_owned()), - config.rpcport.unwrap_or(8332) + config.rpcconnect().unwrap_or(&"localhost".to_owned()), + config.rpcport().unwrap_or(8332) ), config.to_rpc_auth().unwrap(), )?)