diff --git a/.gitignore b/.gitignore index d4d3b3b90..e1d488d12 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ TODO.md .stfolder /charts /price +*\ copy* +/dist + +/start-node.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 975edd6f6..d0706c448 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,11 @@ ![Image of the Satonomics Web App version 0.X.Y](./assets/v0.X.Y.jpg) +### Parser +- Change the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind` +- Use the Bitcoin RPC server for various calls instead of running commands and then parsing the JSON from the output +- Updated the config, run with `-h` to see ## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26 diff --git a/app/src/app/components/frames/settings.tsx b/app/src/app/components/frames/settings.tsx index dc6821ac5..30e1a5409 100644 --- a/app/src/app/components/frames/settings.tsx +++ b/app/src/app/components/frames/settings.tsx @@ -165,6 +165,20 @@ export function SettingsFrame({ url: "https://primal.net/p/npub1mnwjn40hr042rsmzu64rsnwsw07uegg4tjkv620c94p6e797wkvq3qeujc", amount: 5_000, }, + { + name: "lassdas", + url: "https://primal.net/p/npub1gmhctt2hmjqz8ay2x8h5f8fl3h4fpfcezwqneal3usu3u65qca4s8094ea", + amount: 210_000, + }, + { + name: "anon", + amount: 21_000, + }, + { + name: "xplbzx", + url: "https://primal.net/p/npub1e0f808a350rxrhppu4zylzljt3arfpvrrpqdg6ft78xy6u49kq5slf0g92", + amount: 10_110, + }, ] .sort((a, b) => b.amount !== a.amount diff --git a/parser/Cargo.lock b/parser/Cargo.lock index a009a121f..372620352 100644 --- a/parser/Cargo.lock +++ b/parser/Cargo.lock @@ -27,7 +27,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -152,6 +152,12 @@ dependencies = [ "bitcoin_hashes", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.0" @@ -236,6 +242,45 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoincore-rpc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" +dependencies = [ + "bitcoincore-rpc-json", + "jsonrpc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoincore-rpc-json" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" +dependencies = [ + "bitcoin", + "serde", + "serde_json", +] + +[[package]] +name = "biter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2ea6b4c4046feaaf73f3dff1cc648328af021d50be95946e0e7a807c985b47" +dependencies = [ + "bitcoin", + "bitcoincore-rpc", + "crossbeam", + "derived-deref", + "rayon", + "serde", + "serde_json", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -339,15 +384,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" -[[package]] -name = "cmake" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" -dependencies = [ - "cc", -] - [[package]] name = "color-eyre" version = "0.6.3" @@ -477,12 +513,6 @@ dependencies = [ "parking_lot_core 0.9.10", ] -[[package]] -name = "db-key" -version = "0.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72465f46d518f6015d9cf07f7f3013a95dd6b9c2747c3d65ae0cce43929d14f" - [[package]] name = "derive_deref" version = "1.1.1" @@ -494,6 +524,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derived-deref" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "either" version = "1.12.0" @@ -560,12 +601,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" -[[package]] -name = "ffi-opaque" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" - [[package]] name = "fnv" version = "1.0.7" @@ -966,35 +1001,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpc" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +dependencies = [ + "base64 0.13.1", + "minreq", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "leveldb" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32651baaaa5596b3a6e0bee625e73fd0334c167db0ea5ac68750ef9a629a2d6a" -dependencies = [ - "db-key", - "leveldb-sys", - "libc", -] - -[[package]] -name = "leveldb-sys" -version = "2.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd94a4d0242a437e5e41a27c782b69a624469ca1c4d1e5cb3c337f74a8031d4" -dependencies = [ - "cmake", - "ffi-opaque", - "libc", - "num_cpus", -] - [[package]] name = "libc" version = "0.2.155" @@ -1064,14 +1088,26 @@ dependencies = [ ] [[package]] -name = "mio" -version = "0.8.11" +name = "minreq" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0" dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "mio" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1111,16 +1147,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.32.2" @@ -1195,16 +1221,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "par-iter-sync" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa981aaed94bf59211f644922155e8a33bcb01ed662cd63426653187f562790" -dependencies = [ - "crossbeam", - "num_cpus", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -1249,20 +1265,16 @@ version = "0.3.0" dependencies = [ "allocative", "bincode", - "bitcoin", "bitcoin_hashes", - "byteorder", + "biter", "chrono", "clap", "color-eyre", - "db-key", "derive_deref", "inferno", "itertools", - "leveldb", "memory-stats", "ordered-float", - "par-iter-sync", "rayon", "reqwest", "sanakirja", @@ -1315,6 +1327,15 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "ppv-lite86" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +dependencies = [ + "zerocopy 0.6.6", +] + [[package]] name = "proc-macro2" version = "1.0.85" @@ -1342,6 +1363,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.10.0" @@ -1386,7 +1437,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ - "base64", + "base64 0.22.0", "bytes", "encoding_rs", "futures-channel", @@ -1486,7 +1537,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64", + "base64 0.22.0", "rustls-pki-types", ] @@ -1556,6 +1607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3" dependencies = [ "bitcoin_hashes", + "rand", "secp256k1-sys", "serde", ] @@ -1614,11 +1666,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -1810,9 +1863,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", @@ -1820,7 +1873,7 @@ dependencies = [ "mio", "pin-project-lite", "socket2", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2315,13 +2368,34 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "zerocopy" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +dependencies = [ + "byteorder", + "zerocopy-derive 0.6.6", +] + [[package]] name = "zerocopy" version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy-derive" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] [[package]] diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e11a78e56..f73b137d0 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -11,20 +11,16 @@ edition = "2021" [dependencies] allocative = "0.3.3" bincode = { git = "https://github.com/bincode-org/bincode.git" } -bitcoin = { version = "0.32.2", features = ["serde"] } +biter = "0.1.0" bitcoin_hashes = { version = "0.14.0" } -byteorder = "1.5.0" chrono = { version = "0.4.38", features = ["serde"] } clap = { version = "4.5.11", features = ["derive"] } color-eyre = "0.6.3" -db-key = "=0.0.5" derive_deref = "1.1.1" inferno = "0.11.20" itertools = "0.13.0" -leveldb = "0.8.6" memory-stats = "1.2.0" ordered-float = "4.2.1" -par-iter-sync = "0.1.11" rayon = "1.10.0" reqwest = { version = "0.12.5", features = ["blocking", "json"] } sanakirja = "1.4.2" diff --git a/parser/src/actions/iter_blocks.rs b/parser/src/actions/iter_blocks.rs index cb39c76d6..8ada00963 100644 --- a/parser/src/actions/iter_blocks.rs +++ b/parser/src/actions/iter_blocks.rs @@ -8,15 +8,22 @@ use parse::ParseData; use crate::{ actions::{export, find_first_inserted_unsafe_height, parse}, - bitcoin::BitcoinDB, + create_rpc, databases::Databases, datasets::{AllDatasets, ComputeData}, states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates}, structs::{Date, DateData, MapKey}, utils::{generate_allocation_files, log, time}, + Config, Height, }; -pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Result<()> { +pub fn iter_blocks( + config: &Config, + rpc: &biter::bitcoincore_rpc::Client, + approx_block_count: usize, +) -> color_eyre::Result<()> { + let export_dir = "./target/outputs"; + let should_insert = true; let should_export = true; let study_ram_usage = false; @@ -44,11 +51,19 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re log(&format!("Starting parsing at height: {height}")); - let mut block_iter = bitcoin_db.iter_block(height.to_usize(), block_count); - let mut next_block_opt = None; let mut blocks_loop_date = None; + let block_receiver = biter::new( + config.datadir.as_ref().unwrap(), + export_dir, + Some(height.to_usize()), + None, + create_rpc(config).unwrap(), + ); + + let mut block_iter = block_receiver.iter(); + 'parsing: loop { let instant = Instant::now(); @@ -67,15 +82,22 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re next_block_opt = block_iter.next(); - if let Some(current_block) = current_block_opt { + if let Some((_current_block_height, current_block, _current_block_hash)) = + current_block_opt + { let timestamp = current_block.header.time; let current_block_date = Date::from_timestamp(timestamp); - let current_block_height = height + blocks_loop_i; + let current_block_height: Height = height + blocks_loop_i; + + if current_block_height.to_usize() != _current_block_height { + dbg!(current_block_height, _current_block_height); + panic!() + } let next_block_date = next_block_opt .as_ref() - .map(|next_block| Date::from_timestamp(next_block.header.time)); + .map(|(_, next_block, _)| Date::from_timestamp(next_block.header.time)); // Always run for the first block of the loop if blocks_loop_date.is_none() { @@ -139,7 +161,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re } parse(ParseData { - bitcoin_db, + rpc, block: current_block, block_index: blocks_loop_i, compute_addresses, @@ -162,7 +184,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re let is_new_month = next_block_date .map_or(true, |next_block_date| next_block_date.day() == 1); - if is_new_month || height.is_close_to_end(block_count) { + if is_new_month || height.is_close_to_end(approx_block_count) { break 'days; } @@ -196,7 +218,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re } if should_export { - let is_safe = height.is_safe(block_count); + let is_safe = height.is_safe(approx_block_count); export(ExportedData { databases: is_safe.then_some(&mut databases), diff --git a/parser/src/actions/parse.rs b/parser/src/actions/parse.rs index 37e1fdfa2..fa335e026 100644 --- a/parser/src/actions/parse.rs +++ b/parser/src/actions/parse.rs @@ -1,12 +1,14 @@ use std::{collections::BTreeMap, ops::ControlFlow, thread}; -use bitcoin::{Block, Txid}; +use biter::{ + bitcoin::{Block, Txid}, + bitcoincore_rpc::RpcApi, +}; use itertools::Itertools; use rayon::prelude::*; use crate::{ - bitcoin::BitcoinDB, databases::{ AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex, Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount, @@ -23,7 +25,7 @@ use crate::{ }; pub struct ParseData<'a> { - pub bitcoin_db: &'a BitcoinDB, + // pub bitcoin_cli: &'a BitcoinCli, pub block: Block, pub block_index: usize, pub compute_addresses: bool, @@ -33,13 +35,13 @@ pub struct ParseData<'a> { pub first_date_height: Height, pub height: Height, pub is_date_last_block: bool, + pub rpc: &'a biter::bitcoincore_rpc::Client, pub states: &'a mut States, pub timestamp: u32, } pub fn parse( ParseData { - bitcoin_db, block, block_index, compute_addresses, @@ -49,6 +51,7 @@ pub fn parse( first_date_height, height, is_date_last_block, + rpc, states, timestamp, }: ParseData, @@ -334,10 +337,16 @@ pub fn parse( // Can be none because 0 sats inputs happen // https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27 + if input_tx_data.is_none() { if !enable_check_if_txout_value_is_zero_in_db - || bitcoin_db - .check_if_txout_value_is_zero(&input_txid, input_vout as usize) + || rpc + .get_tx_out(&input_txid, input_vout, None) + .unwrap() + .unwrap() + .value + .to_sat() + == 0 { return ControlFlow::Continue::<()>(()); } @@ -374,8 +383,13 @@ pub fn parse( if input_amount_and_address_index.is_none() { if !enable_check_if_txout_value_is_zero_in_db - || bitcoin_db - .check_if_txout_value_is_zero(&input_txid, input_vout as usize) + || rpc + .get_tx_out(&input_txid, input_vout as u32, None) + .unwrap() + .unwrap() + .value + .to_sat() + == 0 { return ControlFlow::Continue::<()>(()); } diff --git a/parser/src/bitcoin/addresses/mod.rs b/parser/src/bitcoin/addresses/mod.rs deleted file mode 100644 index 227f1c636..000000000 --- a/parser/src/bitcoin/addresses/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod multisig; - -pub use multisig::*; diff --git a/parser/src/bitcoin/consts.rs b/parser/src/bitcoin/consts.rs deleted file mode 100644 index e2bf0ed55..000000000 --- a/parser/src/bitcoin/consts.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100; -pub const TARGET_BLOCKS_PER_DAY: usize = 144; diff --git a/parser/src/bitcoin/daemon.rs b/parser/src/bitcoin/daemon.rs deleted file mode 100644 index 9ba837a0e..000000000 --- a/parser/src/bitcoin/daemon.rs +++ /dev/null @@ -1,137 +0,0 @@ -use std::{process::Command, thread::sleep, time::Duration}; - -use color_eyre::eyre::eyre; -use serde_json::Value; - -use crate::{ - structs::Height, - utils::{log, log_output, retry}, - Config, -}; - -struct BlockchainInfo { - pub headers: u64, - pub blocks: u64, -} - -pub struct BitcoinDaemon { - config: Config, -} - -impl BitcoinDaemon { - pub fn new(config: &Config) -> Self { - Self { - config: config.clone(), - } - } - - pub fn start(&self) { - sleep(Duration::from_secs(1)); - - let mut command = Command::new("bitcoind"); - - command - .arg(self.datadir_arg()) - .arg("-txindex=1") - .arg("-daemon"); - - if self.config.blocksonly.unwrap() { - command.arg("-blocksonly"); - } - - if let Some(ip) = self.config.rpcconnect.as_ref() { - command.arg(format!("-rpcconnect={}", ip)); - } - - // bitcoind -datadir=/Users/k/Developer/bitcoin -blocksonly -txindex=1 -daemon - let output = command - .output() - .expect("bitcoind to be able to properly start"); - - log_output(&output); - } - - pub fn stop(&self) { - // bitcoin-cli -datadir=/Users/k/Developer/bitcoin stop - let output = Command::new("bitcoin-cli") - .arg(self.datadir_arg()) - .arg("stop") - .output() - .unwrap(); - - if output.status.success() { - log_output(&output); - - sleep(Duration::from_secs(15)); - } - } - - pub fn wait_sync(&self) { - while !self.check_if_fully_synced() { - sleep(Duration::from_secs(5)) - } - } - - pub fn wait_for_new_block(&self, last_block_height: Height) { - log("Waiting for new block..."); - - while last_block_height == self.get_blockchain_info().headers { - sleep(Duration::from_secs(5)) - } - } - - pub fn check_if_fully_synced(&self) -> bool { - let BlockchainInfo { blocks, headers } = self.get_blockchain_info(); - - let synced = blocks == headers; - - if synced { - log(&format!("Synced ! ({blocks} blocks)")); - } else { - log(&format!("Syncing... ({} remaining)", headers - blocks)); - } - - synced - } - - fn get_blockchain_info(&self) -> BlockchainInfo { - retry( - |_| { - // bitcoin-cli -datadir=/Users/k/Developer/bitcoin getblockchaininfo - let output = Command::new("bitcoin-cli") - .arg(self.datadir_arg()) - .arg("getblockchaininfo") - .output()?; - - let output = String::from_utf8_lossy(&output.stdout); - - let json: Value = serde_json::from_str(&output)?; - let json = json.as_object().ok_or(eyre!("json as object failed"))?; - - let blocks = json - .get("blocks") - .ok_or(eyre!("get field 'blocks' from json failed"))? - .as_u64() - .ok_or(eyre!("blocks to u64 failed"))?; - let headers = json - .get("headers") - .ok_or(eyre!("get field 'headers' from json failed"))? - .as_u64() - .ok_or(eyre!("blocks to u64 failed"))?; - - Ok(BlockchainInfo { headers, blocks }) - }, - 1, - usize::MAX, - ) - .unwrap() - } - - fn datadir_arg(&self) -> String { - if self.config.datadir.is_none() { - unreachable!(); - } - - format!("-datadir={}", self.config.datadir.as_ref().unwrap()) - } -} diff --git a/parser/src/bitcoin/db/blk_files.rs b/parser/src/bitcoin/db/blk_files.rs deleted file mode 100644 index 4429a63f1..000000000 --- a/parser/src/bitcoin/db/blk_files.rs +++ /dev/null @@ -1,152 +0,0 @@ -use std::{ - collections::HashMap, - convert::From, - fs::{self, DirEntry, File}, - io::{self, BufReader, Seek, SeekFrom}, - path::{Path, PathBuf}, -}; - -use bitcoin::{io::Cursor, Block, Transaction}; -use derive_deref::{Deref, DerefMut}; - -use super::{ - errors::{OpError, OpErrorKind, OpResult}, - reader::BlockchainRead, -}; - -/// -/// An index of all blk files found. -/// -#[derive(Debug, Clone, Deref, DerefMut)] -pub struct BlkFiles(HashMap); - -impl BlkFiles { - /// - /// Construct an index of all blk files. - /// - pub fn new(path: &Path) -> OpResult { - Ok(Self(Self::scan_path(path)?)) - } - - /// - /// Read a Block from blk file. - /// - #[inline] - pub fn read_raw_block(&self, n_file: i32, offset: u32) -> OpResult> { - if let Some(blk_path) = self.get(&n_file) { - let mut r = BufReader::new(File::open(blk_path)?); - r.seek(SeekFrom::Start(offset as u64 - 4))?; - let block_size = r.read_u32()?; - let block = r.read_u8_vec(block_size)?; - Ok(block) - } else { - Err(OpError::from("blk file not found, sync with bitcoin core")) - } - } - - /// - /// Read a Block from blk file. - /// - pub fn read_block(&self, n_file: i32, offset: u32) -> OpResult { - Cursor::new(self.read_raw_block(n_file, offset)?).read_block() - } - - /// - /// Read a transaction from blk file. - /// - pub fn read_transaction( - &self, - n_file: i32, - n_pos: u32, - n_tx_offset: u32, - ) -> OpResult { - if let Some(blk_path) = self.get(&n_file) { - let mut r = BufReader::new(File::open(blk_path)?); - // the size of a header is 80. - r.seek(SeekFrom::Start(n_pos as u64 + n_tx_offset as u64 + 80))?; - r.read_transaction() - } else { - Err(OpError::from("blk file not found, sync with bitcoin core")) - } - } - - /// - /// Scan blk folder to build an index of all blk files. - /// - fn scan_path(path: &Path) -> OpResult> { - let mut collected = HashMap::with_capacity(4000); - - for entry in fs::read_dir(path)? { - match entry { - Ok(de) => { - let path = Self::resolve_path(&de)?; - if !path.is_file() { - continue; - }; - if let Some(file_name) = path.as_path().file_name() { - if let Some(file_name) = file_name.to_str() { - if let Some(index) = Self::parse_blk_index(file_name) { - collected.insert(index, path); - } - } - } - } - Err(msg) => { - return Err(OpError::from(msg)); - } - } - } - - collected.shrink_to_fit(); - - if collected.is_empty() { - Err(OpError::new(OpErrorKind::RuntimeError).join_msg("No blk files found!")) - } else { - Ok(collected) - } - } - - /// - /// Resolve symlink. - /// - fn resolve_path(entry: &DirEntry) -> io::Result { - if entry.file_type()?.is_symlink() { - fs::read_link(entry.path()) - } else { - Ok(entry.path()) - } - } - - /// - /// Extract index from block file name. - /// - fn parse_blk_index(file_name: &str) -> Option { - let prefix = "blk"; - let ext = ".dat"; - if file_name.starts_with(prefix) && file_name.ends_with(ext) { - file_name[prefix.len()..(file_name.len() - ext.len())] - .parse::() - .ok() - } else { - None - } - } -} - -// #[cfg(test)] -// mod tests { -// use super::*; - -// #[test] -// fn test_parse_blk_index() { -// assert_eq!(0, BlkFiles::parse_blk_index("blk00000.dat").unwrap()); -// assert_eq!(6, BlkFiles::parse_blk_index("blk6.dat").unwrap()); -// assert_eq!(1202, BlkFiles::parse_blk_index("blk1202.dat").unwrap()); -// assert_eq!( -// 13412451, -// BlkFiles::parse_blk_index("blk13412451.dat").unwrap() -// ); -// assert!(BlkFiles::parse_blk_index("blkindex.dat").is_none()); -// assert!(BlkFiles::parse_blk_index("invalid.dat").is_none()); -// } -// } diff --git a/parser/src/bitcoin/db/block_iter.rs b/parser/src/bitcoin/db/block_iter.rs deleted file mode 100644 index 61611b3d1..000000000 --- a/parser/src/bitcoin/db/block_iter.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! -//! View development note of iter_connected.rs for implementation -//! details of iter_block.rs, which follows similar principles. -//! -use bitcoin::Block; -use par_iter_sync::{IntoParallelIteratorSync, ParIterSync}; - -use super::BitcoinDB; - -pub struct BlockIter(ParIterSync); - -impl BlockIter { - /// the worker threads are dispatched in this `new` constructor! - pub fn new(db: &BitcoinDB, heights: T) -> Self - where - T: IntoIterator + Send + 'static, - ::IntoIter: Send + 'static, - { - let db_ref = db.clone(); - - BlockIter( - heights.into_par_iter_sync(move |h| match db_ref.get_block(h) { - Ok(blk) => Ok(blk), - Err(_) => Err(()), - }), - ) - } - - /// the worker threads are dispatched in this `new` constructor! - pub fn from_range(db: &BitcoinDB, start: usize, end: usize) -> Self { - if end <= start { - BlockIter::new(db, Vec::new()) - } else { - BlockIter::new(db, start..end) - } - } -} - -impl Iterator for BlockIter { - type Item = Block; - - fn next(&mut self) -> Option { - self.0.next() - } -} diff --git a/parser/src/bitcoin/db/blocks_indexes.rs b/parser/src/bitcoin/db/blocks_indexes.rs deleted file mode 100644 index 272d53bc2..000000000 --- a/parser/src/bitcoin/db/blocks_indexes.rs +++ /dev/null @@ -1,211 +0,0 @@ -use std::{collections::BTreeMap, fmt, path::Path}; - -use bitcoin::{block::Header, io::Cursor, BlockHash}; -use derive_deref::{Deref, DerefMut}; -use leveldb::{ - database::{iterator::LevelDBIterator, Database}, - iterator::Iterable, - options::{Options, ReadOptions}, -}; - -use crate::utils::log; - -use super::{BlockchainRead, OpResult}; - -/// -/// See Bitcoin Core repository for definition. -/// -const BLOCK_VALID_HEADER: u32 = 1; -const BLOCK_VALID_TREE: u32 = 2; -const BLOCK_VALID_TRANSACTIONS: u32 = 3; -const BLOCK_VALID_CHAIN: u32 = 4; -const BLOCK_VALID_SCRIPTS: u32 = 5; -const BLOCK_VALID_MASK: u32 = BLOCK_VALID_HEADER - | BLOCK_VALID_TREE - | BLOCK_VALID_TRANSACTIONS - | BLOCK_VALID_CHAIN - | BLOCK_VALID_SCRIPTS; -const BLOCK_HAVE_DATA: u32 = 8; -const BLOCK_HAVE_UNDO: u32 = 16; - -/// -/// - Map from block height to block hash (records) -/// - Map from block hash to block height (hash_to_height) -/// -#[derive(Clone, Deref, DerefMut)] -pub struct BlocksIndexes(Box<[BlockIndexRecord]>); - -/// -/// BLOCK_INDEX RECORD as defined in Bitcoin Core. -/// -#[derive(Clone)] -pub struct BlockIndexRecord { - pub n_version: i32, - pub n_height: i32, - pub n_status: u32, - pub n_tx: u32, - pub n_file: i32, - pub n_data_pos: u32, - pub n_undo_pos: u32, - pub header: Header, -} - -impl BlocksIndexes { - /// - /// Build a collections of block index. - /// - pub(crate) fn new(p: &Path) -> OpResult { - Ok(Self(load_block_index(p)?.into_boxed_slice())) - } -} - -/// -/// Load all block index in memory from leveldb (i.e. `blocks/index` path). -/// -/// Map from block height to block index record. -/// -pub fn load_block_index(path: &Path) -> OpResult> { - let mut block_index_by_block_hash = BTreeMap::new(); - - log("Start loading block_index"); - - let mut options = Options::new(); - options.create_if_missing = false; - let db: Database = Database::open(path, options)?; - let options = ReadOptions::new(); - let mut iter = db.iter(options); - let mut max_height_block_hash = Option::<(BlockHash, i32)>::None; - - while iter.advance() { - let k = iter.key(); - let v = iter.value(); - if is_block_index_record(&k.key) { - let record = BlockIndexRecord::from(&v)?; - // only add valid block index record that has block data. - if record.n_height == 0 - || (record.n_status & BLOCK_VALID_MASK >= BLOCK_VALID_SCRIPTS - && record.n_status & BLOCK_HAVE_DATA > 0) - { - let block_hash = record.header.block_hash(); - // find the block with max height - if let Some((hash, height)) = max_height_block_hash.as_mut() { - if record.n_height > *height { - *hash = block_hash; - *height = record.n_height; - } - } else { - max_height_block_hash = Some((block_hash, record.n_height)); - } - block_index_by_block_hash.insert(block_hash, record); - } - } - } - // build the longest chain - if let Some((hash, height)) = max_height_block_hash { - let mut block_index = Vec::with_capacity(height as usize + 1); - let mut current_hash = hash; - let mut current_height = height; - - // recursively build block index from max height block. - while current_height >= 0 { - let blk = block_index_by_block_hash - .remove(¤t_hash) - .expect("block hash not found in block index!"); - - assert_eq!( - current_height, blk.n_height, - "some block info missing from block index levelDB,\ - delete Bitcoin folder and re-download!" - ); - - current_hash = blk.header.prev_blockhash; - current_height -= 1; - block_index.push(blk); - } - - block_index.reverse(); - - Ok(block_index) - } else { - Ok(Vec::with_capacity(0)) - } -} - -/// levelDB key util -struct BlockKey { - key: Vec, -} - -/// levelDB key util -impl db_key::Key for BlockKey { - fn from_u8(key: &[u8]) -> Self { - BlockKey { - key: Vec::from(key), - } - } - - fn as_slice T>(&self, f: F) -> T { - f(&self.key) - } -} - -impl BlockIndexRecord { - /// - /// Decode levelDB value for Block Index Record. - /// - fn from(values: &[u8]) -> OpResult { - let mut reader = Cursor::new(values); - - let n_version = reader.read_varint()? as i32; - let n_height = reader.read_varint()? as i32; - let n_status = reader.read_varint()? as u32; - let n_tx = reader.read_varint()? as u32; - let n_file = if n_status & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO) > 0 { - reader.read_varint()? as i32 - } else { - -1 - }; - let n_data_pos = if n_status & BLOCK_HAVE_DATA > 0 { - reader.read_varint()? as u32 - } else { - u32::MAX - }; - let n_undo_pos = if n_status & BLOCK_HAVE_UNDO > 0 { - reader.read_varint()? as u32 - } else { - u32::MAX - }; - - let header = reader.read_block_header()?; - - Ok(BlockIndexRecord { - n_version, - n_height, - n_status, - n_tx, - n_file, - n_data_pos, - n_undo_pos, - header, - }) - } -} - -#[inline] -fn is_block_index_record(data: &[u8]) -> bool { - data.first() == Some(&b'b') -} - -impl fmt::Debug for BlockIndexRecord { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BlockIndexRecord") - .field("version", &self.n_version) - .field("height", &self.n_height) - .field("status", &self.n_status) - .field("n_tx", &self.n_tx) - .field("n_file", &self.n_file) - .field("n_data_pos", &self.n_data_pos) - .field("header", &self.header) - .finish() - } -} diff --git a/parser/src/bitcoin/db/errors.rs b/parser/src/bitcoin/db/errors.rs deleted file mode 100644 index 90405f8c3..000000000 --- a/parser/src/bitcoin/db/errors.rs +++ /dev/null @@ -1,135 +0,0 @@ -use std::convert::{self, From}; -use std::error; -use std::fmt; -use std::io; -use std::string; -use std::sync; - -pub type OpResult = Result; - -#[derive(Debug)] -/// Custom error type -pub struct OpError { - pub kind: OpErrorKind, - pub message: String, -} - -impl OpError { - pub fn new(kind: OpErrorKind) -> Self { - OpError { - kind, - message: String::new(), - } - } - - /// Joins the Error with a new message and returns it - pub fn join_msg(mut self, msg: &str) -> Self { - self.message.push_str(msg); - OpError { - kind: self.kind, - message: self.message, - } - } -} - -impl fmt::Display for OpError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.message.is_empty() { - write!(f, "{}", &self.kind) - } else { - write!(f, "{} {}", &self.message, &self.kind) - } - } -} - -impl error::Error for OpError { - fn description(&self) -> &str { - self.message.as_ref() - } - fn cause(&self) -> Option<&dyn error::Error> { - self.kind.source() - } -} - -#[derive(Debug)] -pub enum OpErrorKind { - None, - IoError(io::Error), - Utf8Error(string::FromUtf8Error), - RuntimeError, - PoisonError, - SendError, -} - -impl fmt::Display for OpErrorKind { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - OpErrorKind::IoError(ref err) => write!(f, "I/O Error: {}", err), - OpErrorKind::Utf8Error(ref err) => write!(f, "Utf8 Conversion: {}", err), - ref err @ OpErrorKind::PoisonError => write!(f, "Threading Error: {}", err), - ref err @ OpErrorKind::SendError => write!(f, "Sync: {}", err), - ref err @ OpErrorKind::RuntimeError => write!(f, "RuntimeError: {}", err), - OpErrorKind::None => write!(f, ""), - } - } -} - -impl error::Error for OpErrorKind { - fn source(&self) -> Option<&(dyn error::Error + 'static)> { - match *self { - OpErrorKind::IoError(ref err) => Some(err), - OpErrorKind::Utf8Error(ref err) => Some(err), - ref err @ OpErrorKind::PoisonError => Some(err), - ref err @ OpErrorKind::SendError => Some(err), - _ => None, - } - } -} - -impl From for OpError { - fn from(err: io::Error) -> Self { - Self::new(OpErrorKind::IoError(err)) - } -} - -impl From for OpError { - fn from(_: bitcoin::consensus::encode::Error) -> Self { - Self::from("block decode error") - } -} - -impl convert::From for OpError { - fn from(err_code: i32) -> Self { - Self::from(io::Error::from_raw_os_error(err_code)) - } -} - -impl convert::From<&str> for OpError { - fn from(err: &str) -> Self { - Self::new(OpErrorKind::None).join_msg(err) - } -} - -impl convert::From> for OpError { - fn from(_: sync::PoisonError) -> Self { - Self::new(OpErrorKind::PoisonError) - } -} - -impl convert::From> for OpError { - fn from(_: sync::mpsc::SendError) -> Self { - Self::new(OpErrorKind::SendError) - } -} - -impl convert::From for OpError { - fn from(err: string::FromUtf8Error) -> Self { - Self::new(OpErrorKind::Utf8Error(err)) - } -} - -impl convert::From for OpError { - fn from(err: leveldb::error::Error) -> Self { - Self::from(err.to_string().as_ref()) - } -} diff --git a/parser/src/bitcoin/db/mod.rs b/parser/src/bitcoin/db/mod.rs deleted file mode 100644 index ac2414a0a..000000000 --- a/parser/src/bitcoin/db/mod.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! -//! Mostly a stripped down copy pasta of bitcoin-explorer -//! -//! Huge props to https://github.com/Congyuwang -//! -//! Crates APIs, essential structs, functions, methods are all here! -//! -//! To quickly understand how to use this crate, have a look at the -//! documentation for `bitcoin_explorer::BitcoinDB`!!. -//! - -mod blk_files; -mod block_iter; -mod blocks_indexes; -mod errors; -mod reader; -mod txdb; - -use blk_files::*; -use blocks_indexes::*; -use errors::*; -use reader::*; -use txdb::*; - -use std::ops::Deref; -use std::path::Path; -use std::sync::Arc; - -use bitcoin::{Block, Transaction, Txid}; - -pub use block_iter::BlockIter; - -pub struct InnerDB { - pub blocks_indexes: BlocksIndexes, - pub blk_files: BlkFiles, - pub tx_db: TxDB, -} - -/// -/// This is the main struct of this crate!! Click and read the doc. -/// -/// All queries start from initializing `BitcoinDB`. -/// -/// Note: This is an Arc wrap around `InnerDB`. -/// -#[derive(Clone)] -pub struct BitcoinDB(Arc); - -impl Deref for BitcoinDB { - type Target = InnerDB; - - fn deref(&self) -> &Self::Target { - self.0.deref() - } -} - -impl BitcoinDB { - /// - /// This is the main structure for reading Bitcoin blockchain data. - /// - /// Instantiating this class by passing the `-datadir` directory of - /// Bitcoin core to the `new()` method. - /// `tx_index`: whether to try to open tx_index levelDB. - /// - pub fn new(p: &Path, tx_index: bool) -> OpResult { - if !p.exists() { - return Err(OpError::from("data_dir does not exist")); - } - let blk_path = p.join("blocks"); - let index_path = blk_path.join("index"); - let blocks_indexes = BlocksIndexes::new(index_path.as_path())?; - let tx_db = if tx_index { - let tx_index_path = p.join("indexes").join("txindex"); - TxDB::new(&tx_index_path) - } else { - TxDB::null() - }; - let inner = InnerDB { - blocks_indexes, - blk_files: BlkFiles::new(blk_path.as_path())?, - tx_db, - }; - Ok(BitcoinDB(Arc::new(inner))) - } - - /// - /// Get the maximum number of blocks downloaded. - /// - /// This API guarantee that block 0 to `get_block_count() - 1` - /// have been downloaded and available for query. - /// - pub fn get_block_count(&self) -> usize { - let records = self.blocks_indexes.len(); - for h in 0..records { - // n_tx == 0 indicates that the block is not downloaded - if self.blocks_indexes.get(h).unwrap().n_tx == 0 { - return h; - } - } - records - } - - /// - /// Get a block - /// - pub fn get_block(&self, height: usize) -> OpResult { - if let Some(index) = self.blocks_indexes.get(height) { - Ok(self.blk_files.read_block(index.n_file, index.n_data_pos)?) - } else { - Err(OpError::from("height not found")) - } - } - - /// - /// Get a transaction by providing txid. - /// - /// This function requires `txindex` to be set to `true` for `BitcoinDB`, - /// and requires that flag `txindex=1` has been enabled when - /// running Bitcoin Core. - /// - /// A transaction cannot be found using this function if it is - /// not yet indexed using `txindex`. - /// - pub fn get_transaction(&self, txid: &Txid) -> OpResult { - if !self.tx_db.is_open() { - return Err(OpError::from("TxDB not open")); - } - - // give special treatment for genesis transaction - if self.tx_db.is_genesis_tx(txid) { - return Ok(self.get_block(0)?.txdata.swap_remove(0)); - } - - let record = self.tx_db.get_tx_record(txid)?; - - self.blk_files - .read_transaction(record.n_file, record.n_pos, record.n_tx_offset) - } - - /// - /// Iterate through all blocks from `start` to `end` (excluded). - /// - /// # Performance - /// - /// This iterator is implemented to read the blocks in concurrency, - /// but the result is still produced in sequential order. - /// Results read are stored in a synced queue for `next()` - /// to get. - /// - /// The iterator stops automatically when a block cannot be - /// read (i.e., when the max height in the database met). - /// - /// This is a very efficient implementation. - /// Using SSD and intel core i7 (4 core, 8 threads) - /// Iterating from height 0 to 700000 takes about 10 minutes. - /// - pub fn iter_block(&self, start: usize, end: usize) -> BlockIter { - BlockIter::from_range(self, start, end) - } - - pub fn check_if_txout_value_is_zero(&self, txid: &Txid, vout: usize) -> bool { - self.get_transaction(txid) - .unwrap() - .output - .get(vout) - .unwrap() - .to_owned() - .value - .to_sat() - == 0 - } -} diff --git a/parser/src/bitcoin/db/reader.rs b/parser/src/bitcoin/db/reader.rs deleted file mode 100644 index fd6593e45..000000000 --- a/parser/src/bitcoin/db/reader.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::{fs::File, io::BufReader}; - -use bitcoin::{block::Header, consensus::Decodable, io::Cursor, Block, Transaction}; -use byteorder::{LittleEndian, ReadBytesExt}; - -use super::OpResult; - -/// -/// binary file read utilities. -/// -pub trait BlockchainRead { - #[inline] - fn read_varint(&mut self) -> OpResult - where - Self: bitcoin::io::Read, - { - let mut n = 0; - loop { - let ch_data = self.read_u8()?; - n = (n << 7) | (ch_data & 0x7F) as usize; - if ch_data & 0x80 > 0 { - n += 1; - } else { - break; - } - } - Ok(n) - } - - #[inline] - fn read_u8(&mut self) -> OpResult - where - Self: bitcoin::io::Read, - { - let mut slice = [0u8; 1]; - - self.read_exact(&mut slice).unwrap(); - - Ok(slice[0]) - } - - #[inline] - fn read_u32(&mut self) -> OpResult - where - Self: std::io::Read, - { - let u = ReadBytesExt::read_u32::(self)?; - - Ok(u) - } - - #[inline] - fn read_u8_vec(&mut self, count: u32) -> OpResult> - where - Self: bitcoin::io::Read, - { - let mut arr = vec![0u8; count as usize]; - - self.read_exact(&mut arr).unwrap(); - - Ok(arr) - } - - #[inline] - fn read_block(&mut self) -> OpResult - where - Self: bitcoin::io::BufRead, - { - Ok(Block::consensus_decode(self)?) - } - - #[inline] - fn read_transaction(&mut self) -> OpResult - where - Self: bitcoin::io::BufRead, - { - Ok(Transaction::consensus_decode(self)?) - } - - #[inline] - fn read_block_header(&mut self) -> OpResult
- where - Self: bitcoin::io::BufRead, - { - Ok(Header::consensus_decode(self)?) - } -} - -impl BlockchainRead for Cursor {} -impl BlockchainRead for BufReader {} diff --git a/parser/src/bitcoin/db/txdb.rs b/parser/src/bitcoin/db/txdb.rs deleted file mode 100644 index 89db28380..000000000 --- a/parser/src/bitcoin/db/txdb.rs +++ /dev/null @@ -1,147 +0,0 @@ -use std::{path::Path, str::FromStr}; - -use bitcoin::{hashes::Hash, io::Cursor, Txid}; -use leveldb::{ - database::Database, - kv::KV, - options::{Options, ReadOptions}, -}; - -use crate::utils::log; - -use super::{BlockchainRead, OpError, OpResult}; - -const GENESIS_TXID: &str = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"; - -/// -/// tx-index: looking up transaction position using txid. -/// -/// This is possible if Bitcoin Core has `txindex=1`. -/// -pub struct TxDB { - db: Option>, - genesis_txid: Txid, -} - -/// Records transaction storage on disk -pub struct TransactionRecord { - pub txid: Txid, - pub n_file: i32, - pub n_pos: u32, - pub n_tx_offset: u32, -} - -impl TransactionRecord { - fn from(key: &[u8], values: &[u8]) -> OpResult { - let mut reader = Cursor::new(values); - Ok(TransactionRecord { - txid: Txid::from_slice(key).unwrap(), - n_file: reader.read_varint()? as i32, - n_pos: reader.read_varint()? as u32, - n_tx_offset: reader.read_varint()? as u32, - }) - } -} - -impl TxDB { - /// initialize TxDB for transaction queries - pub fn new(path: &Path) -> TxDB { - let option_db = TxDB::try_open_db(path); - if let Some(db) = option_db { - TxDB { - db: Some(db), - genesis_txid: Txid::from_str(GENESIS_TXID).unwrap(), - } - } else { - TxDB::null() - } - } - - #[inline] - pub fn is_open(&self) -> bool { - self.db.is_some() - } - - #[inline] - pub fn null() -> TxDB { - TxDB { - db: None, - genesis_txid: Txid::from_str(GENESIS_TXID).unwrap(), - } - } - - #[inline] - /// - /// genesis tx is not included in UTXO because of Bitcoin Core Bug - /// - pub fn is_genesis_tx(&self, txid: &Txid) -> bool { - txid == &self.genesis_txid - } - - fn try_open_db(path: &Path) -> Option> { - if !path.exists() { - log("Failed to open tx_index DB: tx_index not built"); - - return None; - } - let options = Options::new(); - match Database::open(path, options) { - Ok(db) => { - log("Successfully opened tx_index DB!"); - - Some(db) - } - Err(e) => { - log(&format!("Failed to open tx_index DB: {:?}", e)); - - None - } - } - } - - /// note that this function cannot find genesis block, which needs special treatment - pub fn get_tx_record(&self, txid: &Txid) -> OpResult { - if let Some(db) = &self.db { - let inner = txid.as_byte_array(); - let mut key = Vec::with_capacity(inner.len() + 1); - key.push(b't'); - key.extend(inner); - let key = TxKey { key }; - let read_options = ReadOptions::new(); - match db.get(read_options, &key) { - Ok(value) => { - if let Some(value) = value { - Ok(TransactionRecord::from(&key.key[1..], value.as_slice())?) - } else { - Err(OpError::from( - format!("value not found for txid: {}", txid).as_str(), - )) - } - } - Err(e) => Err(OpError::from( - format!("value not found for txid: {}", e).as_str(), - )), - } - } else { - Err(OpError::from("TxDB not open")) - } - } -} - -/// levelDB key utility -struct TxKey { - key: Vec, -} - -/// levelDB key utility -impl db_key::Key for TxKey { - fn from_u8(key: &[u8]) -> Self { - TxKey { - key: Vec::from(key), - } - } - - fn as_slice T>(&self, f: F) -> T { - f(&self.key) - } -} diff --git a/parser/src/bitcoin/mod.rs b/parser/src/bitcoin/mod.rs deleted file mode 100644 index 1f3d18a9e..000000000 --- a/parser/src/bitcoin/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod addresses; -mod consts; -mod daemon; -mod db; - -pub use addresses::*; -pub use consts::*; -pub use daemon::*; -pub use db::*; diff --git a/parser/src/databases/txid_to_tx_data.rs b/parser/src/databases/txid_to_tx_data.rs index 3e7ac320b..dc0fe8224 100644 --- a/parser/src/databases/txid_to_tx_data.rs +++ b/parser/src/databases/txid_to_tx_data.rs @@ -5,7 +5,7 @@ use std::{ }; use allocative::Allocative; -use bitcoin::Txid; +use biter::bitcoin::Txid; use rayon::prelude::*; use crate::structs::{Date, Height, TxData}; diff --git a/parser/src/datasets/mining.rs b/parser/src/datasets/mining.rs index 784054742..17f70623c 100644 --- a/parser/src/datasets/mining.rs +++ b/parser/src/datasets/mining.rs @@ -3,13 +3,15 @@ use itertools::Itertools; use ordered_float::OrderedFloat; use crate::{ - bitcoin::TARGET_BLOCKS_PER_DAY, datasets::AnyDataset, structs::{ date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey, }, - utils::{BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, + utils::{ + BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, + TARGET_BLOCKS_PER_DAY, + }, }; use super::{ diff --git a/parser/src/lib.rs b/parser/src/lib.rs index f313e75a4..8fcc22535 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,5 +1,4 @@ mod actions; -mod bitcoin; mod databases; mod datasets; mod io; @@ -10,12 +9,11 @@ mod utils; pub use crate::{ actions::iter_blocks, - bitcoin::{BitcoinDB, BitcoinDaemon}, datasets::OHLC, io::{Binary, Json, Serialization}, structs::{ Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec, HEIGHT_MAP_CHUNK_SIZE, }, - utils::log, + utils::{create_rpc, log}, }; diff --git a/parser/src/main.rs b/parser/src/main.rs index 605748896..52d271bc5 100644 --- a/parser/src/main.rs +++ b/parser/src/main.rs @@ -1,51 +1,30 @@ -use std::{path::Path, thread::sleep, time::Duration}; +use std::{thread::sleep, time::Duration}; -use parser::{iter_blocks, log, BitcoinDB, BitcoinDaemon, Config, Height}; +use biter::bitcoincore_rpc::RpcApi; +use parser::{create_rpc, iter_blocks, log, Config}; fn main() -> color_eyre::Result<()> { color_eyre::install()?; - let config = Config::read(); + let config = Config::import(); - if config.datadir.is_none() { - println!( - "You need to set the --datadir parameter at least once to run the parser.\nRun the program with '-h' for help." - ); - std::process::exit(1); - } - - config.write()?; - - let daemon = BitcoinDaemon::new(&config); + let rpc = create_rpc(&config).unwrap(); loop { - daemon.stop(); + let block_count = rpc.get_blockchain_info().unwrap().blocks as usize; - // Scoped to free bitcoin's lock - let block_count = { - let bitcoin_db = BitcoinDB::new(Path::new(&config.datadir.as_ref().unwrap()), true)?; + log(&format!("{block_count} blocks found.")); - // let block_count = 200_000; - let block_count = bitcoin_db.get_block_count(); - - log(&format!("{block_count} blocks found.")); - - iter_blocks(&bitcoin_db, block_count)?; - - block_count - }; - - daemon.start(); - - if daemon.check_if_fully_synced() { - daemon.wait_for_new_block(Height::new(block_count as u32 - 1)); - } else { - daemon.wait_sync(); - } + iter_blocks(&config, &rpc, block_count)?; if let Some(delay) = config.delay { sleep(Duration::from_secs(delay)) } + + log("Waiting for new block..."); + while block_count == rpc.get_blockchain_info().unwrap().blocks as usize { + sleep(Duration::from_secs(5)) + } } // Ok(()) diff --git a/parser/src/structs/address.rs b/parser/src/structs/address.rs index 6d2603524..2d18f207c 100644 --- a/parser/src/structs/address.rs +++ b/parser/src/structs/address.rs @@ -1,10 +1,10 @@ -use bitcoin::TxOut; use bitcoin_hashes::{hash160, Hash}; +use biter::bitcoin::TxOut; use itertools::Itertools; use crate::{ - bitcoin::multisig_addresses, databases::{U8x19, U8x31, SANAKIRJA_MAX_KEY_SIZE}, + utils::multisig_addresses, }; use super::{AddressType, Counter}; diff --git a/parser/src/structs/amount.rs b/parser/src/structs/amount.rs index 3691f0135..26d43167c 100644 --- a/parser/src/structs/amount.rs +++ b/parser/src/structs/amount.rs @@ -10,7 +10,7 @@ use bincode::{ error::{DecodeError, EncodeError}, BorrowDecode, Decode, Encode, }; -use bitcoin::Amount as BitcoinAmount; +use biter::bitcoin::Amount as BitcoinAmount; use derive_deref::{Deref, DerefMut}; use sanakirja::{direct_repr, Storable, UnsizedStorable}; use serde::{Deserialize, Serialize}; diff --git a/parser/src/structs/bi_map.rs b/parser/src/structs/bi_map.rs index 01d626f23..2847cbed1 100644 --- a/parser/src/structs/bi_map.rs +++ b/parser/src/structs/bi_map.rs @@ -5,7 +5,7 @@ use std::{ use allocative::Allocative; -use crate::{bitcoin::TARGET_BLOCKS_PER_DAY, utils::LossyFrom}; +use crate::utils::{LossyFrom, TARGET_BLOCKS_PER_DAY}; use super::{AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapValue}; diff --git a/parser/src/structs/config.rs b/parser/src/structs/config.rs index ed735ca74..e4b53fe94 100644 --- a/parser/src/structs/config.rs +++ b/parser/src/structs/config.rs @@ -6,17 +6,21 @@ use serde::{Deserialize, Serialize}; #[derive(Parser, Debug, Clone, Default, Serialize, Deserialize)] #[command(version, about, long_about = None)] pub struct Config { - /// bitcoind `-datadir=` argument + /// Bitcoin data directory path #[arg(long, value_name = "DIR")] pub datadir: Option, - /// bitcoind `-blocksonly` argument, default: true - #[arg(long)] - pub blocksonly: Option, + /// Bitcoin RPC port, default: 8332 + #[arg(long, value_name = "PORT")] + pub rpcport: Option, - /// bitcoind `-rpcconnect=` argument - #[arg(long, value_name = "IP")] - pub rpcconnect: Option, + /// Bitcoin RPC username + #[arg(long, value_name = "USERNAME")] + pub rpcuser: Option, + + /// Bitcoin RPC password + #[arg(long, value_name = "PASSWORD")] + pub rpcpassword: Option, /// Delay between runs, default: 0 #[arg(long, value_name = "SECONDS")] @@ -26,7 +30,7 @@ pub struct Config { impl Config { const PATH: &'static str = "config.toml"; - pub fn read() -> Self { + pub fn import() -> Self { let mut config_saved = fs::read_to_string(Self::PATH) .map_or(Config::default(), |contents| { toml::from_str(&contents).unwrap_or_default() @@ -38,24 +42,58 @@ impl Config { config_saved.datadir = Some(datadir); } - if let Some(blocksonly) = config_args.blocksonly { - config_saved.blocksonly = Some(blocksonly); + if let Some(rpcport) = config_args.rpcport { + config_saved.rpcport = Some(rpcport); } else { - config_saved.blocksonly = Some(true); + config_saved.rpcport = Some(8332); } - if let Some(rpcconnect) = config_args.rpcconnect { - config_saved.rpcconnect = Some(rpcconnect); + if let Some(rpcuser) = config_args.rpcuser { + config_saved.rpcuser = Some(rpcuser); + } + + if let Some(rpcpassword) = config_args.rpcpassword { + config_saved.rpcpassword = Some(rpcpassword); } if let Some(delay) = config_args.delay { config_saved.delay = Some(delay); } - config_saved + // Done importing + + let config = config_saved; + + config.check(); + + config.write().unwrap(); + + config } - pub fn write(&self) -> std::io::Result<()> { + fn check(&self) { + if self.datadir.is_none() { + Self::exit("datadir"); + } + + if self.rpcuser.is_none() { + Self::exit("rpcuser"); + } + + if self.rpcpassword.is_none() { + Self::exit("rpcpassword"); + } + } + + fn exit(attribute: &str) { + println!( + "You need to set the --{} parameter at least once to run the parser.\nRun the program with '-h' for help.", attribute + ); + + std::process::exit(1); + } + + fn write(&self) -> std::io::Result<()> { fs::write(Self::PATH, toml::to_string(self).unwrap()) } } diff --git a/parser/src/structs/height.rs b/parser/src/structs/height.rs index 79dcf9b07..b3d811eff 100644 --- a/parser/src/structs/height.rs +++ b/parser/src/structs/height.rs @@ -5,10 +5,11 @@ use std::{ use allocative::Allocative; use bincode::{Decode, Encode}; +use biter::NUMBER_OF_UNSAFE_BLOCKS; use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; -use crate::{bitcoin::NUMBER_OF_UNSAFE_BLOCKS, HEIGHT_MAP_CHUNK_SIZE}; +use crate::HEIGHT_MAP_CHUNK_SIZE; use super::{HeightMapChunkId, MapKey}; diff --git a/parser/src/utils/arr.rs b/parser/src/utils/arr.rs deleted file mode 100644 index 80f91af42..000000000 --- a/parser/src/utils/arr.rs +++ /dev/null @@ -1,219 +0,0 @@ -use std::{ - iter::Sum, - ops::{Add, AddAssign, Div, Mul, Sub, SubAssign}, -}; - -use itertools::Itertools; -use ordered_float::{FloatCore, OrderedFloat}; - -use super::ToF32; - -pub trait ArrayOperations { - fn transform(&self, transform: F) -> Vec - where - T: Copy + Default, - F: Fn((usize, &T, &[T])) -> T; - - fn add(&self, other: &[T]) -> Vec - where - T: Add + Copy + Default; - - fn subtract(&self, other: &[T]) -> Vec - where - T: Sub + Copy + Default; - - fn multiply(&self, other: &[T]) -> Vec - where - T: Mul + Copy + Default; - - fn divide(&self, other: &[T]) -> Vec - where - T: Div + Copy + Default; - - fn match_size<'a>(&'a self, other: &'a [T]) -> &'a [T]; - - fn cumulate(&self) -> Vec - where - T: Sum + Copy + Default + AddAssign; - - fn last_x_sum(&self, x: usize) -> Vec - where - T: Sum + Copy + Default + AddAssign + SubAssign; - - fn moving_average(&self, x: usize) -> Vec - where - T: Sum + Copy + Default + AddAssign + SubAssign + ToF32; - - fn net_change(&self, offset: usize) -> Vec - where - T: Copy + Default + Sub; - - fn median(&self, size: usize) -> Vec> - where - T: FloatCore; -} - -impl ArrayOperations for &[T] { - fn transform(&self, transform: F) -> Vec - where - T: Copy + Default, - F: Fn((usize, &T, &[T])) -> T, - { - self.iter() - .enumerate() - .map(|(index, value)| transform((index, value, self))) - .collect_vec() - } - - fn add(&self, other: &[T]) -> Vec - where - T: Add + Copy + Default, - { - self.match_size(other) - .transform(|(index, value, _)| *value + *other.get(index).unwrap()) - } - - fn subtract(&self, other: &[T]) -> Vec - where - T: Sub + Copy + Default, - { - self.match_size(other) - .transform(|(index, value, _)| *value - *other.get(index).unwrap()) - } - - fn multiply(&self, other: &[T]) -> Vec - where - T: Mul + Copy + Default, - { - self.match_size(other) - .transform(|(index, value, _)| *value * *other.get(index).unwrap()) - } - - fn divide(&self, other: &[T]) -> Vec - where - T: Div + Copy + Default, - { - self.match_size(other) - .transform(|(index, value, _)| *value / *other.get(index).unwrap()) - } - - fn match_size(&self, other: &[T]) -> &[T] { - let len = other.len(); - if self.len() > len { - &self[..len] - } else { - self - } - } - - fn cumulate(&self) -> Vec - where - T: Sum + Copy + Default + AddAssign, - { - let mut sum = T::default(); - - self.iter() - .map(|value| { - sum += *value; - sum - }) - .collect_vec() - } - - fn last_x_sum(&self, x: usize) -> Vec - where - T: Sum + Copy + Default + AddAssign + SubAssign, - { - let mut sum = T::default(); - - self.iter() - .enumerate() - .map(|(index, value)| { - sum += *value; - - if index >= x - 1 { - let previous_index = index + 1 - x; - - sum -= *self.get(previous_index).unwrap() - } - - sum - }) - .collect_vec() - } - - fn moving_average(&self, x: usize) -> Vec - where - T: Sum + Copy + Default + AddAssign + SubAssign + ToF32, - { - let mut sum = T::default(); - - self.iter() - .enumerate() - .map(|(index, value)| { - sum += *value; - - if index >= x - 1 { - sum -= *self.get(index + 1 - x).unwrap() - } - - sum.to_f32() / x as f32 - }) - .collect_vec() - } - - fn net_change(&self, offset: usize) -> Vec - where - T: Copy + Default + Sub, - { - self.transform(|(index, value, arr)| { - let previous = { - if let Some(previous_index) = index.checked_sub(offset) { - *arr.get(previous_index).unwrap() - } else { - T::default() - } - }; - - *value - previous - }) - } - - fn median(&self, size: usize) -> Vec> - where - T: FloatCore, - { - let even = size % 2 == 0; - let median_index = size / 2; - - if size < 3 { - panic!("Computing a median for a size lower than 3 is useless"); - } - - self.iter() - .enumerate() - .map(|(index, _)| { - if index >= size - 1 { - let mut arr = self[index - (size - 1)..index + 1] - .iter() - .map(|value| OrderedFloat(*value)) - .collect_vec(); - - arr.sort_unstable(); - - if even { - Some( - (**arr.get(median_index).unwrap() - + **arr.get(median_index - 1).unwrap()) - / T::from(2.0).unwrap(), - ) - } else { - Some(**arr.get(median_index).unwrap()) - } - } else { - None - } - }) - .collect() - } -} diff --git a/parser/src/utils/bytes.rs b/parser/src/utils/bytes.rs deleted file mode 100644 index 9f51f086b..000000000 --- a/parser/src/utils/bytes.rs +++ /dev/null @@ -1 +0,0 @@ -pub const BYTES_IN_MB: usize = 1_000_000; diff --git a/parser/src/utils/date.rs b/parser/src/utils/consts.rs similarity index 82% rename from parser/src/utils/date.rs rename to parser/src/utils/consts.rs index e9e4dc6e4..a77f88818 100644 --- a/parser/src/utils/date.rs +++ b/parser/src/utils/consts.rs @@ -1,3 +1,7 @@ +pub const BYTES_IN_MB: usize = 1_000_000; + +pub const TARGET_BLOCKS_PER_DAY: usize = 144; + pub const ONE_DAY_IN_DAYS: usize = 1; pub const ONE_WEEK_IN_DAYS: usize = 7; pub const TWO_WEEK_IN_DAYS: usize = 2 * ONE_WEEK_IN_DAYS; diff --git a/parser/src/utils/log.rs b/parser/src/utils/log.rs index 7d8aacfd0..c9695d5bc 100644 --- a/parser/src/utils/log.rs +++ b/parser/src/utils/log.rs @@ -1,4 +1,4 @@ -use std::{fs::OpenOptions, io::Write, process::Output}; +use std::{fs::OpenOptions, io::Write}; use chrono::Local; use color_eyre::owo_colors::OwoColorize; @@ -23,13 +23,3 @@ pub fn log(str: &str) { println!("{} {}", date_time.bright_black(), line); }); } - -pub fn log_output(output: &Output) { - if !output.stdout.is_empty() { - log(&String::from_utf8_lossy(&output.stdout)); - } - - if !output.stderr.is_empty() { - log(&String::from_utf8_lossy(&output.stderr)); - } -} diff --git a/parser/src/utils/mod.rs b/parser/src/utils/mod.rs index 0a9f8a113..eb0bcbd87 100644 --- a/parser/src/utils/mod.rs +++ b/parser/src/utils/mod.rs @@ -1,17 +1,19 @@ -mod bytes; -mod date; +mod consts; mod flamegraph; mod log; mod lossy; +mod multisig; mod percentile; mod retry; +mod rpc; mod time; -pub use bytes::*; -pub use date::*; +pub use consts::*; pub use flamegraph::*; pub use log::*; pub use lossy::*; +pub use multisig::*; pub use percentile::*; pub use retry::*; +pub use rpc::*; pub use time::*; diff --git a/parser/src/bitcoin/addresses/multisig.rs b/parser/src/utils/multisig.rs similarity index 98% rename from parser/src/bitcoin/addresses/multisig.rs rename to parser/src/utils/multisig.rs index cc2292cee..22f8520de 100644 --- a/parser/src/bitcoin/addresses/multisig.rs +++ b/parser/src/utils/multisig.rs @@ -2,7 +2,7 @@ // Code from bitcoin-explorer now deprecated // -use bitcoin::{ +use biter::bitcoin::{ blockdata::{ opcodes::all, script::Instruction::{self, Op, PushBytes}, diff --git a/parser/src/utils/rpc.rs b/parser/src/utils/rpc.rs new file mode 100644 index 000000000..92934127e --- /dev/null +++ b/parser/src/utils/rpc.rs @@ -0,0 +1,13 @@ +use biter::bitcoincore_rpc::{Auth, Client}; + +use crate::Config; + +pub fn create_rpc(config: &Config) -> color_eyre::Result { + Ok(Client::new( + &format!("http://localhost:{}", config.rpcport.unwrap()), + Auth::UserPass( + config.rpcuser.clone().unwrap(), + config.rpcpassword.clone().unwrap(), + ), + )?) +} diff --git a/server/Cargo.lock b/server/Cargo.lock index 1a09858de..6101727b9 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -27,7 +27,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -258,6 +258,12 @@ dependencies = [ "bitcoin_hashes", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.0" @@ -342,6 +348,45 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoincore-rpc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" +dependencies = [ + "bitcoincore-rpc-json", + "jsonrpc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoincore-rpc-json" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" +dependencies = [ + "bitcoin", + "serde", + "serde_json", +] + +[[package]] +name = "biter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2ea6b4c4046feaaf73f3dff1cc648328af021d50be95946e0e7a807c985b47" +dependencies = [ + "bitcoin", + "bitcoincore-rpc", + "crossbeam", + "derived-deref", + "rayon", + "serde", + "serde_json", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -470,15 +515,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" -[[package]] -name = "cmake" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" -dependencies = [ - "cc", -] - [[package]] name = "color-eyre" version = "0.6.3" @@ -617,12 +653,6 @@ dependencies = [ "parking_lot_core 0.9.9", ] -[[package]] -name = "db-key" -version = "0.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72465f46d518f6015d9cf07f7f3013a95dd6b9c2747c3d65ae0cce43929d14f" - [[package]] name = "derive_deref" version = "1.1.1" @@ -634,6 +664,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derived-deref" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "either" version = "1.12.0" @@ -700,12 +741,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" -[[package]] -name = "ffi-opaque" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" - [[package]] name = "flate2" version = "1.0.28" @@ -1132,35 +1167,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpc" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +dependencies = [ + "base64 0.13.1", + "minreq", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "leveldb" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32651baaaa5596b3a6e0bee625e73fd0334c167db0ea5ac68750ef9a629a2d6a" -dependencies = [ - "db-key", - "leveldb-sys", - "libc", -] - -[[package]] -name = "leveldb-sys" -version = "2.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd94a4d0242a437e5e41a27c782b69a624469ca1c4d1e5cb3c337f74a8031d4" -dependencies = [ - "cmake", - "ffi-opaque", - "libc", - "num_cpus", -] - [[package]] name = "libc" version = "0.2.153" @@ -1236,14 +1260,26 @@ dependencies = [ ] [[package]] -name = "mio" -version = "0.8.11" +name = "minreq" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0" dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "mio" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1283,16 +1319,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.32.2" @@ -1367,16 +1393,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "par-iter-sync" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa981aaed94bf59211f644922155e8a33bcb01ed662cd63426653187f562790" -dependencies = [ - "crossbeam", - "num_cpus", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -1431,20 +1447,16 @@ version = "0.3.0" dependencies = [ "allocative", "bincode", - "bitcoin", "bitcoin_hashes", - "byteorder", + "biter", "chrono", "clap", "color-eyre", - "db-key", "derive_deref", "inferno", "itertools", - "leveldb", "memory-stats", "ordered-float", - "par-iter-sync", "rayon", "reqwest", "sanakirja", @@ -1497,6 +1509,15 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "ppv-lite86" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +dependencies = [ + "zerocopy 0.6.6", +] + [[package]] name = "proc-macro2" version = "1.0.81" @@ -1524,6 +1545,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.10.0" @@ -1597,7 +1648,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ - "base64", + "base64 0.22.0", "bytes", "encoding_rs", "futures-channel", @@ -1697,7 +1748,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64", + "base64 0.22.0", "rustls-pki-types", ] @@ -1773,6 +1824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3" dependencies = [ "bitcoin_hashes", + "rand", "secp256k1-sys", "serde", ] @@ -1831,11 +1883,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -2070,28 +2123,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -2610,13 +2662,34 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "zerocopy" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +dependencies = [ + "byteorder", + "zerocopy-derive 0.6.6", +] + [[package]] name = "zerocopy" version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy-derive" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", ] [[package]]