Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-10 06:53:33 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: switch to biter

Browse Source
This commit is contained in:
k
2024-08-03 15:17:57 +02:00
parent afacea3fbb
commit 9067c28d24
35 changed files with 494 additions and 1589 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+4
View File
@@ -11,3 +11,7 @@ TODO.md
.stfolder
/charts
/price
*\ copy*
/dist
/start-node.sh
CHANGELOG.md
+4
View File
@@ -4,7 +4,11 @@
![Image of the Satonomics Web App version 0.X.Y](./assets/v0.X.Y.jpg)
### Parser
- Change the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
- Use the Bitcoin RPC server for various calls instead of running commands and then parsing the JSON from the output
- Updated the config, run with `-h` to see
## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
app/src/app/components/frames/settings.tsx
+14
View File
@@ -165,6 +165,20 @@ export function SettingsFrame({
url: "https://primal.net/p/npub1mnwjn40hr042rsmzu64rsnwsw07uegg4tjkv620c94p6e797wkvq3qeujc",
amount: 5_000,
},
{
name: "lassdas",
url: "https://primal.net/p/npub1gmhctt2hmjqz8ay2x8h5f8fl3h4fpfcezwqneal3usu3u65qca4s8094ea",
amount: 210_000,
},
{
name: "anon",
amount: 21_000,
},
{
name: "xplbzx",
url: "https://primal.net/p/npub1e0f808a350rxrhppu4zylzljt3arfpvrrpqdg6ft78xy6u49kq5slf0g92",
amount: 10_110,
},
]
.sort((a, b) =>
b.amount !== a.amount
parser/Cargo.lock
Generated
+156 -82
View File
@@ -27,7 +27,7 @@ dependencies = [
"getrandom",
"once_cell",
"version_check",
"zerocopy",
"zerocopy 0.7.35",
]
[[package]]
@@ -152,6 +152,12 @@ dependencies = [
"bitcoin_hashes",
]
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.22.0"
@@ -236,6 +242,45 @@ dependencies = [
"serde",
]
[[package]]
name = "bitcoincore-rpc"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee"
dependencies = [
"bitcoincore-rpc-json",
"jsonrpc",
"log",
"serde",
"serde_json",
]
[[package]]
name = "bitcoincore-rpc-json"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a"
dependencies = [
"bitcoin",
"serde",
"serde_json",
]
[[package]]
name = "biter"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd2ea6b4c4046feaaf73f3dff1cc648328af021d50be95946e0e7a807c985b47"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"crossbeam",
"derived-deref",
"rayon",
"serde",
"serde_json",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -339,15 +384,6 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
[[package]]
name = "cmake"
version = "0.1.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
dependencies = [
"cc",
]
[[package]]
name = "color-eyre"
version = "0.6.3"
@@ -477,12 +513,6 @@ dependencies = [
"parking_lot_core 0.9.10",
]
[[package]]
name = "db-key"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b72465f46d518f6015d9cf07f7f3013a95dd6b9c2747c3d65ae0cce43929d14f"
[[package]]
name = "derive_deref"
version = "1.1.1"
@@ -494,6 +524,17 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "derived-deref"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
name = "either"
version = "1.12.0"
@@ -560,12 +601,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
[[package]]
name = "ffi-opaque"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a"
[[package]]
name = "fnv"
version = "1.0.7"
@@ -966,35 +1001,24 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "jsonrpc"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
dependencies = [
"base64 0.13.1",
"minreq",
"serde",
"serde_json",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "leveldb"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32651baaaa5596b3a6e0bee625e73fd0334c167db0ea5ac68750ef9a629a2d6a"
dependencies = [
"db-key",
"leveldb-sys",
"libc",
]
[[package]]
name = "leveldb-sys"
version = "2.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd94a4d0242a437e5e41a27c782b69a624469ca1c4d1e5cb3c337f74a8031d4"
dependencies = [
"cmake",
"ffi-opaque",
"libc",
"num_cpus",
]
[[package]]
name = "libc"
version = "0.2.155"
@@ -1064,14 +1088,26 @@ dependencies = [
]
[[package]]
name = "mio"
version = "0.8.11"
name = "minreq"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0"
dependencies = [
"log",
"serde",
"serde_json",
]
[[package]]
name = "mio"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
dependencies = [
"hermit-abi",
"libc",
"wasi",
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -1111,16 +1147,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "object"
version = "0.32.2"
@@ -1195,16 +1221,6 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "par-iter-sync"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efa981aaed94bf59211f644922155e8a33bcb01ed662cd63426653187f562790"
dependencies = [
"crossbeam",
"num_cpus",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@@ -1249,20 +1265,16 @@ version = "0.3.0"
dependencies = [
"allocative",
"bincode",
"bitcoin",
"bitcoin_hashes",
"byteorder",
"biter",
"chrono",
"clap",
"color-eyre",
"db-key",
"derive_deref",
"inferno",
"itertools",
"leveldb",
"memory-stats",
"ordered-float",
"par-iter-sync",
"rayon",
"reqwest",
"sanakirja",
@@ -1315,6 +1327,15 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "ppv-lite86"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f"
dependencies = [
"zerocopy 0.6.6",
]
[[package]]
name = "proc-macro2"
version = "1.0.85"
@@ -1342,6 +1363,36 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.10.0"
@@ -1386,7 +1437,7 @@ version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
dependencies = [
"base64",
"base64 0.22.0",
"bytes",
"encoding_rs",
"futures-channel",
@@ -1486,7 +1537,7 @@ version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
dependencies = [
"base64",
"base64 0.22.0",
"rustls-pki-types",
]
@@ -1556,6 +1607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3"
dependencies = [
"bitcoin_hashes",
"rand",
"secp256k1-sys",
"serde",
]
@@ -1614,11 +1666,12 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.120"
version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
@@ -1810,9 +1863,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.37.0"
version = "1.39.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787"
checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
dependencies = [
"backtrace",
"bytes",
@@ -1820,7 +1873,7 @@ dependencies = [
"mio",
"pin-project-lite",
"socket2",
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -2315,13 +2368,34 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "zerocopy"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6"
dependencies = [
"byteorder",
"zerocopy-derive 0.6.6",
]
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"zerocopy-derive",
"zerocopy-derive 0.7.35",
]
[[package]]
name = "zerocopy-derive"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
parser/Cargo.toml
+1 -5
View File
@@ -11,20 +11,16 @@ edition = "2021"
[dependencies]
allocative = "0.3.3"
bincode = { git = "https://github.com/bincode-org/bincode.git" }
bitcoin = { version = "0.32.2", features = ["serde"] }
biter = "0.1.0"
bitcoin_hashes = { version = "0.14.0" }
byteorder = "1.5.0"
chrono = { version = "0.4.38", features = ["serde"] }
clap = { version = "4.5.11", features = ["derive"] }
color-eyre = "0.6.3"
db-key = "=0.0.5"
derive_deref = "1.1.1"
inferno = "0.11.20"
itertools = "0.13.0"
leveldb = "0.8.6"
memory-stats = "1.2.0"
ordered-float = "4.2.1"
par-iter-sync = "0.1.11"
rayon = "1.10.0"
reqwest = { version = "0.12.5", features = ["blocking", "json"] }
sanakirja = "1.4.2"
parser/src/actions/iter_blocks.rs
+32 -10
View File
@@ -8,15 +8,22 @@ use parse::ParseData;
use crate::{
actions::{export, find_first_inserted_unsafe_height, parse},
bitcoin::BitcoinDB,
create_rpc,
databases::Databases,
datasets::{AllDatasets, ComputeData},
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{Date, DateData, MapKey},
utils::{generate_allocation_files, log, time},
Config, Height,
};
pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Result<()> {
pub fn iter_blocks(
config: &Config,
rpc: &biter::bitcoincore_rpc::Client,
approx_block_count: usize,
) -> color_eyre::Result<()> {
let export_dir = "./target/outputs";
let should_insert = true;
let should_export = true;
let study_ram_usage = false;
@@ -44,11 +51,19 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
log(&format!("Starting parsing at height: {height}"));
let mut block_iter = bitcoin_db.iter_block(height.to_usize(), block_count);
let mut next_block_opt = None;
let mut blocks_loop_date = None;
let block_receiver = biter::new(
config.datadir.as_ref().unwrap(),
export_dir,
Some(height.to_usize()),
None,
create_rpc(config).unwrap(),
);
let mut block_iter = block_receiver.iter();
'parsing: loop {
let instant = Instant::now();
@@ -67,15 +82,22 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
next_block_opt = block_iter.next();
if let Some(current_block) = current_block_opt {
if let Some((_current_block_height, current_block, _current_block_hash)) =
current_block_opt
{
let timestamp = current_block.header.time;
let current_block_date = Date::from_timestamp(timestamp);
let current_block_height = height + blocks_loop_i;
let current_block_height: Height = height + blocks_loop_i;
if current_block_height.to_usize() != _current_block_height {
dbg!(current_block_height, _current_block_height);
panic!()
}
let next_block_date = next_block_opt
.as_ref()
.map(|next_block| Date::from_timestamp(next_block.header.time));
.map(|(_, next_block, _)| Date::from_timestamp(next_block.header.time));
// Always run for the first block of the loop
if blocks_loop_date.is_none() {
@@ -139,7 +161,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
}
parse(ParseData {
bitcoin_db,
rpc,
block: current_block,
block_index: blocks_loop_i,
compute_addresses,
@@ -162,7 +184,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
let is_new_month = next_block_date
.map_or(true, |next_block_date| next_block_date.day() == 1);
if is_new_month || height.is_close_to_end(block_count) {
if is_new_month || height.is_close_to_end(approx_block_count) {
break 'days;
}
@@ -196,7 +218,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
}
if should_export {
let is_safe = height.is_safe(block_count);
let is_safe = height.is_safe(approx_block_count);
export(ExportedData {
databases: is_safe.then_some(&mut databases),
parser/src/actions/parse.rs
+22 -8
View File
@@ -1,12 +1,14 @@
use std::{collections::BTreeMap, ops::ControlFlow, thread};
use bitcoin::{Block, Txid};
use biter::{
bitcoin::{Block, Txid},
bitcoincore_rpc::RpcApi,
};
use itertools::Itertools;
use rayon::prelude::*;
use crate::{
bitcoin::BitcoinDB,
databases::{
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
@@ -23,7 +25,7 @@ use crate::{
};
pub struct ParseData<'a> {
pub bitcoin_db: &'a BitcoinDB,
// pub bitcoin_cli: &'a BitcoinCli,
pub block: Block,
pub block_index: usize,
pub compute_addresses: bool,
@@ -33,13 +35,13 @@ pub struct ParseData<'a> {
pub first_date_height: Height,
pub height: Height,
pub is_date_last_block: bool,
pub rpc: &'a biter::bitcoincore_rpc::Client,
pub states: &'a mut States,
pub timestamp: u32,
}
pub fn parse(
ParseData {
bitcoin_db,
block,
block_index,
compute_addresses,
@@ -49,6 +51,7 @@ pub fn parse(
first_date_height,
height,
is_date_last_block,
rpc,
states,
timestamp,
}: ParseData,
@@ -334,10 +337,16 @@ pub fn parse(
// Can be none because 0 sats inputs happen
// https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
if input_tx_data.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| bitcoin_db
.check_if_txout_value_is_zero(&input_txid, input_vout as usize)
|| rpc
.get_tx_out(&input_txid, input_vout, None)
.unwrap()
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
@@ -374,8 +383,13 @@ pub fn parse(
if input_amount_and_address_index.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| bitcoin_db
.check_if_txout_value_is_zero(&input_txid, input_vout as usize)
|| rpc
.get_tx_out(&input_txid, input_vout as u32, None)
.unwrap()
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
parser/src/bitcoin/addresses/mod.rs
-3
View File
@@ -1,3 +0,0 @@
mod multisig;
pub use multisig::*;
parser/src/bitcoin/consts.rs
-2
View File
@@ -1,2 +0,0 @@
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
pub const TARGET_BLOCKS_PER_DAY: usize = 144;
parser/src/bitcoin/daemon.rs
-137
View File
@@ -1,137 +0,0 @@
use std::{process::Command, thread::sleep, time::Duration};
use color_eyre::eyre::eyre;
use serde_json::Value;
use crate::{
structs::Height,
utils::{log, log_output, retry},
Config,
};
struct BlockchainInfo {
pub headers: u64,
pub blocks: u64,
}
pub struct BitcoinDaemon {
config: Config,
}
impl BitcoinDaemon {
pub fn new(config: &Config) -> Self {
Self {
config: config.clone(),
}
}
pub fn start(&self) {
sleep(Duration::from_secs(1));
let mut command = Command::new("bitcoind");
command
.arg(self.datadir_arg())
.arg("-txindex=1")
.arg("-daemon");
if self.config.blocksonly.unwrap() {
command.arg("-blocksonly");
}
if let Some(ip) = self.config.rpcconnect.as_ref() {
command.arg(format!("-rpcconnect={}", ip));
}
// bitcoind -datadir=/Users/k/Developer/bitcoin -blocksonly -txindex=1 -daemon
let output = command
.output()
.expect("bitcoind to be able to properly start");
log_output(&output);
}
pub fn stop(&self) {
// bitcoin-cli -datadir=/Users/k/Developer/bitcoin stop
let output = Command::new("bitcoin-cli")
.arg(self.datadir_arg())
.arg("stop")
.output()
.unwrap();
if output.status.success() {
log_output(&output);
sleep(Duration::from_secs(15));
}
}
pub fn wait_sync(&self) {
while !self.check_if_fully_synced() {
sleep(Duration::from_secs(5))
}
}
pub fn wait_for_new_block(&self, last_block_height: Height) {
log("Waiting for new block...");
while last_block_height == self.get_blockchain_info().headers {
sleep(Duration::from_secs(5))
}
}
pub fn check_if_fully_synced(&self) -> bool {
let BlockchainInfo { blocks, headers } = self.get_blockchain_info();
let synced = blocks == headers;
if synced {
log(&format!("Synced ! ({blocks} blocks)"));
} else {
log(&format!("Syncing... ({} remaining)", headers - blocks));
}
synced
}
fn get_blockchain_info(&self) -> BlockchainInfo {
retry(
|_| {
// bitcoin-cli -datadir=/Users/k/Developer/bitcoin getblockchaininfo
let output = Command::new("bitcoin-cli")
.arg(self.datadir_arg())
.arg("getblockchaininfo")
.output()?;
let output = String::from_utf8_lossy(&output.stdout);
let json: Value = serde_json::from_str(&output)?;
let json = json.as_object().ok_or(eyre!("json as object failed"))?;
let blocks = json
.get("blocks")
.ok_or(eyre!("get field 'blocks' from json failed"))?
.as_u64()
.ok_or(eyre!("blocks to u64 failed"))?;
let headers = json
.get("headers")
.ok_or(eyre!("get field 'headers' from json failed"))?
.as_u64()
.ok_or(eyre!("blocks to u64 failed"))?;
Ok(BlockchainInfo { headers, blocks })
},
1,
usize::MAX,
)
.unwrap()
}
fn datadir_arg(&self) -> String {
if self.config.datadir.is_none() {
unreachable!();
}
format!("-datadir={}", self.config.datadir.as_ref().unwrap())
}
}
parser/src/bitcoin/db/blk_files.rs
-152
View File
@@ -1,152 +0,0 @@
use std::{
collections::HashMap,
convert::From,
fs::{self, DirEntry, File},
io::{self, BufReader, Seek, SeekFrom},
path::{Path, PathBuf},
};
use bitcoin::{io::Cursor, Block, Transaction};
use derive_deref::{Deref, DerefMut};
use super::{
errors::{OpError, OpErrorKind, OpResult},
reader::BlockchainRead,
};
///
/// An index of all blk files found.
///
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct BlkFiles(HashMap<i32, PathBuf>);
impl BlkFiles {
///
/// Construct an index of all blk files.
///
pub fn new(path: &Path) -> OpResult<Self> {
Ok(Self(Self::scan_path(path)?))
}
///
/// Read a Block from blk file.
///
#[inline]
pub fn read_raw_block(&self, n_file: i32, offset: u32) -> OpResult<Vec<u8>> {
if let Some(blk_path) = self.get(&n_file) {
let mut r = BufReader::new(File::open(blk_path)?);
r.seek(SeekFrom::Start(offset as u64 - 4))?;
let block_size = r.read_u32()?;
let block = r.read_u8_vec(block_size)?;
Ok(block)
} else {
Err(OpError::from("blk file not found, sync with bitcoin core"))
}
}
///
/// Read a Block from blk file.
///
pub fn read_block(&self, n_file: i32, offset: u32) -> OpResult<Block> {
Cursor::new(self.read_raw_block(n_file, offset)?).read_block()
}
///
/// Read a transaction from blk file.
///
pub fn read_transaction(
&self,
n_file: i32,
n_pos: u32,
n_tx_offset: u32,
) -> OpResult<Transaction> {
if let Some(blk_path) = self.get(&n_file) {
let mut r = BufReader::new(File::open(blk_path)?);
// the size of a header is 80.
r.seek(SeekFrom::Start(n_pos as u64 + n_tx_offset as u64 + 80))?;
r.read_transaction()
} else {
Err(OpError::from("blk file not found, sync with bitcoin core"))
}
}
///
/// Scan blk folder to build an index of all blk files.
///
fn scan_path(path: &Path) -> OpResult<HashMap<i32, PathBuf>> {
let mut collected = HashMap::with_capacity(4000);
for entry in fs::read_dir(path)? {
match entry {
Ok(de) => {
let path = Self::resolve_path(&de)?;
if !path.is_file() {
continue;
};
if let Some(file_name) = path.as_path().file_name() {
if let Some(file_name) = file_name.to_str() {
if let Some(index) = Self::parse_blk_index(file_name) {
collected.insert(index, path);
}
}
}
}
Err(msg) => {
return Err(OpError::from(msg));
}
}
}
collected.shrink_to_fit();
if collected.is_empty() {
Err(OpError::new(OpErrorKind::RuntimeError).join_msg("No blk files found!"))
} else {
Ok(collected)
}
}
///
/// Resolve symlink.
///
fn resolve_path(entry: &DirEntry) -> io::Result<PathBuf> {
if entry.file_type()?.is_symlink() {
fs::read_link(entry.path())
} else {
Ok(entry.path())
}
}
///
/// Extract index from block file name.
///
fn parse_blk_index(file_name: &str) -> Option<i32> {
let prefix = "blk";
let ext = ".dat";
if file_name.starts_with(prefix) && file_name.ends_with(ext) {
file_name[prefix.len()..(file_name.len() - ext.len())]
.parse::<i32>()
.ok()
} else {
None
}
}
}
// #[cfg(test)]
// mod tests {
// use super::*;
// #[test]
// fn test_parse_blk_index() {
// assert_eq!(0, BlkFiles::parse_blk_index("blk00000.dat").unwrap());
// assert_eq!(6, BlkFiles::parse_blk_index("blk6.dat").unwrap());
// assert_eq!(1202, BlkFiles::parse_blk_index("blk1202.dat").unwrap());
// assert_eq!(
// 13412451,
// BlkFiles::parse_blk_index("blk13412451.dat").unwrap()
// );
// assert!(BlkFiles::parse_blk_index("blkindex.dat").is_none());
// assert!(BlkFiles::parse_blk_index("invalid.dat").is_none());
// }
// }
parser/src/bitcoin/db/block_iter.rs
-45
View File
@@ -1,45 +0,0 @@
//!
//! View development note of iter_connected.rs for implementation
//! details of iter_block.rs, which follows similar principles.
//!
use bitcoin::Block;
use par_iter_sync::{IntoParallelIteratorSync, ParIterSync};
use super::BitcoinDB;
pub struct BlockIter(ParIterSync<Block>);
impl BlockIter {
/// the worker threads are dispatched in this `new` constructor!
pub fn new<T>(db: &BitcoinDB, heights: T) -> Self
where
T: IntoIterator<Item = usize> + Send + 'static,
<T as IntoIterator>::IntoIter: Send + 'static,
{
let db_ref = db.clone();
BlockIter(
heights.into_par_iter_sync(move |h| match db_ref.get_block(h) {
Ok(blk) => Ok(blk),
Err(_) => Err(()),
}),
)
}
/// the worker threads are dispatched in this `new` constructor!
pub fn from_range(db: &BitcoinDB, start: usize, end: usize) -> Self {
if end <= start {
BlockIter::new(db, Vec::new())
} else {
BlockIter::new(db, start..end)
}
}
}
impl Iterator for BlockIter {
type Item = Block;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
parser/src/bitcoin/db/blocks_indexes.rs
-211
View File
@@ -1,211 +0,0 @@
use std::{collections::BTreeMap, fmt, path::Path};
use bitcoin::{block::Header, io::Cursor, BlockHash};
use derive_deref::{Deref, DerefMut};
use leveldb::{
database::{iterator::LevelDBIterator, Database},
iterator::Iterable,
options::{Options, ReadOptions},
};
use crate::utils::log;
use super::{BlockchainRead, OpResult};
///
/// See Bitcoin Core repository for definition.
///
const BLOCK_VALID_HEADER: u32 = 1;
const BLOCK_VALID_TREE: u32 = 2;
const BLOCK_VALID_TRANSACTIONS: u32 = 3;
const BLOCK_VALID_CHAIN: u32 = 4;
const BLOCK_VALID_SCRIPTS: u32 = 5;
const BLOCK_VALID_MASK: u32 = BLOCK_VALID_HEADER
| BLOCK_VALID_TREE
| BLOCK_VALID_TRANSACTIONS
| BLOCK_VALID_CHAIN
| BLOCK_VALID_SCRIPTS;
const BLOCK_HAVE_DATA: u32 = 8;
const BLOCK_HAVE_UNDO: u32 = 16;
///
/// - Map from block height to block hash (records)
/// - Map from block hash to block height (hash_to_height)
///
#[derive(Clone, Deref, DerefMut)]
pub struct BlocksIndexes(Box<[BlockIndexRecord]>);
///
/// BLOCK_INDEX RECORD as defined in Bitcoin Core.
///
#[derive(Clone)]
pub struct BlockIndexRecord {
pub n_version: i32,
pub n_height: i32,
pub n_status: u32,
pub n_tx: u32,
pub n_file: i32,
pub n_data_pos: u32,
pub n_undo_pos: u32,
pub header: Header,
}
impl BlocksIndexes {
///
/// Build a collections of block index.
///
pub(crate) fn new(p: &Path) -> OpResult<Self> {
Ok(Self(load_block_index(p)?.into_boxed_slice()))
}
}
///
/// Load all block index in memory from leveldb (i.e. `blocks/index` path).
///
/// Map from block height to block index record.
///
pub fn load_block_index(path: &Path) -> OpResult<Vec<BlockIndexRecord>> {
let mut block_index_by_block_hash = BTreeMap::new();
log("Start loading block_index");
let mut options = Options::new();
options.create_if_missing = false;
let db: Database<BlockKey> = Database::open(path, options)?;
let options = ReadOptions::new();
let mut iter = db.iter(options);
let mut max_height_block_hash = Option::<(BlockHash, i32)>::None;
while iter.advance() {
let k = iter.key();
let v = iter.value();
if is_block_index_record(&k.key) {
let record = BlockIndexRecord::from(&v)?;
// only add valid block index record that has block data.
if record.n_height == 0
|| (record.n_status & BLOCK_VALID_MASK >= BLOCK_VALID_SCRIPTS
&& record.n_status & BLOCK_HAVE_DATA > 0)
{
let block_hash = record.header.block_hash();
// find the block with max height
if let Some((hash, height)) = max_height_block_hash.as_mut() {
if record.n_height > *height {
*hash = block_hash;
*height = record.n_height;
}
} else {
max_height_block_hash = Some((block_hash, record.n_height));
}
block_index_by_block_hash.insert(block_hash, record);
}
}
}
// build the longest chain
if let Some((hash, height)) = max_height_block_hash {
let mut block_index = Vec::with_capacity(height as usize + 1);
let mut current_hash = hash;
let mut current_height = height;
// recursively build block index from max height block.
while current_height >= 0 {
let blk = block_index_by_block_hash
.remove(&current_hash)
.expect("block hash not found in block index!");
assert_eq!(
current_height, blk.n_height,
"some block info missing from block index levelDB,\
delete Bitcoin folder and re-download!"
);
current_hash = blk.header.prev_blockhash;
current_height -= 1;
block_index.push(blk);
}
block_index.reverse();
Ok(block_index)
} else {
Ok(Vec::with_capacity(0))
}
}
/// levelDB key util
struct BlockKey {
key: Vec<u8>,
}
/// levelDB key util
impl db_key::Key for BlockKey {
fn from_u8(key: &[u8]) -> Self {
BlockKey {
key: Vec::from(key),
}
}
fn as_slice<T, F: Fn(&[u8]) -> T>(&self, f: F) -> T {
f(&self.key)
}
}
impl BlockIndexRecord {
///
/// Decode levelDB value for Block Index Record.
///
fn from(values: &[u8]) -> OpResult<Self> {
let mut reader = Cursor::new(values);
let n_version = reader.read_varint()? as i32;
let n_height = reader.read_varint()? as i32;
let n_status = reader.read_varint()? as u32;
let n_tx = reader.read_varint()? as u32;
let n_file = if n_status & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO) > 0 {
reader.read_varint()? as i32
} else {
-1
};
let n_data_pos = if n_status & BLOCK_HAVE_DATA > 0 {
reader.read_varint()? as u32
} else {
u32::MAX
};
let n_undo_pos = if n_status & BLOCK_HAVE_UNDO > 0 {
reader.read_varint()? as u32
} else {
u32::MAX
};
let header = reader.read_block_header()?;
Ok(BlockIndexRecord {
n_version,
n_height,
n_status,
n_tx,
n_file,
n_data_pos,
n_undo_pos,
header,
})
}
}
#[inline]
fn is_block_index_record(data: &[u8]) -> bool {
data.first() == Some(&b'b')
}
impl fmt::Debug for BlockIndexRecord {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BlockIndexRecord")
.field("version", &self.n_version)
.field("height", &self.n_height)
.field("status", &self.n_status)
.field("n_tx", &self.n_tx)
.field("n_file", &self.n_file)
.field("n_data_pos", &self.n_data_pos)
.field("header", &self.header)
.finish()
}
}
parser/src/bitcoin/db/errors.rs
-135
View File
@@ -1,135 +0,0 @@
use std::convert::{self, From};
use std::error;
use std::fmt;
use std::io;
use std::string;
use std::sync;
pub type OpResult<T> = Result<T, OpError>;
#[derive(Debug)]
/// Custom error type
pub struct OpError {
pub kind: OpErrorKind,
pub message: String,
}
impl OpError {
pub fn new(kind: OpErrorKind) -> Self {
OpError {
kind,
message: String::new(),
}
}
/// Joins the Error with a new message and returns it
pub fn join_msg(mut self, msg: &str) -> Self {
self.message.push_str(msg);
OpError {
kind: self.kind,
message: self.message,
}
}
}
impl fmt::Display for OpError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.message.is_empty() {
write!(f, "{}", &self.kind)
} else {
write!(f, "{} {}", &self.message, &self.kind)
}
}
}
impl error::Error for OpError {
fn description(&self) -> &str {
self.message.as_ref()
}
fn cause(&self) -> Option<&dyn error::Error> {
self.kind.source()
}
}
#[derive(Debug)]
pub enum OpErrorKind {
None,
IoError(io::Error),
Utf8Error(string::FromUtf8Error),
RuntimeError,
PoisonError,
SendError,
}
impl fmt::Display for OpErrorKind {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
OpErrorKind::IoError(ref err) => write!(f, "I/O Error: {}", err),
OpErrorKind::Utf8Error(ref err) => write!(f, "Utf8 Conversion: {}", err),
ref err @ OpErrorKind::PoisonError => write!(f, "Threading Error: {}", err),
ref err @ OpErrorKind::SendError => write!(f, "Sync: {}", err),
ref err @ OpErrorKind::RuntimeError => write!(f, "RuntimeError: {}", err),
OpErrorKind::None => write!(f, ""),
}
}
}
impl error::Error for OpErrorKind {
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match *self {
OpErrorKind::IoError(ref err) => Some(err),
OpErrorKind::Utf8Error(ref err) => Some(err),
ref err @ OpErrorKind::PoisonError => Some(err),
ref err @ OpErrorKind::SendError => Some(err),
_ => None,
}
}
}
impl From<io::Error> for OpError {
fn from(err: io::Error) -> Self {
Self::new(OpErrorKind::IoError(err))
}
}
impl From<bitcoin::consensus::encode::Error> for OpError {
fn from(_: bitcoin::consensus::encode::Error) -> Self {
Self::from("block decode error")
}
}
impl convert::From<i32> for OpError {
fn from(err_code: i32) -> Self {
Self::from(io::Error::from_raw_os_error(err_code))
}
}
impl convert::From<&str> for OpError {
fn from(err: &str) -> Self {
Self::new(OpErrorKind::None).join_msg(err)
}
}
impl<T> convert::From<sync::PoisonError<T>> for OpError {
fn from(_: sync::PoisonError<T>) -> Self {
Self::new(OpErrorKind::PoisonError)
}
}
impl<T> convert::From<sync::mpsc::SendError<T>> for OpError {
fn from(_: sync::mpsc::SendError<T>) -> Self {
Self::new(OpErrorKind::SendError)
}
}
impl convert::From<string::FromUtf8Error> for OpError {
fn from(err: string::FromUtf8Error) -> Self {
Self::new(OpErrorKind::Utf8Error(err))
}
}
impl convert::From<leveldb::error::Error> for OpError {
fn from(err: leveldb::error::Error) -> Self {
Self::from(err.to_string().as_ref())
}
}
parser/src/bitcoin/db/mod.rs
-172
View File
@@ -1,172 +0,0 @@
//!
//! Mostly a stripped down copy pasta of bitcoin-explorer
//!
//! Huge props to https://github.com/Congyuwang
//!
//! Crates APIs, essential structs, functions, methods are all here!
//!
//! To quickly understand how to use this crate, have a look at the
//! documentation for `bitcoin_explorer::BitcoinDB`!!.
//!
mod blk_files;
mod block_iter;
mod blocks_indexes;
mod errors;
mod reader;
mod txdb;
use blk_files::*;
use blocks_indexes::*;
use errors::*;
use reader::*;
use txdb::*;
use std::ops::Deref;
use std::path::Path;
use std::sync::Arc;
use bitcoin::{Block, Transaction, Txid};
pub use block_iter::BlockIter;
pub struct InnerDB {
pub blocks_indexes: BlocksIndexes,
pub blk_files: BlkFiles,
pub tx_db: TxDB,
}
///
/// This is the main struct of this crate!! Click and read the doc.
///
/// All queries start from initializing `BitcoinDB`.
///
/// Note: This is an Arc wrap around `InnerDB`.
///
#[derive(Clone)]
pub struct BitcoinDB(Arc<InnerDB>);
impl Deref for BitcoinDB {
type Target = InnerDB;
fn deref(&self) -> &Self::Target {
self.0.deref()
}
}
impl BitcoinDB {
///
/// This is the main structure for reading Bitcoin blockchain data.
///
/// Instantiating this class by passing the `-datadir` directory of
/// Bitcoin core to the `new()` method.
/// `tx_index`: whether to try to open tx_index levelDB.
///
pub fn new(p: &Path, tx_index: bool) -> OpResult<BitcoinDB> {
if !p.exists() {
return Err(OpError::from("data_dir does not exist"));
}
let blk_path = p.join("blocks");
let index_path = blk_path.join("index");
let blocks_indexes = BlocksIndexes::new(index_path.as_path())?;
let tx_db = if tx_index {
let tx_index_path = p.join("indexes").join("txindex");
TxDB::new(&tx_index_path)
} else {
TxDB::null()
};
let inner = InnerDB {
blocks_indexes,
blk_files: BlkFiles::new(blk_path.as_path())?,
tx_db,
};
Ok(BitcoinDB(Arc::new(inner)))
}
///
/// Get the maximum number of blocks downloaded.
///
/// This API guarantee that block 0 to `get_block_count() - 1`
/// have been downloaded and available for query.
///
pub fn get_block_count(&self) -> usize {
let records = self.blocks_indexes.len();
for h in 0..records {
// n_tx == 0 indicates that the block is not downloaded
if self.blocks_indexes.get(h).unwrap().n_tx == 0 {
return h;
}
}
records
}
///
/// Get a block
///
pub fn get_block(&self, height: usize) -> OpResult<Block> {
if let Some(index) = self.blocks_indexes.get(height) {
Ok(self.blk_files.read_block(index.n_file, index.n_data_pos)?)
} else {
Err(OpError::from("height not found"))
}
}
///
/// Get a transaction by providing txid.
///
/// This function requires `txindex` to be set to `true` for `BitcoinDB`,
/// and requires that flag `txindex=1` has been enabled when
/// running Bitcoin Core.
///
/// A transaction cannot be found using this function if it is
/// not yet indexed using `txindex`.
///
pub fn get_transaction(&self, txid: &Txid) -> OpResult<Transaction> {
if !self.tx_db.is_open() {
return Err(OpError::from("TxDB not open"));
}
// give special treatment for genesis transaction
if self.tx_db.is_genesis_tx(txid) {
return Ok(self.get_block(0)?.txdata.swap_remove(0));
}
let record = self.tx_db.get_tx_record(txid)?;
self.blk_files
.read_transaction(record.n_file, record.n_pos, record.n_tx_offset)
}
///
/// Iterate through all blocks from `start` to `end` (excluded).
///
/// # Performance
///
/// This iterator is implemented to read the blocks in concurrency,
/// but the result is still produced in sequential order.
/// Results read are stored in a synced queue for `next()`
/// to get.
///
/// The iterator stops automatically when a block cannot be
/// read (i.e., when the max height in the database met).
///
/// This is a very efficient implementation.
/// Using SSD and intel core i7 (4 core, 8 threads)
/// Iterating from height 0 to 700000 takes about 10 minutes.
///
pub fn iter_block(&self, start: usize, end: usize) -> BlockIter {
BlockIter::from_range(self, start, end)
}
pub fn check_if_txout_value_is_zero(&self, txid: &Txid, vout: usize) -> bool {
self.get_transaction(txid)
.unwrap()
.output
.get(vout)
.unwrap()
.to_owned()
.value
.to_sat()
== 0
}
}
parser/src/bitcoin/db/reader.rs
-90
View File
@@ -1,90 +0,0 @@
use std::{fs::File, io::BufReader};
use bitcoin::{block::Header, consensus::Decodable, io::Cursor, Block, Transaction};
use byteorder::{LittleEndian, ReadBytesExt};
use super::OpResult;
///
/// binary file read utilities.
///
pub trait BlockchainRead {
#[inline]
fn read_varint(&mut self) -> OpResult<usize>
where
Self: bitcoin::io::Read,
{
let mut n = 0;
loop {
let ch_data = self.read_u8()?;
n = (n << 7) | (ch_data & 0x7F) as usize;
if ch_data & 0x80 > 0 {
n += 1;
} else {
break;
}
}
Ok(n)
}
#[inline]
fn read_u8(&mut self) -> OpResult<u8>
where
Self: bitcoin::io::Read,
{
let mut slice = [0u8; 1];
self.read_exact(&mut slice).unwrap();
Ok(slice[0])
}
#[inline]
fn read_u32(&mut self) -> OpResult<u32>
where
Self: std::io::Read,
{
let u = ReadBytesExt::read_u32::<LittleEndian>(self)?;
Ok(u)
}
#[inline]
fn read_u8_vec(&mut self, count: u32) -> OpResult<Vec<u8>>
where
Self: bitcoin::io::Read,
{
let mut arr = vec![0u8; count as usize];
self.read_exact(&mut arr).unwrap();
Ok(arr)
}
#[inline]
fn read_block(&mut self) -> OpResult<Block>
where
Self: bitcoin::io::BufRead,
{
Ok(Block::consensus_decode(self)?)
}
#[inline]
fn read_transaction(&mut self) -> OpResult<Transaction>
where
Self: bitcoin::io::BufRead,
{
Ok(Transaction::consensus_decode(self)?)
}
#[inline]
fn read_block_header(&mut self) -> OpResult<Header>
where
Self: bitcoin::io::BufRead,
{
Ok(Header::consensus_decode(self)?)
}
}
impl<T> BlockchainRead for Cursor<T> {}
impl BlockchainRead for BufReader<File> {}
parser/src/bitcoin/db/txdb.rs
-147
View File
@@ -1,147 +0,0 @@
use std::{path::Path, str::FromStr};
use bitcoin::{hashes::Hash, io::Cursor, Txid};
use leveldb::{
database::Database,
kv::KV,
options::{Options, ReadOptions},
};
use crate::utils::log;
use super::{BlockchainRead, OpError, OpResult};
const GENESIS_TXID: &str = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b";
///
/// tx-index: looking up transaction position using txid.
///
/// This is possible if Bitcoin Core has `txindex=1`.
///
pub struct TxDB {
db: Option<Database<TxKey>>,
genesis_txid: Txid,
}
/// Records transaction storage on disk
pub struct TransactionRecord {
pub txid: Txid,
pub n_file: i32,
pub n_pos: u32,
pub n_tx_offset: u32,
}
impl TransactionRecord {
fn from(key: &[u8], values: &[u8]) -> OpResult<Self> {
let mut reader = Cursor::new(values);
Ok(TransactionRecord {
txid: Txid::from_slice(key).unwrap(),
n_file: reader.read_varint()? as i32,
n_pos: reader.read_varint()? as u32,
n_tx_offset: reader.read_varint()? as u32,
})
}
}
impl TxDB {
/// initialize TxDB for transaction queries
pub fn new(path: &Path) -> TxDB {
let option_db = TxDB::try_open_db(path);
if let Some(db) = option_db {
TxDB {
db: Some(db),
genesis_txid: Txid::from_str(GENESIS_TXID).unwrap(),
}
} else {
TxDB::null()
}
}
#[inline]
pub fn is_open(&self) -> bool {
self.db.is_some()
}
#[inline]
pub fn null() -> TxDB {
TxDB {
db: None,
genesis_txid: Txid::from_str(GENESIS_TXID).unwrap(),
}
}
#[inline]
///
/// genesis tx is not included in UTXO because of Bitcoin Core Bug
///
pub fn is_genesis_tx(&self, txid: &Txid) -> bool {
txid == &self.genesis_txid
}
fn try_open_db(path: &Path) -> Option<Database<TxKey>> {
if !path.exists() {
log("Failed to open tx_index DB: tx_index not built");
return None;
}
let options = Options::new();
match Database::open(path, options) {
Ok(db) => {
log("Successfully opened tx_index DB!");
Some(db)
}
Err(e) => {
log(&format!("Failed to open tx_index DB: {:?}", e));
None
}
}
}
/// note that this function cannot find genesis block, which needs special treatment
pub fn get_tx_record(&self, txid: &Txid) -> OpResult<TransactionRecord> {
if let Some(db) = &self.db {
let inner = txid.as_byte_array();
let mut key = Vec::with_capacity(inner.len() + 1);
key.push(b't');
key.extend(inner);
let key = TxKey { key };
let read_options = ReadOptions::new();
match db.get(read_options, &key) {
Ok(value) => {
if let Some(value) = value {
Ok(TransactionRecord::from(&key.key[1..], value.as_slice())?)
} else {
Err(OpError::from(
format!("value not found for txid: {}", txid).as_str(),
))
}
}
Err(e) => Err(OpError::from(
format!("value not found for txid: {}", e).as_str(),
)),
}
} else {
Err(OpError::from("TxDB not open"))
}
}
}
/// levelDB key utility
struct TxKey {
key: Vec<u8>,
}
/// levelDB key utility
impl db_key::Key for TxKey {
fn from_u8(key: &[u8]) -> Self {
TxKey {
key: Vec::from(key),
}
}
fn as_slice<T, F: Fn(&[u8]) -> T>(&self, f: F) -> T {
f(&self.key)
}
}
parser/src/bitcoin/mod.rs
-9
View File
@@ -1,9 +0,0 @@
mod addresses;
mod consts;
mod daemon;
mod db;
pub use addresses::*;
pub use consts::*;
pub use daemon::*;
pub use db::*;
parser/src/databases/txid_to_tx_data.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use std::{
};
use allocative::Allocative;
use bitcoin::Txid;
use biter::bitcoin::Txid;
use rayon::prelude::*;
use crate::structs::{Date, Height, TxData};
parser/src/datasets/mining.rs
+4 -2
View File
@@ -3,13 +3,15 @@ use itertools::Itertools;
use ordered_float::OrderedFloat;
use crate::{
bitcoin::TARGET_BLOCKS_PER_DAY,
datasets::AnyDataset,
structs::{
date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap,
AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey,
},
utils::{BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
utils::{
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
TARGET_BLOCKS_PER_DAY,
},
};
use super::{
parser/src/lib.rs
+1 -3
View File
@@ -1,5 +1,4 @@
mod actions;
mod bitcoin;
mod databases;
mod datasets;
mod io;
@@ -10,12 +9,11 @@ mod utils;
pub use crate::{
actions::iter_blocks,
bitcoin::{BitcoinDB, BitcoinDaemon},
datasets::OHLC,
io::{Binary, Json, Serialization},
structs::{
Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
HEIGHT_MAP_CHUNK_SIZE,
},
utils::log,
utils::{create_rpc, log},
};
parser/src/main.rs
+13 -34
View File
@@ -1,51 +1,30 @@
use std::{path::Path, thread::sleep, time::Duration};
use std::{thread::sleep, time::Duration};
use parser::{iter_blocks, log, BitcoinDB, BitcoinDaemon, Config, Height};
use biter::bitcoincore_rpc::RpcApi;
use parser::{create_rpc, iter_blocks, log, Config};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let config = Config::read();
let config = Config::import();
if config.datadir.is_none() {
println!(
"You need to set the --datadir parameter at least once to run the parser.\nRun the program with '-h' for help."
);
std::process::exit(1);
}
config.write()?;
let daemon = BitcoinDaemon::new(&config);
let rpc = create_rpc(&config).unwrap();
loop {
daemon.stop();
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
// Scoped to free bitcoin's lock
let block_count = {
let bitcoin_db = BitcoinDB::new(Path::new(&config.datadir.as_ref().unwrap()), true)?;
log(&format!("{block_count} blocks found."));
// let block_count = 200_000;
let block_count = bitcoin_db.get_block_count();
log(&format!("{block_count} blocks found."));
iter_blocks(&bitcoin_db, block_count)?;
block_count
};
daemon.start();
if daemon.check_if_fully_synced() {
daemon.wait_for_new_block(Height::new(block_count as u32 - 1));
} else {
daemon.wait_sync();
}
iter_blocks(&config, &rpc, block_count)?;
if let Some(delay) = config.delay {
sleep(Duration::from_secs(delay))
}
log("Waiting for new block...");
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
sleep(Duration::from_secs(5))
}
}
// Ok(())
parser/src/structs/address.rs
+2 -2
View File
@@ -1,10 +1,10 @@
use bitcoin::TxOut;
use bitcoin_hashes::{hash160, Hash};
use biter::bitcoin::TxOut;
use itertools::Itertools;
use crate::{
bitcoin::multisig_addresses,
databases::{U8x19, U8x31, SANAKIRJA_MAX_KEY_SIZE},
utils::multisig_addresses,
};
use super::{AddressType, Counter};
parser/src/structs/amount.rs
+1 -1
View File
@@ -10,7 +10,7 @@ use bincode::{
error::{DecodeError, EncodeError},
BorrowDecode, Decode, Encode,
};
use bitcoin::Amount as BitcoinAmount;
use biter::bitcoin::Amount as BitcoinAmount;
use derive_deref::{Deref, DerefMut};
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use serde::{Deserialize, Serialize};
parser/src/structs/bi_map.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use std::{
use allocative::Allocative;
use crate::{bitcoin::TARGET_BLOCKS_PER_DAY, utils::LossyFrom};
use crate::utils::{LossyFrom, TARGET_BLOCKS_PER_DAY};
use super::{AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapValue};
parser/src/structs/config.rs
+53 -15
View File
@@ -6,17 +6,21 @@ use serde::{Deserialize, Serialize};
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize)]
#[command(version, about, long_about = None)]
pub struct Config {
/// bitcoind `-datadir=<dir>` argument
/// Bitcoin data directory path
#[arg(long, value_name = "DIR")]
pub datadir: Option<String>,
/// bitcoind `-blocksonly` argument, default: true
#[arg(long)]
pub blocksonly: Option<bool>,
/// Bitcoin RPC port, default: 8332
#[arg(long, value_name = "PORT")]
pub rpcport: Option<u16>,
/// bitcoind `-rpcconnect=<ip>` argument
#[arg(long, value_name = "IP")]
pub rpcconnect: Option<String>,
/// Bitcoin RPC username
#[arg(long, value_name = "USERNAME")]
pub rpcuser: Option<String>,
/// Bitcoin RPC password
#[arg(long, value_name = "PASSWORD")]
pub rpcpassword: Option<String>,
/// Delay between runs, default: 0
#[arg(long, value_name = "SECONDS")]
@@ -26,7 +30,7 @@ pub struct Config {
impl Config {
const PATH: &'static str = "config.toml";
pub fn read() -> Self {
pub fn import() -> Self {
let mut config_saved = fs::read_to_string(Self::PATH)
.map_or(Config::default(), |contents| {
toml::from_str(&contents).unwrap_or_default()
@@ -38,24 +42,58 @@ impl Config {
config_saved.datadir = Some(datadir);
}
if let Some(blocksonly) = config_args.blocksonly {
config_saved.blocksonly = Some(blocksonly);
if let Some(rpcport) = config_args.rpcport {
config_saved.rpcport = Some(rpcport);
} else {
config_saved.blocksonly = Some(true);
config_saved.rpcport = Some(8332);
}
if let Some(rpcconnect) = config_args.rpcconnect {
config_saved.rpcconnect = Some(rpcconnect);
if let Some(rpcuser) = config_args.rpcuser {
config_saved.rpcuser = Some(rpcuser);
}
if let Some(rpcpassword) = config_args.rpcpassword {
config_saved.rpcpassword = Some(rpcpassword);
}
if let Some(delay) = config_args.delay {
config_saved.delay = Some(delay);
}
config_saved
// Done importing
let config = config_saved;
config.check();
config.write().unwrap();
config
}
pub fn write(&self) -> std::io::Result<()> {
fn check(&self) {
if self.datadir.is_none() {
Self::exit("datadir");
}
if self.rpcuser.is_none() {
Self::exit("rpcuser");
}
if self.rpcpassword.is_none() {
Self::exit("rpcpassword");
}
}
fn exit(attribute: &str) {
println!(
"You need to set the --{} parameter at least once to run the parser.\nRun the program with '-h' for help.", attribute
);
std::process::exit(1);
}
fn write(&self) -> std::io::Result<()> {
fs::write(Self::PATH, toml::to_string(self).unwrap())
}
}
parser/src/structs/height.rs
+2 -1
View File
@@ -5,10 +5,11 @@ use std::{
use allocative::Allocative;
use bincode::{Decode, Encode};
use biter::NUMBER_OF_UNSAFE_BLOCKS;
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use crate::{bitcoin::NUMBER_OF_UNSAFE_BLOCKS, HEIGHT_MAP_CHUNK_SIZE};
use crate::HEIGHT_MAP_CHUNK_SIZE;
use super::{HeightMapChunkId, MapKey};
parser/src/utils/arr.rs
-219
View File
@@ -1,219 +0,0 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Div, Mul, Sub, SubAssign},
};
use itertools::Itertools;
use ordered_float::{FloatCore, OrderedFloat};
use super::ToF32;
pub trait ArrayOperations<T> {
fn transform<F>(&self, transform: F) -> Vec<T>
where
T: Copy + Default,
F: Fn((usize, &T, &[T])) -> T;
fn add(&self, other: &[T]) -> Vec<T>
where
T: Add<Output = T> + Copy + Default;
fn subtract(&self, other: &[T]) -> Vec<T>
where
T: Sub<Output = T> + Copy + Default;
fn multiply(&self, other: &[T]) -> Vec<T>
where
T: Mul<Output = T> + Copy + Default;
fn divide(&self, other: &[T]) -> Vec<T>
where
T: Div<Output = T> + Copy + Default;
fn match_size<'a>(&'a self, other: &'a [T]) -> &'a [T];
fn cumulate(&self) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign;
fn last_x_sum(&self, x: usize) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign + SubAssign;
fn moving_average(&self, x: usize) -> Vec<f32>
where
T: Sum + Copy + Default + AddAssign + SubAssign + ToF32;
fn net_change(&self, offset: usize) -> Vec<T>
where
T: Copy + Default + Sub<Output = T>;
fn median(&self, size: usize) -> Vec<Option<T>>
where
T: FloatCore;
}
impl<T> ArrayOperations<T> for &[T] {
fn transform<F>(&self, transform: F) -> Vec<T>
where
T: Copy + Default,
F: Fn((usize, &T, &[T])) -> T,
{
self.iter()
.enumerate()
.map(|(index, value)| transform((index, value, self)))
.collect_vec()
}
fn add(&self, other: &[T]) -> Vec<T>
where
T: Add<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value + *other.get(index).unwrap())
}
fn subtract(&self, other: &[T]) -> Vec<T>
where
T: Sub<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value - *other.get(index).unwrap())
}
fn multiply(&self, other: &[T]) -> Vec<T>
where
T: Mul<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value * *other.get(index).unwrap())
}
fn divide(&self, other: &[T]) -> Vec<T>
where
T: Div<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value / *other.get(index).unwrap())
}
fn match_size(&self, other: &[T]) -> &[T] {
let len = other.len();
if self.len() > len {
&self[..len]
} else {
self
}
}
fn cumulate(&self) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign,
{
let mut sum = T::default();
self.iter()
.map(|value| {
sum += *value;
sum
})
.collect_vec()
}
fn last_x_sum(&self, x: usize) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign + SubAssign,
{
let mut sum = T::default();
self.iter()
.enumerate()
.map(|(index, value)| {
sum += *value;
if index >= x - 1 {
let previous_index = index + 1 - x;
sum -= *self.get(previous_index).unwrap()
}
sum
})
.collect_vec()
}
fn moving_average(&self, x: usize) -> Vec<f32>
where
T: Sum + Copy + Default + AddAssign + SubAssign + ToF32,
{
let mut sum = T::default();
self.iter()
.enumerate()
.map(|(index, value)| {
sum += *value;
if index >= x - 1 {
sum -= *self.get(index + 1 - x).unwrap()
}
sum.to_f32() / x as f32
})
.collect_vec()
}
fn net_change(&self, offset: usize) -> Vec<T>
where
T: Copy + Default + Sub<Output = T>,
{
self.transform(|(index, value, arr)| {
let previous = {
if let Some(previous_index) = index.checked_sub(offset) {
*arr.get(previous_index).unwrap()
} else {
T::default()
}
};
*value - previous
})
}
fn median(&self, size: usize) -> Vec<Option<T>>
where
T: FloatCore,
{
let even = size % 2 == 0;
let median_index = size / 2;
if size < 3 {
panic!("Computing a median for a size lower than 3 is useless");
}
self.iter()
.enumerate()
.map(|(index, _)| {
if index >= size - 1 {
let mut arr = self[index - (size - 1)..index + 1]
.iter()
.map(|value| OrderedFloat(*value))
.collect_vec();
arr.sort_unstable();
if even {
Some(
(**arr.get(median_index).unwrap()
+ **arr.get(median_index - 1).unwrap())
/ T::from(2.0).unwrap(),
)
} else {
Some(**arr.get(median_index).unwrap())
}
} else {
None
}
})
.collect()
}
}
parser/src/utils/bytes.rs
-1
View File
@@ -1 +0,0 @@
pub const BYTES_IN_MB: usize = 1_000_000;
parser/src/utils/date.rs → parser/src/utils/consts.rs
+4
View File
@@ -1,3 +1,7 @@
pub const BYTES_IN_MB: usize = 1_000_000;
pub const TARGET_BLOCKS_PER_DAY: usize = 144;
pub const ONE_DAY_IN_DAYS: usize = 1;
pub const ONE_WEEK_IN_DAYS: usize = 7;
pub const TWO_WEEK_IN_DAYS: usize = 2 * ONE_WEEK_IN_DAYS;
parser/src/utils/log.rs
+1 -11
View File
@@ -1,4 +1,4 @@
use std::{fs::OpenOptions, io::Write, process::Output};
use std::{fs::OpenOptions, io::Write};
use chrono::Local;
use color_eyre::owo_colors::OwoColorize;
@@ -23,13 +23,3 @@ pub fn log(str: &str) {
println!("{} {}", date_time.bright_black(), line);
});
}
pub fn log_output(output: &Output) {
if !output.stdout.is_empty() {
log(&String::from_utf8_lossy(&output.stdout));
}
if !output.stderr.is_empty() {
log(&String::from_utf8_lossy(&output.stderr));
}
}
parser/src/utils/mod.rs
+6 -4
View File
@@ -1,17 +1,19 @@
mod bytes;
mod date;
mod consts;
mod flamegraph;
mod log;
mod lossy;
mod multisig;
mod percentile;
mod retry;
mod rpc;
mod time;
pub use bytes::*;
pub use date::*;
pub use consts::*;
pub use flamegraph::*;
pub use log::*;
pub use lossy::*;
pub use multisig::*;
pub use percentile::*;
pub use retry::*;
pub use rpc::*;
pub use time::*;
parser/src/bitcoin/addresses/multisig.rs → parser/src/utils/multisig.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// Code from bitcoin-explorer now deprecated
//
use bitcoin::{
use biter::bitcoin::{
blockdata::{
opcodes::all,
script::Instruction::{self, Op, PushBytes},
parser/src/utils/rpc.rs
+13
View File
@@ -0,0 +1,13 @@
use biter::bitcoincore_rpc::{Auth, Client};
use crate::Config;
pub fn create_rpc(config: &Config) -> color_eyre::Result<Client> {
Ok(Client::new(
&format!("http://localhost:{}", config.rpcport.unwrap()),
Auth::UserPass(
config.rpcuser.clone().unwrap(),
config.rpcpassword.clone().unwrap(),
),
)?)
}
server/Cargo.lock
Generated
+158 -85
View File
@@ -27,7 +27,7 @@ dependencies = [
"getrandom",
"once_cell",
"version_check",
"zerocopy",
"zerocopy 0.7.35",
]
[[package]]
@@ -258,6 +258,12 @@ dependencies = [
"bitcoin_hashes",
]
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.22.0"
@@ -342,6 +348,45 @@ dependencies = [
"serde",
]
[[package]]
name = "bitcoincore-rpc"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee"
dependencies = [
"bitcoincore-rpc-json",
"jsonrpc",
"log",
"serde",
"serde_json",
]
[[package]]
name = "bitcoincore-rpc-json"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a"
dependencies = [
"bitcoin",
"serde",
"serde_json",
]
[[package]]
name = "biter"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd2ea6b4c4046feaaf73f3dff1cc648328af021d50be95946e0e7a807c985b47"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"crossbeam",
"derived-deref",
"rayon",
"serde",
"serde_json",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -470,15 +515,6 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
[[package]]
name = "cmake"
version = "0.1.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
dependencies = [
"cc",
]
[[package]]
name = "color-eyre"
version = "0.6.3"
@@ -617,12 +653,6 @@ dependencies = [
"parking_lot_core 0.9.9",
]
[[package]]
name = "db-key"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b72465f46d518f6015d9cf07f7f3013a95dd6b9c2747c3d65ae0cce43929d14f"
[[package]]
name = "derive_deref"
version = "1.1.1"
@@ -634,6 +664,17 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "derived-deref"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.60",
]
[[package]]
name = "either"
version = "1.12.0"
@@ -700,12 +741,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
[[package]]
name = "ffi-opaque"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a"
[[package]]
name = "flate2"
version = "1.0.28"
@@ -1132,35 +1167,24 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "jsonrpc"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
dependencies = [
"base64 0.13.1",
"minreq",
"serde",
"serde_json",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "leveldb"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32651baaaa5596b3a6e0bee625e73fd0334c167db0ea5ac68750ef9a629a2d6a"
dependencies = [
"db-key",
"leveldb-sys",
"libc",
]
[[package]]
name = "leveldb-sys"
version = "2.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd94a4d0242a437e5e41a27c782b69a624469ca1c4d1e5cb3c337f74a8031d4"
dependencies = [
"cmake",
"ffi-opaque",
"libc",
"num_cpus",
]
[[package]]
name = "libc"
version = "0.2.153"
@@ -1236,14 +1260,26 @@ dependencies = [
]
[[package]]
name = "mio"
version = "0.8.11"
name = "minreq"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0"
dependencies = [
"log",
"serde",
"serde_json",
]
[[package]]
name = "mio"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
dependencies = [
"hermit-abi",
"libc",
"wasi",
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -1283,16 +1319,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "object"
version = "0.32.2"
@@ -1367,16 +1393,6 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "par-iter-sync"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efa981aaed94bf59211f644922155e8a33bcb01ed662cd63426653187f562790"
dependencies = [
"crossbeam",
"num_cpus",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@@ -1431,20 +1447,16 @@ version = "0.3.0"
dependencies = [
"allocative",
"bincode",
"bitcoin",
"bitcoin_hashes",
"byteorder",
"biter",
"chrono",
"clap",
"color-eyre",
"db-key",
"derive_deref",
"inferno",
"itertools",
"leveldb",
"memory-stats",
"ordered-float",
"par-iter-sync",
"rayon",
"reqwest",
"sanakirja",
@@ -1497,6 +1509,15 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "ppv-lite86"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f"
dependencies = [
"zerocopy 0.6.6",
]
[[package]]
name = "proc-macro2"
version = "1.0.81"
@@ -1524,6 +1545,36 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.10.0"
@@ -1597,7 +1648,7 @@ version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
dependencies = [
"base64",
"base64 0.22.0",
"bytes",
"encoding_rs",
"futures-channel",
@@ -1697,7 +1748,7 @@ version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
dependencies = [
"base64",
"base64 0.22.0",
"rustls-pki-types",
]
@@ -1773,6 +1824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3"
dependencies = [
"bitcoin_hashes",
"rand",
"secp256k1-sys",
"serde",
]
@@ -1831,11 +1883,12 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.120"
version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
@@ -2070,28 +2123,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.38.0"
version = "1.39.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"num_cpus",
"parking_lot 0.12.1",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
name = "tokio-macros"
version = "2.3.0"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
dependencies = [
"proc-macro2",
"quote",
@@ -2610,13 +2662,34 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "zerocopy"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6"
dependencies = [
"byteorder",
"zerocopy-derive 0.6.6",
]
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"zerocopy-derive",
"zerocopy-derive 0.7.35",
]
[[package]]
name = "zerocopy-derive"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.60",
]
[[package]]