Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-12 16:03:31 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: switch to biter

Browse Source
This commit is contained in:
k
2024-08-03 15:17:57 +02:00
parent afacea3fbb
commit 9067c28d24
35 changed files with 494 additions and 1589 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parser/src/actions/iter_blocks.rs
+32 -10
View File
@@ -8,15 +8,22 @@ use parse::ParseData;
use crate::{
actions::{export, find_first_inserted_unsafe_height, parse},
bitcoin::BitcoinDB,
create_rpc,
databases::Databases,
datasets::{AllDatasets, ComputeData},
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{Date, DateData, MapKey},
utils::{generate_allocation_files, log, time},
Config, Height,
};
pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Result<()> {
pub fn iter_blocks(
config: &Config,
rpc: &biter::bitcoincore_rpc::Client,
approx_block_count: usize,
) -> color_eyre::Result<()> {
let export_dir = "./target/outputs";
let should_insert = true;
let should_export = true;
let study_ram_usage = false;
@@ -44,11 +51,19 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
log(&format!("Starting parsing at height: {height}"));
let mut block_iter = bitcoin_db.iter_block(height.to_usize(), block_count);
let mut next_block_opt = None;
let mut blocks_loop_date = None;
let block_receiver = biter::new(
config.datadir.as_ref().unwrap(),
export_dir,
Some(height.to_usize()),
None,
create_rpc(config).unwrap(),
);
let mut block_iter = block_receiver.iter();
'parsing: loop {
let instant = Instant::now();
@@ -67,15 +82,22 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
next_block_opt = block_iter.next();
if let Some(current_block) = current_block_opt {
if let Some((_current_block_height, current_block, _current_block_hash)) =
current_block_opt
{
let timestamp = current_block.header.time;
let current_block_date = Date::from_timestamp(timestamp);
let current_block_height = height + blocks_loop_i;
let current_block_height: Height = height + blocks_loop_i;
if current_block_height.to_usize() != _current_block_height {
dbg!(current_block_height, _current_block_height);
panic!()
}
let next_block_date = next_block_opt
.as_ref()
.map(|next_block| Date::from_timestamp(next_block.header.time));
.map(|(_, next_block, _)| Date::from_timestamp(next_block.header.time));
// Always run for the first block of the loop
if blocks_loop_date.is_none() {
@@ -139,7 +161,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
}
parse(ParseData {
bitcoin_db,
rpc,
block: current_block,
block_index: blocks_loop_i,
compute_addresses,
@@ -162,7 +184,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
let is_new_month = next_block_date
.map_or(true, |next_block_date| next_block_date.day() == 1);
if is_new_month || height.is_close_to_end(block_count) {
if is_new_month || height.is_close_to_end(approx_block_count) {
break 'days;
}
@@ -196,7 +218,7 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
}
if should_export {
let is_safe = height.is_safe(block_count);
let is_safe = height.is_safe(approx_block_count);
export(ExportedData {
databases: is_safe.then_some(&mut databases),
parser/src/actions/parse.rs
+22 -8
View File
@@ -1,12 +1,14 @@
use std::{collections::BTreeMap, ops::ControlFlow, thread};
use bitcoin::{Block, Txid};
use biter::{
bitcoin::{Block, Txid},
bitcoincore_rpc::RpcApi,
};
use itertools::Itertools;
use rayon::prelude::*;
use crate::{
bitcoin::BitcoinDB,
databases::{
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
@@ -23,7 +25,7 @@ use crate::{
};
pub struct ParseData<'a> {
pub bitcoin_db: &'a BitcoinDB,
// pub bitcoin_cli: &'a BitcoinCli,
pub block: Block,
pub block_index: usize,
pub compute_addresses: bool,
@@ -33,13 +35,13 @@ pub struct ParseData<'a> {
pub first_date_height: Height,
pub height: Height,
pub is_date_last_block: bool,
pub rpc: &'a biter::bitcoincore_rpc::Client,
pub states: &'a mut States,
pub timestamp: u32,
}
pub fn parse(
ParseData {
bitcoin_db,
block,
block_index,
compute_addresses,
@@ -49,6 +51,7 @@ pub fn parse(
first_date_height,
height,
is_date_last_block,
rpc,
states,
timestamp,
}: ParseData,
@@ -334,10 +337,16 @@ pub fn parse(
// Can be none because 0 sats inputs happen
// https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
if input_tx_data.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| bitcoin_db
.check_if_txout_value_is_zero(&input_txid, input_vout as usize)
|| rpc
.get_tx_out(&input_txid, input_vout, None)
.unwrap()
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
@@ -374,8 +383,13 @@ pub fn parse(
if input_amount_and_address_index.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| bitcoin_db
.check_if_txout_value_is_zero(&input_txid, input_vout as usize)
|| rpc
.get_tx_out(&input_txid, input_vout as u32, None)
.unwrap()
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
parser/src/bitcoin/addresses/mod.rs
-3
View File
@@ -1,3 +0,0 @@
mod multisig;
pub use multisig::*;
parser/src/bitcoin/consts.rs
-2
View File
@@ -1,2 +0,0 @@
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
pub const TARGET_BLOCKS_PER_DAY: usize = 144;
parser/src/bitcoin/daemon.rs
-137
View File
@@ -1,137 +0,0 @@
use std::{process::Command, thread::sleep, time::Duration};
use color_eyre::eyre::eyre;
use serde_json::Value;
use crate::{
structs::Height,
utils::{log, log_output, retry},
Config,
};
struct BlockchainInfo {
pub headers: u64,
pub blocks: u64,
}
pub struct BitcoinDaemon {
config: Config,
}
impl BitcoinDaemon {
pub fn new(config: &Config) -> Self {
Self {
config: config.clone(),
}
}
pub fn start(&self) {
sleep(Duration::from_secs(1));
let mut command = Command::new("bitcoind");
command
.arg(self.datadir_arg())
.arg("-txindex=1")
.arg("-daemon");
if self.config.blocksonly.unwrap() {
command.arg("-blocksonly");
}
if let Some(ip) = self.config.rpcconnect.as_ref() {
command.arg(format!("-rpcconnect={}", ip));
}
// bitcoind -datadir=/Users/k/Developer/bitcoin -blocksonly -txindex=1 -daemon
let output = command
.output()
.expect("bitcoind to be able to properly start");
log_output(&output);
}
pub fn stop(&self) {
// bitcoin-cli -datadir=/Users/k/Developer/bitcoin stop
let output = Command::new("bitcoin-cli")
.arg(self.datadir_arg())
.arg("stop")
.output()
.unwrap();
if output.status.success() {
log_output(&output);
sleep(Duration::from_secs(15));
}
}
pub fn wait_sync(&self) {
while !self.check_if_fully_synced() {
sleep(Duration::from_secs(5))
}
}
pub fn wait_for_new_block(&self, last_block_height: Height) {
log("Waiting for new block...");
while last_block_height == self.get_blockchain_info().headers {
sleep(Duration::from_secs(5))
}
}
pub fn check_if_fully_synced(&self) -> bool {
let BlockchainInfo { blocks, headers } = self.get_blockchain_info();
let synced = blocks == headers;
if synced {
log(&format!("Synced ! ({blocks} blocks)"));
} else {
log(&format!("Syncing... ({} remaining)", headers - blocks));
}
synced
}
fn get_blockchain_info(&self) -> BlockchainInfo {
retry(
|_| {
// bitcoin-cli -datadir=/Users/k/Developer/bitcoin getblockchaininfo
let output = Command::new("bitcoin-cli")
.arg(self.datadir_arg())
.arg("getblockchaininfo")
.output()?;
let output = String::from_utf8_lossy(&output.stdout);
let json: Value = serde_json::from_str(&output)?;
let json = json.as_object().ok_or(eyre!("json as object failed"))?;
let blocks = json
.get("blocks")
.ok_or(eyre!("get field 'blocks' from json failed"))?
.as_u64()
.ok_or(eyre!("blocks to u64 failed"))?;
let headers = json
.get("headers")
.ok_or(eyre!("get field 'headers' from json failed"))?
.as_u64()
.ok_or(eyre!("blocks to u64 failed"))?;
Ok(BlockchainInfo { headers, blocks })
},
1,
usize::MAX,
)
.unwrap()
}
fn datadir_arg(&self) -> String {
if self.config.datadir.is_none() {
unreachable!();
}
format!("-datadir={}", self.config.datadir.as_ref().unwrap())
}
}
parser/src/bitcoin/db/blk_files.rs
-152
View File
@@ -1,152 +0,0 @@
use std::{
collections::HashMap,
convert::From,
fs::{self, DirEntry, File},
io::{self, BufReader, Seek, SeekFrom},
path::{Path, PathBuf},
};
use bitcoin::{io::Cursor, Block, Transaction};
use derive_deref::{Deref, DerefMut};
use super::{
errors::{OpError, OpErrorKind, OpResult},
reader::BlockchainRead,
};
///
/// An index of all blk files found.
///
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct BlkFiles(HashMap<i32, PathBuf>);
impl BlkFiles {
///
/// Construct an index of all blk files.
///
pub fn new(path: &Path) -> OpResult<Self> {
Ok(Self(Self::scan_path(path)?))
}
///
/// Read a Block from blk file.
///
#[inline]
pub fn read_raw_block(&self, n_file: i32, offset: u32) -> OpResult<Vec<u8>> {
if let Some(blk_path) = self.get(&n_file) {
let mut r = BufReader::new(File::open(blk_path)?);
r.seek(SeekFrom::Start(offset as u64 - 4))?;
let block_size = r.read_u32()?;
let block = r.read_u8_vec(block_size)?;
Ok(block)
} else {
Err(OpError::from("blk file not found, sync with bitcoin core"))
}
}
///
/// Read a Block from blk file.
///
pub fn read_block(&self, n_file: i32, offset: u32) -> OpResult<Block> {
Cursor::new(self.read_raw_block(n_file, offset)?).read_block()
}
///
/// Read a transaction from blk file.
///
pub fn read_transaction(
&self,
n_file: i32,
n_pos: u32,
n_tx_offset: u32,
) -> OpResult<Transaction> {
if let Some(blk_path) = self.get(&n_file) {
let mut r = BufReader::new(File::open(blk_path)?);
// the size of a header is 80.
r.seek(SeekFrom::Start(n_pos as u64 + n_tx_offset as u64 + 80))?;
r.read_transaction()
} else {
Err(OpError::from("blk file not found, sync with bitcoin core"))
}
}
///
/// Scan blk folder to build an index of all blk files.
///
fn scan_path(path: &Path) -> OpResult<HashMap<i32, PathBuf>> {
let mut collected = HashMap::with_capacity(4000);
for entry in fs::read_dir(path)? {
match entry {
Ok(de) => {
let path = Self::resolve_path(&de)?;
if !path.is_file() {
continue;
};
if let Some(file_name) = path.as_path().file_name() {
if let Some(file_name) = file_name.to_str() {
if let Some(index) = Self::parse_blk_index(file_name) {
collected.insert(index, path);
}
}
}
}
Err(msg) => {
return Err(OpError::from(msg));
}
}
}
collected.shrink_to_fit();
if collected.is_empty() {
Err(OpError::new(OpErrorKind::RuntimeError).join_msg("No blk files found!"))
} else {
Ok(collected)
}
}
///
/// Resolve symlink.
///
fn resolve_path(entry: &DirEntry) -> io::Result<PathBuf> {
if entry.file_type()?.is_symlink() {
fs::read_link(entry.path())
} else {
Ok(entry.path())
}
}
///
/// Extract index from block file name.
///
fn parse_blk_index(file_name: &str) -> Option<i32> {
let prefix = "blk";
let ext = ".dat";
if file_name.starts_with(prefix) && file_name.ends_with(ext) {
file_name[prefix.len()..(file_name.len() - ext.len())]
.parse::<i32>()
.ok()
} else {
None
}
}
}
// #[cfg(test)]
// mod tests {
// use super::*;
// #[test]
// fn test_parse_blk_index() {
// assert_eq!(0, BlkFiles::parse_blk_index("blk00000.dat").unwrap());
// assert_eq!(6, BlkFiles::parse_blk_index("blk6.dat").unwrap());
// assert_eq!(1202, BlkFiles::parse_blk_index("blk1202.dat").unwrap());
// assert_eq!(
// 13412451,
// BlkFiles::parse_blk_index("blk13412451.dat").unwrap()
// );
// assert!(BlkFiles::parse_blk_index("blkindex.dat").is_none());
// assert!(BlkFiles::parse_blk_index("invalid.dat").is_none());
// }
// }
parser/src/bitcoin/db/block_iter.rs
-45
View File
@@ -1,45 +0,0 @@
//!
//! View development note of iter_connected.rs for implementation
//! details of iter_block.rs, which follows similar principles.
//!
use bitcoin::Block;
use par_iter_sync::{IntoParallelIteratorSync, ParIterSync};
use super::BitcoinDB;
pub struct BlockIter(ParIterSync<Block>);
impl BlockIter {
/// the worker threads are dispatched in this `new` constructor!
pub fn new<T>(db: &BitcoinDB, heights: T) -> Self
where
T: IntoIterator<Item = usize> + Send + 'static,
<T as IntoIterator>::IntoIter: Send + 'static,
{
let db_ref = db.clone();
BlockIter(
heights.into_par_iter_sync(move |h| match db_ref.get_block(h) {
Ok(blk) => Ok(blk),
Err(_) => Err(()),
}),
)
}
/// the worker threads are dispatched in this `new` constructor!
pub fn from_range(db: &BitcoinDB, start: usize, end: usize) -> Self {
if end <= start {
BlockIter::new(db, Vec::new())
} else {
BlockIter::new(db, start..end)
}
}
}
impl Iterator for BlockIter {
type Item = Block;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
parser/src/bitcoin/db/blocks_indexes.rs
-211
View File
@@ -1,211 +0,0 @@
use std::{collections::BTreeMap, fmt, path::Path};
use bitcoin::{block::Header, io::Cursor, BlockHash};
use derive_deref::{Deref, DerefMut};
use leveldb::{
database::{iterator::LevelDBIterator, Database},
iterator::Iterable,
options::{Options, ReadOptions},
};
use crate::utils::log;
use super::{BlockchainRead, OpResult};
///
/// See Bitcoin Core repository for definition.
///
const BLOCK_VALID_HEADER: u32 = 1;
const BLOCK_VALID_TREE: u32 = 2;
const BLOCK_VALID_TRANSACTIONS: u32 = 3;
const BLOCK_VALID_CHAIN: u32 = 4;
const BLOCK_VALID_SCRIPTS: u32 = 5;
const BLOCK_VALID_MASK: u32 = BLOCK_VALID_HEADER
| BLOCK_VALID_TREE
| BLOCK_VALID_TRANSACTIONS
| BLOCK_VALID_CHAIN
| BLOCK_VALID_SCRIPTS;
const BLOCK_HAVE_DATA: u32 = 8;
const BLOCK_HAVE_UNDO: u32 = 16;
///
/// - Map from block height to block hash (records)
/// - Map from block hash to block height (hash_to_height)
///
#[derive(Clone, Deref, DerefMut)]
pub struct BlocksIndexes(Box<[BlockIndexRecord]>);
///
/// BLOCK_INDEX RECORD as defined in Bitcoin Core.
///
#[derive(Clone)]
pub struct BlockIndexRecord {
pub n_version: i32,
pub n_height: i32,
pub n_status: u32,
pub n_tx: u32,
pub n_file: i32,
pub n_data_pos: u32,
pub n_undo_pos: u32,
pub header: Header,
}
impl BlocksIndexes {
///
/// Build a collections of block index.
///
pub(crate) fn new(p: &Path) -> OpResult<Self> {
Ok(Self(load_block_index(p)?.into_boxed_slice()))
}
}
///
/// Load all block index in memory from leveldb (i.e. `blocks/index` path).
///
/// Map from block height to block index record.
///
pub fn load_block_index(path: &Path) -> OpResult<Vec<BlockIndexRecord>> {
let mut block_index_by_block_hash = BTreeMap::new();
log("Start loading block_index");
let mut options = Options::new();
options.create_if_missing = false;
let db: Database<BlockKey> = Database::open(path, options)?;
let options = ReadOptions::new();
let mut iter = db.iter(options);
let mut max_height_block_hash = Option::<(BlockHash, i32)>::None;
while iter.advance() {
let k = iter.key();
let v = iter.value();
if is_block_index_record(&k.key) {
let record = BlockIndexRecord::from(&v)?;
// only add valid block index record that has block data.
if record.n_height == 0
|| (record.n_status & BLOCK_VALID_MASK >= BLOCK_VALID_SCRIPTS
&& record.n_status & BLOCK_HAVE_DATA > 0)
{
let block_hash = record.header.block_hash();
// find the block with max height
if let Some((hash, height)) = max_height_block_hash.as_mut() {
if record.n_height > *height {
*hash = block_hash;
*height = record.n_height;
}
} else {
max_height_block_hash = Some((block_hash, record.n_height));
}
block_index_by_block_hash.insert(block_hash, record);
}
}
}
// build the longest chain
if let Some((hash, height)) = max_height_block_hash {
let mut block_index = Vec::with_capacity(height as usize + 1);
let mut current_hash = hash;
let mut current_height = height;
// recursively build block index from max height block.
while current_height >= 0 {
let blk = block_index_by_block_hash
.remove(&current_hash)
.expect("block hash not found in block index!");
assert_eq!(
current_height, blk.n_height,
"some block info missing from block index levelDB,\
delete Bitcoin folder and re-download!"
);
current_hash = blk.header.prev_blockhash;
current_height -= 1;
block_index.push(blk);
}
block_index.reverse();
Ok(block_index)
} else {
Ok(Vec::with_capacity(0))
}
}
/// levelDB key util
struct BlockKey {
key: Vec<u8>,
}
/// levelDB key util
impl db_key::Key for BlockKey {
fn from_u8(key: &[u8]) -> Self {
BlockKey {
key: Vec::from(key),
}
}
fn as_slice<T, F: Fn(&[u8]) -> T>(&self, f: F) -> T {
f(&self.key)
}
}
impl BlockIndexRecord {
///
/// Decode levelDB value for Block Index Record.
///
fn from(values: &[u8]) -> OpResult<Self> {
let mut reader = Cursor::new(values);
let n_version = reader.read_varint()? as i32;
let n_height = reader.read_varint()? as i32;
let n_status = reader.read_varint()? as u32;
let n_tx = reader.read_varint()? as u32;
let n_file = if n_status & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO) > 0 {
reader.read_varint()? as i32
} else {
-1
};
let n_data_pos = if n_status & BLOCK_HAVE_DATA > 0 {
reader.read_varint()? as u32
} else {
u32::MAX
};
let n_undo_pos = if n_status & BLOCK_HAVE_UNDO > 0 {
reader.read_varint()? as u32
} else {
u32::MAX
};
let header = reader.read_block_header()?;
Ok(BlockIndexRecord {
n_version,
n_height,
n_status,
n_tx,
n_file,
n_data_pos,
n_undo_pos,
header,
})
}
}
#[inline]
fn is_block_index_record(data: &[u8]) -> bool {
data.first() == Some(&b'b')
}
impl fmt::Debug for BlockIndexRecord {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BlockIndexRecord")
.field("version", &self.n_version)
.field("height", &self.n_height)
.field("status", &self.n_status)
.field("n_tx", &self.n_tx)
.field("n_file", &self.n_file)
.field("n_data_pos", &self.n_data_pos)
.field("header", &self.header)
.finish()
}
}
parser/src/bitcoin/db/errors.rs
-135
View File
@@ -1,135 +0,0 @@
use std::convert::{self, From};
use std::error;
use std::fmt;
use std::io;
use std::string;
use std::sync;
pub type OpResult<T> = Result<T, OpError>;
#[derive(Debug)]
/// Custom error type
pub struct OpError {
pub kind: OpErrorKind,
pub message: String,
}
impl OpError {
pub fn new(kind: OpErrorKind) -> Self {
OpError {
kind,
message: String::new(),
}
}
/// Joins the Error with a new message and returns it
pub fn join_msg(mut self, msg: &str) -> Self {
self.message.push_str(msg);
OpError {
kind: self.kind,
message: self.message,
}
}
}
impl fmt::Display for OpError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.message.is_empty() {
write!(f, "{}", &self.kind)
} else {
write!(f, "{} {}", &self.message, &self.kind)
}
}
}
impl error::Error for OpError {
fn description(&self) -> &str {
self.message.as_ref()
}
fn cause(&self) -> Option<&dyn error::Error> {
self.kind.source()
}
}
#[derive(Debug)]
pub enum OpErrorKind {
None,
IoError(io::Error),
Utf8Error(string::FromUtf8Error),
RuntimeError,
PoisonError,
SendError,
}
impl fmt::Display for OpErrorKind {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
OpErrorKind::IoError(ref err) => write!(f, "I/O Error: {}", err),
OpErrorKind::Utf8Error(ref err) => write!(f, "Utf8 Conversion: {}", err),
ref err @ OpErrorKind::PoisonError => write!(f, "Threading Error: {}", err),
ref err @ OpErrorKind::SendError => write!(f, "Sync: {}", err),
ref err @ OpErrorKind::RuntimeError => write!(f, "RuntimeError: {}", err),
OpErrorKind::None => write!(f, ""),
}
}
}
impl error::Error for OpErrorKind {
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match *self {
OpErrorKind::IoError(ref err) => Some(err),
OpErrorKind::Utf8Error(ref err) => Some(err),
ref err @ OpErrorKind::PoisonError => Some(err),
ref err @ OpErrorKind::SendError => Some(err),
_ => None,
}
}
}
impl From<io::Error> for OpError {
fn from(err: io::Error) -> Self {
Self::new(OpErrorKind::IoError(err))
}
}
impl From<bitcoin::consensus::encode::Error> for OpError {
fn from(_: bitcoin::consensus::encode::Error) -> Self {
Self::from("block decode error")
}
}
impl convert::From<i32> for OpError {
fn from(err_code: i32) -> Self {
Self::from(io::Error::from_raw_os_error(err_code))
}
}
impl convert::From<&str> for OpError {
fn from(err: &str) -> Self {
Self::new(OpErrorKind::None).join_msg(err)
}
}
impl<T> convert::From<sync::PoisonError<T>> for OpError {
fn from(_: sync::PoisonError<T>) -> Self {
Self::new(OpErrorKind::PoisonError)
}
}
impl<T> convert::From<sync::mpsc::SendError<T>> for OpError {
fn from(_: sync::mpsc::SendError<T>) -> Self {
Self::new(OpErrorKind::SendError)
}
}
impl convert::From<string::FromUtf8Error> for OpError {
fn from(err: string::FromUtf8Error) -> Self {
Self::new(OpErrorKind::Utf8Error(err))
}
}
impl convert::From<leveldb::error::Error> for OpError {
fn from(err: leveldb::error::Error) -> Self {
Self::from(err.to_string().as_ref())
}
}
parser/src/bitcoin/db/mod.rs
-172
View File
@@ -1,172 +0,0 @@
//!
//! Mostly a stripped down copy pasta of bitcoin-explorer
//!
//! Huge props to https://github.com/Congyuwang
//!
//! Crates APIs, essential structs, functions, methods are all here!
//!
//! To quickly understand how to use this crate, have a look at the
//! documentation for `bitcoin_explorer::BitcoinDB`!!.
//!
mod blk_files;
mod block_iter;
mod blocks_indexes;
mod errors;
mod reader;
mod txdb;
use blk_files::*;
use blocks_indexes::*;
use errors::*;
use reader::*;
use txdb::*;
use std::ops::Deref;
use std::path::Path;
use std::sync::Arc;
use bitcoin::{Block, Transaction, Txid};
pub use block_iter::BlockIter;
pub struct InnerDB {
pub blocks_indexes: BlocksIndexes,
pub blk_files: BlkFiles,
pub tx_db: TxDB,
}
///
/// This is the main struct of this crate!! Click and read the doc.
///
/// All queries start from initializing `BitcoinDB`.
///
/// Note: This is an Arc wrap around `InnerDB`.
///
#[derive(Clone)]
pub struct BitcoinDB(Arc<InnerDB>);
impl Deref for BitcoinDB {
type Target = InnerDB;
fn deref(&self) -> &Self::Target {
self.0.deref()
}
}
impl BitcoinDB {
///
/// This is the main structure for reading Bitcoin blockchain data.
///
/// Instantiating this class by passing the `-datadir` directory of
/// Bitcoin core to the `new()` method.
/// `tx_index`: whether to try to open tx_index levelDB.
///
pub fn new(p: &Path, tx_index: bool) -> OpResult<BitcoinDB> {
if !p.exists() {
return Err(OpError::from("data_dir does not exist"));
}
let blk_path = p.join("blocks");
let index_path = blk_path.join("index");
let blocks_indexes = BlocksIndexes::new(index_path.as_path())?;
let tx_db = if tx_index {
let tx_index_path = p.join("indexes").join("txindex");
TxDB::new(&tx_index_path)
} else {
TxDB::null()
};
let inner = InnerDB {
blocks_indexes,
blk_files: BlkFiles::new(blk_path.as_path())?,
tx_db,
};
Ok(BitcoinDB(Arc::new(inner)))
}
///
/// Get the maximum number of blocks downloaded.
///
/// This API guarantee that block 0 to `get_block_count() - 1`
/// have been downloaded and available for query.
///
pub fn get_block_count(&self) -> usize {
let records = self.blocks_indexes.len();
for h in 0..records {
// n_tx == 0 indicates that the block is not downloaded
if self.blocks_indexes.get(h).unwrap().n_tx == 0 {
return h;
}
}
records
}
///
/// Get a block
///
pub fn get_block(&self, height: usize) -> OpResult<Block> {
if let Some(index) = self.blocks_indexes.get(height) {
Ok(self.blk_files.read_block(index.n_file, index.n_data_pos)?)
} else {
Err(OpError::from("height not found"))
}
}
///
/// Get a transaction by providing txid.
///
/// This function requires `txindex` to be set to `true` for `BitcoinDB`,
/// and requires that flag `txindex=1` has been enabled when
/// running Bitcoin Core.
///
/// A transaction cannot be found using this function if it is
/// not yet indexed using `txindex`.
///
pub fn get_transaction(&self, txid: &Txid) -> OpResult<Transaction> {
if !self.tx_db.is_open() {
return Err(OpError::from("TxDB not open"));
}
// give special treatment for genesis transaction
if self.tx_db.is_genesis_tx(txid) {
return Ok(self.get_block(0)?.txdata.swap_remove(0));
}
let record = self.tx_db.get_tx_record(txid)?;
self.blk_files
.read_transaction(record.n_file, record.n_pos, record.n_tx_offset)
}
///
/// Iterate through all blocks from `start` to `end` (excluded).
///
/// # Performance
///
/// This iterator is implemented to read the blocks in concurrency,
/// but the result is still produced in sequential order.
/// Results read are stored in a synced queue for `next()`
/// to get.
///
/// The iterator stops automatically when a block cannot be
/// read (i.e., when the max height in the database met).
///
/// This is a very efficient implementation.
/// Using SSD and intel core i7 (4 core, 8 threads)
/// Iterating from height 0 to 700000 takes about 10 minutes.
///
pub fn iter_block(&self, start: usize, end: usize) -> BlockIter {
BlockIter::from_range(self, start, end)
}
pub fn check_if_txout_value_is_zero(&self, txid: &Txid, vout: usize) -> bool {
self.get_transaction(txid)
.unwrap()
.output
.get(vout)
.unwrap()
.to_owned()
.value
.to_sat()
== 0
}
}
parser/src/bitcoin/db/reader.rs
-90
View File
@@ -1,90 +0,0 @@
use std::{fs::File, io::BufReader};
use bitcoin::{block::Header, consensus::Decodable, io::Cursor, Block, Transaction};
use byteorder::{LittleEndian, ReadBytesExt};
use super::OpResult;
///
/// binary file read utilities.
///
pub trait BlockchainRead {
#[inline]
fn read_varint(&mut self) -> OpResult<usize>
where
Self: bitcoin::io::Read,
{
let mut n = 0;
loop {
let ch_data = self.read_u8()?;
n = (n << 7) | (ch_data & 0x7F) as usize;
if ch_data & 0x80 > 0 {
n += 1;
} else {
break;
}
}
Ok(n)
}
#[inline]
fn read_u8(&mut self) -> OpResult<u8>
where
Self: bitcoin::io::Read,
{
let mut slice = [0u8; 1];
self.read_exact(&mut slice).unwrap();
Ok(slice[0])
}
#[inline]
fn read_u32(&mut self) -> OpResult<u32>
where
Self: std::io::Read,
{
let u = ReadBytesExt::read_u32::<LittleEndian>(self)?;
Ok(u)
}
#[inline]
fn read_u8_vec(&mut self, count: u32) -> OpResult<Vec<u8>>
where
Self: bitcoin::io::Read,
{
let mut arr = vec![0u8; count as usize];
self.read_exact(&mut arr).unwrap();
Ok(arr)
}
#[inline]
fn read_block(&mut self) -> OpResult<Block>
where
Self: bitcoin::io::BufRead,
{
Ok(Block::consensus_decode(self)?)
}
#[inline]
fn read_transaction(&mut self) -> OpResult<Transaction>
where
Self: bitcoin::io::BufRead,
{
Ok(Transaction::consensus_decode(self)?)
}
#[inline]
fn read_block_header(&mut self) -> OpResult<Header>
where
Self: bitcoin::io::BufRead,
{
Ok(Header::consensus_decode(self)?)
}
}
impl<T> BlockchainRead for Cursor<T> {}
impl BlockchainRead for BufReader<File> {}
parser/src/bitcoin/db/txdb.rs
-147
View File
@@ -1,147 +0,0 @@
use std::{path::Path, str::FromStr};
use bitcoin::{hashes::Hash, io::Cursor, Txid};
use leveldb::{
database::Database,
kv::KV,
options::{Options, ReadOptions},
};
use crate::utils::log;
use super::{BlockchainRead, OpError, OpResult};
const GENESIS_TXID: &str = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b";
///
/// tx-index: looking up transaction position using txid.
///
/// This is possible if Bitcoin Core has `txindex=1`.
///
pub struct TxDB {
db: Option<Database<TxKey>>,
genesis_txid: Txid,
}
/// Records transaction storage on disk
pub struct TransactionRecord {
pub txid: Txid,
pub n_file: i32,
pub n_pos: u32,
pub n_tx_offset: u32,
}
impl TransactionRecord {
fn from(key: &[u8], values: &[u8]) -> OpResult<Self> {
let mut reader = Cursor::new(values);
Ok(TransactionRecord {
txid: Txid::from_slice(key).unwrap(),
n_file: reader.read_varint()? as i32,
n_pos: reader.read_varint()? as u32,
n_tx_offset: reader.read_varint()? as u32,
})
}
}
impl TxDB {
/// initialize TxDB for transaction queries
pub fn new(path: &Path) -> TxDB {
let option_db = TxDB::try_open_db(path);
if let Some(db) = option_db {
TxDB {
db: Some(db),
genesis_txid: Txid::from_str(GENESIS_TXID).unwrap(),
}
} else {
TxDB::null()
}
}
#[inline]
pub fn is_open(&self) -> bool {
self.db.is_some()
}
#[inline]
pub fn null() -> TxDB {
TxDB {
db: None,
genesis_txid: Txid::from_str(GENESIS_TXID).unwrap(),
}
}
#[inline]
///
/// genesis tx is not included in UTXO because of Bitcoin Core Bug
///
pub fn is_genesis_tx(&self, txid: &Txid) -> bool {
txid == &self.genesis_txid
}
fn try_open_db(path: &Path) -> Option<Database<TxKey>> {
if !path.exists() {
log("Failed to open tx_index DB: tx_index not built");
return None;
}
let options = Options::new();
match Database::open(path, options) {
Ok(db) => {
log("Successfully opened tx_index DB!");
Some(db)
}
Err(e) => {
log(&format!("Failed to open tx_index DB: {:?}", e));
None
}
}
}
/// note that this function cannot find genesis block, which needs special treatment
pub fn get_tx_record(&self, txid: &Txid) -> OpResult<TransactionRecord> {
if let Some(db) = &self.db {
let inner = txid.as_byte_array();
let mut key = Vec::with_capacity(inner.len() + 1);
key.push(b't');
key.extend(inner);
let key = TxKey { key };
let read_options = ReadOptions::new();
match db.get(read_options, &key) {
Ok(value) => {
if let Some(value) = value {
Ok(TransactionRecord::from(&key.key[1..], value.as_slice())?)
} else {
Err(OpError::from(
format!("value not found for txid: {}", txid).as_str(),
))
}
}
Err(e) => Err(OpError::from(
format!("value not found for txid: {}", e).as_str(),
)),
}
} else {
Err(OpError::from("TxDB not open"))
}
}
}
/// levelDB key utility
struct TxKey {
key: Vec<u8>,
}
/// levelDB key utility
impl db_key::Key for TxKey {
fn from_u8(key: &[u8]) -> Self {
TxKey {
key: Vec::from(key),
}
}
fn as_slice<T, F: Fn(&[u8]) -> T>(&self, f: F) -> T {
f(&self.key)
}
}
parser/src/bitcoin/mod.rs
-9
View File
@@ -1,9 +0,0 @@
mod addresses;
mod consts;
mod daemon;
mod db;
pub use addresses::*;
pub use consts::*;
pub use daemon::*;
pub use db::*;
parser/src/databases/txid_to_tx_data.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use std::{
};
use allocative::Allocative;
use bitcoin::Txid;
use biter::bitcoin::Txid;
use rayon::prelude::*;
use crate::structs::{Date, Height, TxData};
parser/src/datasets/mining.rs
+4 -2
View File
@@ -3,13 +3,15 @@ use itertools::Itertools;
use ordered_float::OrderedFloat;
use crate::{
bitcoin::TARGET_BLOCKS_PER_DAY,
datasets::AnyDataset,
structs::{
date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap,
AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey,
},
utils::{BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
utils::{
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
TARGET_BLOCKS_PER_DAY,
},
};
use super::{
parser/src/lib.rs
+1 -3
View File
@@ -1,5 +1,4 @@
mod actions;
mod bitcoin;
mod databases;
mod datasets;
mod io;
@@ -10,12 +9,11 @@ mod utils;
pub use crate::{
actions::iter_blocks,
bitcoin::{BitcoinDB, BitcoinDaemon},
datasets::OHLC,
io::{Binary, Json, Serialization},
structs::{
Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
HEIGHT_MAP_CHUNK_SIZE,
},
utils::log,
utils::{create_rpc, log},
};
parser/src/main.rs
+13 -34
View File
@@ -1,51 +1,30 @@
use std::{path::Path, thread::sleep, time::Duration};
use std::{thread::sleep, time::Duration};
use parser::{iter_blocks, log, BitcoinDB, BitcoinDaemon, Config, Height};
use biter::bitcoincore_rpc::RpcApi;
use parser::{create_rpc, iter_blocks, log, Config};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let config = Config::read();
let config = Config::import();
if config.datadir.is_none() {
println!(
"You need to set the --datadir parameter at least once to run the parser.\nRun the program with '-h' for help."
);
std::process::exit(1);
}
config.write()?;
let daemon = BitcoinDaemon::new(&config);
let rpc = create_rpc(&config).unwrap();
loop {
daemon.stop();
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
// Scoped to free bitcoin's lock
let block_count = {
let bitcoin_db = BitcoinDB::new(Path::new(&config.datadir.as_ref().unwrap()), true)?;
log(&format!("{block_count} blocks found."));
// let block_count = 200_000;
let block_count = bitcoin_db.get_block_count();
log(&format!("{block_count} blocks found."));
iter_blocks(&bitcoin_db, block_count)?;
block_count
};
daemon.start();
if daemon.check_if_fully_synced() {
daemon.wait_for_new_block(Height::new(block_count as u32 - 1));
} else {
daemon.wait_sync();
}
iter_blocks(&config, &rpc, block_count)?;
if let Some(delay) = config.delay {
sleep(Duration::from_secs(delay))
}
log("Waiting for new block...");
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
sleep(Duration::from_secs(5))
}
}
// Ok(())
parser/src/structs/address.rs
+2 -2
View File
@@ -1,10 +1,10 @@
use bitcoin::TxOut;
use bitcoin_hashes::{hash160, Hash};
use biter::bitcoin::TxOut;
use itertools::Itertools;
use crate::{
bitcoin::multisig_addresses,
databases::{U8x19, U8x31, SANAKIRJA_MAX_KEY_SIZE},
utils::multisig_addresses,
};
use super::{AddressType, Counter};
parser/src/structs/amount.rs
+1 -1
View File
@@ -10,7 +10,7 @@ use bincode::{
error::{DecodeError, EncodeError},
BorrowDecode, Decode, Encode,
};
use bitcoin::Amount as BitcoinAmount;
use biter::bitcoin::Amount as BitcoinAmount;
use derive_deref::{Deref, DerefMut};
use sanakirja::{direct_repr, Storable, UnsizedStorable};
use serde::{Deserialize, Serialize};
parser/src/structs/bi_map.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use std::{
use allocative::Allocative;
use crate::{bitcoin::TARGET_BLOCKS_PER_DAY, utils::LossyFrom};
use crate::utils::{LossyFrom, TARGET_BLOCKS_PER_DAY};
use super::{AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapValue};
parser/src/structs/config.rs
+53 -15
View File
@@ -6,17 +6,21 @@ use serde::{Deserialize, Serialize};
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize)]
#[command(version, about, long_about = None)]
pub struct Config {
/// bitcoind `-datadir=<dir>` argument
/// Bitcoin data directory path
#[arg(long, value_name = "DIR")]
pub datadir: Option<String>,
/// bitcoind `-blocksonly` argument, default: true
#[arg(long)]
pub blocksonly: Option<bool>,
/// Bitcoin RPC port, default: 8332
#[arg(long, value_name = "PORT")]
pub rpcport: Option<u16>,
/// bitcoind `-rpcconnect=<ip>` argument
#[arg(long, value_name = "IP")]
pub rpcconnect: Option<String>,
/// Bitcoin RPC username
#[arg(long, value_name = "USERNAME")]
pub rpcuser: Option<String>,
/// Bitcoin RPC password
#[arg(long, value_name = "PASSWORD")]
pub rpcpassword: Option<String>,
/// Delay between runs, default: 0
#[arg(long, value_name = "SECONDS")]
@@ -26,7 +30,7 @@ pub struct Config {
impl Config {
const PATH: &'static str = "config.toml";
pub fn read() -> Self {
pub fn import() -> Self {
let mut config_saved = fs::read_to_string(Self::PATH)
.map_or(Config::default(), |contents| {
toml::from_str(&contents).unwrap_or_default()
@@ -38,24 +42,58 @@ impl Config {
config_saved.datadir = Some(datadir);
}
if let Some(blocksonly) = config_args.blocksonly {
config_saved.blocksonly = Some(blocksonly);
if let Some(rpcport) = config_args.rpcport {
config_saved.rpcport = Some(rpcport);
} else {
config_saved.blocksonly = Some(true);
config_saved.rpcport = Some(8332);
}
if let Some(rpcconnect) = config_args.rpcconnect {
config_saved.rpcconnect = Some(rpcconnect);
if let Some(rpcuser) = config_args.rpcuser {
config_saved.rpcuser = Some(rpcuser);
}
if let Some(rpcpassword) = config_args.rpcpassword {
config_saved.rpcpassword = Some(rpcpassword);
}
if let Some(delay) = config_args.delay {
config_saved.delay = Some(delay);
}
config_saved
// Done importing
let config = config_saved;
config.check();
config.write().unwrap();
config
}
pub fn write(&self) -> std::io::Result<()> {
fn check(&self) {
if self.datadir.is_none() {
Self::exit("datadir");
}
if self.rpcuser.is_none() {
Self::exit("rpcuser");
}
if self.rpcpassword.is_none() {
Self::exit("rpcpassword");
}
}
fn exit(attribute: &str) {
println!(
"You need to set the --{} parameter at least once to run the parser.\nRun the program with '-h' for help.", attribute
);
std::process::exit(1);
}
fn write(&self) -> std::io::Result<()> {
fs::write(Self::PATH, toml::to_string(self).unwrap())
}
}
parser/src/structs/height.rs
+2 -1
View File
@@ -5,10 +5,11 @@ use std::{
use allocative::Allocative;
use bincode::{Decode, Encode};
use biter::NUMBER_OF_UNSAFE_BLOCKS;
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use crate::{bitcoin::NUMBER_OF_UNSAFE_BLOCKS, HEIGHT_MAP_CHUNK_SIZE};
use crate::HEIGHT_MAP_CHUNK_SIZE;
use super::{HeightMapChunkId, MapKey};
parser/src/utils/arr.rs
-219
View File
@@ -1,219 +0,0 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Div, Mul, Sub, SubAssign},
};
use itertools::Itertools;
use ordered_float::{FloatCore, OrderedFloat};
use super::ToF32;
pub trait ArrayOperations<T> {
fn transform<F>(&self, transform: F) -> Vec<T>
where
T: Copy + Default,
F: Fn((usize, &T, &[T])) -> T;
fn add(&self, other: &[T]) -> Vec<T>
where
T: Add<Output = T> + Copy + Default;
fn subtract(&self, other: &[T]) -> Vec<T>
where
T: Sub<Output = T> + Copy + Default;
fn multiply(&self, other: &[T]) -> Vec<T>
where
T: Mul<Output = T> + Copy + Default;
fn divide(&self, other: &[T]) -> Vec<T>
where
T: Div<Output = T> + Copy + Default;
fn match_size<'a>(&'a self, other: &'a [T]) -> &'a [T];
fn cumulate(&self) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign;
fn last_x_sum(&self, x: usize) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign + SubAssign;
fn moving_average(&self, x: usize) -> Vec<f32>
where
T: Sum + Copy + Default + AddAssign + SubAssign + ToF32;
fn net_change(&self, offset: usize) -> Vec<T>
where
T: Copy + Default + Sub<Output = T>;
fn median(&self, size: usize) -> Vec<Option<T>>
where
T: FloatCore;
}
impl<T> ArrayOperations<T> for &[T] {
fn transform<F>(&self, transform: F) -> Vec<T>
where
T: Copy + Default,
F: Fn((usize, &T, &[T])) -> T,
{
self.iter()
.enumerate()
.map(|(index, value)| transform((index, value, self)))
.collect_vec()
}
fn add(&self, other: &[T]) -> Vec<T>
where
T: Add<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value + *other.get(index).unwrap())
}
fn subtract(&self, other: &[T]) -> Vec<T>
where
T: Sub<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value - *other.get(index).unwrap())
}
fn multiply(&self, other: &[T]) -> Vec<T>
where
T: Mul<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value * *other.get(index).unwrap())
}
fn divide(&self, other: &[T]) -> Vec<T>
where
T: Div<Output = T> + Copy + Default,
{
self.match_size(other)
.transform(|(index, value, _)| *value / *other.get(index).unwrap())
}
fn match_size(&self, other: &[T]) -> &[T] {
let len = other.len();
if self.len() > len {
&self[..len]
} else {
self
}
}
fn cumulate(&self) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign,
{
let mut sum = T::default();
self.iter()
.map(|value| {
sum += *value;
sum
})
.collect_vec()
}
fn last_x_sum(&self, x: usize) -> Vec<T>
where
T: Sum + Copy + Default + AddAssign + SubAssign,
{
let mut sum = T::default();
self.iter()
.enumerate()
.map(|(index, value)| {
sum += *value;
if index >= x - 1 {
let previous_index = index + 1 - x;
sum -= *self.get(previous_index).unwrap()
}
sum
})
.collect_vec()
}
fn moving_average(&self, x: usize) -> Vec<f32>
where
T: Sum + Copy + Default + AddAssign + SubAssign + ToF32,
{
let mut sum = T::default();
self.iter()
.enumerate()
.map(|(index, value)| {
sum += *value;
if index >= x - 1 {
sum -= *self.get(index + 1 - x).unwrap()
}
sum.to_f32() / x as f32
})
.collect_vec()
}
fn net_change(&self, offset: usize) -> Vec<T>
where
T: Copy + Default + Sub<Output = T>,
{
self.transform(|(index, value, arr)| {
let previous = {
if let Some(previous_index) = index.checked_sub(offset) {
*arr.get(previous_index).unwrap()
} else {
T::default()
}
};
*value - previous
})
}
fn median(&self, size: usize) -> Vec<Option<T>>
where
T: FloatCore,
{
let even = size % 2 == 0;
let median_index = size / 2;
if size < 3 {
panic!("Computing a median for a size lower than 3 is useless");
}
self.iter()
.enumerate()
.map(|(index, _)| {
if index >= size - 1 {
let mut arr = self[index - (size - 1)..index + 1]
.iter()
.map(|value| OrderedFloat(*value))
.collect_vec();
arr.sort_unstable();
if even {
Some(
(**arr.get(median_index).unwrap()
+ **arr.get(median_index - 1).unwrap())
/ T::from(2.0).unwrap(),
)
} else {
Some(**arr.get(median_index).unwrap())
}
} else {
None
}
})
.collect()
}
}
parser/src/utils/bytes.rs
-1
View File
@@ -1 +0,0 @@
pub const BYTES_IN_MB: usize = 1_000_000;
parser/src/utils/date.rs → parser/src/utils/consts.rs
+4
View File
@@ -1,3 +1,7 @@
pub const BYTES_IN_MB: usize = 1_000_000;
pub const TARGET_BLOCKS_PER_DAY: usize = 144;
pub const ONE_DAY_IN_DAYS: usize = 1;
pub const ONE_WEEK_IN_DAYS: usize = 7;
pub const TWO_WEEK_IN_DAYS: usize = 2 * ONE_WEEK_IN_DAYS;
parser/src/utils/log.rs
+1 -11
View File
@@ -1,4 +1,4 @@
use std::{fs::OpenOptions, io::Write, process::Output};
use std::{fs::OpenOptions, io::Write};
use chrono::Local;
use color_eyre::owo_colors::OwoColorize;
@@ -23,13 +23,3 @@ pub fn log(str: &str) {
println!("{} {}", date_time.bright_black(), line);
});
}
pub fn log_output(output: &Output) {
if !output.stdout.is_empty() {
log(&String::from_utf8_lossy(&output.stdout));
}
if !output.stderr.is_empty() {
log(&String::from_utf8_lossy(&output.stderr));
}
}
parser/src/utils/mod.rs
+6 -4
View File
@@ -1,17 +1,19 @@
mod bytes;
mod date;
mod consts;
mod flamegraph;
mod log;
mod lossy;
mod multisig;
mod percentile;
mod retry;
mod rpc;
mod time;
pub use bytes::*;
pub use date::*;
pub use consts::*;
pub use flamegraph::*;
pub use log::*;
pub use lossy::*;
pub use multisig::*;
pub use percentile::*;
pub use retry::*;
pub use rpc::*;
pub use time::*;
parser/src/bitcoin/addresses/multisig.rs → parser/src/utils/multisig.rs
+1 -1
View File
@@ -2,7 +2,7 @@
// Code from bitcoin-explorer now deprecated
//
use bitcoin::{
use biter::bitcoin::{
blockdata::{
opcodes::all,
script::Instruction::{self, Op, PushBytes},
parser/src/utils/rpc.rs
+13
View File
@@ -0,0 +1,13 @@
use biter::bitcoincore_rpc::{Auth, Client};
use crate::Config;
pub fn create_rpc(config: &Config) -> color_eyre::Result<Client> {
Ok(Client::new(
&format!("http://localhost:{}", config.rpcport.unwrap()),
Auth::UserPass(
config.rpcuser.clone().unwrap(),
config.rpcpassword.clone().unwrap(),
),
)?)
}