Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-28 00:29:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

general: snapshot

Browse Source
This commit is contained in:
k
2024-10-02 10:48:05 +02:00
parent 9d2c2f7945
commit 669205aa4d
22 changed files with 236 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
parser/src/actions/export.rs
View File

@@ -1,4 +1,4 @@
use std::thread;
use std::thread::{self};
use crate::{
databases::Databases,
@@ -6,6 +6,7 @@ use crate::{
states::States,
structs::{Date, Height},
utils::{log, time},
Exit,
};
pub struct ExportedData<'a> {
@@ -14,6 +15,7 @@ pub struct ExportedData<'a> {
pub date: Date,
pub height: Height,
pub states: Option<&'a States>,
pub exit: Exit,
}
pub fn export(
@@ -23,10 +25,17 @@ pub fn export(
states,
height,
date,
exit,
}: ExportedData,
) -> color_eyre::Result<()> {
if exit.active() {
log("Exit in progress, skipping export");
return Ok(());
}
exit.block();
log("Exporting...");
log("WARNING: NOT SAFE TO STOP !!!");
time("Total save time", || -> color_eyre::Result<()> {
time("Datasets saved", || datasets.export())?;
@@ -44,7 +53,7 @@ pub fn export(
Ok(())
})?;
log("Export done - Safe to stop now");
exit.unblock();
Ok(())
}

14
parser/src/actions/iter_blocks.rs
View File

@@ -15,18 +15,15 @@ use crate::{
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{Date, DateData, MapKey},
utils::{generate_allocation_files, log, time},
Config, Height,
Config, Exit, Height,
};
pub fn iter_blocks(
config: &Config,
rpc: &biter::bitcoincore_rpc::Client,
approx_block_count: usize,
exit: Exit,
) -> color_eyre::Result<()> {
let should_insert = true;
let should_export = true;
let study_ram_usage = false;
log("Starting...");
let mut datasets = AllDatasets::import()?;
@@ -132,7 +129,7 @@ pub fn iter_blocks(
processed_heights.insert(current_block_height);
if should_insert && first_unsafe_heights.inserted <= current_block_height {
if first_unsafe_heights.inserted <= current_block_height {
let compute_addresses = databases.check_if_needs_to_compute_addresses(
current_block_height,
blocks_loop_date,
@@ -216,7 +213,7 @@ pub fn iter_blocks(
});
}
if should_export {
if !config.dry_run {
let is_safe = height.is_safe(approx_block_count);
export(ExportedData {
@@ -225,9 +222,10 @@ pub fn iter_blocks(
date: blocks_loop_date.unwrap(),
height: last_height,
states: is_safe.then_some(&states),
exit: exit.clone(),
})?;
if study_ram_usage {
if config.record_ram_usage {
time("Exporing allocation files", || {
generate_allocation_files(&datasets, &databases, &states, last_height)
})?;

6
parser/src/actions/parse.rs
View File

@@ -811,6 +811,7 @@ fn pre_process_outputs(
// Op Return
// https://mempool.space/tx/139c004f477101c468767983536caaeef568613fab9c2ed9237521f5ff530afd
// Provably unspendable https://mempool.space/tx/8a68c461a2473653fe0add786f0ca6ebb99b257286166dfb00707be24716af3a#flow=&vout=0
#[allow(deprecated)]
if script.is_op_return() {
// TODO: Count fee paid to write said OP_RETURN, beware of coinbase transactions
// For coinbase transactions, count miners
@@ -818,9 +819,8 @@ fn pre_process_outputs(
provably_unspendable += amount;
// return None;
}
// https://mempool.space/tx/8a68c461a2473653fe0add786f0ca6ebb99b257286166dfb00707be24716af3a#flow=&vout=0
else if script.is_provably_unspendable() {
// https://mempool.space/tx/8a68c461a2473653fe0add786f0ca6ebb99b257286166dfb00707be24716af3a#flow=&vout=0
} else if script.is_provably_unspendable() {
provably_unspendable += amount;
// return None;
}

4
parser/src/lib.rs
View File

@@ -12,8 +12,8 @@ pub use crate::{
datasets::OHLC,
io::{Binary, Json, Serialization, COMPRESSED_BIN_EXTENSION, JSON_EXTENSION},
structs::{
Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
HEIGHT_MAP_CHUNK_SIZE,
Config, Date, DateMap, Exit, Height, HeightMap, MapChunkId, SerializedBTreeMap,
SerializedVec, HEIGHT_MAP_CHUNK_SIZE,
},
utils::{create_rpc, log, reset_logs},
};

8
parser/src/main.rs
View File

@@ -1,7 +1,7 @@
use std::{thread::sleep, time::Duration};
use biter::bitcoincore_rpc::RpcApi;
use parser::{create_rpc, iter_blocks, log, reset_logs, Config};
use parser::{create_rpc, iter_blocks, log, reset_logs, Config, Exit};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -12,12 +12,14 @@ fn main() -> color_eyre::Result<()> {
let rpc = create_rpc(&config).unwrap();
let exit = Exit::new();
loop {
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
log(&format!("{block_count} blocks found."));
iter_blocks(&config, &rpc, block_count)?;
iter_blocks(&config, &rpc, block_count, exit.clone())?;
if let Some(delay) = config.delay {
sleep(Duration::from_secs(delay))
@@ -25,7 +27,7 @@ fn main() -> color_eyre::Result<()> {
log("Waiting for new block...");
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
sleep(Duration::from_secs(5))
sleep(Duration::from_secs(1))
}
}

7
parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs
View File

@@ -128,7 +128,12 @@ impl UTXOCohortsDurableStates {
state
.decrement(amount, utxo_count, block_data.price)
.unwrap_or_else(|report| {
dbg!(report, block_data, sent_data, previous_last_block_data);
dbg!(
report.to_string(),
block_data,
sent_data,
previous_last_block_data
);
panic!()
});
})

8
parser/src/structs/config.rs
View File

@@ -29,6 +29,14 @@ pub struct Config {
/// Delay between runs, default: 0, saved
#[arg(long, value_name = "SECONDS")]
pub delay: Option<u64>,
/// Start a dry run, default: false, not saved
#[arg(long, default_value_t = false)]
pub dry_run: bool,
/// Record ram usage, default: false, not saved
#[arg(long, default_value_t = false)]
pub record_ram_usage: bool,
}
impl Config {

8
parser/src/structs/date.rs
View File

@@ -45,6 +45,14 @@ impl Date {
.date_naive(),
)
}
pub fn today() -> Self {
Self(chrono::offset::Utc::now().date_naive())
}
pub fn yesterday() -> Self {
Self(Self::today().checked_sub_days(Days::new(1)).unwrap())
}
}
impl MapKey<DateMapChunkId> for Date {

62
parser/src/structs/exit.rs Normal file
View File

@@ -0,0 +1,62 @@
use std::{
process::exit,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
thread::sleep,
time::Duration,
};
use super::super::log;
#[derive(Default, Clone)]
pub struct Exit {
blocked: Arc<AtomicBool>,
active: Arc<AtomicBool>,
}
impl Exit {
pub fn new() -> Self {
let s = Self {
active: Arc::new(AtomicBool::new(false)),
blocked: Arc::new(AtomicBool::new(false)),
};
let active = s.active.clone();
let _blocked = s.blocked.clone();
let blocked = move || _blocked.load(Ordering::SeqCst);
ctrlc::set_handler(move || {
log("Exitting...");
active.store(true, Ordering::SeqCst);
if blocked() {
log("Waiting to exit safely");
while blocked() {
sleep(Duration::from_millis(50));
}
}
exit(0);
})
.expect("Error setting Ctrl-C handler");
s
}
pub fn block(&self) {
self.blocked.store(true, Ordering::SeqCst);
}
pub fn unblock(&self) {
self.blocked.store(false, Ordering::SeqCst);
}
pub fn active(&self) -> bool {
self.active.load(Ordering::SeqCst)
}
}

47
parser/src/structs/liquidity.rs
View File

@@ -11,7 +11,6 @@ use super::Amount;
pub struct LiquidityClassification {
illiquid: f64,
liquid: f64,
// highly_liquid: f64,
}
impl LiquidityClassification {
@@ -54,11 +53,7 @@ impl LiquidityClassification {
unreachable!()
}
Self {
illiquid,
liquid,
// highly_liquid: 1.0 - liquid - illiquid,
}
Self { illiquid, liquid }
}
#[inline(always)]
@@ -136,43 +131,3 @@ where
self.highly_liquid -= rhs.highly_liquid;
}
}
// impl<T> SplitByLiquidity<T>
// where
// T: Default,
// {
// // pub fn get(&self, id: &LiquidityId) -> &T {
// // match id {
// // LiquidityId::All => &self.all,
// // LiquidityId::Illiquid => &self.illiquid,
// // LiquidityId::Liquid => &self.liquid,
// // LiquidityId::HighlyLiquid => &self.highly_liquid,
// // }
// // }
// pub fn get_mut(&mut self, id: &LiquidityId) -> &mut T {
// match id {
// LiquidityId::All => &mut self.all,
// LiquidityId::Illiquid => &mut self.illiquid,
// LiquidityId::Liquid => &mut self.liquid,
// LiquidityId::HighlyLiquid => &mut self.highly_liquid,
// }
// }
// pub fn as_vec(&self) -> Vec<(&T, LiquidityId)> {
// vec![
// (&self.all, LiquidityId::All),
// (&self.illiquid, LiquidityId::Illiquid),
// (&self.liquid, LiquidityId::Liquid),
// (&self.highly_liquid, LiquidityId::HighlyLiquid),
// ]
// }
// }
// #[derive(Debug, Clone, Copy)]
// pub enum LiquidityId {
// All,
// Illiquid,
// Liquid,
// HighlyLiquid,
// }

2
parser/src/structs/mod.rs
View File

@@ -16,6 +16,7 @@ mod date_data;
mod date_map;
mod date_map_chunk_id;
mod empty_address_data;
mod exit;
mod generic_map;
mod height;
mod height_map;
@@ -48,6 +49,7 @@ pub use date_data::*;
pub use date_map::*;
pub use date_map_chunk_id::*;
pub use empty_address_data::*;
pub use exit::*;
pub use generic_map::*;
pub use height::*;
pub use height_map::*;