Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 14:11:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

general: snapshot

Browse Source
This commit is contained in:
k
2024-10-02 10:48:05 +02:00
parent 9d2c2f7945
commit 669205aa4d
22 changed files with 236 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -14,6 +14,7 @@ in
out
.log
/datasets
/datasets2
/price
# Sync
@@ -46,4 +47,4 @@ benches
snapshots*/
# Docker
docker/kibo
docker/kibo
CHANGELOG.md
+3 -2
View File
@@ -5,9 +5,9 @@
![Image of the kibō Web App version 0.X.Y](./assets/v0.X.Y.jpg)
-->
## v. 0.4.1 | WIP
## v. 0.5.0 | WIP
<!-- ![Image of the kibō Web App version 0.4.1](./assets/v0.4.1.jpg) -->
<!-- ![Image of the kibō Web App version 0.5.0](./assets/v0.5.0.jpg) -->
## Datasets
@@ -26,6 +26,7 @@
- Added a `/datasets/last` json file with all the latest values
- Added `--rpcconnect` parameter to the config
- Added handling of SIGINT and SIGTERM terminal signals which menas you can now safely CTRL+C or kill the parser while it's exporting
## Server
parser/Cargo.lock
Generated
+30 -1
View File
@@ -340,6 +340,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.38"
@@ -510,6 +516,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "ctrlc"
version = "3.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
dependencies = [
"nix",
"windows-sys 0.59.0",
]
[[package]]
name = "dashmap"
version = "6.0.1"
@@ -1147,6 +1163,18 @@ dependencies = [
"tempfile",
]
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.6.0",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "num-format"
version = "0.4.4"
@@ -1280,7 +1308,7 @@ dependencies = [
[[package]]
name = "parser"
version = "0.4.1"
version = "0.5.0"
dependencies = [
"allocative",
"bincode",
@@ -1289,6 +1317,7 @@ dependencies = [
"chrono",
"clap",
"color-eyre",
"ctrlc",
"derive_deref",
"inferno",
"itertools",
parser/Cargo.toml
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "parser"
version = "0.4.1"
version = "0.5.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -15,6 +15,7 @@ bitcoin_hashes = { version = "0.14.0" }
chrono = { version = "0.4.38", features = ["serde"] }
clap = { version = "4.5.16", features = ["derive"] }
color-eyre = "0.6.3"
ctrlc = { version = "3.4.5", features = ["termination"] }
derive_deref = "1.1.1"
inferno = "0.11.21"
itertools = "0.13.0"
parser/reset.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
echo "Deleting datasets..."
rm -r ../datasets
echo "Deleting states and datasets..."
rm -r ./out
echo "Done."
parser/samply.sh
+2
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env bash
echo "Increasing limit of opened files..."
ulimit -n 1000000
parser/src/actions/export.rs
+12 -3
View File
@@ -1,4 +1,4 @@
use std::thread;
use std::thread::{self};
use crate::{
databases::Databases,
@@ -6,6 +6,7 @@ use crate::{
states::States,
structs::{Date, Height},
utils::{log, time},
Exit,
};
pub struct ExportedData<'a> {
@@ -14,6 +15,7 @@ pub struct ExportedData<'a> {
pub date: Date,
pub height: Height,
pub states: Option<&'a States>,
pub exit: Exit,
}
pub fn export(
@@ -23,10 +25,17 @@ pub fn export(
states,
height,
date,
exit,
}: ExportedData,
) -> color_eyre::Result<()> {
if exit.active() {
log("Exit in progress, skipping export");
return Ok(());
}
exit.block();
log("Exporting...");
log("WARNING: NOT SAFE TO STOP !!!");
time("Total save time", || -> color_eyre::Result<()> {
time("Datasets saved", || datasets.export())?;
@@ -44,7 +53,7 @@ pub fn export(
Ok(())
})?;
log("Export done - Safe to stop now");
exit.unblock();
Ok(())
}
parser/src/actions/iter_blocks.rs
+6 -8
View File
@@ -15,18 +15,15 @@ use crate::{
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{Date, DateData, MapKey},
utils::{generate_allocation_files, log, time},
Config, Height,
Config, Exit, Height,
};
pub fn iter_blocks(
config: &Config,
rpc: &biter::bitcoincore_rpc::Client,
approx_block_count: usize,
exit: Exit,
) -> color_eyre::Result<()> {
let should_insert = true;
let should_export = true;
let study_ram_usage = false;
log("Starting...");
let mut datasets = AllDatasets::import()?;
@@ -132,7 +129,7 @@ pub fn iter_blocks(
processed_heights.insert(current_block_height);
if should_insert && first_unsafe_heights.inserted <= current_block_height {
if first_unsafe_heights.inserted <= current_block_height {
let compute_addresses = databases.check_if_needs_to_compute_addresses(
current_block_height,
blocks_loop_date,
@@ -216,7 +213,7 @@ pub fn iter_blocks(
});
}
if should_export {
if !config.dry_run {
let is_safe = height.is_safe(approx_block_count);
export(ExportedData {
@@ -225,9 +222,10 @@ pub fn iter_blocks(
date: blocks_loop_date.unwrap(),
height: last_height,
states: is_safe.then_some(&states),
exit: exit.clone(),
})?;
if study_ram_usage {
if config.record_ram_usage {
time("Exporing allocation files", || {
generate_allocation_files(&datasets, &databases, &states, last_height)
})?;
parser/src/actions/parse.rs
+3 -3
View File
@@ -811,6 +811,7 @@ fn pre_process_outputs(
// Op Return
// https://mempool.space/tx/139c004f477101c468767983536caaeef568613fab9c2ed9237521f5ff530afd
// Provably unspendable https://mempool.space/tx/8a68c461a2473653fe0add786f0ca6ebb99b257286166dfb00707be24716af3a#flow=&vout=0
#[allow(deprecated)]
if script.is_op_return() {
// TODO: Count fee paid to write said OP_RETURN, beware of coinbase transactions
// For coinbase transactions, count miners
@@ -818,9 +819,8 @@ fn pre_process_outputs(
provably_unspendable += amount;
// return None;
}
// https://mempool.space/tx/8a68c461a2473653fe0add786f0ca6ebb99b257286166dfb00707be24716af3a#flow=&vout=0
else if script.is_provably_unspendable() {
// https://mempool.space/tx/8a68c461a2473653fe0add786f0ca6ebb99b257286166dfb00707be24716af3a#flow=&vout=0
} else if script.is_provably_unspendable() {
provably_unspendable += amount;
// return None;
}
parser/src/lib.rs
+2 -2
View File
@@ -12,8 +12,8 @@ pub use crate::{
datasets::OHLC,
io::{Binary, Json, Serialization, COMPRESSED_BIN_EXTENSION, JSON_EXTENSION},
structs::{
Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
HEIGHT_MAP_CHUNK_SIZE,
Config, Date, DateMap, Exit, Height, HeightMap, MapChunkId, SerializedBTreeMap,
SerializedVec, HEIGHT_MAP_CHUNK_SIZE,
},
utils::{create_rpc, log, reset_logs},
};
parser/src/main.rs
+5 -3
View File
@@ -1,7 +1,7 @@
use std::{thread::sleep, time::Duration};
use biter::bitcoincore_rpc::RpcApi;
use parser::{create_rpc, iter_blocks, log, reset_logs, Config};
use parser::{create_rpc, iter_blocks, log, reset_logs, Config, Exit};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -12,12 +12,14 @@ fn main() -> color_eyre::Result<()> {
let rpc = create_rpc(&config).unwrap();
let exit = Exit::new();
loop {
let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
log(&format!("{block_count} blocks found."));
iter_blocks(&config, &rpc, block_count)?;
iter_blocks(&config, &rpc, block_count, exit.clone())?;
if let Some(delay) = config.delay {
sleep(Duration::from_secs(delay))
@@ -25,7 +27,7 @@ fn main() -> color_eyre::Result<()> {
log("Waiting for new block...");
while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
sleep(Duration::from_secs(5))
sleep(Duration::from_secs(1))
}
}
parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs
+6 -1
View File
@@ -128,7 +128,12 @@ impl UTXOCohortsDurableStates {
state
.decrement(amount, utxo_count, block_data.price)
.unwrap_or_else(|report| {
dbg!(report, block_data, sent_data, previous_last_block_data);
dbg!(
report.to_string(),
block_data,
sent_data,
previous_last_block_data
);
panic!()
});
})
parser/src/structs/config.rs
+8
View File
@@ -29,6 +29,14 @@ pub struct Config {
/// Delay between runs, default: 0, saved
#[arg(long, value_name = "SECONDS")]
pub delay: Option<u64>,
/// Start a dry run, default: false, not saved
#[arg(long, default_value_t = false)]
pub dry_run: bool,
/// Record ram usage, default: false, not saved
#[arg(long, default_value_t = false)]
pub record_ram_usage: bool,
}
impl Config {
parser/src/structs/date.rs
+8
View File
@@ -45,6 +45,14 @@ impl Date {
.date_naive(),
)
}
pub fn today() -> Self {
Self(chrono::offset::Utc::now().date_naive())
}
pub fn yesterday() -> Self {
Self(Self::today().checked_sub_days(Days::new(1)).unwrap())
}
}
impl MapKey<DateMapChunkId> for Date {
parser/src/structs/exit.rs
+62
View File
@@ -0,0 +1,62 @@
use std::{
process::exit,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
thread::sleep,
time::Duration,
};
use super::super::log;
#[derive(Default, Clone)]
pub struct Exit {
blocked: Arc<AtomicBool>,
active: Arc<AtomicBool>,
}
impl Exit {
pub fn new() -> Self {
let s = Self {
active: Arc::new(AtomicBool::new(false)),
blocked: Arc::new(AtomicBool::new(false)),
};
let active = s.active.clone();
let _blocked = s.blocked.clone();
let blocked = move || _blocked.load(Ordering::SeqCst);
ctrlc::set_handler(move || {
log("Exitting...");
active.store(true, Ordering::SeqCst);
if blocked() {
log("Waiting to exit safely");
while blocked() {
sleep(Duration::from_millis(50));
}
}
exit(0);
})
.expect("Error setting Ctrl-C handler");
s
}
pub fn block(&self) {
self.blocked.store(true, Ordering::SeqCst);
}
pub fn unblock(&self) {
self.blocked.store(false, Ordering::SeqCst);
}
pub fn active(&self) -> bool {
self.active.load(Ordering::SeqCst)
}
}
parser/src/structs/liquidity.rs
+1 -46
View File
@@ -11,7 +11,6 @@ use super::Amount;
pub struct LiquidityClassification {
illiquid: f64,
liquid: f64,
// highly_liquid: f64,
}
impl LiquidityClassification {
@@ -54,11 +53,7 @@ impl LiquidityClassification {
unreachable!()
}
Self {
illiquid,
liquid,
// highly_liquid: 1.0 - liquid - illiquid,
}
Self { illiquid, liquid }
}
#[inline(always)]
@@ -136,43 +131,3 @@ where
self.highly_liquid -= rhs.highly_liquid;
}
}
// impl<T> SplitByLiquidity<T>
// where
// T: Default,
// {
// // pub fn get(&self, id: &LiquidityId) -> &T {
// // match id {
// // LiquidityId::All => &self.all,
// // LiquidityId::Illiquid => &self.illiquid,
// // LiquidityId::Liquid => &self.liquid,
// // LiquidityId::HighlyLiquid => &self.highly_liquid,
// // }
// // }
// pub fn get_mut(&mut self, id: &LiquidityId) -> &mut T {
// match id {
// LiquidityId::All => &mut self.all,
// LiquidityId::Illiquid => &mut self.illiquid,
// LiquidityId::Liquid => &mut self.liquid,
// LiquidityId::HighlyLiquid => &mut self.highly_liquid,
// }
// }
// pub fn as_vec(&self) -> Vec<(&T, LiquidityId)> {
// vec![
// (&self.all, LiquidityId::All),
// (&self.illiquid, LiquidityId::Illiquid),
// (&self.liquid, LiquidityId::Liquid),
// (&self.highly_liquid, LiquidityId::HighlyLiquid),
// ]
// }
// }
// #[derive(Debug, Clone, Copy)]
// pub enum LiquidityId {
// All,
// Illiquid,
// Liquid,
// HighlyLiquid,
// }
parser/src/structs/mod.rs
+2
View File
@@ -16,6 +16,7 @@ mod date_data;
mod date_map;
mod date_map_chunk_id;
mod empty_address_data;
mod exit;
mod generic_map;
mod height;
mod height_map;
@@ -48,6 +49,7 @@ pub use date_data::*;
pub use date_map::*;
pub use date_map_chunk_id::*;
pub use empty_address_data::*;
pub use exit::*;
pub use generic_map::*;
pub use height::*;
pub use height_map::*;
server/Cargo.lock
Generated
+31 -2
View File
@@ -579,6 +579,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.38"
@@ -777,6 +783,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "ctrlc"
version = "3.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
dependencies = [
"nix",
"windows-sys 0.59.0",
]
[[package]]
name = "dashmap"
version = "5.5.3"
@@ -1610,6 +1626,18 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "nix"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.6.0",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "nom"
version = "7.1.3"
@@ -1814,7 +1842,7 @@ dependencies = [
[[package]]
name = "parser"
version = "0.4.1"
version = "0.5.0"
dependencies = [
"allocative",
"bincode",
@@ -1823,6 +1851,7 @@ dependencies = [
"chrono",
"clap",
"color-eyre",
"ctrlc",
"derive_deref",
"inferno",
"itertools",
@@ -2478,7 +2507,7 @@ dependencies = [
[[package]]
name = "server"
version = "0.4.1"
version = "0.5.0"
dependencies = [
"axum",
"bincode",
server/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "server"
version = "0.4.1"
version = "0.5.0"
edition = "2021"
[dependencies]
website/index.html
+6 -6
View File
@@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
@@ -1038,8 +1038,8 @@
border-top: 0;
padding: 0.25rem 0;
margin: -0.5px;
font-size: var(--font-size-xs);
line-height: var(--line-height-xs);
/* font-size: var(--font-size-xs);
line-height: var(--line-height-xs); */
caption {
border-bottom-style: dashed !important;
@@ -1386,7 +1386,7 @@
localStorage.getItem(settingsThemeLocalStorageKey)
);
const preferredColorSchemeMatchMedia = window.matchMedia(
"(prefers-color-scheme: dark)"
"(prefers-color-scheme: dark)",
);
if (
theme === "dark" ||
@@ -1398,7 +1398,7 @@
}
const backgroundColor = getComputedStyle(
window.document.documentElement
window.document.documentElement,
).getPropertyValue("--background-color");
const meta = window.document.createElement("meta");
meta.name = "theme-color";
@@ -2093,7 +2093,7 @@
rel="noopener noreferrer"
target="_blank"
>
0.4.1
0.5.0
</a>
</p>
</footer>
website/script.js
+34
View File
@@ -5896,6 +5896,40 @@ signalsPromise.then((signals) => {
},
],
},
{
name: "Blockchain",
values: [
{
name: "Height",
path: "last-height",
},
{
name: "Cumulative size",
path: "cumulative-block-size",
},
],
},
{
name: "Blocks Mined",
values: [
{
name: "Since UTC",
path: "blocks-mined",
},
{
name: "Weekly Avg.",
path: "blocks-mined-1w-sma",
},
{
name: "Monthly Avg.",
path: "blocks-mined-1m-sma",
},
{
name: "Target",
path: "blocks-mined-1d-target",
},
],
},
],
},
],
website/types/paths.d.ts
Vendored
+3 -3
View File
File diff suppressed because one or more lines are too long