Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-05 03:39:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: small fixes

Browse Source
This commit is contained in:
nym21
2024-12-17 10:39:28 +01:00
parent e0a378cb81
commit 8fabbde13b
23 changed files with 179 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/crates/biter/Cargo.lock generated
View File

@@ -110,7 +110,7 @@ dependencies = [
[[package]]
name = "biter"
version = "0.2.1"
version = "0.2.2"
dependencies = [
"bitcoin",
"bitcoincore-rpc",

2
src/crates/snkrj/Cargo.lock generated
View File

@@ -177,7 +177,7 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "snkrj"
version = "0.1.0"
version = "0.1.1"
dependencies = [
"sanakirja",
]

34
src/parser/actions/export.rs
View File

@@ -1,5 +1,3 @@
use std::thread::{self};
use log::info;
use crate::{
@@ -38,27 +36,23 @@ pub fn export(
exit.block();
info!("Exporting...");
if defragment {
info!("Will also defragment databases, please be patient it might take a while")
}
let text = if defragment {
"export and defragmentation"
} else {
"export"
};
info!("Starting {text}");
time("Total save time", || -> color_eyre::Result<()> {
time("Datasets saved", || datasets.export(config))?;
time(&format!("Finished {text}"), || -> color_eyre::Result<()> {
datasets.export(config)?;
thread::scope(|s| {
if let Some(databases) = databases {
s.spawn(|| {
time("Databases saved", || {
databases.export(height, date, defragment)
})
});
}
if let Some(databases) = databases {
databases.export(height, date, defragment)?;
}
if let Some(states) = states {
s.spawn(|| time("States saved", || states.export(config)));
}
});
if let Some(states) = states {
states.export(config)?;
}
Ok(())
})?;

26
src/parser/actions/iter_blocks.rs
View File

@@ -15,7 +15,7 @@ use crate::{
datasets::{ComputeData, Datasets},
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
},
structs::{Config, DateData, Exit, Height, MapKey, Timestamp},
structs::{Config, DateData, DisplayInstant, Exit, Height, MapKey, Timestamp},
utils::{generate_allocation_files, time},
};
@@ -52,8 +52,6 @@ pub fn iter_blocks(
let mut block_iter = block_receiver.iter();
'parsing: loop {
let instant = Instant::now();
let mut processed_heights = BTreeSet::new();
let mut processed_dates = BTreeSet::new();
@@ -64,6 +62,8 @@ pub fn iter_blocks(
blocks_loop_date.take();
}
let instant = Instant::now();
'blocks: loop {
let current_block_opt = next_block_opt.take().or_else(|| block_iter.next());
@@ -88,8 +88,6 @@ pub fn iter_blocks(
// Always run for the first block of the loop
if blocks_loop_date.is_none() {
info!("Processing {current_block_date} (height: {height})...");
blocks_loop_date.replace(current_block_date);
if states
@@ -164,15 +162,18 @@ pub fn iter_blocks(
blocks_loop_i += 1;
if is_date_last_block {
info!(
"Processed {current_block_date} ({height} - {current_block_height}) {}",
instant.display()
);
height += blocks_loop_i;
let is_check_point = next_date_opt
.as_ref()
.map_or(true, |date| date.is_first_of_month());
let ran_for_at_least_a_minute = instant.elapsed().as_secs() >= 60;
if (is_check_point && ran_for_at_least_a_minute)
if (is_check_point && instant.elapsed().as_secs() >= 2)
|| height.is_close_to_end(approx_block_count)
{
break 'days;
@@ -189,11 +190,6 @@ pub fn iter_blocks(
// Don't remember why -1
let last_height = height - 1_u32;
info!(
"Parsing group took {} seconds (last height: {last_height})",
instant.elapsed().as_secs_f32(),
);
if first_unsafe_heights.computed <= last_height {
info!("Computing datasets...");
time("Computing datasets", || {
@@ -214,7 +210,7 @@ pub fn iter_blocks(
let defragment = is_safe
&& next_date_opt.is_some_and(|date| {
(date.year() >= 2020 && date.is_january()
|| date.year() >= 2022 && date.is_june())
|| date.year() >= 2022 && date.is_july())
&& date.is_first_of_month()
});
@@ -237,8 +233,6 @@ pub fn iter_blocks(
} else {
info!("Skipping export");
}
println!();
}
Ok(())

2
src/parser/actions/min_height.rs
View File

@@ -123,6 +123,8 @@ pub fn find_first_inserted_unsafe_height(
// panic!("");
// }
if true {panic!()}
states.reset(config, include_addresses);
databases.reset(include_addresses);

31
src/parser/databases/address_index_to_address_data.rs
View File

@@ -13,7 +13,6 @@ use snkrj::{AnyDatabase, Database as _Database};
use crate::{
parser::states::AddressCohortsDurableStates,
structs::{AddressData, Config},
utils::time,
};
use super::{AnyDatabaseGroup, Metadata};
@@ -91,24 +90,24 @@ impl AddressIndexToAddressData {
}
pub fn compute_addres_cohorts_durable_states(&mut self) -> AddressCohortsDurableStates {
time("Iter through address_index_to_address_data", || {
self.open_all();
// time("Iter through address_index_to_address_data", || {
self.open_all();
// MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
mem::take(&mut self.map)
.par_iter()
.map(|(_, database)| {
let mut s = AddressCohortsDurableStates::default();
// MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
mem::take(&mut self.map)
.par_iter()
.map(|(_, database)| {
let mut s = AddressCohortsDurableStates::default();
database
.iter_disk()
.map(|r| r.unwrap().1)
.for_each(|address_data| s.increment(address_data).unwrap());
database
.iter_disk()
.map(|r| r.unwrap().1)
.for_each(|address_data| s.increment(address_data).unwrap());
s
})
.sum()
})
s
})
.sum()
// })
}
fn db_index(key: &Key) -> usize {

7
src/parser/datasets/_traits/min_initial_state.rs
View File

@@ -45,13 +45,6 @@ enum Mode {
}
impl MinInitialState {
// pub fn consume(&mut self, other: Self) {
// self.first_unsafe_date = other.first_unsafe_date;
// self.first_unsafe_height = other.first_unsafe_height;
// self.last_date = other.last_date;
// self.last_height = other.last_height;
// }
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode, config: &Config) -> Self {
match mode {
Mode::Inserted => {

5
src/parser/states/_trait.rs
View File

@@ -9,7 +9,6 @@ use serde::{de::DeserializeOwned, Serialize};
use crate::{io::Serialization, structs::Config};
// https://github.com/djkoloski/rust_serialization_benchmark
pub trait AnyState
where
Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
@@ -26,9 +25,7 @@ where
}
fn import(config: &Config) -> color_eyre::Result<Self> {
let path = Self::path(config);
fs::create_dir_all(&path)?;
Serialization::Binary.import(&path)
Serialization::Binary.import(&Self::path(config))
}
fn export(&self, config: &Config) -> color_eyre::Result<()> {

4
src/parser/states/mod.rs
View File

@@ -1,4 +1,4 @@
use std::thread;
use std::{fs, thread};
mod _trait;
mod cohorts_states;
@@ -25,6 +25,8 @@ pub struct States {
impl States {
pub fn import(config: &Config) -> color_eyre::Result<Self> {
fs::create_dir_all(config.path_states())?;
let date_data_vec = DateDataVec::import(config)?;
let address_counters = Counters::import(config)?;

32
src/server/api/structs/routes.rs
View File

@@ -1,5 +1,6 @@
use std::{
collections::{BTreeMap, HashMap},
fs,
path::PathBuf,
};
@@ -31,17 +32,30 @@ impl Routes {
pub fn build(paths_to_type: BTreeMap<PathBuf, String>, config: &Config) -> Self {
let mut routes = Routes::default();
paths_to_type.into_iter().for_each(|(file_path, value)| {
let serialization =
Serialization::try_from(&file_path).unwrap_or(Serialization::Binary);
paths_to_type.into_iter().for_each(|(path, value)| {
let try_from_path = if path.is_file() {
path.clone()
} else {
fs::read_dir(&path)
.unwrap_or_else(|_| {
dbg!(&path);
panic!();
})
.map(|e| e.unwrap().path())
.find(|e| e.is_file())
.unwrap()
};
let file_path_ser = file_path.to_str().unwrap().to_owned();
let serialization =
Serialization::try_from(&try_from_path).unwrap_or(Serialization::Binary);
let file_path_ser = path.to_str().unwrap().to_owned();
let split_key = file_path_ser.replace(
&format!("{}/", config.path_datasets().to_str().unwrap()),
"",
);
let split_key =
split_key.replace(&format!("{}/", config.path_price().to_str().unwrap()), "");
split_key.replace(&format!("{}/", config.path_kibodir().to_str().unwrap()), "");
let mut split_key = split_key.split('/').collect_vec();
let last = split_key.pop().unwrap().to_owned();
let last = last.split('.').next().unwrap();
@@ -63,7 +77,7 @@ impl Routes {
map_key,
Route {
url_path: format!("date-to-{url_path}"),
file_path,
file_path: path,
values_type,
serialization,
},
@@ -74,7 +88,7 @@ impl Routes {
map_key,
Route {
url_path: format!("height-to-{url_path}"),
file_path,
file_path: path,
values_type,
serialization,
},
@@ -85,14 +99,14 @@ impl Routes {
map_key,
Route {
url_path,
file_path,
file_path: path,
values_type,
serialization,
},
);
}
_ => {
dbg!(&file_path, value, &last, &split_key);
dbg!(&path, value, &last, &split_key);
panic!("")
}
}

2
src/structs/bi_map.rs
View File

@@ -11,7 +11,7 @@ use super::{
AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapKind, MapPath, MapValue,
};
#[derive(Allocative)]
#[derive(Allocative, Debug)]
pub struct BiMap<Value>
where
Value: MapValue,

61
src/structs/config.rs
View File

@@ -1,5 +1,6 @@
use std::{
fs::{self},
mem,
path::{Path, PathBuf},
};
@@ -46,28 +47,30 @@ pub struct Config {
#[arg(long, value_name = "SECONDS")]
delay: Option<u64>,
// Maximum ram you want the program to use in GB, default: 50% of total, not saved
// #[arg(long, value_name = "GB")]
// pub max_ram: Option<f64>,
/// Enable or disable the parser, default: true, not saved
#[arg(long, value_name = "BOOL")]
parser: Option<bool>,
/// Disable the parser, not saved
#[serde(default)]
#[arg(long, default_value_t = false)]
no_parser: bool,
/// Enable or disable the server, default: true, not saved
#[arg(long, value_name = "BOOL")]
server: Option<bool>,
/// Disable the server, not saved
#[serde(default)]
#[arg(long, default_value_t = false)]
no_server: bool,
/// Start a dry run, default: true, not saved
#[arg(long, value_name = "BOOL")]
dry_run: Option<bool>,
/// Run without saving, not saved
#[serde(default)]
#[arg(long, default_value_t = false)]
dry_run: bool,
/// Record ram usage, default: false, not saved
#[arg(long, value_name = "BOOL")]
record_ram_usage: Option<bool>,
/// Record ram usage, not saved
#[serde(default)]
#[arg(long, default_value_t = false)]
record_ram_usage: bool,
/// Recompute all computed datasets, default: false, not saved
#[arg(long, value_name = "BOOL")]
recompute_computed: Option<bool>,
/// Recompute all computed datasets, not saved
#[serde(default)]
#[arg(long, default_value_t = false)]
recompute_computed: bool,
}
impl Config {
@@ -125,11 +128,11 @@ impl Config {
config.write(&path)?;
config.parser = config_args.parser.take();
config.server = config_args.server.take();
config.dry_run = config_args.dry_run.take();
config.record_ram_usage = config_args.record_ram_usage.take();
config.recompute_computed = config_args.recompute_computed.take();
config.no_parser = mem::take(&mut config_args.no_parser);
config.no_server = mem::take(&mut config_args.no_server);
config.dry_run = mem::take(&mut config_args.dry_run);
config.record_ram_usage = mem::take(&mut config_args.record_ram_usage);
config.recompute_computed = mem::take(&mut config_args.recompute_computed);
info!("Configuration {{");
info!(" bitcoindir: {:?}", config.bitcoindir);
@@ -233,22 +236,22 @@ impl Config {
}
pub fn dry_run(&self) -> bool {
self.dry_run.is_some_and(|b| b)
self.dry_run
}
pub fn record_ram_usage(&self) -> bool {
self.record_ram_usage.is_some_and(|b| b)
self.record_ram_usage
}
pub fn recompute_computed(&self) -> bool {
self.recompute_computed.is_some_and(|b| b)
self.recompute_computed
}
pub fn path_bitcoindir(&self) -> PathBuf {
Self::fix_user_path(self.bitcoindir.as_ref().unwrap().as_ref())
}
fn path_kibodir(&self) -> PathBuf {
pub fn path_kibodir(&self) -> PathBuf {
Self::fix_user_path(self.kibodir.as_ref().unwrap().as_ref())
}
@@ -311,10 +314,10 @@ impl Config {
}
pub fn parser(&self) -> bool {
self.parser.is_none_or(|b| b)
!self.no_parser
}
pub fn server(&self) -> bool {
self.server.is_none_or(|b| b)
!self.no_server
}
}

4
src/structs/date.rs
View File

@@ -74,8 +74,8 @@ impl Date {
self.month() == 1
}
pub fn is_june(&self) -> bool {
self.month() == 6
pub fn is_july(&self) -> bool {
self.month() == 7
}
pub fn is_first_of_month(&self) -> bool {

15
src/structs/instant.rs Normal file
View File

@@ -0,0 +1,15 @@
use std::time::Instant;
use color_eyre::owo_colors::OwoColorize;
pub trait DisplayInstant {
fn display(&self) -> String;
}
impl DisplayInstant for Instant {
fn display(&self) -> String {
format!("{:.2}s", self.elapsed().as_secs_f32())
.bright_black()
.to_string()
}
}

4
src/structs/mod.rs
View File

@@ -24,12 +24,14 @@ mod generic_map;
mod height;
mod height_map;
mod height_map_chunk_id;
mod instant;
mod liquidity;
mod map_path;
mod map_value;
mod ohlc;
mod partial_txout_data;
mod price;
mod rpc;
mod sent_data;
mod serialized_btreemap;
mod serialized_vec;
@@ -63,12 +65,14 @@ pub use generic_map::*;
pub use height::*;
pub use height_map::*;
pub use height_map_chunk_id::*;
pub use instant::*;
pub use liquidity::*;
pub use map_path::*;
pub use map_value::*;
pub use ohlc::*;
pub use partial_txout_data::*;
pub use price::*;
pub use rpc::*;
pub use sent_data::*;
pub use serialized_btreemap::*;
pub use serialized_vec::*;

17
src/structs/rpc.rs Normal file
View File

@@ -0,0 +1,17 @@
use biter::bitcoincore_rpc::Client;
use crate::structs::Config;
impl From<&Config> for Client {
fn from(config: &Config) -> Self {
Client::new(
&format!(
"http://{}:{}",
config.rpcconnect().unwrap_or(&"localhost".to_owned()),
config.rpcport().unwrap_or(8332)
),
config.to_rpc_auth().unwrap(),
)
.unwrap()
}
}

1
src/utils/mod.rs
View File

@@ -4,7 +4,6 @@ mod log;
mod lossy;
mod percentile;
mod retry;
mod rpc;
mod time;
pub use consts::*;

20
src/utils/rpc.rs
View File

@@ -1,20 +0,0 @@
use biter::bitcoincore_rpc::Client;
use crate::structs::Config;
impl From<&Config> for Client {
fn from(config: &Config) -> Self {
create_rpc(config).unwrap()
}
}
fn create_rpc(config: &Config) -> color_eyre::Result<Client> {
Ok(Client::new(
&format!(
"http://{}:{}",
config.rpcconnect().unwrap_or(&"localhost".to_owned()),
config.rpcport().unwrap_or(8332)
),
config.to_rpc_auth().unwrap(),
)?)
}

6
src/utils/time.rs
View File

@@ -2,7 +2,9 @@ use std::time::Instant;
use log::info;
pub fn time<F, T>(name: &str, function: F) -> T
use crate::structs::DisplayInstant;
pub fn time<F, T>(text: &str, function: F) -> T
where
F: FnOnce() -> T,
{
@@ -10,7 +12,7 @@ where
let returned = function();
info!("{name}: {} seconds", time.elapsed().as_secs_f32());
info!("{text} {}", time.display());
returned
}

1
src/website/index.html
View File

@@ -1518,6 +1518,7 @@
"
>希望</small
>
希望.お金
</a>
<small style="display: block">
<strong>Bitcoin</strong> is

8
src/website/scripts/main.js
View File

@@ -1311,10 +1311,10 @@ function initFrameSelectors() {
function setAsideParent() {
const { clientWidth } = window.document.documentElement;
const { aside, body, main } = elements;
if (clientWidth >= consts.MEDIUM_WIDTH && aside.parentElement !== body) {
body.append(aside);
} else if (aside.parentElement !== main) {
main.append(aside);
if (clientWidth >= consts.MEDIUM_WIDTH) {
aside.parentElement !== body && body.append(aside);
} else {
aside.parentElement !== main && main.append(aside);
}
}

52
src/website/scripts/options.js
View File

@@ -1501,28 +1501,23 @@ function createPartialOptions(colors) {
name: "Market",
tree: [
{
name: "Price",
tree: [
scale,
name: "Dollars Per Bitcoin",
title: "Dollars Per Bitcoin",
description: "",
unit: "US Dollars",
},
{
scale,
name: "Satoshis Per Dollar",
title: "Satoshis Per Dollar",
description: "",
unit: "Satoshis",
bottom: [
{
scale,
name: "Dollars Per Bitcoin",
title: "Dollars Per Bitcoin",
description: "",
unit: "US Dollars",
},
{
scale,
name: "Sats Per Dollar",
title: "Satoshis Per Dollar",
description: "",
unit: "Satoshis",
bottom: [
{
title: "Sats",
datasetPath: `${scale}-to-sats-per-dollar`,
color: colors.bitcoin,
},
],
title: "Satoshis",
datasetPath: `${scale}-to-sats-per-dollar`,
color: colors.bitcoin,
},
],
},
@@ -5128,18 +5123,9 @@ function createPartialOptions(colors) {
url: () => window.location.href,
},
{
name: "Socials",
tree: [
{
name: "Bluesky",
url: () => "https://bsky.app/profile/kibo.money",
},
{
name: "Nostr",
url: () =>
"https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44",
},
],
name: "Social",
url: () =>
"https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44",
},
{
name: "Developers",