diff --git a/CHANGELOG.md b/CHANGELOG.md index d2224721e..dc60dcab1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,11 @@ ### Parser - Global - - Fixed an incredibly annoying bug that made the program panic because of a wrong utxo/address durable state after a or many new datasets were added/changed after a first successful parse of the chain - - Fixed bug that would crash program if launched for the first time ever + - Improved self-hosting by: + - Fixing an incredibly annoying bug that made the program panic because of a wrong utxo/address durable state after a or many new datasets were added/changed after a first successful parse of the chain + - Fixing a bug that would crash the program if launched for the first time ever - Merged the core of `HeightMap` and `DateMap` structs into `GenericMap` - - Added `Height` struct + - Added `Height` struct and many others - CLI - Added an argument parser for improved UX with several options - Datasets @@ -34,6 +35,11 @@ - {X} 1% Bottom Probability - {X} 0.5% Bottom Probability - {X} 0.1% Bottom Probability + - Added block metadatasets and their variants (raw/sum/average/min/max/percentiles): + - Block size + - Block weight + - Block VBytes + - Block interval - Price - Improved error message when price cannot be found diff --git a/app/src/scripts/presets/blocks/index.ts b/app/src/scripts/presets/blocks/index.ts index b1dc02ef8..349b8ce42 100644 --- a/app/src/scripts/presets/blocks/index.ts +++ b/app/src/scripts/presets/blocks/index.ts @@ -1,4 +1,5 @@ import { colors } from "../../utils/colors"; +import { createRecapPresets } from "../templates/recap"; export function createPresets(scale: ResourceScale) { return { @@ -141,42 +142,72 @@ export function createPresets(scale: ResourceScale) { }, { scale, - name: "Mined", - tree: [ - { - scale, - icon: IconTablerCube, - name: "Daily Sum", - title: "Daily Sum Of Blocks Mined", - description: "", - bottom: [ - { - title: "Target", - color: colors.white, - datasetPath: `/date-to-blocks-mined-1d-target`, - options: { - lineStyle: 3, - }, - }, - { - title: "1W Avg.", - color: colors.momentumYellow, - datasetPath: `/date-to-blocks-mined-1w-sma`, - defaultVisible: false, - }, - { - title: "1M Avg.", - color: colors.bitcoin, - datasetPath: `/date-to-blocks-mined-1m-sma`, - }, - { - title: "Mined", - color: colors.darkBitcoin, - datasetPath: `/date-to-blocks-mined`, - }, - ], - }, - ], + name: "Size", + tree: createRecapPresets({ + scale, + title: "Block Size", + color: colors.darkWhite, + keySum: "/date-to-block-size-1d-sum", + keyAverage: "/date-to-block-size-1d-average", + keyMax: "/date-to-block-size-1d-max", + key90p: "/date-to-block-size-1d-90p", + key75p: "/date-to-block-size-1d-75p", + keyMedian: "/date-to-block-size-1d-median", + key25p: "/date-to-block-size-1d-25p", + key10p: "/date-to-block-size-1d-10p", + keyMin: "/date-to-block-size-1d-min", + }), + }, + { + scale, + name: "Weight", + tree: createRecapPresets({ + scale, + title: "Block Weight", + color: colors.darkWhite, + keyAverage: "/date-to-block-weight-1d-average", + keyMax: "/date-to-block-weight-1d-max", + key90p: "/date-to-block-weight-1d-90p", + key75p: "/date-to-block-weight-1d-75p", + keyMedian: "/date-to-block-weight-1d-median", + key25p: "/date-to-block-weight-1d-25p", + key10p: "/date-to-block-weight-1d-10p", + keyMin: "/date-to-block-weight-1d-min", + }), + }, + { + scale, + name: "VBytes", + tree: createRecapPresets({ + scale, + title: "Block VBytes", + color: colors.darkWhite, + keyAverage: "/date-to-block-vbytes-1d-average", + keyMax: "/date-to-block-vbytes-1d-max", + key90p: "/date-to-block-vbytes-1d-90p", + key75p: "/date-to-block-vbytes-1d-75p", + keyMedian: "/date-to-block-vbytes-1d-median", + key25p: "/date-to-block-vbytes-1d-25p", + key10p: "/date-to-block-vbytes-1d-10p", + keyMin: "/date-to-block-vbytes-1d-min", + }), + }, + { + scale, + name: "Interval", + tree: createRecapPresets({ + scale, + title: "Block Interval", + color: colors.darkWhite, + keyAverage: "/date-to-block-interval-1d-average", + keyMax: "/date-to-block-interval-1d-max", + key90p: "/date-to-block-interval-1d-90p", + key75p: "/date-to-block-interval-1d-75p", + keyMedian: "/date-to-block-interval-1d-median", + key25p: "/date-to-block-interval-1d-25p", + key10p: "/date-to-block-interval-1d-10p", + keyMin: "/date-to-block-interval-1d-min", + }), }, ] : [ diff --git a/app/src/scripts/presets/templates/recap.ts b/app/src/scripts/presets/templates/recap.ts new file mode 100644 index 000000000..f0a2c409a --- /dev/null +++ b/app/src/scripts/presets/templates/recap.ts @@ -0,0 +1,268 @@ +export function createRecapPresets({ + scale, + title, + keyAverage, + color, + keySum, + keyMax, + key90p, + key75p, + keyMedian, + key25p, + key10p, + keyMin, +}: { + scale: ResourceScale; + title: string; + color: Color; + keySum?: AnyDatasetPath; + keyAverage?: AnyDatasetPath; + keyMax?: AnyDatasetPath; + key90p?: AnyDatasetPath; + key75p?: AnyDatasetPath; + keyMedian?: AnyDatasetPath; + key25p?: AnyDatasetPath; + key10p?: AnyDatasetPath; + keyMin?: AnyDatasetPath; +}): PartialPreset[] { + return [ + ...(keySum + ? ([ + { + scale, + icon: IconTablerSum, + name: "Daily Sum", + title: `${title} Daily Sum`, + description: "", + bottom: [ + { + title: "Sum", + color, + datasetPath: keySum, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(keyAverage + ? [ + { + scale, + icon: IconTablerMathAvg, + name: "Daily Average", + title: `${title} Daily Average`, + description: "", + bottom: [ + { + title: "Average", + color, + datasetPath: keyAverage, + }, + ], + }, + ] + : []), + ...(keyMax || key90p || key75p || keyMedian || key25p || key10p || keyMin + ? ([ + { + scale, + icon: IconTablerPercentage, + name: "Daily Percentiles", + title: `${title} Daily Percentiles`, + description: "", + bottom: [ + ...(keyMax + ? [ + { + title: "Max", + color, + datasetPath: keyMax, + }, + ] + : []), + ...(key90p + ? [ + { + title: "90%", + color, + datasetPath: key90p, + }, + ] + : []), + ...(key75p + ? [ + { + title: "75%", + color, + datasetPath: key75p, + }, + ] + : []), + ...(keyMedian + ? [ + { + title: "Median", + color, + datasetPath: keyMedian, + }, + ] + : []), + ...(key25p + ? [ + { + title: "25%", + color, + datasetPath: key25p, + }, + ] + : []), + ...(key10p + ? [ + { + title: "10%", + color, + datasetPath: key10p, + }, + ] + : []), + ...(keyMin + ? [ + { + title: "Min", + color, + datasetPath: keyMin, + }, + ] + : []), + ], + }, + ] as PartialPreset[]) + : []), + ...(keyMax + ? ([ + { + scale, + icon: IconTablerArrowBarToUp, + name: "Daily Max", + title: `${title} Daily Max`, + description: "", + bottom: [ + { + title: "Max", + color, + datasetPath: keyMax, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(key90p + ? ([ + { + scale, + icon: IconTablerPercentage90, + name: "Daily 90th Percentile", + title: `${title} Daily 90th Percentile`, + description: "", + bottom: [ + { + title: "90%", + color, + datasetPath: key90p, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(key75p + ? ([ + { + scale, + icon: IconTablerPercentage75, + name: "Daily 75th Percentile", + title: `${title} Size 75th Percentile`, + description: "", + bottom: [ + { + title: "75%", + color, + datasetPath: key75p, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(keyMedian + ? ([ + { + scale, + icon: IconTablerPercentage50, + name: "Daily Median", + title: `${title} Daily Median`, + description: "", + bottom: [ + { + title: "Median", + color, + datasetPath: keyMedian, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(key25p + ? ([ + { + scale, + icon: IconTablerPercentage25, + name: "Daily 25th Percentile", + title: `${title} Daily 25th Percentile`, + description: "", + bottom: [ + { + title: "25%", + color, + datasetPath: key25p, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(key10p + ? ([ + { + scale, + icon: IconTablerPercentage10, + name: "Daily 10th Percentile", + title: `${title} Daily 10th Percentile`, + description: "", + bottom: [ + { + title: "10%", + color, + datasetPath: key10p, + }, + ], + }, + ] as PartialPreset[]) + : []), + ...(keyMin + ? ([ + { + scale, + icon: IconTablerArrowBarToDown, + name: "Daily Min", + title: `${title} Daily Min`, + description: "", + bottom: [ + { + title: "Min", + color, + datasetPath: keyMin, + }, + ], + }, + ] as PartialPreset[]) + : []), + ]; +} diff --git a/app/src/types/auto-imports.d.ts b/app/src/types/auto-imports.d.ts index 808ba9309..46d943e7c 100644 --- a/app/src/types/auto-imports.d.ts +++ b/app/src/types/auto-imports.d.ts @@ -18,6 +18,8 @@ declare global { const IconTablerArchive: typeof import('~icons/tabler/archive.jsx')['default'] const IconTablerArrowBack: typeof import('~icons/tabler/arrow-back.jsx')['default'] const IconTablerArrowBackUp: (typeof import("~icons/tabler/arrow-back-up.jsx"))["default"] + const IconTablerArrowBarToDown: typeof import('~icons/tabler/arrow-bar-to-down.jsx')['default'] + const IconTablerArrowBarToUp: typeof import('~icons/tabler/arrow-bar-to-up.jsx')['default'] const IconTablerArrowCross: typeof import('~icons/tabler/arrow-cross.jsx')['default'] const IconTablerArrowForward: typeof import('~icons/tabler/arrow-forward.jsx')['default'] const IconTablerArrowRight: (typeof import("~icons/tabler/arrow-right.jsx"))["default"] @@ -182,6 +184,11 @@ declare global { const IconTablerMotorbike: (typeof import("~icons/tabler/motorbike.jsx"))["default"] const IconTablerNumber123: typeof import('~icons/tabler/number123.jsx')['default'] const IconTablerPercentage: typeof import('~icons/tabler/percentage.jsx')['default'] + const IconTablerPercentage10: typeof import('~icons/tabler/percentage10.jsx')['default'] + const IconTablerPercentage25: typeof import('~icons/tabler/percentage25.jsx')['default'] + const IconTablerPercentage50: typeof import('~icons/tabler/percentage50.jsx')['default'] + const IconTablerPercentage75: typeof import('~icons/tabler/percentage75.jsx')['default'] + const IconTablerPercentage90: typeof import('~icons/tabler/percentage90.jsx')['default'] const IconTablerPick: typeof import('~icons/tabler/pick.jsx')['default'] const IconTablerPigMoney: typeof import('~icons/tabler/pig-money.jsx')['default'] const IconTablerPlay: (typeof import("~icons/tabler/play.jsx"))["default"] diff --git a/parser/src/actions/parse.rs b/parser/src/actions/parse.rs index ded87a1df..37e1fdfa2 100644 --- a/parser/src/actions/parse.rs +++ b/parser/src/actions/parse.rs @@ -61,14 +61,10 @@ pub fn parse( let date_index = states.date_data_vec.len() - 1; - let previous_timestamp = if let Some(previous_height) = height.checked_sub(1) { - datasets - .block_metadata - .timestamp - .get_or_import(&Height::new(previous_height)) - } else { - None - }; + let previous_timestamp = height + .checked_sub(1) + .map(Height::new) + .and_then(|height| datasets.block_metadata.timestamp.get_or_import(&height)); let block_price = Price::from_dollar( datasets @@ -91,7 +87,7 @@ pub fn parse( let block_weight = block.weight().to_wu(); let block_vbytes = block.weight().to_vbytes_floor(); let block_interval = previous_timestamp.map_or(0, |previous_timestamp| { - if previous_timestamp <= timestamp { + if previous_timestamp >= timestamp { 0 } else { timestamp - previous_timestamp diff --git a/parser/src/datasets/mining.rs b/parser/src/datasets/mining.rs index bad7c8bfb..784054742 100644 --- a/parser/src/datasets/mining.rs +++ b/parser/src/datasets/mining.rs @@ -185,6 +185,7 @@ impl MiningDataset { block_size_recap: RecapDataset::import( &f("block_size_1d"), RecapOptions::default() + .add_sum() .add_average() .add_max() .add_90p() @@ -548,7 +549,12 @@ impl MiningDataset { self.block_size_recap.compute( *date, - &mut self.block_vbytes.get_or_import_range_inclusive(first, last), + &mut self + .block_size + .get_or_import_range_inclusive(first, last) + .into_iter() + .map(OrderedFloat) + .collect_vec(), ); self.block_weight_recap.compute( diff --git a/parser/src/datasets/mod.rs b/parser/src/datasets/mod.rs index 12a23bd57..93f6750aa 100644 --- a/parser/src/datasets/mod.rs +++ b/parser/src/datasets/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, ops::RangeInclusive}; +use std::{collections::BTreeMap, fs, ops::RangeInclusive}; use allocative::Allocative; @@ -140,7 +140,7 @@ impl AllDatasets { s.min_initial_states .consume(MinInitialStates::compute_from_datasets(&s)); - s.export_path_to_type()?; + s.export_meta_files()?; Ok(s) } @@ -263,7 +263,7 @@ impl AllDatasets { } } - pub fn export_path_to_type(&self) -> color_eyre::Result<()> { + pub fn export_meta_files(&self) -> color_eyre::Result<()> { let path_to_type: BTreeMap<&str, &str> = self .to_any_dataset_vec() .into_iter() @@ -275,7 +275,27 @@ impl AllDatasets { }) .collect(); - Json::export("../datasets/disk_path_to_type.json", &path_to_type) + let datasets_len = path_to_type.len(); + + Json::export("../datasets/disk_path_to_type.json", &path_to_type)?; + + let server_trigger_path = "../server/.trigger"; + + fs::create_dir_all(server_trigger_path)?; + + let datasets_len_path = format!("{server_trigger_path}/datasets_len.txt"); + + if let Ok(len) = fs::read_to_string(&datasets_len_path) { + if let Ok(len) = len.parse::() { + if datasets_len == len { + return Ok(()); + } + } + } + + fs::write(datasets_len_path, datasets_len.to_string())?; + + Ok(()) } pub fn export(&mut self) -> color_eyre::Result<()> { diff --git a/server/.gitignore b/server/.gitignore index adc710f2f..12c3a4b28 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,3 +1,4 @@ /target .DS_Store /parser.log +.trigger diff --git a/server/run.sh b/server/run.sh index ea7b3817d..4f525d71b 100755 --- a/server/run.sh +++ b/server/run.sh @@ -1,6 +1,5 @@ if cargo watch --help &> /dev/null; then - cargo watch -w "./src" -w "./run.sh" -w "../datasets/disk_path_to_type.json" -x "run -r" - # cargo watch -w "./src" -w "./run.sh" -x "run -r" + cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w ".trigger" -x "run -r" else cargo run -r fi