diff --git a/crates/brk_binder/src/javascript.rs b/crates/brk_binder/src/javascript.rs index 622862b3d..6b6e258b1 100644 --- a/crates/brk_binder/src/javascript.rs +++ b/crates/brk_binder/src/javascript.rs @@ -1,11 +1,11 @@ use std::{collections::HashSet, fmt::Write as FmtWrite, fs, io, path::Path}; -use brk_types::{Index, TreeNode}; +use brk_types::{pools, Index, TreeNode}; use serde_json::Value; use crate::{ ClientMetadata, Endpoint, FieldNamePosition, IndexSetPattern, PatternField, StructuralPattern, - TypeSchemas, extract_inner_type, get_fields_with_child_info, get_first_leaf_name, + TypeSchemas, VERSION, extract_inner_type, get_fields_with_child_info, get_first_leaf_name, get_node_fields, get_pattern_instance_base, to_camel_case, to_pascal_case, }; @@ -23,6 +23,7 @@ pub fn generate_javascript_client( writeln!(output, "// Auto-generated BRK JavaScript client").unwrap(); writeln!(output, "// Do not edit manually\n").unwrap(); + generate_constants(&mut output); generate_type_definitions(&mut output, schemas); generate_base_client(&mut output); generate_index_accessors(&mut output, &metadata.index_set_patterns); @@ -35,6 +36,32 @@ pub fn generate_javascript_client( Ok(()) } +fn generate_constants(output: &mut String) { + writeln!(output, "// Constants\n").unwrap(); + + // VERSION + writeln!(output, "export const VERSION = \"v{VERSION}\";\n").unwrap(); + + // INDEXES + let indexes = Index::all(); + writeln!(output, "export const INDEXES = /** @type {{const}} */ ([").unwrap(); + for index in &indexes { + writeln!(output, " \"{}\",", index.serialize_long()).unwrap(); + } + writeln!(output, "]);\n").unwrap(); + + // POOL_ID_TO_POOL_NAME + let pools = pools(); + let mut sorted_pools: Vec<_> = pools.iter().collect(); + sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())); + + writeln!(output, "export const POOL_ID_TO_POOL_NAME = /** @type {{const}} */ ({{").unwrap(); + for pool in &sorted_pools { + writeln!(output, " {}: \"{}\",", pool.slug(), pool.name).unwrap(); + } + writeln!(output, "}});\n").unwrap(); +} + fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) { if schemas.is_empty() { return; @@ -246,7 +273,7 @@ class MetricNode {{ /** * Fetch all data points for this metric. * @param {{(value: T[]) => void}} [onUpdate] - Called when data is available (may be called twice: cache then fresh) - * @returns {{Promise}} + * @returns {{Promise}} */ get(onUpdate) {{ return this._client.get(this._path, onUpdate); @@ -257,7 +284,7 @@ class MetricNode {{ * @param {{string | number}} from * @param {{string | number}} to * @param {{(value: T[]) => void}} [onUpdate] - Called when data is available (may be called twice: cache then fresh) - * @returns {{Promise}} + * @returns {{Promise}} */ getRange(from, to, onUpdate) {{ return this._client.get(`${{this._path}}?from=${{from}}&to=${{to}}`, onUpdate); @@ -282,7 +309,7 @@ class BrkClientBase {{ * @template T * @param {{string}} path * @param {{(value: T) => void}} [onUpdate] - Called when data is available - * @returns {{Promise}} + * @returns {{Promise}} */ async get(path, onUpdate) {{ const url = `${{this.baseUrl}}${{path}}`; @@ -291,7 +318,10 @@ class BrkClientBase {{ const cachedJson = cachedRes ? await cachedRes.json() : null; if (cachedJson) onUpdate?.(cachedJson); - if (!globalThis.navigator?.onLine) return cachedJson; + if (!globalThis.navigator?.onLine) {{ + if (cachedJson) return cachedJson; + throw new BrkError('Offline and no cached data available'); + }} try {{ const res = await fetch(url, {{ signal: AbortSignal.timeout(this.timeout) }}); diff --git a/crates/brk_binder/src/python.rs b/crates/brk_binder/src/python.rs index 86bed8125..0e311506f 100644 --- a/crates/brk_binder/src/python.rs +++ b/crates/brk_binder/src/python.rs @@ -1,11 +1,11 @@ use std::{collections::HashSet, fmt::Write as FmtWrite, fs, io, path::Path}; -use brk_types::{Index, TreeNode}; +use brk_types::{pools, Index, TreeNode}; use serde_json::Value; use crate::{ ClientMetadata, Endpoint, FieldNamePosition, IndexSetPattern, PatternField, StructuralPattern, - TypeSchemas, extract_inner_type, get_fields_with_child_info, get_node_fields, + TypeSchemas, VERSION, extract_inner_type, get_fields_with_child_info, get_node_fields, get_pattern_instance_base, to_pascal_case, to_snake_case, }; @@ -25,12 +25,13 @@ pub fn generate_python_client( writeln!(output, "from __future__ import annotations").unwrap(); writeln!( output, - "from typing import TypeVar, Generic, Any, Optional, List, Literal, TypedDict" + "from typing import TypeVar, Generic, Any, Optional, List, Literal, TypedDict, Final" ) .unwrap(); writeln!(output, "import httpx\n").unwrap(); writeln!(output, "T = TypeVar('T')\n").unwrap(); + generate_constants(&mut output); generate_type_definitions(&mut output, schemas); generate_base_client(&mut output); generate_metric_node(&mut output); @@ -44,6 +45,32 @@ pub fn generate_python_client( Ok(()) } +fn generate_constants(output: &mut String) { + writeln!(output, "# Constants\n").unwrap(); + + // VERSION + writeln!(output, "VERSION: Final[str] = \"v{VERSION}\"\n").unwrap(); + + // INDEXES + let indexes = Index::all(); + writeln!(output, "INDEXES: Final[tuple[str, ...]] = (").unwrap(); + for index in &indexes { + writeln!(output, " \"{}\",", index.serialize_long()).unwrap(); + } + writeln!(output, ")\n").unwrap(); + + // POOL_ID_TO_POOL_NAME + let pools = pools(); + let mut sorted_pools: Vec<_> = pools.iter().collect(); + sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())); + + writeln!(output, "POOL_ID_TO_POOL_NAME: Final[dict[str, str]] = {{").unwrap(); + for pool in &sorted_pools { + writeln!(output, " \"{}\": \"{}\",", pool.slug(), pool.name).unwrap(); + } + writeln!(output, "}}\n").unwrap(); +} + fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) { if schemas.is_empty() { return; diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs index 9434f7aa7..bde94c376 100644 --- a/crates/brk_cli/src/main.rs +++ b/crates/brk_cli/src/main.rs @@ -8,6 +8,7 @@ use std::{ time::Duration, }; +use brk_alloc::Mimalloc; use brk_binder::generate_js_files; use brk_bundler::bundle; use brk_computer::Computer; @@ -17,7 +18,6 @@ use brk_iterator::Blocks; use brk_mempool::Mempool; use brk_query::AsyncQuery; use brk_reader::Reader; -use brk_alloc::Mimalloc; use brk_server::{Server, VERSION}; use log::info; use vecdb::Exit; @@ -60,7 +60,7 @@ pub fn run() -> color_eyre::Result<()> { // Pre-run indexer if too far behind, then drop and reimport to reduce memory let chain_height = client.get_last_height()?; let indexed_height = indexer.vecs.starting_height(); - if u32::from(chain_height) - u32::from(indexed_height) > 1000 { + if chain_height.saturating_sub(*indexed_height) > 1000 { indexer.index(&blocks, &client, &exit)?; drop(indexer); Mimalloc::collect(); diff --git a/crates/brk_computer/src/chain/import.rs b/crates/brk_computer/src/chain/import.rs index 79a6c880e..5cf18ddf7 100644 --- a/crates/brk_computer/src/chain/import.rs +++ b/crates/brk_computer/src/chain/import.rs @@ -210,6 +210,9 @@ impl Vecs { |_, _| Some(StoredU64::from(TARGET_BLOCKS_PER_DECADE)), ); + let height_to_interval = eager!("interval"); + let height_to_vbytes = eager!("vbytes"); + let this = Self { dateindex_to_block_count_target, weekindex_to_block_count_target, @@ -218,34 +221,62 @@ impl Vecs { semesterindex_to_block_count_target, yearindex_to_block_count_target, decadeindex_to_block_count_target, - height_to_interval: eager!("interval"), timeindexes_to_timestamp: computed_di!( "timestamp", VecBuilderOptions::default().add_first() ), - indexes_to_block_interval: computed_h!("block_interval", Source::None, stats()), + indexes_to_block_interval: computed_h!( + "block_interval", + Source::Vec(height_to_interval.boxed_clone()), + stats() + ), indexes_to_block_count: computed_h!("block_count", Source::Compute, sum_cum()), indexes_to_1w_block_count: computed_di!("1w_block_count", last()), indexes_to_1m_block_count: computed_di!("1m_block_count", last()), indexes_to_1y_block_count: computed_di!("1y_block_count", last()), - indexes_to_block_weight: computed_h!("block_weight", Source::None, full_stats()), - indexes_to_block_size: computed_h!("block_size", Source::None, full_stats()), - height_to_vbytes: eager!("vbytes"), + indexes_to_block_weight: computed_h!( + "block_weight", + Source::Vec(indexer.vecs.block.height_to_weight.boxed_clone()), + full_stats() + ), + indexes_to_block_size: computed_h!( + "block_size", + Source::Vec(indexer.vecs.block.height_to_total_size.boxed_clone()), + full_stats() + ), height_to_24h_block_count: eager!("24h_block_count"), height_to_24h_coinbase_sum: eager!("24h_coinbase_sum"), height_to_24h_coinbase_usd_sum: eager!("24h_coinbase_usd_sum"), - indexes_to_block_vbytes: computed_h!("block_vbytes", Source::None, full_stats()), + indexes_to_block_vbytes: computed_h!( + "block_vbytes", + Source::Vec(height_to_vbytes.boxed_clone()), + full_stats() + ), difficultyepoch_to_timestamp: eager!("timestamp"), halvingepoch_to_timestamp: eager!("timestamp"), dateindex_to_fee_dominance: eager!("fee_dominance"), dateindex_to_subsidy_dominance: eager!("subsidy_dominance"), - indexes_to_difficulty: computed_h!("difficulty", Source::None, last()), + indexes_to_difficulty: computed_h!( + "difficulty", + Source::Vec(indexer.vecs.block.height_to_difficulty.boxed_clone()), + last() + ), + height_to_interval, + height_to_vbytes, indexes_to_difficultyepoch: computed_di!("difficultyepoch", last()), indexes_to_halvingepoch: computed_di!("halvingepoch", last()), indexes_to_tx_count: computed_h!("tx_count", Source::Compute, full_stats()), - indexes_to_input_count: computed_tx!("input_count", Source::None, full_stats()), - indexes_to_output_count: computed_tx!("output_count", Source::None, full_stats()), + indexes_to_input_count: computed_tx!( + "input_count", + Source::Vec(indexes.txindex_to_input_count.boxed_clone()), + full_stats() + ), + indexes_to_output_count: computed_tx!( + "output_count", + Source::Vec(indexes.txindex_to_output_count.boxed_clone()), + full_stats() + ), indexes_to_tx_v1: computed_h!("tx_v1", Source::Compute, sum_cum()), indexes_to_tx_v2: computed_h!("tx_v2", Source::Compute, sum_cum()), indexes_to_tx_v3: computed_h!("tx_v3", Source::Compute, sum_cum()), @@ -273,9 +304,21 @@ impl Vecs { .add_minmax() .add_average(), )?, - indexes_to_fee_rate: computed_tx!("fee_rate", Source::None, stats()), - indexes_to_tx_vsize: computed_tx!("tx_vsize", Source::None, stats()), - indexes_to_tx_weight: computed_tx!("tx_weight", Source::None, stats()), + indexes_to_fee_rate: computed_tx!( + "fee_rate", + Source::Vec(txindex_to_fee_rate.boxed_clone()), + stats() + ), + indexes_to_tx_vsize: computed_tx!( + "tx_vsize", + Source::Vec(txindex_to_vsize.boxed_clone()), + stats() + ), + indexes_to_tx_weight: computed_tx!( + "tx_weight", + Source::Vec(txindex_to_weight.boxed_clone()), + stats() + ), indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( &db, "subsidy", diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index 816e8f00d..c44f86e80 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -3,7 +3,7 @@ use std::path::Path; use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Bitcoin, CheckedSub, Dollars, StoredF32, StoredF64, Version}; -use vecdb::{Database, Exit, PAGE_SIZE, TypedVecIterator}; +use vecdb::{Database, Exit, IterableCloneableVec, PAGE_SIZE, TypedVecIterator}; use crate::{grouped::ComputedVecsFromDateIndex, utils::OptionExt}; @@ -103,11 +103,11 @@ impl Vecs { }; } macro_rules! ratio_di { - ($name:expr) => { + ($name:expr, $source:expr) => { ComputedRatioVecsFromDateIndex::forced_import( &db, $name, - Source::None, + Source::Vec($source.dateindex.unwrap_last().boxed_clone()), v0, indexes, true, @@ -128,6 +128,12 @@ impl Vecs { }; } + // Extract price vecs before struct literal so they can be used as sources for ratios + let indexes_to_vaulted_price = computed_h!("vaulted_price", last()); + let indexes_to_active_price = computed_h!("active_price", last()); + let indexes_to_true_market_mean = computed_h!("true_market_mean", last()); + let indexes_to_cointime_price = computed_h!("cointime_price", last()); + let this = Self { indexes_to_coinblocks_created: computed_h!("coinblocks_created", sum_cum()), indexes_to_coinblocks_stored: computed_h!("coinblocks_stored", sum_cum()), @@ -143,18 +149,24 @@ impl Vecs { indexes_to_investor_cap: computed_h!("investor_cap", v1, last()), indexes_to_vaulted_cap: computed_h!("vaulted_cap", v1, last()), indexes_to_active_cap: computed_h!("active_cap", v1, last()), - indexes_to_vaulted_price: computed_h!("vaulted_price", last()), - indexes_to_vaulted_price_ratio: ratio_di!("vaulted_price"), - indexes_to_active_price: computed_h!("active_price", last()), - indexes_to_active_price_ratio: ratio_di!("active_price"), - indexes_to_true_market_mean: computed_h!("true_market_mean", last()), - indexes_to_true_market_mean_ratio: ratio_di!("true_market_mean"), + indexes_to_vaulted_price_ratio: ratio_di!("vaulted_price", &indexes_to_vaulted_price), + indexes_to_vaulted_price, + indexes_to_active_price_ratio: ratio_di!("active_price", &indexes_to_active_price), + indexes_to_active_price, + indexes_to_true_market_mean_ratio: ratio_di!( + "true_market_mean", + &indexes_to_true_market_mean + ), + indexes_to_true_market_mean, indexes_to_cointime_value_destroyed: computed_h!("cointime_value_destroyed", sum_cum()), indexes_to_cointime_value_created: computed_h!("cointime_value_created", sum_cum()), indexes_to_cointime_value_stored: computed_h!("cointime_value_stored", sum_cum()), - indexes_to_cointime_price: computed_h!("cointime_price", last()), indexes_to_cointime_cap: computed_h!("cointime_cap", last()), - indexes_to_cointime_price_ratio: ratio_di!("cointime_price"), + indexes_to_cointime_price_ratio: ratio_di!( + "cointime_price", + &indexes_to_cointime_price + ), + indexes_to_cointime_price, indexes_to_cointime_adj_inflation_rate: computed_di!( "cointime_adj_inflation_rate", last() diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index db2686de4..73b437147 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -6,6 +6,8 @@ use super::{ indexes, }; +pub const DB_NAME: &str = "constants"; + #[derive(Clone, Traversable)] pub struct Vecs { pub constant_0: ConstantVecs, diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index 0624718a5..50b076cf0 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -12,6 +12,8 @@ use vecdb::{ use super::{Indexes, indexes, utils::OptionExt}; +pub const DB_NAME: &str = "fetched"; + #[derive(Clone, Traversable)] pub struct Vecs { db: Database, @@ -23,7 +25,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result { - let db = Database::open(&parent.join("fetched"))?; + let db = Database::open(&parent.join(DB_NAME))?; db.set_min_len(PAGE_SIZE * 1_000_000)?; let this = Self { diff --git a/crates/brk_computer/src/grouped/computed_from_height.rs b/crates/brk_computer/src/grouped/computed_from_height.rs index 528f6d539..737c92162 100644 --- a/crates/brk_computer/src/grouped/computed_from_height.rs +++ b/crates/brk_computer/src/grouped/computed_from_height.rs @@ -29,7 +29,7 @@ where pub height_extra: EagerVecsBuilder, pub dateindex: EagerVecsBuilder, pub weekindex: LazyVecsBuilder, - pub difficultyepoch: EagerVecsBuilder, + pub difficultyepoch: LazyVecsBuilder, pub monthindex: LazyVecsBuilder, pub quarterindex: LazyVecsBuilder, pub semesterindex: LazyVecsBuilder, @@ -70,6 +70,8 @@ where let options = options.remove_percentiles(); + let height_source = source.vec().or(height.as_ref().map(|v| v.boxed_clone())); + Ok(Self { weekindex: LazyVecsBuilder::forced_import( name, @@ -120,15 +122,17 @@ where options.into(), ), // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, + difficultyepoch: LazyVecsBuilder::forced_import( + name, + version + VERSION + Version::ZERO, + height_source, + &height_extra, + indexes.difficultyepoch_to_difficultyepoch.boxed_clone(), + options.into(), + ), height, height_extra, dateindex, - difficultyepoch: EagerVecsBuilder::forced_import( - db, - name, - version + VERSION + Version::ZERO, - options, - )?, }) } @@ -166,14 +170,6 @@ where &indexes.dateindex_to_height_count, exit, )?; - - self.difficultyepoch.compute( - starting_indexes.difficultyepoch, - height, - &indexes.difficultyepoch_to_first_height, - &indexes.difficultyepoch_to_height_count, - exit, - )?; } else { let height = self.height.u(); @@ -187,14 +183,6 @@ where &indexes.dateindex_to_height_count, exit, )?; - - self.difficultyepoch.compute( - starting_indexes.difficultyepoch, - height, - &indexes.difficultyepoch_to_first_height, - &indexes.difficultyepoch_to_height_count, - exit, - )?; } Ok(()) diff --git a/crates/brk_computer/src/grouped/computed_from_height_strict.rs b/crates/brk_computer/src/grouped/computed_from_height_strict.rs index 6196abcea..a0df92424 100644 --- a/crates/brk_computer/src/grouped/computed_from_height_strict.rs +++ b/crates/brk_computer/src/grouped/computed_from_height_strict.rs @@ -3,11 +3,13 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{DifficultyEpoch, Height, Version}; use schemars::JsonSchema; -use vecdb::{AnyExportableVec, Database, EagerVec, Exit, ImportableVec, PcoVec}; +use vecdb::{ + AnyExportableVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, PcoVec, +}; use crate::{Indexes, indexes}; -use super::{ComputedVecValue, EagerVecsBuilder, VecBuilderOptions}; +use super::{ComputedVecValue, EagerVecsBuilder, LazyVecsBuilder, VecBuilderOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeightStrict @@ -16,7 +18,7 @@ where { pub height: EagerVec>, pub height_extra: EagerVecsBuilder, - pub difficultyepoch: EagerVecsBuilder, + pub difficultyepoch: LazyVecsBuilder, // TODO: pub halvingepoch: StorableVecGeneator, } @@ -31,6 +33,7 @@ where db: &Database, name: &str, version: Version, + indexes: &indexes::Vecs, options: VecBuilderOptions, ) -> Result { let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO)?; @@ -45,21 +48,22 @@ where let options = options.remove_percentiles(); Ok(Self { - height, - height_extra, - difficultyepoch: EagerVecsBuilder::forced_import( - db, + difficultyepoch: LazyVecsBuilder::forced_import( name, version + VERSION + Version::ZERO, - options, - )?, + Some(height.boxed_clone()), + &height_extra, + indexes.difficultyepoch_to_difficultyepoch.boxed_clone(), + options.into(), + ), + height, + height_extra, // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, }) } pub fn compute( &mut self, - indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, mut compute: F, @@ -72,14 +76,6 @@ where self.height_extra .extend(starting_indexes.height, &self.height, exit)?; - self.difficultyepoch.compute( - starting_indexes.difficultyepoch, - &self.height, - &indexes.difficultyepoch_to_first_height, - &indexes.difficultyepoch_to_height_count, - exit, - )?; - Ok(()) } } diff --git a/crates/brk_computer/src/grouped/computed_from_txindex.rs b/crates/brk_computer/src/grouped/computed_from_txindex.rs index c4eab3a43..ccaebccbe 100644 --- a/crates/brk_computer/src/grouped/computed_from_txindex.rs +++ b/crates/brk_computer/src/grouped/computed_from_txindex.rs @@ -29,7 +29,7 @@ where pub height: EagerVecsBuilder, pub dateindex: EagerVecsBuilder, pub weekindex: LazyVecsBuilder, - pub difficultyepoch: EagerVecsBuilder, + pub difficultyepoch: LazyVecsBuilder, pub monthindex: LazyVecsBuilder, pub quarterindex: LazyVecsBuilder, pub semesterindex: LazyVecsBuilder, @@ -75,6 +75,14 @@ where indexes.weekindex_to_weekindex.boxed_clone(), options.into(), ), + difficultyepoch: LazyVecsBuilder::forced_import( + name, + version + VERSION + Version::ZERO, + None, + &height, + indexes.difficultyepoch_to_difficultyepoch.boxed_clone(), + options.into(), + ), monthindex: LazyVecsBuilder::forced_import( name, version + VERSION + Version::ZERO, @@ -119,12 +127,6 @@ where txindex, height, dateindex, - difficultyepoch: EagerVecsBuilder::forced_import( - db, - name, - version + VERSION + Version::ZERO, - options, - )?, // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, }) } @@ -205,14 +207,6 @@ where exit, )?; - self.difficultyepoch.from_aligned( - starting_indexes.difficultyepoch, - &self.height, - &indexes.difficultyepoch_to_first_height, - &indexes.difficultyepoch_to_height_count, - exit, - )?; - Ok(()) } } diff --git a/crates/brk_computer/src/grouped/lazy2_from_height.rs b/crates/brk_computer/src/grouped/lazy2_from_height.rs index 8a86530ba..901349156 100644 --- a/crates/brk_computer/src/grouped/lazy2_from_height.rs +++ b/crates/brk_computer/src/grouped/lazy2_from_height.rs @@ -67,7 +67,7 @@ where &source1.weekindex, &source2.weekindex, ), - difficultyepoch: LazyTransform2Builder::from_eager::( + difficultyepoch: LazyTransform2Builder::from_lazy::( name, v, &source1.difficultyepoch, diff --git a/crates/brk_computer/src/grouped/lazy_from_height.rs b/crates/brk_computer/src/grouped/lazy_from_height.rs index db1af7590..89ce71f3b 100644 --- a/crates/brk_computer/src/grouped/lazy_from_height.rs +++ b/crates/brk_computer/src/grouped/lazy_from_height.rs @@ -49,7 +49,7 @@ where height_extra: LazyTransformBuilder::from_eager::(name, v, &source.height_extra), dateindex: LazyTransformBuilder::from_eager::(name, v, &source.dateindex), weekindex: LazyTransformBuilder::from_lazy::(name, v, &source.weekindex), - difficultyepoch: LazyTransformBuilder::from_eager::( + difficultyepoch: LazyTransformBuilder::from_lazy::( name, v, &source.difficultyepoch, diff --git a/crates/brk_computer/src/grouped/lazy_from_height_strict.rs b/crates/brk_computer/src/grouped/lazy_from_height_strict.rs index f7a320177..4d49f5ce5 100644 --- a/crates/brk_computer/src/grouped/lazy_from_height_strict.rs +++ b/crates/brk_computer/src/grouped/lazy_from_height_strict.rs @@ -34,7 +34,7 @@ where let v = version + VERSION; Self { height: LazyVecFrom1::transformed::(name, v, height_source), - difficultyepoch: LazyTransformBuilder::from_eager::(name, v, &source.difficultyepoch), + difficultyepoch: LazyTransformBuilder::from_lazy::(name, v, &source.difficultyepoch), } } } diff --git a/crates/brk_computer/src/grouped/source.rs b/crates/brk_computer/src/grouped/source.rs index 71fc7785d..095e5314d 100644 --- a/crates/brk_computer/src/grouped/source.rs +++ b/crates/brk_computer/src/grouped/source.rs @@ -3,7 +3,6 @@ use vecdb::IterableBoxedVec; #[derive(Clone)] pub enum Source { Compute, - None, Vec(IterableBoxedVec), } @@ -12,10 +11,6 @@ impl Source { matches!(self, Self::Compute) } - pub fn is_none(&self) -> bool { - matches!(self, Self::None) - } - pub fn is_vec(&self) -> bool { matches!(self, Self::Vec(_)) } @@ -28,27 +23,9 @@ impl Source { } } -impl From for Source { - #[inline] - fn from(value: bool) -> Self { - if value { Self::Compute } else { Self::None } - } -} - impl From> for Source { #[inline] fn from(value: IterableBoxedVec) -> Self { Self::Vec(value) } } - -impl From>> for Source { - #[inline] - fn from(value: Option>) -> Self { - if let Some(v) = value { - Self::Vec(v) - } else { - Self::None - } - } -} diff --git a/crates/brk_computer/src/grouped/transforms.rs b/crates/brk_computer/src/grouped/transforms.rs index a56425cb5..529fc3377 100644 --- a/crates/brk_computer/src/grouped/transforms.rs +++ b/crates/brk_computer/src/grouped/transforms.rs @@ -1,4 +1,4 @@ -use brk_types::{Bitcoin, Close, Dollars, Sats, StoredF32, StoredF64}; +use brk_types::{Bitcoin, Close, Dollars, Sats, StoredF32, StoredF64, StoredU32}; use vecdb::{BinaryTransform, UnaryTransform}; /// (Dollars, Dollars) -> Dollars addition @@ -212,3 +212,14 @@ impl BinaryTransform for PercentageSatsF64 { StoredF64::from((*numerator as f64 / *denominator as f64) * 100.0) } } + +/// (StoredU32, StoredU32) -> StoredF32 percentage (a/b × 100) +/// Used for pool dominance calculations (pool_blocks / total_blocks × 100) +pub struct PercentageU32F32; + +impl BinaryTransform for PercentageU32F32 { + #[inline(always)] + fn apply(numerator: StoredU32, denominator: StoredU32) -> StoredF32 { + StoredF32::from((*numerator as f64 / *denominator as f64) * 100.0) + } +} diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index c230ac892..786013cbd 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -37,8 +37,6 @@ impl ComputedValueVecsFromTxindex { price: Option<&price::Vecs>, options: VecBuilderOptions, ) -> Result { - let compute_dollars = price.is_some(); - let name_btc = format!("{name}_btc"); let name_usd = format!("{name}_usd"); @@ -63,7 +61,7 @@ impl ComputedValueVecsFromTxindex { let bitcoin = ComputedVecsFromTxindex::forced_import( db, &name_btc, - Source::None, + Source::Vec(bitcoin_txindex.boxed_clone()), version + VERSION, indexes, options, @@ -96,18 +94,18 @@ impl ComputedValueVecsFromTxindex { sats, bitcoin_txindex, bitcoin, - dollars_txindex, - dollars: compute_dollars.then(|| { + dollars: dollars_txindex.as_ref().map(|dtx| { ComputedVecsFromTxindex::forced_import( db, &name_usd, - Source::None, + Source::Vec(dtx.boxed_clone()), version + VERSION, indexes, options, ) .unwrap() }), + dollars_txindex, }) } diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 7b2c744a6..b0662b370 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -124,15 +124,14 @@ impl Computer { ) })?; - let pools_handle = big_thread().spawn_scoped(s, || { - pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref()) - })?; - let cointime = cointime::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())?; let chain = chain_handle.join().unwrap()?; - let pools = pools_handle.join().unwrap()?; + + // pools depends on chain for lazy dominance vecs + let pools = + pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref(), &chain)?; Ok((chain, pools, cointime)) })?; @@ -225,9 +224,7 @@ impl Computer { info!("Computing prices..."); let i = Instant::now(); - self.price - .um() - .compute(&self.indexes, &starting_indexes, fetched, exit)?; + self.price.um().compute(&starting_indexes, fetched, exit)?; info!("Computed prices in {:?}", i.elapsed()); } @@ -330,6 +327,71 @@ impl Computer { info!("Total compute time: {:?}", compute_start.elapsed()); Ok(()) } + + /// Iterate over all exportable vecs with their database name. + pub fn iter_named_exportable( + &self, + ) -> impl Iterator { + use brk_traversable::Traversable; + + std::iter::empty() + .chain(self.blks.iter_any_exportable().map(|v| (blks::DB_NAME, v))) + .chain( + self.chain + .iter_any_exportable() + .map(|v| (chain::DB_NAME, v)), + ) + .chain( + self.cointime + .iter_any_exportable() + .map(|v| (cointime::DB_NAME, v)), + ) + .chain( + self.constants + .iter_any_exportable() + .map(|v| (constants::DB_NAME, v)), + ) + .chain( + self.fetched + .iter_any_exportable() + .map(|v| (fetched::DB_NAME, v)), + ) + .chain( + self.indexes + .iter_any_exportable() + .map(|v| (indexes::DB_NAME, v)), + ) + .chain( + self.market + .iter_any_exportable() + .map(|v| (market::DB_NAME, v)), + ) + .chain( + self.pools + .iter_any_exportable() + .map(|v| (pools::DB_NAME, v)), + ) + .chain( + self.price + .iter_any_exportable() + .map(|v| (price::DB_NAME, v)), + ) + .chain( + self.stateful + .iter_any_exportable() + .map(|v| (stateful::DB_NAME, v)), + ) + .chain( + self.txins + .iter_any_exportable() + .map(|v| (txins::DB_NAME, v)), + ) + .chain( + self.txouts + .iter_any_exportable() + .map(|v| (txouts::DB_NAME, v)), + ) + } } // pub fn generate_allocation_files(monitored: &pools::Vecs) -> Result<()> { diff --git a/crates/brk_computer/src/pools/mod.rs b/crates/brk_computer/src/pools/mod.rs index 68b0455f3..cf10e54f9 100644 --- a/crates/brk_computer/src/pools/mod.rs +++ b/crates/brk_computer/src/pools/mod.rs @@ -36,6 +36,7 @@ impl Vecs { parent_version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, + chain: &chain::Vecs, ) -> Result { let db = Database::open(&parent_path.join(DB_NAME))?; db.set_min_len(PAGE_SIZE * 1_000_000)?; @@ -54,6 +55,7 @@ impl Vecs { version + Version::ZERO, indexes, price, + chain, ) .map(|vecs| (pool.slug, vecs)) }) diff --git a/crates/brk_computer/src/pools/vecs.rs b/crates/brk_computer/src/pools/vecs.rs index 396f1c17e..678e3367c 100644 --- a/crates/brk_computer/src/pools/vecs.rs +++ b/crates/brk_computer/src/pools/vecs.rs @@ -1,12 +1,13 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Height, PoolSlug, Sats, StoredF32, StoredU16, StoredU32}; -use vecdb::{Database, Exit, GenericStoredVec, IterableVec, VecIndex, Version}; +use vecdb::{Database, Exit, GenericStoredVec, IterableCloneableVec, IterableVec, VecIndex, Version}; use crate::{ chain, grouped::{ - ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, + ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, + LazyVecsFrom2FromDateIndex, LazyVecsFrom2FromHeight, PercentageU32F32, Source, VecBuilderOptions, }, indexes::{self, Indexes}, @@ -25,11 +26,11 @@ pub struct Vecs { pub indexes_to_subsidy: ComputedValueVecsFromHeight, pub indexes_to_fee: ComputedValueVecsFromHeight, pub indexes_to_coinbase: ComputedValueVecsFromHeight, - pub indexes_to_dominance: ComputedVecsFromDateIndex, - pub indexes_to_1d_dominance: ComputedVecsFromDateIndex, - pub indexes_to_1w_dominance: ComputedVecsFromDateIndex, - pub indexes_to_1m_dominance: ComputedVecsFromDateIndex, - pub indexes_to_1y_dominance: ComputedVecsFromDateIndex, + pub indexes_to_dominance: LazyVecsFrom2FromHeight, + pub indexes_to_1d_dominance: LazyVecsFrom2FromHeight, + pub indexes_to_1w_dominance: LazyVecsFrom2FromDateIndex, + pub indexes_to_1m_dominance: LazyVecsFrom2FromDateIndex, + pub indexes_to_1y_dominance: LazyVecsFrom2FromDateIndex, pub indexes_to_days_since_block: ComputedVecsFromDateIndex, } @@ -40,6 +41,7 @@ impl Vecs { parent_version: Version, indexes: &indexes::Vecs, price: Option<&price::Vecs>, + chain: &chain::Vecs, ) -> Result { let suffix = |s: &str| format!("{}_{s}", slug); let compute_dollars = price.is_some(); @@ -61,19 +63,59 @@ impl Vecs { }; } + let indexes_to_blocks_mined = ComputedVecsFromHeight::forced_import( + db, + &suffix("blocks_mined"), + Source::Compute, + version, + indexes, + sum_cum, + )?; + + let indexes_to_1w_blocks_mined = import_di!("1w_blocks_mined"); + let indexes_to_1m_blocks_mined = import_di!("1m_blocks_mined"); + let indexes_to_1y_blocks_mined = import_di!("1y_blocks_mined"); + Ok(Self { - slug, - indexes_to_blocks_mined: ComputedVecsFromHeight::forced_import( - db, - &suffix("blocks_mined"), - Source::Compute, + indexes_to_dominance: LazyVecsFrom2FromHeight::from_computed::( + &suffix("dominance"), version, - indexes, - sum_cum, - )?, - indexes_to_1w_blocks_mined: import_di!("1w_blocks_mined"), - indexes_to_1m_blocks_mined: import_di!("1m_blocks_mined"), - indexes_to_1y_blocks_mined: import_di!("1y_blocks_mined"), + indexes_to_blocks_mined.height.as_ref().unwrap().boxed_clone(), + chain.indexes_to_block_count.height.as_ref().unwrap().boxed_clone(), + &indexes_to_blocks_mined, + &chain.indexes_to_block_count, + ), + indexes_to_1d_dominance: LazyVecsFrom2FromHeight::from_computed::( + &suffix("1d_dominance"), + version, + indexes_to_blocks_mined.height.as_ref().unwrap().boxed_clone(), + chain.indexes_to_block_count.height.as_ref().unwrap().boxed_clone(), + &indexes_to_blocks_mined, + &chain.indexes_to_block_count, + ), + indexes_to_1w_dominance: LazyVecsFrom2FromDateIndex::from_computed::( + &suffix("1w_dominance"), + version, + &indexes_to_1w_blocks_mined, + &chain.indexes_to_1w_block_count, + ), + indexes_to_1m_dominance: LazyVecsFrom2FromDateIndex::from_computed::( + &suffix("1m_dominance"), + version, + &indexes_to_1m_blocks_mined, + &chain.indexes_to_1m_block_count, + ), + indexes_to_1y_dominance: LazyVecsFrom2FromDateIndex::from_computed::( + &suffix("1y_dominance"), + version, + &indexes_to_1y_blocks_mined, + &chain.indexes_to_1y_block_count, + ), + slug, + indexes_to_blocks_mined, + indexes_to_1w_blocks_mined, + indexes_to_1m_blocks_mined, + indexes_to_1y_blocks_mined, indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( db, &suffix("subsidy"), @@ -101,11 +143,6 @@ impl Vecs { compute_dollars, indexes, )?, - indexes_to_dominance: import_di!("dominance"), - indexes_to_1d_dominance: import_di!("1d_dominance"), - indexes_to_1w_dominance: import_di!("1w_dominance"), - indexes_to_1m_dominance: import_di!("1m_dominance"), - indexes_to_1y_dominance: import_di!("1y_dominance"), indexes_to_days_since_block: import_di!("days_since_block"), }) } @@ -238,61 +275,6 @@ impl Vecs { Ok(()) })?; - self.indexes_to_dominance - .compute_all(starting_indexes, exit, |vec| { - vec.compute_percentage( - starting_indexes.dateindex, - self.indexes_to_blocks_mined.dateindex.unwrap_cumulative(), - chain.indexes_to_block_count.dateindex.unwrap_cumulative(), - exit, - )?; - Ok(()) - })?; - - self.indexes_to_1d_dominance - .compute_all(starting_indexes, exit, |vec| { - vec.compute_percentage( - starting_indexes.dateindex, - self.indexes_to_blocks_mined.dateindex.unwrap_sum(), - chain.indexes_to_block_count.dateindex.unwrap_sum(), - exit, - )?; - Ok(()) - })?; - - self.indexes_to_1w_dominance - .compute_all(starting_indexes, exit, |vec| { - vec.compute_percentage( - starting_indexes.dateindex, - self.indexes_to_1w_blocks_mined.dateindex.u(), - chain.indexes_to_1w_block_count.dateindex.u(), - exit, - )?; - Ok(()) - })?; - - self.indexes_to_1m_dominance - .compute_all(starting_indexes, exit, |vec| { - vec.compute_percentage( - starting_indexes.dateindex, - self.indexes_to_1m_blocks_mined.dateindex.u(), - chain.indexes_to_1m_block_count.dateindex.u(), - exit, - )?; - Ok(()) - })?; - - self.indexes_to_1y_dominance - .compute_all(starting_indexes, exit, |vec| { - vec.compute_percentage( - starting_indexes.dateindex, - self.indexes_to_1y_blocks_mined.dateindex.u(), - chain.indexes_to_1y_block_count.dateindex.u(), - exit, - )?; - Ok(()) - })?; - self.indexes_to_days_since_block .compute_all(starting_indexes, exit, |v| { let mut prev = None; diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs index fd29d16ad..03f11064e 100644 --- a/crates/brk_computer/src/price.rs +++ b/crates/brk_computer/src/price.rs @@ -115,12 +115,12 @@ impl Vecs { } macro_rules! computed_h { ($name:expr, $opts:expr) => { - ComputedVecsFromHeightStrict::forced_import(&db, $name, v, $opts)? + ComputedVecsFromHeightStrict::forced_import(&db, $name, v, indexes, $opts)? }; } macro_rules! computed_h_sats { ($name:expr, $opts:expr) => { - ComputedVecsFromHeightStrict::forced_import(&db, $name, v_sats, $opts)? + ComputedVecsFromHeightStrict::forced_import(&db, $name, v_sats, indexes, $opts)? }; } let first = || VecBuilderOptions::default().add_first(); @@ -189,12 +189,11 @@ impl Vecs { pub fn compute( &mut self, - indexes: &indexes::Vecs, starting_indexes: &Indexes, fetched: &fetched::Vecs, exit: &Exit, ) -> Result<()> { - self.compute_(indexes, starting_indexes, fetched, exit)?; + self.compute_(starting_indexes, fetched, exit)?; let _lock = exit.lock(); self.db.compact()?; Ok(()) @@ -202,7 +201,6 @@ impl Vecs { fn compute_( &mut self, - indexes: &indexes::Vecs, starting_indexes: &Indexes, fetched: &fetched::Vecs, exit: &Exit, @@ -322,7 +320,7 @@ impl Vecs { })?; self.chainindexes_to_price_close - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.height_to_price_ohlc, @@ -333,7 +331,7 @@ impl Vecs { })?; self.chainindexes_to_price_high - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.height_to_price_ohlc, @@ -344,7 +342,7 @@ impl Vecs { })?; self.chainindexes_to_price_low - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.height_to_price_ohlc, @@ -355,7 +353,7 @@ impl Vecs { })?; self.chainindexes_to_price_open - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.height_to_price_ohlc, @@ -510,7 +508,7 @@ impl Vecs { )?; self.chainindexes_to_price_open_in_sats - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.chainindexes_to_price_open.height, @@ -521,7 +519,7 @@ impl Vecs { })?; self.chainindexes_to_price_high_in_sats - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.chainindexes_to_price_low.height, @@ -532,7 +530,7 @@ impl Vecs { })?; self.chainindexes_to_price_low_in_sats - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.chainindexes_to_price_high.height, @@ -543,7 +541,7 @@ impl Vecs { })?; self.chainindexes_to_price_close_in_sats - .compute(indexes, starting_indexes, exit, |v| { + .compute(starting_indexes, exit, |v| { v.compute_transform( starting_indexes.height, &self.chainindexes_to_price_close.height, diff --git a/crates/brk_computer/src/stateful/address/address_count.rs b/crates/brk_computer/src/stateful/address/address_count.rs index 7b7363cf3..c6caeb18d 100644 --- a/crates/brk_computer/src/stateful/address/address_count.rs +++ b/crates/brk_computer/src/stateful/address/address_count.rs @@ -5,8 +5,8 @@ use brk_types::{Height, StoredU64, Version}; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; use vecdb::{ - AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec, - TypedVecIterator, + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, + IterableCloneableVec, PcoVec, TypedVecIterator, }; use crate::{ @@ -170,17 +170,23 @@ impl AddressTypeToIndexesToAddressCount { name: &str, version: Version, indexes: &indexes::Vecs, + sources: &AddressTypeToHeightToAddressCount, ) -> Result { - Ok(Self::from(ByAddressType::new_with_name(|type_name| { - ComputedVecsFromHeight::forced_import( - db, - &format!("{type_name}_{name}"), - Source::None, - version, - indexes, - VecBuilderOptions::default().add_last(), - ) - })?)) + Ok(Self::from(ByAddressType::< + ComputedVecsFromHeight, + >::try_zip_with_name( + sources, + |type_name, source| { + ComputedVecsFromHeight::forced_import( + db, + &format!("{type_name}_{name}"), + Source::Vec(source.boxed_clone()), + version, + indexes, + VecBuilderOptions::default().add_last(), + ) + }, + )?)) } pub fn compute( diff --git a/crates/brk_computer/src/stateful/cohorts/address.rs b/crates/brk_computer/src/stateful/cohorts/address.rs index ff39af06e..becb707b1 100644 --- a/crates/brk_computer/src/stateful/cohorts/address.rs +++ b/crates/brk_computer/src/stateful/cohorts/address.rs @@ -6,8 +6,8 @@ use brk_traversable::Traversable; use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version}; use rayon::prelude::*; use vecdb::{ - AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec, - PcoVec, + AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, + IterableCloneableVec, IterableVec, PcoVec, }; use crate::{ @@ -71,6 +71,12 @@ impl AddressCohortVecs { price, }; + let height_to_addr_count = EagerVec::forced_import( + db, + &cfg.name("addr_count"), + version + VERSION + Version::ZERO, + )?; + Ok(Self { starting_height: None, @@ -79,20 +85,15 @@ impl AddressCohortVecs { metrics: CohortMetrics::forced_import(&cfg, all_supply)?, - height_to_addr_count: EagerVec::forced_import( - db, - &cfg.name("addr_count"), - version + VERSION + Version::ZERO, - )?, - indexes_to_addr_count: ComputedVecsFromHeight::forced_import( db, &cfg.name("addr_count"), - Source::None, + Source::Vec(height_to_addr_count.boxed_clone()), version + VERSION + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, + height_to_addr_count, }) } diff --git a/crates/brk_computer/src/stateful/compute/block_loop.rs b/crates/brk_computer/src/stateful/compute/block_loop.rs index fa198fcbd..b9790eb15 100644 --- a/crates/brk_computer/src/stateful/compute/block_loop.rs +++ b/crates/brk_computer/src/stateful/compute/block_loop.rs @@ -456,23 +456,22 @@ pub fn process_blocks( } // Final write - always save changes for rollback support - { - let _lock = exit.lock(); - drop(vr); - let (empty_updates, loaded_updates) = cache.take(); + let _lock = exit.lock(); + drop(vr); - // Process address updates (mutations) - process_address_updates( - &mut vecs.addresses_data, - &mut vecs.any_address_indexes, - empty_updates, - loaded_updates, - )?; + let (empty_updates, loaded_updates) = cache.take(); - // Write to disk (pure I/O) - save changes for rollback - write(vecs, last_height, chain_state, true)?; - } + // Process address updates (mutations) + process_address_updates( + &mut vecs.addresses_data, + &mut vecs.any_address_indexes, + empty_updates, + loaded_updates, + )?; + + // Write to disk (pure I/O) - save changes for rollback + write(vecs, last_height, chain_state, true)?; Ok(()) } diff --git a/crates/brk_computer/src/stateful/compute/context.rs b/crates/brk_computer/src/stateful/compute/context.rs index 4360e64bf..0666d7335 100644 --- a/crates/brk_computer/src/stateful/compute/context.rs +++ b/crates/brk_computer/src/stateful/compute/context.rs @@ -4,19 +4,13 @@ use vecdb::VecIndex; use crate::{indexes, price}; /// Context shared across block processing. -pub struct ComputeContext<'a> { +pub struct ComputeContext { /// Starting height for this computation run pub starting_height: Height, /// Last height to process pub last_height: Height, - /// Whether price data is available - pub compute_dollars: bool, - - /// Price data (optional) - pub price: Option<&'a price::Vecs>, - /// Pre-computed height -> timestamp mapping pub height_to_timestamp: Vec, @@ -24,13 +18,13 @@ pub struct ComputeContext<'a> { pub height_to_price: Option>, } -impl<'a> ComputeContext<'a> { +impl ComputeContext { /// Create a new computation context. pub fn new( starting_height: Height, last_height: Height, indexes: &indexes::Vecs, - price: Option<&'a price::Vecs>, + price: Option<&price::Vecs>, ) -> Self { let height_to_timestamp: Vec = indexes.height_to_timestamp_fixed.into_iter().collect(); @@ -42,8 +36,6 @@ impl<'a> ComputeContext<'a> { Self { starting_height, last_height, - compute_dollars: price.is_some(), - price, height_to_timestamp, height_to_price, } diff --git a/crates/brk_computer/src/stateful/metrics/price_paid.rs b/crates/brk_computer/src/stateful/metrics/price_paid.rs index 18c1709d4..3638286a5 100644 --- a/crates/brk_computer/src/stateful/metrics/price_paid.rs +++ b/crates/brk_computer/src/stateful/metrics/price_paid.rs @@ -2,7 +2,9 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{DateIndex, Dollars, Height, Version}; use rayon::prelude::*; -use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec}; +use vecdb::{ + AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec, +}; use crate::{ Indexes, @@ -34,33 +36,31 @@ impl PricePaidMetrics { let extended = cfg.extended(); let last = VecBuilderOptions::default().add_last(); + let height_to_min_price_paid = + EagerVec::forced_import(cfg.db, &cfg.name("min_price_paid"), cfg.version + v0)?; + + let height_to_max_price_paid = + EagerVec::forced_import(cfg.db, &cfg.name("max_price_paid"), cfg.version + v0)?; + Ok(Self { - height_to_min_price_paid: EagerVec::forced_import( - cfg.db, - &cfg.name("min_price_paid"), - cfg.version + v0, - )?, indexes_to_min_price_paid: ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("min_price_paid"), - Source::None, + Source::Vec(height_to_min_price_paid.boxed_clone()), cfg.version + v0, cfg.indexes, last, )?, - height_to_max_price_paid: EagerVec::forced_import( - cfg.db, - &cfg.name("max_price_paid"), - cfg.version + v0, - )?, indexes_to_max_price_paid: ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("max_price_paid"), - Source::None, + Source::Vec(height_to_max_price_paid.boxed_clone()), cfg.version + v0, cfg.indexes, last, )?, + height_to_min_price_paid, + height_to_max_price_paid, price_percentiles: extended .then(|| { PricePercentiles::forced_import( diff --git a/crates/brk_computer/src/stateful/metrics/realized.rs b/crates/brk_computer/src/stateful/metrics/realized.rs index ad904463d..6f9b4dd34 100644 --- a/crates/brk_computer/src/stateful/metrics/realized.rs +++ b/crates/brk_computer/src/stateful/metrics/realized.rs @@ -104,7 +104,7 @@ impl RealizedMetrics { let indexes_to_realized_loss = ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("realized_loss"), - Source::None, + Source::Vec(height_to_realized_loss.boxed_clone()), cfg.version + v0, cfg.indexes, sum_cum, @@ -146,7 +146,7 @@ impl RealizedMetrics { let indexes_to_realized_cap = ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("realized_cap"), - Source::None, + Source::Vec(height_to_realized_cap.boxed_clone()), cfg.version + v0, cfg.indexes, last, @@ -158,7 +158,7 @@ impl RealizedMetrics { let indexes_to_realized_profit = ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("realized_profit"), - Source::None, + Source::Vec(height_to_realized_profit.boxed_clone()), cfg.version + v0, cfg.indexes, sum_cum, @@ -198,32 +198,68 @@ impl RealizedMetrics { LazyVecsFrom2FromHeight::from_computed::( &cfg.name("net_realized_pnl_rel_to_realized_cap"), cfg.version + v1, - indexes_to_net_realized_pnl.height.as_ref().unwrap().boxed_clone(), + indexes_to_net_realized_pnl + .height + .as_ref() + .unwrap() + .boxed_clone(), height_to_realized_cap.boxed_clone(), &indexes_to_net_realized_pnl, &indexes_to_realized_cap, ); + let indexes_to_realized_price = ComputedVecsFromHeight::forced_import( + cfg.db, + &cfg.name("realized_price"), + Source::Compute, + cfg.version + v0, + cfg.indexes, + last, + )?; + + let height_to_value_created = + EagerVec::forced_import(cfg.db, &cfg.name("value_created"), cfg.version + v0)?; + let height_to_value_destroyed = + EagerVec::forced_import(cfg.db, &cfg.name("value_destroyed"), cfg.version + v0)?; + + let height_to_adjusted_value_created = compute_adjusted + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("adjusted_value_created"), + cfg.version + v0, + ) + }) + .transpose()?; + let height_to_adjusted_value_destroyed = compute_adjusted + .then(|| { + EagerVec::forced_import( + cfg.db, + &cfg.name("adjusted_value_destroyed"), + cfg.version + v0, + ) + }) + .transpose()?; + Ok(Self { // === Realized Cap === height_to_realized_cap, indexes_to_realized_cap, - indexes_to_realized_price: ComputedVecsFromHeight::forced_import( - cfg.db, - &cfg.name("realized_price"), - Source::Compute, - cfg.version + v0, - cfg.indexes, - last, - )?, + indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex::forced_import( cfg.db, &cfg.name("realized_price"), - Source::None, + Source::Vec( + indexes_to_realized_price + .dateindex + .unwrap_last() + .boxed_clone(), + ), cfg.version + v0, cfg.indexes, extended, )?, + indexes_to_realized_price, indexes_to_realized_cap_rel_to_own_market_cap: extended .then(|| { ComputedVecsFromHeight::forced_import( @@ -272,76 +308,62 @@ impl RealizedMetrics { .transpose()?, // === Value Created/Destroyed === - height_to_value_created: EagerVec::forced_import( - cfg.db, - &cfg.name("value_created"), - cfg.version + v0, - )?, indexes_to_value_created: ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("value_created"), - Source::None, + Source::Vec(height_to_value_created.boxed_clone()), cfg.version + v0, cfg.indexes, sum, )?, - height_to_value_destroyed: EagerVec::forced_import( - cfg.db, - &cfg.name("value_destroyed"), - cfg.version + v0, - )?, indexes_to_value_destroyed: ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("value_destroyed"), - Source::None, + Source::Vec(height_to_value_destroyed.boxed_clone()), cfg.version + v0, cfg.indexes, sum, )?, + height_to_value_created, + height_to_value_destroyed, // === Adjusted Value (optional) === - height_to_adjusted_value_created: compute_adjusted - .then(|| { - EagerVec::forced_import( - cfg.db, - &cfg.name("adjusted_value_created"), - cfg.version + v0, - ) - }) - .transpose()?, indexes_to_adjusted_value_created: compute_adjusted .then(|| { ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("adjusted_value_created"), - Source::None, + Source::Vec( + height_to_adjusted_value_created + .as_ref() + .unwrap() + .boxed_clone(), + ), cfg.version + v0, cfg.indexes, sum, ) }) .transpose()?, - height_to_adjusted_value_destroyed: compute_adjusted - .then(|| { - EagerVec::forced_import( - cfg.db, - &cfg.name("adjusted_value_destroyed"), - cfg.version + v0, - ) - }) - .transpose()?, indexes_to_adjusted_value_destroyed: compute_adjusted .then(|| { ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("adjusted_value_destroyed"), - Source::None, + Source::Vec( + height_to_adjusted_value_destroyed + .as_ref() + .unwrap() + .boxed_clone(), + ), cfg.version + v0, cfg.indexes, sum, ) }) .transpose()?, + height_to_adjusted_value_created, + height_to_adjusted_value_destroyed, // === SOPR === dateindex_to_sopr: EagerVec::forced_import( diff --git a/crates/brk_computer/src/stateful/metrics/supply.rs b/crates/brk_computer/src/stateful/metrics/supply.rs index e92925c43..249ca0229 100644 --- a/crates/brk_computer/src/stateful/metrics/supply.rs +++ b/crates/brk_computer/src/stateful/metrics/supply.rs @@ -96,26 +96,22 @@ impl SupplyMetrics { cfg.version + v0, ); + let height_to_utxo_count = + EagerVec::forced_import(cfg.db, &cfg.name("utxo_count"), cfg.version + v0)?; + Ok(Self { - height_to_supply, - height_to_supply_value, - indexes_to_supply, - - height_to_utxo_count: EagerVec::forced_import( - cfg.db, - &cfg.name("utxo_count"), - cfg.version + v0, - )?, - indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( cfg.db, &cfg.name("utxo_count"), - Source::None, + Source::Vec(height_to_utxo_count.boxed_clone()), cfg.version + v0, cfg.indexes, last, )?, - + height_to_supply, + height_to_supply_value, + indexes_to_supply, + height_to_utxo_count, height_to_supply_half_value, indexes_to_supply_half, }) diff --git a/crates/brk_computer/src/stateful/vecs.rs b/crates/brk_computer/src/stateful/vecs.rs index ade8d9f6a..d3ce9d190 100644 --- a/crates/brk_computer/src/stateful/vecs.rs +++ b/crates/brk_computer/src/stateful/vecs.rs @@ -147,6 +147,12 @@ impl Vecs { indexes, )?; + // Extract address type height vecs before struct literal to use as sources + let addresstype_to_height_to_addr_count = + AddressTypeToHeightToAddressCount::forced_import(&db, "addr_count", v0)?; + let addresstype_to_height_to_empty_addr_count = + AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?; + let this = Self { chain_state: BytesVec::forced_import_with( vecdb::ImportOptions::new(&db, "chain", v0) @@ -204,22 +210,23 @@ impl Vecs { .unwrap() }), - addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::forced_import( + addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::forced_import( &db, "addr_count", v0, + indexes, + &addresstype_to_height_to_addr_count, )?, - addresstype_to_height_to_empty_addr_count: - AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?, - addresstype_to_indexes_to_addr_count: - AddressTypeToIndexesToAddressCount::forced_import(&db, "addr_count", v0, indexes)?, addresstype_to_indexes_to_empty_addr_count: AddressTypeToIndexesToAddressCount::forced_import( &db, "empty_addr_count", v0, indexes, + &addresstype_to_height_to_empty_addr_count, )?, + addresstype_to_height_to_addr_count, + addresstype_to_height_to_empty_addr_count, utxo_cohorts, address_cohorts, diff --git a/crates/brk_grouper/src/by_address_type.rs b/crates/brk_grouper/src/by_address_type.rs index d1fdf5959..560840656 100644 --- a/crates/brk_grouper/src/by_address_type.rs +++ b/crates/brk_grouper/src/by_address_type.rs @@ -78,6 +78,22 @@ impl ByAddressType { }) } + pub fn try_zip_with_name(other: &ByAddressType, f: F) -> Result> + where + F: Fn(&'static str, &S) -> Result, + { + Ok(ByAddressType { + p2pk65: f(P2PK65, &other.p2pk65)?, + p2pk33: f(P2PK33, &other.p2pk33)?, + p2pkh: f(P2PKH, &other.p2pkh)?, + p2sh: f(P2SH, &other.p2sh)?, + p2wpkh: f(P2WPKH, &other.p2wpkh)?, + p2wsh: f(P2WSH, &other.p2wsh)?, + p2tr: f(P2TR, &other.p2tr)?, + p2a: f(P2A, &other.p2a)?, + }) + } + #[inline] pub fn get_unwrap(&self, addresstype: OutputType) -> &T { self.get(addresstype).unwrap() diff --git a/crates/brk_query/src/impl/metrics.rs b/crates/brk_query/src/impl/metrics.rs index 855a66b7e..99c830c3e 100644 --- a/crates/brk_query/src/impl/metrics.rs +++ b/crates/brk_query/src/impl/metrics.rs @@ -3,8 +3,8 @@ use std::collections::BTreeMap; use brk_error::{Error, Result}; use brk_traversable::TreeNode; use brk_types::{ - Format, Index, IndexInfo, Limit, Metric, MetricCount, MetricData, PaginatedMetrics, Pagination, - PaginationIndex, + DetailedMetricCount, Format, Index, IndexInfo, Limit, Metric, MetricData, PaginatedMetrics, + Pagination, PaginationIndex, }; use vecdb::AnyExportableVec; @@ -171,9 +171,18 @@ impl Query { Ok(vecs) } - /// Calculate total weight of the vecs for the given range + /// Calculate total weight of the vecs for the given range. + /// Applies index-specific cost multipliers for rate limiting. pub fn weight(vecs: &[&dyn AnyExportableVec], from: Option, to: Option) -> usize { - vecs.iter().map(|v| v.range_weight(from, to)).sum() + vecs.iter() + .map(|v| { + let base = v.range_weight(from, to); + let multiplier = Index::try_from(v.index_type_to_string()) + .map(|i| i.cost_multiplier()) + .unwrap_or(1); + base * multiplier + }) + .sum() } /// Search and format single metric @@ -235,29 +244,13 @@ impl Query { &self.vecs().index_to_metric_to_vec } - pub fn metric_count(&self) -> MetricCount { - let total = self.total_metric_count(); - let lazy = self.lazy_metric_count(); - MetricCount { - distinct_metrics: self.distinct_metric_count(), - total_endpoints: total, - lazy_endpoints: lazy, - stored_endpoints: total - lazy, + pub fn metric_count(&self) -> DetailedMetricCount { + DetailedMetricCount { + total: self.vecs().counts.clone(), + by_db: self.vecs().counts_by_db.clone(), } } - pub fn distinct_metric_count(&self) -> usize { - self.vecs().distinct_metric_count - } - - pub fn total_metric_count(&self) -> usize { - self.vecs().total_metric_count - } - - pub fn lazy_metric_count(&self) -> usize { - self.vecs().lazy_metric_count - } - pub fn indexes(&self) -> &[IndexInfo] { &self.vecs().indexes } diff --git a/crates/brk_query/src/vecs.rs b/crates/brk_query/src/vecs.rs index 29bf4188c..40a1657ef 100644 --- a/crates/brk_query/src/vecs.rs +++ b/crates/brk_query/src/vecs.rs @@ -3,7 +3,7 @@ use std::{borrow::Cow, collections::BTreeMap}; use brk_computer::Computer; use brk_indexer::Indexer; use brk_traversable::{Traversable, TreeNode}; -use brk_types::{Index, IndexInfo, Limit, Metric, PaginatedMetrics, Pagination, PaginationIndex}; +use brk_types::{Index, IndexInfo, Limit, Metric, MetricCount, PaginatedMetrics, Pagination, PaginationIndex}; use derive_deref::{Deref, DerefMut}; use quickmatch::{QuickMatch, QuickMatchConfig}; use vecdb::AnyExportableVec; @@ -14,9 +14,8 @@ pub struct Vecs<'a> { pub index_to_metric_to_vec: BTreeMap>, pub metrics: Vec<&'a str>, pub indexes: Vec, - pub distinct_metric_count: usize, - pub total_metric_count: usize, - pub lazy_metric_count: usize, + pub counts: MetricCount, + pub counts_by_db: BTreeMap, catalog: Option, matcher: Option>, metric_to_indexes: BTreeMap<&'a str, Vec>, @@ -30,11 +29,11 @@ impl<'a> Vecs<'a> { indexer .vecs .iter_any_exportable() - .for_each(|vec| this.insert(vec)); + .for_each(|vec| this.insert(vec, "indexed")); computer - .iter_any_exportable() - .for_each(|vec| this.insert(vec)); + .iter_named_exportable() + .for_each(|(db, vec)| this.insert(vec, db)); let mut ids = this .metric_to_index_to_vec @@ -56,18 +55,20 @@ impl<'a> Vecs<'a> { sort_ids(&mut ids); this.metrics = ids; - this.distinct_metric_count = this.metric_to_index_to_vec.keys().count(); - this.total_metric_count = this + this.counts.distinct_metrics = this.metric_to_index_to_vec.keys().count(); + this.counts.total_endpoints = this .index_to_metric_to_vec .values() .map(|tree| tree.len()) .sum::(); - this.lazy_metric_count = this + this.counts.lazy_endpoints = this .index_to_metric_to_vec .values() .flat_map(|tree| tree.values()) .filter(|vec| vec.region_names().is_empty()) .count(); + this.counts.stored_endpoints = + this.counts.total_endpoints - this.counts.lazy_endpoints; this.indexes = this .index_to_metric_to_vec .keys() @@ -109,7 +110,7 @@ impl<'a> Vecs<'a> { this } - fn insert(&mut self, vec: &'a dyn AnyExportableVec) { + fn insert(&mut self, vec: &'a dyn AnyExportableVec, db: &str) { let name = vec.name(); let serialized_index = vec.index_type_to_string(); let index = Index::try_from(serialized_index) @@ -134,6 +135,13 @@ impl<'a> Vecs<'a> { prev.is_none(), "Duplicate metric: {name} for index {index:?}" ); + + // Track per-db counts + let is_lazy = vec.region_names().is_empty(); + self.counts_by_db + .entry(db.to_string()) + .or_default() + .add_endpoint(is_lazy); } pub fn metrics(&'static self, pagination: Pagination) -> PaginatedMetrics { diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 685d5fe4d..1c4c97015 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -81,7 +81,6 @@ where fs::create_dir_all(path)?; let (meta, keyspace) = StoreMeta::checked_open( - db, &path.join(format!("meta/{name}")), MAJOR_FJALL_VERSION + version, || { diff --git a/crates/brk_store/src/meta.rs b/crates/brk_store/src/meta.rs index 2ee788673..f11ab0baa 100644 --- a/crates/brk_store/src/meta.rs +++ b/crates/brk_store/src/meta.rs @@ -5,7 +5,7 @@ use std::{ use brk_error::{Error, Result}; use brk_types::Version; -use fjall::{Database, Keyspace}; +use fjall::Keyspace; use super::Height; @@ -18,7 +18,6 @@ pub struct StoreMeta { impl StoreMeta { pub fn checked_open( - _database: &Database, path: &Path, version: Version, open_partition_handle: F, diff --git a/crates/brk_types/src/height.rs b/crates/brk_types/src/height.rs index 4304b9055..adbe2bca8 100644 --- a/crates/brk_types/src/height.rs +++ b/crates/brk_types/src/height.rs @@ -7,7 +7,9 @@ use byteview::ByteView; use derive_deref::Deref; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use vecdb::{Bytes, CheckedSub, Formattable, Pco, PrintableIndex, Stamp}; +use vecdb::{Bytes, CheckedSub, Formattable, FromCoarserIndex, Pco, PrintableIndex, Stamp}; + +use super::DifficultyEpoch; use crate::{BLOCKS_PER_DIFF_EPOCHS, BLOCKS_PER_HALVING}; @@ -289,3 +291,13 @@ impl Formattable for Height { false } } + +impl FromCoarserIndex for Height { + fn min_from(coarser: DifficultyEpoch) -> usize { + usize::from(coarser) * BLOCKS_PER_DIFF_EPOCHS as usize + } + + fn max_from_(coarser: DifficultyEpoch) -> usize { + (usize::from(coarser) + 1) * BLOCKS_PER_DIFF_EPOCHS as usize - 1 + } +} diff --git a/crates/brk_types/src/index.rs b/crates/brk_types/src/index.rs index f712d4c21..dee562528 100644 --- a/crates/brk_types/src/index.rs +++ b/crates/brk_types/src/index.rs @@ -130,6 +130,15 @@ impl Index { pub fn serialize_long(&self) -> &'static str { self.possible_values().last().unwrap() } + + /// Returns the query cost multiplier for this index type. + /// Used for rate limiting to account for expensive lazy computations. + pub const fn cost_multiplier(&self) -> usize { + match self { + Self::DifficultyEpoch => 60, + _ => 1, + } + } } impl TryFrom<&str> for Index { diff --git a/crates/brk_types/src/metriccount.rs b/crates/brk_types/src/metriccount.rs index 24de55578..a12b2fdb3 100644 --- a/crates/brk_types/src/metriccount.rs +++ b/crates/brk_types/src/metriccount.rs @@ -1,8 +1,10 @@ +use std::collections::BTreeMap; + use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Metric count statistics - distinct metrics and total metric-index combinations -#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)] pub struct MetricCount { /// Number of unique metrics available (e.g., realized_price, market_cap) #[schemars(example = 3141)] @@ -17,3 +19,24 @@ pub struct MetricCount { #[schemars(example = 16000)] pub stored_endpoints: usize, } + +impl MetricCount { + pub fn add_endpoint(&mut self, is_lazy: bool) { + self.total_endpoints += 1; + if is_lazy { + self.lazy_endpoints += 1; + } else { + self.stored_endpoints += 1; + } + } +} + +/// Detailed metric count with per-database breakdown +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct DetailedMetricCount { + /// Aggregate counts + #[serde(flatten)] + pub total: MetricCount, + /// Per-database breakdown of counts + pub by_db: BTreeMap, +} diff --git a/websites/bitview/scripts/main.js b/websites/bitview/scripts/main.js index 70c3468ab..984902be0 100644 --- a/websites/bitview/scripts/main.js +++ b/websites/bitview/scripts/main.js @@ -111,13 +111,13 @@ Promise.all([ modules.brkClient(), modules.brkResources(), modules.options(), -]).then(([signals, { createClient }, { createResources }, { initOptions }]) => +]).then(([signals, { BrkClient, VERSION }, { createResources }, { initOptions }]) => signals.createRoot(() => { - const brk = createClient("/"); + const brk = new BrkClient("/"); const resources = createResources(brk, signals); const owner = signals.getOwner(); - console.log(`VERSION = ${brk.VERSION}`); + console.log(`VERSION = ${VERSION}`); function initDark() { const preferredColorSchemeMatchMedia = window.matchMedia(