Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-01 02:33:39 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: snapshot

Browse Source
This commit is contained in:
nym21
2025-12-29 00:14:54 +01:00
parent 236b4097c5
commit 31d2f8ef37
37 changed files with 593 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_binder/src/javascript.rs
+36 -6
View File
@@ -1,11 +1,11 @@
use std::{collections::HashSet, fmt::Write as FmtWrite, fs, io, path::Path};
use brk_types::{Index, TreeNode};
use brk_types::{pools, Index, TreeNode};
use serde_json::Value;
use crate::{
ClientMetadata, Endpoint, FieldNamePosition, IndexSetPattern, PatternField, StructuralPattern,
TypeSchemas, extract_inner_type, get_fields_with_child_info, get_first_leaf_name,
TypeSchemas, VERSION, extract_inner_type, get_fields_with_child_info, get_first_leaf_name,
get_node_fields, get_pattern_instance_base, to_camel_case, to_pascal_case,
};
@@ -23,6 +23,7 @@ pub fn generate_javascript_client(
writeln!(output, "// Auto-generated BRK JavaScript client").unwrap();
writeln!(output, "// Do not edit manually\n").unwrap();
generate_constants(&mut output);
generate_type_definitions(&mut output, schemas);
generate_base_client(&mut output);
generate_index_accessors(&mut output, &metadata.index_set_patterns);
@@ -35,6 +36,32 @@ pub fn generate_javascript_client(
Ok(())
}
fn generate_constants(output: &mut String) {
writeln!(output, "// Constants\n").unwrap();
// VERSION
writeln!(output, "export const VERSION = \"v{VERSION}\";\n").unwrap();
// INDEXES
let indexes = Index::all();
writeln!(output, "export const INDEXES = /** @type {{const}} */ ([").unwrap();
for index in &indexes {
writeln!(output, " \"{}\",", index.serialize_long()).unwrap();
}
writeln!(output, "]);\n").unwrap();
// POOL_ID_TO_POOL_NAME
let pools = pools();
let mut sorted_pools: Vec<_> = pools.iter().collect();
sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
writeln!(output, "export const POOL_ID_TO_POOL_NAME = /** @type {{const}} */ ({{").unwrap();
for pool in &sorted_pools {
writeln!(output, " {}: \"{}\",", pool.slug(), pool.name).unwrap();
}
writeln!(output, "}});\n").unwrap();
}
fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) {
if schemas.is_empty() {
return;
@@ -246,7 +273,7 @@ class MetricNode {{
/**
* Fetch all data points for this metric.
* @param {{(value: T[]) => void}} [onUpdate] - Called when data is available (may be called twice: cache then fresh)
* @returns {{Promise<T[] | null>}}
* @returns {{Promise<T[]>}}
*/
get(onUpdate) {{
return this._client.get(this._path, onUpdate);
@@ -257,7 +284,7 @@ class MetricNode {{
* @param {{string | number}} from
* @param {{string | number}} to
* @param {{(value: T[]) => void}} [onUpdate] - Called when data is available (may be called twice: cache then fresh)
* @returns {{Promise<T[] | null>}}
* @returns {{Promise<T[]>}}
*/
getRange(from, to, onUpdate) {{
return this._client.get(`${{this._path}}?from=${{from}}&to=${{to}}`, onUpdate);
@@ -282,7 +309,7 @@ class BrkClientBase {{
* @template T
* @param {{string}} path
* @param {{(value: T) => void}} [onUpdate] - Called when data is available
* @returns {{Promise<T | null>}}
* @returns {{Promise<T>}}
*/
async get(path, onUpdate) {{
const url = `${{this.baseUrl}}${{path}}`;
@@ -291,7 +318,10 @@ class BrkClientBase {{
const cachedJson = cachedRes ? await cachedRes.json() : null;
if (cachedJson) onUpdate?.(cachedJson);
if (!globalThis.navigator?.onLine) return cachedJson;
if (!globalThis.navigator?.onLine) {{
if (cachedJson) return cachedJson;
throw new BrkError('Offline and no cached data available');
}}
try {{
const res = await fetch(url, {{ signal: AbortSignal.timeout(this.timeout) }});
crates/brk_binder/src/python.rs
+30 -3
View File
@@ -1,11 +1,11 @@
use std::{collections::HashSet, fmt::Write as FmtWrite, fs, io, path::Path};
use brk_types::{Index, TreeNode};
use brk_types::{pools, Index, TreeNode};
use serde_json::Value;
use crate::{
ClientMetadata, Endpoint, FieldNamePosition, IndexSetPattern, PatternField, StructuralPattern,
TypeSchemas, extract_inner_type, get_fields_with_child_info, get_node_fields,
TypeSchemas, VERSION, extract_inner_type, get_fields_with_child_info, get_node_fields,
get_pattern_instance_base, to_pascal_case, to_snake_case,
};
@@ -25,12 +25,13 @@ pub fn generate_python_client(
writeln!(output, "from __future__ import annotations").unwrap();
writeln!(
output,
"from typing import TypeVar, Generic, Any, Optional, List, Literal, TypedDict"
"from typing import TypeVar, Generic, Any, Optional, List, Literal, TypedDict, Final"
)
.unwrap();
writeln!(output, "import httpx\n").unwrap();
writeln!(output, "T = TypeVar('T')\n").unwrap();
generate_constants(&mut output);
generate_type_definitions(&mut output, schemas);
generate_base_client(&mut output);
generate_metric_node(&mut output);
@@ -44,6 +45,32 @@ pub fn generate_python_client(
Ok(())
}
fn generate_constants(output: &mut String) {
writeln!(output, "# Constants\n").unwrap();
// VERSION
writeln!(output, "VERSION: Final[str] = \"v{VERSION}\"\n").unwrap();
// INDEXES
let indexes = Index::all();
writeln!(output, "INDEXES: Final[tuple[str, ...]] = (").unwrap();
for index in &indexes {
writeln!(output, " \"{}\",", index.serialize_long()).unwrap();
}
writeln!(output, ")\n").unwrap();
// POOL_ID_TO_POOL_NAME
let pools = pools();
let mut sorted_pools: Vec<_> = pools.iter().collect();
sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
writeln!(output, "POOL_ID_TO_POOL_NAME: Final[dict[str, str]] = {{").unwrap();
for pool in &sorted_pools {
writeln!(output, " \"{}\": \"{}\",", pool.slug(), pool.name).unwrap();
}
writeln!(output, "}}\n").unwrap();
}
fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) {
if schemas.is_empty() {
return;
crates/brk_cli/src/main.rs
+2 -2
View File
@@ -8,6 +8,7 @@ use std::{
time::Duration,
};
use brk_alloc::Mimalloc;
use brk_binder::generate_js_files;
use brk_bundler::bundle;
use brk_computer::Computer;
@@ -17,7 +18,6 @@ use brk_iterator::Blocks;
use brk_mempool::Mempool;
use brk_query::AsyncQuery;
use brk_reader::Reader;
use brk_alloc::Mimalloc;
use brk_server::{Server, VERSION};
use log::info;
use vecdb::Exit;
@@ -60,7 +60,7 @@ pub fn run() -> color_eyre::Result<()> {
// Pre-run indexer if too far behind, then drop and reimport to reduce memory
let chain_height = client.get_last_height()?;
let indexed_height = indexer.vecs.starting_height();
if u32::from(chain_height) - u32::from(indexed_height) > 1000 {
if chain_height.saturating_sub(*indexed_height) > 1000 {
indexer.index(&blocks, &client, &exit)?;
drop(indexer);
Mimalloc::collect();
crates/brk_computer/src/chain/import.rs
+55 -12
View File
@@ -210,6 +210,9 @@ impl Vecs {
|_, _| Some(StoredU64::from(TARGET_BLOCKS_PER_DECADE)),
);
let height_to_interval = eager!("interval");
let height_to_vbytes = eager!("vbytes");
let this = Self {
dateindex_to_block_count_target,
weekindex_to_block_count_target,
@@ -218,34 +221,62 @@ impl Vecs {
semesterindex_to_block_count_target,
yearindex_to_block_count_target,
decadeindex_to_block_count_target,
height_to_interval: eager!("interval"),
timeindexes_to_timestamp: computed_di!(
"timestamp",
VecBuilderOptions::default().add_first()
),
indexes_to_block_interval: computed_h!("block_interval", Source::None, stats()),
indexes_to_block_interval: computed_h!(
"block_interval",
Source::Vec(height_to_interval.boxed_clone()),
stats()
),
indexes_to_block_count: computed_h!("block_count", Source::Compute, sum_cum()),
indexes_to_1w_block_count: computed_di!("1w_block_count", last()),
indexes_to_1m_block_count: computed_di!("1m_block_count", last()),
indexes_to_1y_block_count: computed_di!("1y_block_count", last()),
indexes_to_block_weight: computed_h!("block_weight", Source::None, full_stats()),
indexes_to_block_size: computed_h!("block_size", Source::None, full_stats()),
height_to_vbytes: eager!("vbytes"),
indexes_to_block_weight: computed_h!(
"block_weight",
Source::Vec(indexer.vecs.block.height_to_weight.boxed_clone()),
full_stats()
),
indexes_to_block_size: computed_h!(
"block_size",
Source::Vec(indexer.vecs.block.height_to_total_size.boxed_clone()),
full_stats()
),
height_to_24h_block_count: eager!("24h_block_count"),
height_to_24h_coinbase_sum: eager!("24h_coinbase_sum"),
height_to_24h_coinbase_usd_sum: eager!("24h_coinbase_usd_sum"),
indexes_to_block_vbytes: computed_h!("block_vbytes", Source::None, full_stats()),
indexes_to_block_vbytes: computed_h!(
"block_vbytes",
Source::Vec(height_to_vbytes.boxed_clone()),
full_stats()
),
difficultyepoch_to_timestamp: eager!("timestamp"),
halvingepoch_to_timestamp: eager!("timestamp"),
dateindex_to_fee_dominance: eager!("fee_dominance"),
dateindex_to_subsidy_dominance: eager!("subsidy_dominance"),
indexes_to_difficulty: computed_h!("difficulty", Source::None, last()),
indexes_to_difficulty: computed_h!(
"difficulty",
Source::Vec(indexer.vecs.block.height_to_difficulty.boxed_clone()),
last()
),
height_to_interval,
height_to_vbytes,
indexes_to_difficultyepoch: computed_di!("difficultyepoch", last()),
indexes_to_halvingepoch: computed_di!("halvingepoch", last()),
indexes_to_tx_count: computed_h!("tx_count", Source::Compute, full_stats()),
indexes_to_input_count: computed_tx!("input_count", Source::None, full_stats()),
indexes_to_output_count: computed_tx!("output_count", Source::None, full_stats()),
indexes_to_input_count: computed_tx!(
"input_count",
Source::Vec(indexes.txindex_to_input_count.boxed_clone()),
full_stats()
),
indexes_to_output_count: computed_tx!(
"output_count",
Source::Vec(indexes.txindex_to_output_count.boxed_clone()),
full_stats()
),
indexes_to_tx_v1: computed_h!("tx_v1", Source::Compute, sum_cum()),
indexes_to_tx_v2: computed_h!("tx_v2", Source::Compute, sum_cum()),
indexes_to_tx_v3: computed_h!("tx_v3", Source::Compute, sum_cum()),
@@ -273,9 +304,21 @@ impl Vecs {
.add_minmax()
.add_average(),
)?,
indexes_to_fee_rate: computed_tx!("fee_rate", Source::None, stats()),
indexes_to_tx_vsize: computed_tx!("tx_vsize", Source::None, stats()),
indexes_to_tx_weight: computed_tx!("tx_weight", Source::None, stats()),
indexes_to_fee_rate: computed_tx!(
"fee_rate",
Source::Vec(txindex_to_fee_rate.boxed_clone()),
stats()
),
indexes_to_tx_vsize: computed_tx!(
"tx_vsize",
Source::Vec(txindex_to_vsize.boxed_clone()),
stats()
),
indexes_to_tx_weight: computed_tx!(
"tx_weight",
Source::Vec(txindex_to_weight.boxed_clone()),
stats()
),
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
&db,
"subsidy",
crates/brk_computer/src/cointime.rs
+23 -11
View File
@@ -3,7 +3,7 @@ use std::path::Path;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, CheckedSub, Dollars, StoredF32, StoredF64, Version};
use vecdb::{Database, Exit, PAGE_SIZE, TypedVecIterator};
use vecdb::{Database, Exit, IterableCloneableVec, PAGE_SIZE, TypedVecIterator};
use crate::{grouped::ComputedVecsFromDateIndex, utils::OptionExt};
@@ -103,11 +103,11 @@ impl Vecs {
};
}
macro_rules! ratio_di {
($name:expr) => {
($name:expr, $source:expr) => {
ComputedRatioVecsFromDateIndex::forced_import(
&db,
$name,
Source::None,
Source::Vec($source.dateindex.unwrap_last().boxed_clone()),
v0,
indexes,
true,
@@ -128,6 +128,12 @@ impl Vecs {
};
}
// Extract price vecs before struct literal so they can be used as sources for ratios
let indexes_to_vaulted_price = computed_h!("vaulted_price", last());
let indexes_to_active_price = computed_h!("active_price", last());
let indexes_to_true_market_mean = computed_h!("true_market_mean", last());
let indexes_to_cointime_price = computed_h!("cointime_price", last());
let this = Self {
indexes_to_coinblocks_created: computed_h!("coinblocks_created", sum_cum()),
indexes_to_coinblocks_stored: computed_h!("coinblocks_stored", sum_cum()),
@@ -143,18 +149,24 @@ impl Vecs {
indexes_to_investor_cap: computed_h!("investor_cap", v1, last()),
indexes_to_vaulted_cap: computed_h!("vaulted_cap", v1, last()),
indexes_to_active_cap: computed_h!("active_cap", v1, last()),
indexes_to_vaulted_price: computed_h!("vaulted_price", last()),
indexes_to_vaulted_price_ratio: ratio_di!("vaulted_price"),
indexes_to_active_price: computed_h!("active_price", last()),
indexes_to_active_price_ratio: ratio_di!("active_price"),
indexes_to_true_market_mean: computed_h!("true_market_mean", last()),
indexes_to_true_market_mean_ratio: ratio_di!("true_market_mean"),
indexes_to_vaulted_price_ratio: ratio_di!("vaulted_price", &indexes_to_vaulted_price),
indexes_to_vaulted_price,
indexes_to_active_price_ratio: ratio_di!("active_price", &indexes_to_active_price),
indexes_to_active_price,
indexes_to_true_market_mean_ratio: ratio_di!(
"true_market_mean",
&indexes_to_true_market_mean
),
indexes_to_true_market_mean,
indexes_to_cointime_value_destroyed: computed_h!("cointime_value_destroyed", sum_cum()),
indexes_to_cointime_value_created: computed_h!("cointime_value_created", sum_cum()),
indexes_to_cointime_value_stored: computed_h!("cointime_value_stored", sum_cum()),
indexes_to_cointime_price: computed_h!("cointime_price", last()),
indexes_to_cointime_cap: computed_h!("cointime_cap", last()),
indexes_to_cointime_price_ratio: ratio_di!("cointime_price"),
indexes_to_cointime_price_ratio: ratio_di!(
"cointime_price",
&indexes_to_cointime_price
),
indexes_to_cointime_price,
indexes_to_cointime_adj_inflation_rate: computed_di!(
"cointime_adj_inflation_rate",
last()
crates/brk_computer/src/constants.rs
+2
View File
@@ -6,6 +6,8 @@ use super::{
indexes,
};
pub const DB_NAME: &str = "constants";
#[derive(Clone, Traversable)]
pub struct Vecs {
pub constant_0: ConstantVecs<StoredU16>,
crates/brk_computer/src/fetched.rs
+3 -1
View File
@@ -12,6 +12,8 @@ use vecdb::{
use super::{Indexes, indexes, utils::OptionExt};
pub const DB_NAME: &str = "fetched";
#[derive(Clone, Traversable)]
pub struct Vecs {
db: Database,
@@ -23,7 +25,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result<Self> {
let db = Database::open(&parent.join("fetched"))?;
let db = Database::open(&parent.join(DB_NAME))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let this = Self {
crates/brk_computer/src/grouped/computed_from_height.rs
+11 -23
View File
@@ -29,7 +29,7 @@ where
pub height_extra: EagerVecsBuilder<Height, T>,
pub dateindex: EagerVecsBuilder<DateIndex, T>,
pub weekindex: LazyVecsBuilder<WeekIndex, T, DateIndex, WeekIndex>,
pub difficultyepoch: EagerVecsBuilder<DifficultyEpoch, T>,
pub difficultyepoch: LazyVecsBuilder<DifficultyEpoch, T, Height, DifficultyEpoch>,
pub monthindex: LazyVecsBuilder<MonthIndex, T, DateIndex, MonthIndex>,
pub quarterindex: LazyVecsBuilder<QuarterIndex, T, DateIndex, QuarterIndex>,
pub semesterindex: LazyVecsBuilder<SemesterIndex, T, DateIndex, SemesterIndex>,
@@ -70,6 +70,8 @@ where
let options = options.remove_percentiles();
let height_source = source.vec().or(height.as_ref().map(|v| v.boxed_clone()));
Ok(Self {
weekindex: LazyVecsBuilder::forced_import(
name,
@@ -120,15 +122,17 @@ where
options.into(),
),
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
difficultyepoch: LazyVecsBuilder::forced_import(
name,
version + VERSION + Version::ZERO,
height_source,
&height_extra,
indexes.difficultyepoch_to_difficultyepoch.boxed_clone(),
options.into(),
),
height,
height_extra,
dateindex,
difficultyepoch: EagerVecsBuilder::forced_import(
db,
name,
version + VERSION + Version::ZERO,
options,
)?,
})
}
@@ -166,14 +170,6 @@ where
&indexes.dateindex_to_height_count,
exit,
)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
height,
&indexes.difficultyepoch_to_first_height,
&indexes.difficultyepoch_to_height_count,
exit,
)?;
} else {
let height = self.height.u();
@@ -187,14 +183,6 @@ where
&indexes.dateindex_to_height_count,
exit,
)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
height,
&indexes.difficultyepoch_to_first_height,
&indexes.difficultyepoch_to_height_count,
exit,
)?;
}
Ok(())
crates/brk_computer/src/grouped/computed_from_height_strict.rs
+14 -18
View File
@@ -3,11 +3,13 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DifficultyEpoch, Height, Version};
use schemars::JsonSchema;
use vecdb::{AnyExportableVec, Database, EagerVec, Exit, ImportableVec, PcoVec};
use vecdb::{
AnyExportableVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, PcoVec,
};
use crate::{Indexes, indexes};
use super::{ComputedVecValue, EagerVecsBuilder, VecBuilderOptions};
use super::{ComputedVecValue, EagerVecsBuilder, LazyVecsBuilder, VecBuilderOptions};
#[derive(Clone)]
pub struct ComputedVecsFromHeightStrict<T>
@@ -16,7 +18,7 @@ where
{
pub height: EagerVec<PcoVec<Height, T>>,
pub height_extra: EagerVecsBuilder<Height, T>,
pub difficultyepoch: EagerVecsBuilder<DifficultyEpoch, T>,
pub difficultyepoch: LazyVecsBuilder<DifficultyEpoch, T, Height, DifficultyEpoch>,
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
}
@@ -31,6 +33,7 @@ where
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> Result<Self> {
let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO)?;
@@ -45,21 +48,22 @@ where
let options = options.remove_percentiles();
Ok(Self {
height,
height_extra,
difficultyepoch: EagerVecsBuilder::forced_import(
db,
difficultyepoch: LazyVecsBuilder::forced_import(
name,
version + VERSION + Version::ZERO,
options,
)?,
Some(height.boxed_clone()),
&height_extra,
indexes.difficultyepoch_to_difficultyepoch.boxed_clone(),
options.into(),
),
height,
height_extra,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}
pub fn compute<F>(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
mut compute: F,
@@ -72,14 +76,6 @@ where
self.height_extra
.extend(starting_indexes.height, &self.height, exit)?;
self.difficultyepoch.compute(
starting_indexes.difficultyepoch,
&self.height,
&indexes.difficultyepoch_to_first_height,
&indexes.difficultyepoch_to_height_count,
exit,
)?;
Ok(())
}
}
crates/brk_computer/src/grouped/computed_from_txindex.rs
+9 -15
View File
@@ -29,7 +29,7 @@ where
pub height: EagerVecsBuilder<Height, T>,
pub dateindex: EagerVecsBuilder<DateIndex, T>,
pub weekindex: LazyVecsBuilder<WeekIndex, T, DateIndex, WeekIndex>,
pub difficultyepoch: EagerVecsBuilder<DifficultyEpoch, T>,
pub difficultyepoch: LazyVecsBuilder<DifficultyEpoch, T, Height, DifficultyEpoch>,
pub monthindex: LazyVecsBuilder<MonthIndex, T, DateIndex, MonthIndex>,
pub quarterindex: LazyVecsBuilder<QuarterIndex, T, DateIndex, QuarterIndex>,
pub semesterindex: LazyVecsBuilder<SemesterIndex, T, DateIndex, SemesterIndex>,
@@ -75,6 +75,14 @@ where
indexes.weekindex_to_weekindex.boxed_clone(),
options.into(),
),
difficultyepoch: LazyVecsBuilder::forced_import(
name,
version + VERSION + Version::ZERO,
None,
&height,
indexes.difficultyepoch_to_difficultyepoch.boxed_clone(),
options.into(),
),
monthindex: LazyVecsBuilder::forced_import(
name,
version + VERSION + Version::ZERO,
@@ -119,12 +127,6 @@ where
txindex,
height,
dateindex,
difficultyepoch: EagerVecsBuilder::forced_import(
db,
name,
version + VERSION + Version::ZERO,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}
@@ -205,14 +207,6 @@ where
exit,
)?;
self.difficultyepoch.from_aligned(
starting_indexes.difficultyepoch,
&self.height,
&indexes.difficultyepoch_to_first_height,
&indexes.difficultyepoch_to_height_count,
exit,
)?;
Ok(())
}
}
crates/brk_computer/src/grouped/lazy2_from_height.rs
+1 -1
View File
@@ -67,7 +67,7 @@ where
&source1.weekindex,
&source2.weekindex,
),
difficultyepoch: LazyTransform2Builder::from_eager::<F>(
difficultyepoch: LazyTransform2Builder::from_lazy::<F, _, _, _, _>(
name,
v,
&source1.difficultyepoch,
crates/brk_computer/src/grouped/lazy_from_height.rs
+1 -1
View File
@@ -49,7 +49,7 @@ where
height_extra: LazyTransformBuilder::from_eager::<F>(name, v, &source.height_extra),
dateindex: LazyTransformBuilder::from_eager::<F>(name, v, &source.dateindex),
weekindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.weekindex),
difficultyepoch: LazyTransformBuilder::from_eager::<F>(
difficultyepoch: LazyTransformBuilder::from_lazy::<F, _, _>(
name,
v,
&source.difficultyepoch,
crates/brk_computer/src/grouped/lazy_from_height_strict.rs
+1 -1
View File
@@ -34,7 +34,7 @@ where
let v = version + VERSION;
Self {
height: LazyVecFrom1::transformed::<F>(name, v, height_source),
difficultyepoch: LazyTransformBuilder::from_eager::<F>(name, v, &source.difficultyepoch),
difficultyepoch: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.difficultyepoch),
}
}
}
crates/brk_computer/src/grouped/source.rs
-23
View File
@@ -3,7 +3,6 @@ use vecdb::IterableBoxedVec;
#[derive(Clone)]
pub enum Source<I, T> {
Compute,
None,
Vec(IterableBoxedVec<I, T>),
}
@@ -12,10 +11,6 @@ impl<I, T> Source<I, T> {
matches!(self, Self::Compute)
}
pub fn is_none(&self) -> bool {
matches!(self, Self::None)
}
pub fn is_vec(&self) -> bool {
matches!(self, Self::Vec(_))
}
@@ -28,27 +23,9 @@ impl<I, T> Source<I, T> {
}
}
impl<I, T> From<bool> for Source<I, T> {
#[inline]
fn from(value: bool) -> Self {
if value { Self::Compute } else { Self::None }
}
}
impl<I, T> From<IterableBoxedVec<I, T>> for Source<I, T> {
#[inline]
fn from(value: IterableBoxedVec<I, T>) -> Self {
Self::Vec(value)
}
}
impl<I, T> From<Option<IterableBoxedVec<I, T>>> for Source<I, T> {
#[inline]
fn from(value: Option<IterableBoxedVec<I, T>>) -> Self {
if let Some(v) = value {
Self::Vec(v)
} else {
Self::None
}
}
}
crates/brk_computer/src/grouped/transforms.rs
+12 -1
View File
@@ -1,4 +1,4 @@
use brk_types::{Bitcoin, Close, Dollars, Sats, StoredF32, StoredF64};
use brk_types::{Bitcoin, Close, Dollars, Sats, StoredF32, StoredF64, StoredU32};
use vecdb::{BinaryTransform, UnaryTransform};
/// (Dollars, Dollars) -> Dollars addition
@@ -212,3 +212,14 @@ impl BinaryTransform<Sats, Sats, StoredF64> for PercentageSatsF64 {
StoredF64::from((*numerator as f64 / *denominator as f64) * 100.0)
}
}
/// (StoredU32, StoredU32) -> StoredF32 percentage (a/b × 100)
/// Used for pool dominance calculations (pool_blocks / total_blocks × 100)
pub struct PercentageU32F32;
impl BinaryTransform<StoredU32, StoredU32, StoredF32> for PercentageU32F32 {
#[inline(always)]
fn apply(numerator: StoredU32, denominator: StoredU32) -> StoredF32 {
StoredF32::from((*numerator as f64 / *denominator as f64) * 100.0)
}
}
crates/brk_computer/src/grouped/value_from_txindex.rs
+4 -6
View File
@@ -37,8 +37,6 @@ impl ComputedValueVecsFromTxindex {
price: Option<&price::Vecs>,
options: VecBuilderOptions,
) -> Result<Self> {
let compute_dollars = price.is_some();
let name_btc = format!("{name}_btc");
let name_usd = format!("{name}_usd");
@@ -63,7 +61,7 @@ impl ComputedValueVecsFromTxindex {
let bitcoin = ComputedVecsFromTxindex::forced_import(
db,
&name_btc,
Source::None,
Source::Vec(bitcoin_txindex.boxed_clone()),
version + VERSION,
indexes,
options,
@@ -96,18 +94,18 @@ impl ComputedValueVecsFromTxindex {
sats,
bitcoin_txindex,
bitcoin,
dollars_txindex,
dollars: compute_dollars.then(|| {
dollars: dollars_txindex.as_ref().map(|dtx| {
ComputedVecsFromTxindex::forced_import(
db,
&name_usd,
Source::None,
Source::Vec(dtx.boxed_clone()),
version + VERSION,
indexes,
options,
)
.unwrap()
}),
dollars_txindex,
})
}
crates/brk_computer/src/lib.rs
+70 -8
View File
@@ -124,15 +124,14 @@ impl Computer {
)
})?;
let pools_handle = big_thread().spawn_scoped(s, || {
pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())
})?;
let cointime =
cointime::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())?;
let chain = chain_handle.join().unwrap()?;
let pools = pools_handle.join().unwrap()?;
// pools depends on chain for lazy dominance vecs
let pools =
pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref(), &chain)?;
Ok((chain, pools, cointime))
})?;
@@ -225,9 +224,7 @@ impl Computer {
info!("Computing prices...");
let i = Instant::now();
self.price
.um()
.compute(&self.indexes, &starting_indexes, fetched, exit)?;
self.price.um().compute(&starting_indexes, fetched, exit)?;
info!("Computed prices in {:?}", i.elapsed());
}
@@ -330,6 +327,71 @@ impl Computer {
info!("Total compute time: {:?}", compute_start.elapsed());
Ok(())
}
/// Iterate over all exportable vecs with their database name.
pub fn iter_named_exportable(
&self,
) -> impl Iterator<Item = (&'static str, &dyn vecdb::AnyExportableVec)> {
use brk_traversable::Traversable;
std::iter::empty()
.chain(self.blks.iter_any_exportable().map(|v| (blks::DB_NAME, v)))
.chain(
self.chain
.iter_any_exportable()
.map(|v| (chain::DB_NAME, v)),
)
.chain(
self.cointime
.iter_any_exportable()
.map(|v| (cointime::DB_NAME, v)),
)
.chain(
self.constants
.iter_any_exportable()
.map(|v| (constants::DB_NAME, v)),
)
.chain(
self.fetched
.iter_any_exportable()
.map(|v| (fetched::DB_NAME, v)),
)
.chain(
self.indexes
.iter_any_exportable()
.map(|v| (indexes::DB_NAME, v)),
)
.chain(
self.market
.iter_any_exportable()
.map(|v| (market::DB_NAME, v)),
)
.chain(
self.pools
.iter_any_exportable()
.map(|v| (pools::DB_NAME, v)),
)
.chain(
self.price
.iter_any_exportable()
.map(|v| (price::DB_NAME, v)),
)
.chain(
self.stateful
.iter_any_exportable()
.map(|v| (stateful::DB_NAME, v)),
)
.chain(
self.txins
.iter_any_exportable()
.map(|v| (txins::DB_NAME, v)),
)
.chain(
self.txouts
.iter_any_exportable()
.map(|v| (txouts::DB_NAME, v)),
)
}
}
// pub fn generate_allocation_files(monitored: &pools::Vecs) -> Result<()> {
crates/brk_computer/src/pools/mod.rs
+2
View File
@@ -36,6 +36,7 @@ impl Vecs {
parent_version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
chain: &chain::Vecs,
) -> Result<Self> {
let db = Database::open(&parent_path.join(DB_NAME))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
@@ -54,6 +55,7 @@ impl Vecs {
version + Version::ZERO,
indexes,
price,
chain,
)
.map(|vecs| (pool.slug, vecs))
})
crates/brk_computer/src/pools/vecs.rs
+60 -78
View File
@@ -1,12 +1,13 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, PoolSlug, Sats, StoredF32, StoredU16, StoredU32};
use vecdb::{Database, Exit, GenericStoredVec, IterableVec, VecIndex, Version};
use vecdb::{Database, Exit, GenericStoredVec, IterableCloneableVec, IterableVec, VecIndex, Version};
use crate::{
chain,
grouped::{
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source,
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
LazyVecsFrom2FromDateIndex, LazyVecsFrom2FromHeight, PercentageU32F32, Source,
VecBuilderOptions,
},
indexes::{self, Indexes},
@@ -25,11 +26,11 @@ pub struct Vecs {
pub indexes_to_subsidy: ComputedValueVecsFromHeight,
pub indexes_to_fee: ComputedValueVecsFromHeight,
pub indexes_to_coinbase: ComputedValueVecsFromHeight,
pub indexes_to_dominance: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_1d_dominance: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_1w_dominance: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_1m_dominance: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_1y_dominance: ComputedVecsFromDateIndex<StoredF32>,
pub indexes_to_dominance: LazyVecsFrom2FromHeight<StoredF32, StoredU32, StoredU32>,
pub indexes_to_1d_dominance: LazyVecsFrom2FromHeight<StoredF32, StoredU32, StoredU32>,
pub indexes_to_1w_dominance: LazyVecsFrom2FromDateIndex<StoredF32, StoredU32, StoredU32>,
pub indexes_to_1m_dominance: LazyVecsFrom2FromDateIndex<StoredF32, StoredU32, StoredU32>,
pub indexes_to_1y_dominance: LazyVecsFrom2FromDateIndex<StoredF32, StoredU32, StoredU32>,
pub indexes_to_days_since_block: ComputedVecsFromDateIndex<StoredU16>,
}
@@ -40,6 +41,7 @@ impl Vecs {
parent_version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
chain: &chain::Vecs,
) -> Result<Self> {
let suffix = |s: &str| format!("{}_{s}", slug);
let compute_dollars = price.is_some();
@@ -61,19 +63,59 @@ impl Vecs {
};
}
let indexes_to_blocks_mined = ComputedVecsFromHeight::forced_import(
db,
&suffix("blocks_mined"),
Source::Compute,
version,
indexes,
sum_cum,
)?;
let indexes_to_1w_blocks_mined = import_di!("1w_blocks_mined");
let indexes_to_1m_blocks_mined = import_di!("1m_blocks_mined");
let indexes_to_1y_blocks_mined = import_di!("1y_blocks_mined");
Ok(Self {
slug,
indexes_to_blocks_mined: ComputedVecsFromHeight::forced_import(
db,
&suffix("blocks_mined"),
Source::Compute,
indexes_to_dominance: LazyVecsFrom2FromHeight::from_computed::<PercentageU32F32>(
&suffix("dominance"),
version,
indexes,
sum_cum,
)?,
indexes_to_1w_blocks_mined: import_di!("1w_blocks_mined"),
indexes_to_1m_blocks_mined: import_di!("1m_blocks_mined"),
indexes_to_1y_blocks_mined: import_di!("1y_blocks_mined"),
indexes_to_blocks_mined.height.as_ref().unwrap().boxed_clone(),
chain.indexes_to_block_count.height.as_ref().unwrap().boxed_clone(),
&indexes_to_blocks_mined,
&chain.indexes_to_block_count,
),
indexes_to_1d_dominance: LazyVecsFrom2FromHeight::from_computed::<PercentageU32F32>(
&suffix("1d_dominance"),
version,
indexes_to_blocks_mined.height.as_ref().unwrap().boxed_clone(),
chain.indexes_to_block_count.height.as_ref().unwrap().boxed_clone(),
&indexes_to_blocks_mined,
&chain.indexes_to_block_count,
),
indexes_to_1w_dominance: LazyVecsFrom2FromDateIndex::from_computed::<PercentageU32F32>(
&suffix("1w_dominance"),
version,
&indexes_to_1w_blocks_mined,
&chain.indexes_to_1w_block_count,
),
indexes_to_1m_dominance: LazyVecsFrom2FromDateIndex::from_computed::<PercentageU32F32>(
&suffix("1m_dominance"),
version,
&indexes_to_1m_blocks_mined,
&chain.indexes_to_1m_block_count,
),
indexes_to_1y_dominance: LazyVecsFrom2FromDateIndex::from_computed::<PercentageU32F32>(
&suffix("1y_dominance"),
version,
&indexes_to_1y_blocks_mined,
&chain.indexes_to_1y_block_count,
),
slug,
indexes_to_blocks_mined,
indexes_to_1w_blocks_mined,
indexes_to_1m_blocks_mined,
indexes_to_1y_blocks_mined,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
db,
&suffix("subsidy"),
@@ -101,11 +143,6 @@ impl Vecs {
compute_dollars,
indexes,
)?,
indexes_to_dominance: import_di!("dominance"),
indexes_to_1d_dominance: import_di!("1d_dominance"),
indexes_to_1w_dominance: import_di!("1w_dominance"),
indexes_to_1m_dominance: import_di!("1m_dominance"),
indexes_to_1y_dominance: import_di!("1y_dominance"),
indexes_to_days_since_block: import_di!("days_since_block"),
})
}
@@ -238,61 +275,6 @@ impl Vecs {
Ok(())
})?;
self.indexes_to_dominance
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_cumulative(),
chain.indexes_to_block_count.dateindex.unwrap_cumulative(),
exit,
)?;
Ok(())
})?;
self.indexes_to_1d_dominance
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_sum(),
chain.indexes_to_block_count.dateindex.unwrap_sum(),
exit,
)?;
Ok(())
})?;
self.indexes_to_1w_dominance
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_1w_blocks_mined.dateindex.u(),
chain.indexes_to_1w_block_count.dateindex.u(),
exit,
)?;
Ok(())
})?;
self.indexes_to_1m_dominance
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_1m_blocks_mined.dateindex.u(),
chain.indexes_to_1m_block_count.dateindex.u(),
exit,
)?;
Ok(())
})?;
self.indexes_to_1y_dominance
.compute_all(starting_indexes, exit, |vec| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_1y_blocks_mined.dateindex.u(),
chain.indexes_to_1y_block_count.dateindex.u(),
exit,
)?;
Ok(())
})?;
self.indexes_to_days_since_block
.compute_all(starting_indexes, exit, |v| {
let mut prev = None;
crates/brk_computer/src/price.rs
+11 -13
View File
@@ -115,12 +115,12 @@ impl Vecs {
}
macro_rules! computed_h {
($name:expr, $opts:expr) => {
ComputedVecsFromHeightStrict::forced_import(&db, $name, v, $opts)?
ComputedVecsFromHeightStrict::forced_import(&db, $name, v, indexes, $opts)?
};
}
macro_rules! computed_h_sats {
($name:expr, $opts:expr) => {
ComputedVecsFromHeightStrict::forced_import(&db, $name, v_sats, $opts)?
ComputedVecsFromHeightStrict::forced_import(&db, $name, v_sats, indexes, $opts)?
};
}
let first = || VecBuilderOptions::default().add_first();
@@ -189,12 +189,11 @@ impl Vecs {
pub fn compute(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
fetched: &fetched::Vecs,
exit: &Exit,
) -> Result<()> {
self.compute_(indexes, starting_indexes, fetched, exit)?;
self.compute_(starting_indexes, fetched, exit)?;
let _lock = exit.lock();
self.db.compact()?;
Ok(())
@@ -202,7 +201,6 @@ impl Vecs {
fn compute_(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
fetched: &fetched::Vecs,
exit: &Exit,
@@ -322,7 +320,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_close
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_price_ohlc,
@@ -333,7 +331,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_high
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_price_ohlc,
@@ -344,7 +342,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_low
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_price_ohlc,
@@ -355,7 +353,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_open
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.height_to_price_ohlc,
@@ -510,7 +508,7 @@ impl Vecs {
)?;
self.chainindexes_to_price_open_in_sats
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.chainindexes_to_price_open.height,
@@ -521,7 +519,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_high_in_sats
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.chainindexes_to_price_low.height,
@@ -532,7 +530,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_low_in_sats
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.chainindexes_to_price_high.height,
@@ -543,7 +541,7 @@ impl Vecs {
})?;
self.chainindexes_to_price_close_in_sats
.compute(indexes, starting_indexes, exit, |v| {
.compute(starting_indexes, exit, |v| {
v.compute_transform(
starting_indexes.height,
&self.chainindexes_to_price_close.height,
crates/brk_computer/src/stateful/address/address_count.rs
+18 -12
View File
@@ -5,8 +5,8 @@ use brk_types::{Height, StoredU64, Version};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec,
TypedVecIterator,
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
IterableCloneableVec, PcoVec, TypedVecIterator,
};
use crate::{
@@ -170,17 +170,23 @@ impl AddressTypeToIndexesToAddressCount {
name: &str,
version: Version,
indexes: &indexes::Vecs,
sources: &AddressTypeToHeightToAddressCount,
) -> Result<Self> {
Ok(Self::from(ByAddressType::new_with_name(|type_name| {
ComputedVecsFromHeight::forced_import(
db,
&format!("{type_name}_{name}"),
Source::None,
version,
indexes,
VecBuilderOptions::default().add_last(),
)
})?))
Ok(Self::from(ByAddressType::<
ComputedVecsFromHeight<StoredU64>,
>::try_zip_with_name(
sources,
|type_name, source| {
ComputedVecsFromHeight::forced_import(
db,
&format!("{type_name}_{name}"),
Source::Vec(source.boxed_clone()),
version,
indexes,
VecBuilderOptions::default().add_last(),
)
},
)?))
}
pub fn compute(
crates/brk_computer/src/stateful/cohorts/address.rs
+10 -9
View File
@@ -6,8 +6,8 @@ use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableVec,
PcoVec,
AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
IterableCloneableVec, IterableVec, PcoVec,
};
use crate::{
@@ -71,6 +71,12 @@ impl AddressCohortVecs {
price,
};
let height_to_addr_count = EagerVec::forced_import(
db,
&cfg.name("addr_count"),
version + VERSION + Version::ZERO,
)?;
Ok(Self {
starting_height: None,
@@ -79,20 +85,15 @@ impl AddressCohortVecs {
metrics: CohortMetrics::forced_import(&cfg, all_supply)?,
height_to_addr_count: EagerVec::forced_import(
db,
&cfg.name("addr_count"),
version + VERSION + Version::ZERO,
)?,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
db,
&cfg.name("addr_count"),
Source::None,
Source::Vec(height_to_addr_count.boxed_clone()),
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
height_to_addr_count,
})
}
crates/brk_computer/src/stateful/compute/block_loop.rs
+13 -14
View File
@@ -456,23 +456,22 @@ pub fn process_blocks(
}
// Final write - always save changes for rollback support
{
let _lock = exit.lock();
drop(vr);
let (empty_updates, loaded_updates) = cache.take();
let _lock = exit.lock();
drop(vr);
// Process address updates (mutations)
process_address_updates(
&mut vecs.addresses_data,
&mut vecs.any_address_indexes,
empty_updates,
loaded_updates,
)?;
let (empty_updates, loaded_updates) = cache.take();
// Write to disk (pure I/O) - save changes for rollback
write(vecs, last_height, chain_state, true)?;
}
// Process address updates (mutations)
process_address_updates(
&mut vecs.addresses_data,
&mut vecs.any_address_indexes,
empty_updates,
loaded_updates,
)?;
// Write to disk (pure I/O) - save changes for rollback
write(vecs, last_height, chain_state, true)?;
Ok(())
}
crates/brk_computer/src/stateful/compute/context.rs
+3 -11
View File
@@ -4,19 +4,13 @@ use vecdb::VecIndex;
use crate::{indexes, price};
/// Context shared across block processing.
pub struct ComputeContext<'a> {
pub struct ComputeContext {
/// Starting height for this computation run
pub starting_height: Height,
/// Last height to process
pub last_height: Height,
/// Whether price data is available
pub compute_dollars: bool,
/// Price data (optional)
pub price: Option<&'a price::Vecs>,
/// Pre-computed height -> timestamp mapping
pub height_to_timestamp: Vec<Timestamp>,
@@ -24,13 +18,13 @@ pub struct ComputeContext<'a> {
pub height_to_price: Option<Vec<Dollars>>,
}
impl<'a> ComputeContext<'a> {
impl ComputeContext {
/// Create a new computation context.
pub fn new(
starting_height: Height,
last_height: Height,
indexes: &indexes::Vecs,
price: Option<&'a price::Vecs>,
price: Option<&price::Vecs>,
) -> Self {
let height_to_timestamp: Vec<Timestamp> =
indexes.height_to_timestamp_fixed.into_iter().collect();
@@ -42,8 +36,6 @@ impl<'a> ComputeContext<'a> {
Self {
starting_height,
last_height,
compute_dollars: price.is_some(),
price,
height_to_timestamp,
height_to_price,
}
crates/brk_computer/src/stateful/metrics/price_paid.rs
+13 -13
View File
@@ -2,7 +2,9 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{DateIndex, Dollars, Height, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use vecdb::{
AnyStoredVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, PcoVec,
};
use crate::{
Indexes,
@@ -34,33 +36,31 @@ impl PricePaidMetrics {
let extended = cfg.extended();
let last = VecBuilderOptions::default().add_last();
let height_to_min_price_paid =
EagerVec::forced_import(cfg.db, &cfg.name("min_price_paid"), cfg.version + v0)?;
let height_to_max_price_paid =
EagerVec::forced_import(cfg.db, &cfg.name("max_price_paid"), cfg.version + v0)?;
Ok(Self {
height_to_min_price_paid: EagerVec::forced_import(
cfg.db,
&cfg.name("min_price_paid"),
cfg.version + v0,
)?,
indexes_to_min_price_paid: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("min_price_paid"),
Source::None,
Source::Vec(height_to_min_price_paid.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_max_price_paid: EagerVec::forced_import(
cfg.db,
&cfg.name("max_price_paid"),
cfg.version + v0,
)?,
indexes_to_max_price_paid: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("max_price_paid"),
Source::None,
Source::Vec(height_to_max_price_paid.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_min_price_paid,
height_to_max_price_paid,
price_percentiles: extended
.then(|| {
PricePercentiles::forced_import(
crates/brk_computer/src/stateful/metrics/realized.rs
+67 -45
View File
@@ -104,7 +104,7 @@ impl RealizedMetrics {
let indexes_to_realized_loss = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_loss"),
Source::None,
Source::Vec(height_to_realized_loss.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum_cum,
@@ -146,7 +146,7 @@ impl RealizedMetrics {
let indexes_to_realized_cap = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_cap"),
Source::None,
Source::Vec(height_to_realized_cap.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
@@ -158,7 +158,7 @@ impl RealizedMetrics {
let indexes_to_realized_profit = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_profit"),
Source::None,
Source::Vec(height_to_realized_profit.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum_cum,
@@ -198,32 +198,68 @@ impl RealizedMetrics {
LazyVecsFrom2FromHeight::from_computed::<PercentageDollarsF32>(
&cfg.name("net_realized_pnl_rel_to_realized_cap"),
cfg.version + v1,
indexes_to_net_realized_pnl.height.as_ref().unwrap().boxed_clone(),
indexes_to_net_realized_pnl
.height
.as_ref()
.unwrap()
.boxed_clone(),
height_to_realized_cap.boxed_clone(),
&indexes_to_net_realized_pnl,
&indexes_to_realized_cap,
);
let indexes_to_realized_price = ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_price"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?;
let height_to_value_created =
EagerVec::forced_import(cfg.db, &cfg.name("value_created"), cfg.version + v0)?;
let height_to_value_destroyed =
EagerVec::forced_import(cfg.db, &cfg.name("value_destroyed"), cfg.version + v0)?;
let height_to_adjusted_value_created = compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
cfg.version + v0,
)
})
.transpose()?;
let height_to_adjusted_value_destroyed = compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
cfg.version + v0,
)
})
.transpose()?;
Ok(Self {
// === Realized Cap ===
height_to_realized_cap,
indexes_to_realized_cap,
indexes_to_realized_price: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("realized_price"),
Source::Compute,
cfg.version + v0,
cfg.indexes,
last,
)?,
indexes_to_realized_price_extra: ComputedRatioVecsFromDateIndex::forced_import(
cfg.db,
&cfg.name("realized_price"),
Source::None,
Source::Vec(
indexes_to_realized_price
.dateindex
.unwrap_last()
.boxed_clone(),
),
cfg.version + v0,
cfg.indexes,
extended,
)?,
indexes_to_realized_price,
indexes_to_realized_cap_rel_to_own_market_cap: extended
.then(|| {
ComputedVecsFromHeight::forced_import(
@@ -272,76 +308,62 @@ impl RealizedMetrics {
.transpose()?,
// === Value Created/Destroyed ===
height_to_value_created: EagerVec::forced_import(
cfg.db,
&cfg.name("value_created"),
cfg.version + v0,
)?,
indexes_to_value_created: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("value_created"),
Source::None,
Source::Vec(height_to_value_created.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum,
)?,
height_to_value_destroyed: EagerVec::forced_import(
cfg.db,
&cfg.name("value_destroyed"),
cfg.version + v0,
)?,
indexes_to_value_destroyed: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("value_destroyed"),
Source::None,
Source::Vec(height_to_value_destroyed.boxed_clone()),
cfg.version + v0,
cfg.indexes,
sum,
)?,
height_to_value_created,
height_to_value_destroyed,
// === Adjusted Value (optional) ===
height_to_adjusted_value_created: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
cfg.version + v0,
)
})
.transpose()?,
indexes_to_adjusted_value_created: compute_adjusted
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("adjusted_value_created"),
Source::None,
Source::Vec(
height_to_adjusted_value_created
.as_ref()
.unwrap()
.boxed_clone(),
),
cfg.version + v0,
cfg.indexes,
sum,
)
})
.transpose()?,
height_to_adjusted_value_destroyed: compute_adjusted
.then(|| {
EagerVec::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
cfg.version + v0,
)
})
.transpose()?,
indexes_to_adjusted_value_destroyed: compute_adjusted
.then(|| {
ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("adjusted_value_destroyed"),
Source::None,
Source::Vec(
height_to_adjusted_value_destroyed
.as_ref()
.unwrap()
.boxed_clone(),
),
cfg.version + v0,
cfg.indexes,
sum,
)
})
.transpose()?,
height_to_adjusted_value_created,
height_to_adjusted_value_destroyed,
// === SOPR ===
dateindex_to_sopr: EagerVec::forced_import(
crates/brk_computer/src/stateful/metrics/supply.rs
+8 -12
View File
@@ -96,26 +96,22 @@ impl SupplyMetrics {
cfg.version + v0,
);
let height_to_utxo_count =
EagerVec::forced_import(cfg.db, &cfg.name("utxo_count"), cfg.version + v0)?;
Ok(Self {
height_to_supply,
height_to_supply_value,
indexes_to_supply,
height_to_utxo_count: EagerVec::forced_import(
cfg.db,
&cfg.name("utxo_count"),
cfg.version + v0,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
cfg.db,
&cfg.name("utxo_count"),
Source::None,
Source::Vec(height_to_utxo_count.boxed_clone()),
cfg.version + v0,
cfg.indexes,
last,
)?,
height_to_supply,
height_to_supply_value,
indexes_to_supply,
height_to_utxo_count,
height_to_supply_half_value,
indexes_to_supply_half,
})
crates/brk_computer/src/stateful/vecs.rs
+12 -5
View File
@@ -147,6 +147,12 @@ impl Vecs {
indexes,
)?;
// Extract address type height vecs before struct literal to use as sources
let addresstype_to_height_to_addr_count =
AddressTypeToHeightToAddressCount::forced_import(&db, "addr_count", v0)?;
let addresstype_to_height_to_empty_addr_count =
AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?;
let this = Self {
chain_state: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "chain", v0)
@@ -204,22 +210,23 @@ impl Vecs {
.unwrap()
}),
addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::forced_import(
addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::forced_import(
&db,
"addr_count",
v0,
indexes,
&addresstype_to_height_to_addr_count,
)?,
addresstype_to_height_to_empty_addr_count:
AddressTypeToHeightToAddressCount::forced_import(&db, "empty_addr_count", v0)?,
addresstype_to_indexes_to_addr_count:
AddressTypeToIndexesToAddressCount::forced_import(&db, "addr_count", v0, indexes)?,
addresstype_to_indexes_to_empty_addr_count:
AddressTypeToIndexesToAddressCount::forced_import(
&db,
"empty_addr_count",
v0,
indexes,
&addresstype_to_height_to_empty_addr_count,
)?,
addresstype_to_height_to_addr_count,
addresstype_to_height_to_empty_addr_count,
utxo_cohorts,
address_cohorts,
crates/brk_grouper/src/by_address_type.rs
+16
View File
@@ -78,6 +78,22 @@ impl<T> ByAddressType<T> {
})
}
pub fn try_zip_with_name<S, R, F>(other: &ByAddressType<S>, f: F) -> Result<ByAddressType<R>>
where
F: Fn(&'static str, &S) -> Result<R>,
{
Ok(ByAddressType {
p2pk65: f(P2PK65, &other.p2pk65)?,
p2pk33: f(P2PK33, &other.p2pk33)?,
p2pkh: f(P2PKH, &other.p2pkh)?,
p2sh: f(P2SH, &other.p2sh)?,
p2wpkh: f(P2WPKH, &other.p2wpkh)?,
p2wsh: f(P2WSH, &other.p2wsh)?,
p2tr: f(P2TR, &other.p2tr)?,
p2a: f(P2A, &other.p2a)?,
})
}
#[inline]
pub fn get_unwrap(&self, addresstype: OutputType) -> &T {
self.get(addresstype).unwrap()
crates/brk_query/src/impl/metrics.rs
+17 -24
View File
@@ -3,8 +3,8 @@ use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_traversable::TreeNode;
use brk_types::{
Format, Index, IndexInfo, Limit, Metric, MetricCount, MetricData, PaginatedMetrics, Pagination,
PaginationIndex,
DetailedMetricCount, Format, Index, IndexInfo, Limit, Metric, MetricData, PaginatedMetrics,
Pagination, PaginationIndex,
};
use vecdb::AnyExportableVec;
@@ -171,9 +171,18 @@ impl Query {
Ok(vecs)
}
/// Calculate total weight of the vecs for the given range
/// Calculate total weight of the vecs for the given range.
/// Applies index-specific cost multipliers for rate limiting.
pub fn weight(vecs: &[&dyn AnyExportableVec], from: Option<i64>, to: Option<i64>) -> usize {
vecs.iter().map(|v| v.range_weight(from, to)).sum()
vecs.iter()
.map(|v| {
let base = v.range_weight(from, to);
let multiplier = Index::try_from(v.index_type_to_string())
.map(|i| i.cost_multiplier())
.unwrap_or(1);
base * multiplier
})
.sum()
}
/// Search and format single metric
@@ -235,29 +244,13 @@ impl Query {
&self.vecs().index_to_metric_to_vec
}
pub fn metric_count(&self) -> MetricCount {
let total = self.total_metric_count();
let lazy = self.lazy_metric_count();
MetricCount {
distinct_metrics: self.distinct_metric_count(),
total_endpoints: total,
lazy_endpoints: lazy,
stored_endpoints: total - lazy,
pub fn metric_count(&self) -> DetailedMetricCount {
DetailedMetricCount {
total: self.vecs().counts.clone(),
by_db: self.vecs().counts_by_db.clone(),
}
}
pub fn distinct_metric_count(&self) -> usize {
self.vecs().distinct_metric_count
}
pub fn total_metric_count(&self) -> usize {
self.vecs().total_metric_count
}
pub fn lazy_metric_count(&self) -> usize {
self.vecs().lazy_metric_count
}
pub fn indexes(&self) -> &[IndexInfo] {
&self.vecs().indexes
}
crates/brk_query/src/vecs.rs
+19 -11
View File
@@ -3,7 +3,7 @@ use std::{borrow::Cow, collections::BTreeMap};
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_traversable::{Traversable, TreeNode};
use brk_types::{Index, IndexInfo, Limit, Metric, PaginatedMetrics, Pagination, PaginationIndex};
use brk_types::{Index, IndexInfo, Limit, Metric, MetricCount, PaginatedMetrics, Pagination, PaginationIndex};
use derive_deref::{Deref, DerefMut};
use quickmatch::{QuickMatch, QuickMatchConfig};
use vecdb::AnyExportableVec;
@@ -14,9 +14,8 @@ pub struct Vecs<'a> {
pub index_to_metric_to_vec: BTreeMap<Index, MetricToVec<'a>>,
pub metrics: Vec<&'a str>,
pub indexes: Vec<IndexInfo>,
pub distinct_metric_count: usize,
pub total_metric_count: usize,
pub lazy_metric_count: usize,
pub counts: MetricCount,
pub counts_by_db: BTreeMap<String, MetricCount>,
catalog: Option<TreeNode>,
matcher: Option<QuickMatch<'a>>,
metric_to_indexes: BTreeMap<&'a str, Vec<Index>>,
@@ -30,11 +29,11 @@ impl<'a> Vecs<'a> {
indexer
.vecs
.iter_any_exportable()
.for_each(|vec| this.insert(vec));
.for_each(|vec| this.insert(vec, "indexed"));
computer
.iter_any_exportable()
.for_each(|vec| this.insert(vec));
.iter_named_exportable()
.for_each(|(db, vec)| this.insert(vec, db));
let mut ids = this
.metric_to_index_to_vec
@@ -56,18 +55,20 @@ impl<'a> Vecs<'a> {
sort_ids(&mut ids);
this.metrics = ids;
this.distinct_metric_count = this.metric_to_index_to_vec.keys().count();
this.total_metric_count = this
this.counts.distinct_metrics = this.metric_to_index_to_vec.keys().count();
this.counts.total_endpoints = this
.index_to_metric_to_vec
.values()
.map(|tree| tree.len())
.sum::<usize>();
this.lazy_metric_count = this
this.counts.lazy_endpoints = this
.index_to_metric_to_vec
.values()
.flat_map(|tree| tree.values())
.filter(|vec| vec.region_names().is_empty())
.count();
this.counts.stored_endpoints =
this.counts.total_endpoints - this.counts.lazy_endpoints;
this.indexes = this
.index_to_metric_to_vec
.keys()
@@ -109,7 +110,7 @@ impl<'a> Vecs<'a> {
this
}
fn insert(&mut self, vec: &'a dyn AnyExportableVec) {
fn insert(&mut self, vec: &'a dyn AnyExportableVec, db: &str) {
let name = vec.name();
let serialized_index = vec.index_type_to_string();
let index = Index::try_from(serialized_index)
@@ -134,6 +135,13 @@ impl<'a> Vecs<'a> {
prev.is_none(),
"Duplicate metric: {name} for index {index:?}"
);
// Track per-db counts
let is_lazy = vec.region_names().is_empty();
self.counts_by_db
.entry(db.to_string())
.or_default()
.add_endpoint(is_lazy);
}
pub fn metrics(&'static self, pagination: Pagination) -> PaginatedMetrics {
crates/brk_store/src/lib.rs
-1
View File
@@ -81,7 +81,6 @@ where
fs::create_dir_all(path)?;
let (meta, keyspace) = StoreMeta::checked_open(
db,
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
|| {
crates/brk_store/src/meta.rs
+1 -2
View File
@@ -5,7 +5,7 @@ use std::{
use brk_error::{Error, Result};
use brk_types::Version;
use fjall::{Database, Keyspace};
use fjall::Keyspace;
use super::Height;
@@ -18,7 +18,6 @@ pub struct StoreMeta {
impl StoreMeta {
pub fn checked_open<F>(
_database: &Database,
path: &Path,
version: Version,
open_partition_handle: F,
crates/brk_types/src/height.rs
+13 -1
View File
@@ -7,7 +7,9 @@ use byteview::ByteView;
use derive_deref::Deref;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use vecdb::{Bytes, CheckedSub, Formattable, Pco, PrintableIndex, Stamp};
use vecdb::{Bytes, CheckedSub, Formattable, FromCoarserIndex, Pco, PrintableIndex, Stamp};
use super::DifficultyEpoch;
use crate::{BLOCKS_PER_DIFF_EPOCHS, BLOCKS_PER_HALVING};
@@ -289,3 +291,13 @@ impl Formattable for Height {
false
}
}
impl FromCoarserIndex<DifficultyEpoch> for Height {
fn min_from(coarser: DifficultyEpoch) -> usize {
usize::from(coarser) * BLOCKS_PER_DIFF_EPOCHS as usize
}
fn max_from_(coarser: DifficultyEpoch) -> usize {
(usize::from(coarser) + 1) * BLOCKS_PER_DIFF_EPOCHS as usize - 1
}
}
crates/brk_types/src/index.rs
+9
View File
@@ -130,6 +130,15 @@ impl Index {
pub fn serialize_long(&self) -> &'static str {
self.possible_values().last().unwrap()
}
/// Returns the query cost multiplier for this index type.
/// Used for rate limiting to account for expensive lazy computations.
pub const fn cost_multiplier(&self) -> usize {
match self {
Self::DifficultyEpoch => 60,
_ => 1,
}
}
}
impl TryFrom<&str> for Index {
crates/brk_types/src/metriccount.rs
+24 -1
View File
@@ -1,8 +1,10 @@
use std::collections::BTreeMap;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
/// Metric count statistics - distinct metrics and total metric-index combinations
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
pub struct MetricCount {
/// Number of unique metrics available (e.g., realized_price, market_cap)
#[schemars(example = 3141)]
@@ -17,3 +19,24 @@ pub struct MetricCount {
#[schemars(example = 16000)]
pub stored_endpoints: usize,
}
impl MetricCount {
pub fn add_endpoint(&mut self, is_lazy: bool) {
self.total_endpoints += 1;
if is_lazy {
self.lazy_endpoints += 1;
} else {
self.stored_endpoints += 1;
}
}
}
/// Detailed metric count with per-database breakdown
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct DetailedMetricCount {
/// Aggregate counts
#[serde(flatten)]
pub total: MetricCount,
/// Per-database breakdown of counts
pub by_db: BTreeMap<String, MetricCount>,
}