Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: snapshot

Browse Source
This commit is contained in:
nym21
2025-12-28 10:25:55 +01:00
parent 5d6325ae30
commit e77d338357
16 changed files with 197 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/brk_computer/examples/full_bench.rs
View File

@@ -8,7 +8,7 @@ use std::{
use brk_bencher::Bencher;
use brk_computer::Computer;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_fetcher::{Fetcher, PriceSource};
use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_reader::Reader;
@@ -56,6 +56,8 @@ fn run() -> Result<()> {
let fetcher = Fetcher::import(true, None)?;
info!("Ping: {:?}", fetcher.brk.ping()?);
let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
let mut bencher = Bencher::from_cargo_env("brk", &outputs_dir)?;

6
crates/brk_computer/src/cointime.rs
View File

@@ -459,11 +459,7 @@ impl Vecs {
vec.compute_divide(
starting_indexes.height,
self.indexes_to_investor_cap.height.u(),
self.indexes_to_active_supply
.bitcoin
.height
.as_ref()
.unwrap(),
&self.indexes_to_active_supply.bitcoin.height,
exit,
)?;
Ok(())

15
crates/brk_computer/src/grouped/builder_transform.rs
View File

@@ -132,6 +132,21 @@ where
}
}
impl<I, T, S1T> LazyTransformBuilder<I, T, S1T>
where
I: VecIndex,
T: ComputedVecValue + JsonSchema,
S1T: ComputedVecValue,
{
pub fn unwrap_sum(&self) -> &LazyVecFrom1<I, T, I, S1T> {
self.sum.as_ref().unwrap()
}
pub fn unwrap_cumulative(&self) -> &LazyVecFrom1<I, T, I, S1T> {
self.cumulative.as_ref().unwrap()
}
}
impl<I, T, S1T> LazyTransformBuilder<I, T, S1T>
where
I: VecIndex,

9
crates/brk_computer/src/grouped/lazy_from_dateindex.rs
View File

@@ -17,6 +17,7 @@ where
S1T: ComputedVecValue,
{
pub dateindex: Option<LazyVecFrom1<DateIndex, T, DateIndex, S1T>>,
pub dateindex_extra: LazyTransformBuilder<DateIndex, T, S1T>,
pub weekindex: LazyTransformBuilder<WeekIndex, T, S1T>,
pub monthindex: LazyTransformBuilder<MonthIndex, T, S1T>,
pub quarterindex: LazyTransformBuilder<QuarterIndex, T, S1T>,
@@ -41,6 +42,7 @@ where
let v = version + VERSION;
Self {
dateindex: dateindex_source.map(|s| LazyVecFrom1::transformed::<F>(name, v, s)),
dateindex_extra: LazyTransformBuilder::from_eager::<F>(name, v, &source.dateindex_extra),
weekindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.weekindex),
monthindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.monthindex),
quarterindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.quarterindex),
@@ -57,11 +59,17 @@ where
S1T: ComputedVecValue,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
let dateindex_extra_node = self.dateindex_extra.to_tree_node();
brk_traversable::TreeNode::Branch(
[
self.dateindex
.as_ref()
.map(|v| ("dateindex".to_string(), v.to_tree_node())),
if dateindex_extra_node.is_empty() {
None
} else {
Some(("dateindex_extra".to_string(), dateindex_extra_node))
},
Some(("weekindex".to_string(), self.weekindex.to_tree_node())),
Some(("monthindex".to_string(), self.monthindex.to_tree_node())),
Some(("quarterindex".to_string(), self.quarterindex.to_tree_node())),
@@ -86,6 +94,7 @@ where
if let Some(ref dateindex) = self.dateindex {
regular_iter = Box::new(regular_iter.chain(dateindex.iter_any_exportable()));
}
regular_iter = Box::new(regular_iter.chain(self.dateindex_extra.iter_any_exportable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_exportable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_exportable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_exportable()));

21
crates/brk_computer/src/grouped/lazy_from_height.rs
View File

@@ -19,6 +19,7 @@ where
S1T: ComputedVecValue,
{
pub height: LazyVecFrom1<Height, T, Height, S1T>,
pub height_extra: LazyTransformBuilder<Height, T, S1T>,
pub dateindex: LazyTransformBuilder<DateIndex, T, S1T>,
pub weekindex: LazyTransformBuilder<WeekIndex, T, S1T>,
pub difficultyepoch: LazyTransformBuilder<DifficultyEpoch, T, S1T>,
@@ -45,12 +46,21 @@ where
let v = version + VERSION;
Self {
height: LazyVecFrom1::transformed::<F>(name, v, height_source),
height_extra: LazyTransformBuilder::from_eager::<F>(name, v, &source.height_extra),
dateindex: LazyTransformBuilder::from_eager::<F>(name, v, &source.dateindex),
weekindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.weekindex),
difficultyepoch: LazyTransformBuilder::from_eager::<F>(name, v, &source.difficultyepoch),
difficultyepoch: LazyTransformBuilder::from_eager::<F>(
name,
v,
&source.difficultyepoch,
),
monthindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.monthindex),
quarterindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.quarterindex),
semesterindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.semesterindex),
semesterindex: LazyTransformBuilder::from_lazy::<F, _, _>(
name,
v,
&source.semesterindex,
),
yearindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.yearindex),
decadeindex: LazyTransformBuilder::from_lazy::<F, _, _>(name, v, &source.decadeindex),
}
@@ -63,9 +73,15 @@ where
S1T: ComputedVecValue,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
let height_extra_node = self.height_extra.to_tree_node();
brk_traversable::TreeNode::Branch(
[
Some(("height".to_string(), self.height.to_tree_node())),
if height_extra_node.is_empty() {
None
} else {
Some(("height_extra".to_string(), height_extra_node))
},
Some(("dateindex".to_string(), self.dateindex.to_tree_node())),
Some(("weekindex".to_string(), self.weekindex.to_tree_node())),
Some((
@@ -92,6 +108,7 @@ where
fn iter_any_exportable(&self) -> impl Iterator<Item = &dyn AnyExportableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn AnyExportableVec>> =
Box::new(self.height.iter_any_exportable());
regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_exportable()));
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_exportable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_exportable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_exportable()));

60
crates/brk_computer/src/grouped/value_from_dateindex.rs
View File

@@ -1,13 +1,13 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Sats, Version};
use vecdb::{CollectableVec, Database, EagerVec, Exit, PcoVec};
use vecdb::{CollectableVec, Database, EagerVec, Exit, IterableCloneableVec, PcoVec};
use crate::{
Indexes,
grouped::ComputedVecsFromDateIndex,
grouped::{ComputedVecsFromDateIndex, LazyVecsFromDateIndex, SatsToBitcoin},
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
traits::ComputeFromBitcoin,
utils::OptionExt,
};
@@ -16,7 +16,7 @@ use super::{Source, VecBuilderOptions};
#[derive(Clone, Traversable)]
pub struct ComputedValueVecsFromDateIndex {
pub sats: ComputedVecsFromDateIndex<Sats>,
pub bitcoin: ComputedVecsFromDateIndex<Bitcoin>,
pub bitcoin: LazyVecsFromDateIndex<Bitcoin, Sats>,
pub dollars: Option<ComputedVecsFromDateIndex<Dollars>>,
}
@@ -33,23 +33,27 @@ impl ComputedValueVecsFromDateIndex {
compute_dollars: bool,
indexes: &indexes::Vecs,
) -> Result<Self> {
let sats = ComputedVecsFromDateIndex::forced_import(
db,
name,
source.clone(),
version + VERSION,
indexes,
options,
)?;
let sats_source = source.vec().or(sats.dateindex.as_ref().map(|v| v.boxed_clone()));
let bitcoin = LazyVecsFromDateIndex::from_computed::<SatsToBitcoin>(
&format!("{name}_btc"),
version + VERSION,
sats_source,
&sats,
);
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
db,
name,
source,
version + VERSION,
indexes,
options,
)?,
bitcoin: ComputedVecsFromDateIndex::forced_import(
db,
&format!("{name}_btc"),
Source::Compute,
version + VERSION,
indexes,
options,
)?,
sats,
bitcoin,
dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
db,
@@ -92,28 +96,14 @@ impl ComputedValueVecsFromDateIndex {
if let Some(dateindex) = dateindex {
self.sats
.compute_rest(starting_indexes, exit, Some(dateindex))?;
self.bitcoin.compute_all(starting_indexes, exit, |v| {
v.compute_from_sats(starting_indexes.dateindex, dateindex, exit)
})?;
} else {
let dateindex: Option<&PcoVec<DateIndex, Sats>> = None;
self.sats.compute_rest(starting_indexes, exit, dateindex)?;
self.bitcoin.compute_all(starting_indexes, exit, |v| {
v.compute_from_sats(
starting_indexes.dateindex,
self.sats.dateindex.u(),
exit,
)
})?;
}
let dateindex_to_bitcoin = self.bitcoin.dateindex.u();
let dateindex_to_price_close = price
.as_ref()
.unwrap()
.u()
.timeindexes_to_price_close
.dateindex
.as_ref()

63
crates/brk_computer/src/grouped/value_from_height.rs
View File

@@ -1,13 +1,13 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Dollars, Height, Sats, Version};
use vecdb::{CollectableVec, Database, EagerVec, Exit, PcoVec};
use vecdb::{CollectableVec, Database, EagerVec, Exit, IterableCloneableVec, PcoVec};
use crate::{
Indexes,
grouped::Source,
grouped::{LazyVecsFromHeight, SatsToBitcoin, Source},
indexes, price,
traits::{ComputeFromBitcoin, ComputeFromSats},
traits::ComputeFromBitcoin,
utils::OptionExt,
};
@@ -16,7 +16,7 @@ use super::{ComputedVecsFromHeight, VecBuilderOptions};
#[derive(Clone, Traversable)]
pub struct ComputedValueVecsFromHeight {
pub sats: ComputedVecsFromHeight<Sats>,
pub bitcoin: ComputedVecsFromHeight<Bitcoin>,
pub bitcoin: LazyVecsFromHeight<Bitcoin, Sats>,
pub dollars: Option<ComputedVecsFromHeight<Dollars>>,
}
@@ -33,23 +33,29 @@ impl ComputedValueVecsFromHeight {
compute_dollars: bool,
indexes: &indexes::Vecs,
) -> Result<Self> {
let sats = ComputedVecsFromHeight::forced_import(
db,
name,
source.clone(),
version + VERSION,
indexes,
options,
)?;
let sats_source = source
.vec()
.unwrap_or_else(|| sats.height.as_ref().unwrap().boxed_clone());
let bitcoin = LazyVecsFromHeight::from_computed::<SatsToBitcoin>(
&format!("{name}_btc"),
version + VERSION,
sats_source,
&sats,
);
Ok(Self {
sats: ComputedVecsFromHeight::forced_import(
db,
name,
source,
version + VERSION,
indexes,
options,
)?,
bitcoin: ComputedVecsFromHeight::forced_import(
db,
&format!("{name}_btc"),
Source::Compute,
version + VERSION,
indexes,
options,
)?,
sats,
bitcoin,
dollars: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
db,
@@ -94,28 +100,13 @@ impl ComputedValueVecsFromHeight {
if let Some(height) = height {
self.sats
.compute_rest(indexes, starting_indexes, exit, Some(height))?;
self.bitcoin
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_from_sats(starting_indexes.height, height, exit)
})?;
} else {
let height: Option<&PcoVec<Height, Sats>> = None;
self.sats
.compute_rest(indexes, starting_indexes, exit, height)?;
self.bitcoin
.compute_all(indexes, starting_indexes, exit, |v| {
v.compute_from_sats(
starting_indexes.height,
self.sats.height.u(),
exit,
)
})?;
}
let height_to_bitcoin = self.bitcoin.height.u();
let height_to_bitcoin = &self.bitcoin.height;
let height_to_price_close = &price.u().chainindexes_to_price_close.height;
if let Some(dollars) = self.dollars.as_mut() {

30
crates/brk_computer/src/stateful/metrics/activity.rs
View File

@@ -2,7 +2,10 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Height, Sats, StoredF64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, PcoVec};
use vecdb::{
AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec,
PcoVec,
};
use crate::{
Indexes,
@@ -41,18 +44,21 @@ impl ActivityMetrics {
let compute_dollars = cfg.compute_dollars();
let sum_cum = VecBuilderOptions::default().add_sum().add_cumulative();
Ok(Self {
height_to_sent: EagerVec::forced_import(cfg.db, &cfg.name("sent"), cfg.version + v0)?,
let height_to_sent: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(cfg.db, &cfg.name("sent"), cfg.version + v0)?;
let indexes_to_sent = ComputedValueVecsFromHeight::forced_import(
cfg.db,
&cfg.name("sent"),
Source::Vec(height_to_sent.boxed_clone()),
cfg.version + v0,
sum_cum,
compute_dollars,
cfg.indexes,
)?;
indexes_to_sent: ComputedValueVecsFromHeight::forced_import(
cfg.db,
&cfg.name("sent"),
Source::None,
cfg.version + v0,
sum_cum,
compute_dollars,
cfg.indexes,
)?,
Ok(Self {
height_to_sent,
indexes_to_sent,
height_to_satblocks_destroyed: EagerVec::forced_import(
cfg.db,

48
crates/brk_computer/src/stateful/vecs.rs
View File

@@ -113,32 +113,40 @@ impl Vecs {
|index, _| Some(index),
);
let height_to_unspendable_supply: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(&db, "unspendable_supply", v0)?;
let indexes_to_unspendable_supply = ComputedValueVecsFromHeight::forced_import(
&db,
"unspendable_supply",
Source::Vec(height_to_unspendable_supply.boxed_clone()),
v0,
VecBuilderOptions::default().add_last(),
compute_dollars,
indexes,
)?;
let height_to_opreturn_supply: EagerVec<PcoVec<Height, Sats>> =
EagerVec::forced_import(&db, "opreturn_supply", v0)?;
let indexes_to_opreturn_supply = ComputedValueVecsFromHeight::forced_import(
&db,
"opreturn_supply",
Source::Vec(height_to_opreturn_supply.boxed_clone()),
v0,
VecBuilderOptions::default().add_last(),
compute_dollars,
indexes,
)?;
let this = Self {
chain_state: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
&db,
"unspendable_supply",
Source::None,
v0,
VecBuilderOptions::default().add_last(),
compute_dollars,
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
&db,
"opreturn_supply",
Source::None,
v0,
VecBuilderOptions::default().add_last(),
compute_dollars,
indexes,
)?,
height_to_unspendable_supply,
indexes_to_unspendable_supply,
height_to_opreturn_supply,
indexes_to_opreturn_supply,
indexes_to_addr_count: ComputedVecsFromHeight::forced_import(
&db,

29
crates/brk_computer/src/traits.rs
View File

@@ -214,35 +214,6 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<PcoVec<DateIndex, Dollars>> {
}
}
pub trait ComputeFromSats<I> {
fn compute_from_sats(
&mut self,
max_from: I,
sats: &impl IterableVec<I, Sats>,
exit: &Exit,
) -> Result<()>;
}
impl<I> ComputeFromSats<I> for EagerVec<PcoVec<I, Bitcoin>>
where
I: VecIndex,
{
fn compute_from_sats(
&mut self,
max_from: I,
sats: &impl IterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.compute_transform(
max_from,
sats,
|(i, sats, _)| (i, Bitcoin::from(sats)),
exit,
)?;
Ok(())
}
}
pub trait ComputeFromBitcoin<I> {
fn compute_from_bitcoin(
&mut self,

9
crates/brk_fetcher/src/binance.rs
View File

@@ -203,6 +203,11 @@ impl Binance {
format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}")
}
pub fn ping() -> Result<()> {
minreq::get("https://api.binance.com/api/v3/ping")
.send()?;
Ok(())
}
}
impl PriceSource for Binance {
@@ -226,6 +231,10 @@ impl PriceSource for Binance {
None // Binance doesn't support height-based queries
}
fn ping(&self) -> Result<()> {
Self::ping()
}
fn clear(&mut self) {
self._1d.take();
self._1mn.take();

9
crates/brk_fetcher/src/brk.rs
View File

@@ -118,6 +118,11 @@ impl BRK {
)))
}
pub fn ping() -> Result<()> {
minreq::get(API_URL)
.send()?;
Ok(())
}
}
impl PriceSource for BRK {
@@ -141,6 +146,10 @@ impl PriceSource for BRK {
Some(self.get_from_height(height))
}
fn ping(&self) -> Result<()> {
Self::ping()
}
fn clear(&mut self) {
self.height_to_ohlc.clear();
self.dateindex_to_ohlc.clear();

10
crates/brk_fetcher/src/kraken.rs
View File

@@ -93,6 +93,12 @@ impl Kraken {
fn url(interval: usize) -> String {
format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}")
}
pub fn ping() -> Result<()> {
minreq::get("https://api.kraken.com/0/public/Time")
.send()?;
Ok(())
}
}
impl PriceSource for Kraken {
@@ -116,6 +122,10 @@ impl PriceSource for Kraken {
None // Kraken doesn't support height-based queries
}
fn ping(&self) -> Result<()> {
Self::ping()
}
fn clear(&mut self) {
self._1d.take();
self._1mn.take();

15
crates/brk_fetcher/src/lib.rs
View File

@@ -166,4 +166,19 @@ How to fix this:
self.brk.clear();
self.brk.reset_health();
}
/// Ping all sources and return results for each
pub fn ping(&self) -> Vec<(&'static str, Result<()>)> {
let mut results = Vec::new();
if let Some(binance) = &self.binance {
results.push((binance.name(), binance.ping()));
}
if let Some(kraken) = &self.kraken {
results.push((kraken.name(), kraken.ping()));
}
results.push((self.brk.name(), self.brk.ping()));
results
}
}

7
crates/brk_fetcher/src/source.rs
View File

@@ -24,6 +24,9 @@ pub trait PriceSource {
/// Fetch OHLC by block height. Returns None if unsupported.
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCents>>;
/// Check if the source is reachable
fn ping(&self) -> Result<()>;
/// Clear cached data
fn clear(&mut self);
}
@@ -128,6 +131,10 @@ impl<T: PriceSource> PriceSource for TrackedSource<T> {
self.try_fetch(|s| s.get_height(height))
}
fn ping(&self) -> Result<()> {
self.source.ping()
}
fn clear(&mut self) {
self.source.clear();
}

7
crates/brk_grouper/src/filter.rs
View File

@@ -109,15 +109,12 @@ impl Filter {
}
/// Whether to compute extended metrics (realized cap ratios, profit/loss ratios, percentiles)
/// For UTXO context: false for Type, Amount, Year, and Epoch filters
/// For UTXO context: false for Type and Amount filters
/// For Address context: always false
pub fn is_extended(&self, context: CohortContext) -> bool {
match context {
CohortContext::Address => false,
CohortContext::Utxo => !matches!(
self,
Filter::Type(_) | Filter::Amount(_) | Filter::Year(_) | Filter::Epoch(_)
),
CohortContext::Utxo => !matches!(self, Filter::Type(_) | Filter::Amount(_)),
}
}