Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-28 11:18:17 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot + pools + fixes

Browse Source
This commit is contained in:
nym21
2025-09-07 17:01:34 +02:00
parent e155a3dacf
commit 4049d694f7
69 changed files with 2768 additions and 1444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/chain.rs
+80 -1
View File
@@ -1,5 +1,6 @@
use std::path::Path;
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
@@ -30,7 +31,7 @@ const TARGET_BLOCKS_PER_SEMESTER: u64 = 2 * TARGET_BLOCKS_PER_QUARTER;
const TARGET_BLOCKS_PER_YEAR: u64 = 2 * TARGET_BLOCKS_PER_SEMESTER;
const TARGET_BLOCKS_PER_DECADE: u64 = 10 * TARGET_BLOCKS_PER_YEAR;
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct Vecs {
db: Database,
@@ -51,6 +52,9 @@ pub struct Vecs {
pub halvingepoch_to_timestamp: EagerVec<HalvingEpoch, Timestamp>,
pub timeindexes_to_timestamp: ComputedVecsFromDateIndex<Timestamp>,
pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_1w_block_count: ComputedVecsFromDateIndex<StoredU32>,
pub indexes_to_1m_block_count: ComputedVecsFromDateIndex<StoredU32>,
pub indexes_to_1y_block_count: ComputedVecsFromDateIndex<StoredU32>,
pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
pub indexes_to_block_size: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
@@ -389,6 +393,30 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_1w_block_count: ComputedVecsFromDateIndex::forced_import(
&db,
"1w_block_count",
Source::Compute,
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1m_block_count: ComputedVecsFromDateIndex::forced_import(
&db,
"1m_block_count",
Source::Compute,
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1y_block_count: ComputedVecsFromDateIndex::forced_import(
&db,
"1y_block_count",
Source::Compute,
version + VERSION + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
&db,
"block_weight",
@@ -937,6 +965,54 @@ impl Vecs {
},
)?;
self.indexes_to_1w_block_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_block_count.dateindex.unwrap_sum(),
7,
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1m_block_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_block_count.dateindex.unwrap_sum(),
30,
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1y_block_count.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_block_count.dateindex.unwrap_sum(),
365,
exit,
)?;
Ok(())
},
)?;
let mut height_to_timestamp_iter = indexer.vecs.height_to_timestamp.iter();
self.height_to_interval.compute_transform(
starting_indexes.height,
@@ -1769,6 +1845,9 @@ impl Vecs {
self.indexes_to_hash_rate_1y_sma.vecs(),
self.timeindexes_to_timestamp.vecs(),
self.indexes_to_block_count.vecs(),
self.indexes_to_1w_block_count.vecs(),
self.indexes_to_1m_block_count.vecs(),
self.indexes_to_1y_block_count.vecs(),
self.indexes_to_block_interval.vecs(),
self.indexes_to_block_size.vecs(),
self.indexes_to_block_vbytes.vecs(),
crates/brk_computer/src/grouped/builder_eager.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_error::{Error, Result};
use brk_structs::{CheckedSub, StoredU64, Version};
use vecdb::{
@@ -9,7 +10,7 @@ use crate::utils::get_percentile;
use super::ComputedType;
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Allocative)]
pub struct EagerVecBuilder<I, T>
where
I: StoredIndex,
crates/brk_computer/src/grouped/builder_lazy.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_structs::Version;
use vecdb::{
AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, FromCoarserIndex,
@@ -9,7 +10,7 @@ use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
use super::ComputedType;
#[allow(clippy::type_complexity)]
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct LazyVecBuilder<I, T, S1I, S2T>
where
I: StoredIndex,
crates/brk_computer/src/grouped/from_dateindex.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -10,7 +11,7 @@ use crate::{Indexes, grouped::LazyVecBuilder, indexes};
use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions};
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct ComputedVecsFromDateIndex<T>
where
T: ComputedType + PartialOrd,
crates/brk_computer/src/grouped/from_height.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -15,7 +16,7 @@ use crate::{
use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
crates/brk_computer/src/grouped/from_txindex.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
@@ -17,7 +18,7 @@ use crate::{
use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct ComputedVecsFromTxindex<T>
where
T: ComputedType + PartialOrd,
crates/brk_computer/src/grouped/value_from_height.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
@@ -12,7 +13,7 @@ use crate::{
use super::{ComputedVecsFromHeight, VecBuilderOptions};
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct ComputedValueVecsFromHeight {
pub sats: ComputedVecsFromHeight<Sats>,
pub bitcoin: ComputedVecsFromHeight<Bitcoin>,
crates/brk_computer/src/grouped/value_from_txindex.rs
+2 -1
View File
@@ -1,3 +1,4 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
@@ -10,7 +11,7 @@ use crate::{Indexes, grouped::Source, indexes, price};
use super::{ComputedVecsFromTxindex, VecBuilderOptions};
#[derive(Clone)]
#[derive(Clone, Allocative)]
pub struct ComputedValueVecsFromTxindex {
pub sats: ComputedVecsFromTxindex<Sats>,
pub bitcoin_txindex: LazyVecFrom1<TxIndex, Bitcoin, TxIndex, Sats>,
crates/brk_computer/src/lib.rs
+65 -14
View File
@@ -1,6 +1,6 @@
#![doc = include_str!("../README.md")]
use std::{path::Path, thread};
use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
@@ -34,6 +34,7 @@ pub struct Computer {
pub indexes: indexes::Vecs,
pub constants: constants::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub price: Option<price::Vecs>,
pub chain: chain::Vecs,
pub stateful: stateful::Vecs,
@@ -41,7 +42,7 @@ pub struct Computer {
pub cointime: cointime::Vecs,
}
const VERSION: Version = Version::TWO;
const VERSION: Version = Version::new(4);
impl Computer {
/// Do NOT import multiple times or things will break !!!
@@ -86,6 +87,12 @@ impl Computer {
&indexes,
price.as_ref(),
)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
&indexes,
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
@@ -125,8 +132,8 @@ impl Computer {
)?;
}
thread::scope(|scope| -> Result<()> {
let chain = scope.spawn(|| {
std::thread::scope(|scope| -> Result<()> {
let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
self.chain.compute(
indexer,
@@ -134,23 +141,33 @@ impl Computer {
&starting_indexes,
self.price.as_ref(),
exit,
)
});
let market = scope.spawn(|| -> Result<()> {
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market
.compute(indexer, &self.indexes, price, &starting_indexes, exit)?;
}
)?;
Ok(())
});
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market
.compute(indexer, &self.indexes, price, &starting_indexes, exit)?;
}
// let _ = generate_allocation_files(&self.pools);
chain.join().unwrap()?;
market.join().unwrap()?;
Ok(())
})?;
self.pools.compute(
indexer,
&self.indexes,
&starting_indexes,
&self.chain,
self.price.as_ref(),
exit,
)?;
return Ok(());
info!("Computing stateful...");
self.stateful.compute(
indexer,
@@ -183,6 +200,7 @@ impl Computer {
self.chain.vecs(),
self.stateful.vecs(),
self.cointime.vecs(),
self.pools.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
self.price.as_ref().map_or(vec![], |v| v.vecs()),
]
@@ -195,3 +213,36 @@ impl Computer {
Box::leak(Box::new(self.clone()))
}
}
// pub fn generate_allocation_files(monitored: &pools::Vecs) -> Result<()> {
// info!("Generating Allocative files...");
// let mut flamegraph = allocative::FlameGraphBuilder::default();
// flamegraph.visit_root(monitored);
// let output = flamegraph.finish();
// let folder = format!(
// "at-{}",
// jiff::Timestamp::now().strftime("%Y-%m-%d_%Hh%Mm%Ss"),
// );
// let path = std::path::PathBuf::from(&format!("./target/flamegraph/{folder}"));
// std::fs::create_dir_all(&path)?;
// // fs::write(path.join("flamegraph.src"), &output.flamegraph())?;
// let mut fg_svg = Vec::new();
// inferno::flamegraph::from_reader(
// &mut inferno::flamegraph::Options::default(),
// output.flamegraph().write().as_bytes(),
// &mut fg_svg,
// )?;
// std::fs::write(path.join("flamegraph.svg"), &fg_svg)?;
// std::fs::write(path.join("warnings.txt"), output.warnings())?;
// info!("Successfully generated Allocative files");
// Ok(())
// }
crates/brk_computer/src/pools/id.rs
+187 -75
View File
@@ -1,101 +1,123 @@
use allocative::Allocative;
use num_enum::{FromPrimitive, IntoPrimitive};
use serde::{Deserialize, Serialize};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
// Created from the list in `pools.rs`
// Can be used as index for said list
#[allow(clippy::upper_case_acronyms)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, FromPrimitive, IntoPrimitive)]
#[repr(u16)]
#[derive(
Debug,
Copy,
Clone,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
FromPrimitive,
IntoPrimitive,
FromBytes,
IntoBytes,
Immutable,
KnownLayout,
Allocative,
)]
#[serde(rename_all = "lowercase")]
#[repr(u8)]
pub enum PoolId {
#[default]
Unknown,
BlockFills,
Ultimuspool,
UltimusPool,
TerraPool,
Luxor,
OneTHash,
BTCCom,
OneThash,
BtcCom,
Bitfarms,
HuobiPool,
WayiCn,
CanoePool,
BTCTop,
BtcTop,
BitcoinCom,
OneSevenFiveBtc,
GBMiners,
Pool175btc,
GbMiners,
AXbt,
ASICMiner,
AsicMiner,
BitMinter,
BitcoinRussia,
BTCServ,
BtcServ,
SimplecoinUs,
BTCGuild,
BtcGuild,
Eligius,
OzCoin,
EclipseMC,
MaxBTC,
EclipseMc,
MaxBtc,
TripleMining,
CoinLab,
FiftyBTC,
GHashIO,
STMiningCorp,
Pool50btc,
GhashIo,
StMiningCorp,
Bitparking,
MMPool,
Mmpool,
Polmine,
KnCMiner,
KncMiner,
Bitalo,
F2Pool,
HHTT,
Hhtt,
MegaBigPower,
MtRed,
NMCbit,
NmcBit,
YourbtcNet,
GiveMeCoins,
BraiinsPool,
AntPool,
MultiCoinCo,
BCPoolIo,
BcpoolIo,
Cointerra,
KanoPool,
SoloCK,
CKPool,
SoloCk,
CkPool,
NiceHash,
BitClub,
BitcoinAffiliateNetwork,
BTCC,
BWPool,
EXXAndBW,
Btcc,
BwPool,
ExxBw,
Bitsolo,
BitFury,
TwentyOneInc,
DigitalBTC,
DigitalBtc,
EightBaochi,
MyBTCcoinPool,
TBDice,
HASHPOOL,
MyBtcCoinPool,
TbDice,
HashPool,
Nexious,
BravoMining,
HotPool,
OKExPool,
BCMonster,
OkExPool,
BcMonster,
OneHash,
Bixin,
TATMASPool,
ViaBTC,
ConnectBTC,
BATPOOL,
TatmasPool,
ViaBtc,
ConnectBtc,
BatPool,
Waterhole,
DCExploration,
DCEX,
BTPOOL,
DcExploration,
Dcex,
BtPool,
FiftyEightCoin,
BitcoinIndiaLowercase,
BitcoinIndia,
ShawnP0wers,
PHashIO,
PHashIo,
RigPool,
HAOZHUZHU,
HaoZhuZhu,
SevenPool,
MiningKings,
HashBX,
DPOOL,
HashBx,
DPool,
Rawpool,
Haominer,
Helix,
@@ -114,62 +136,152 @@ pub enum PoolId {
BinancePool,
Minerium,
LubianCom,
OKKONG,
AAOPool,
EMCDPool,
FoundryUSA,
SBICrypto,
Okkong,
AaoPool,
EmcdPool,
FoundryUsa,
SbiCrypto,
ArkPool,
PureBTCCom,
MARAPool,
PureBtcCom,
MaraPool,
KuCoinPool,
EntrustCharityPool,
OKMINER,
OkMiner,
Titan,
PEGAPool,
BTCNuggets,
PegaPool,
BtcNuggets,
CloudHashing,
DigitalXMintsy,
Telco214,
BTCPoolParty,
BtcPoolParty,
Multipool,
TransactionCoinMining,
BTCDig,
TrickysBTCPool,
BTCMP,
BtcDig,
TrickysBtcPool,
BtcMp,
Eobot,
UNOMP,
Unomp,
Patels,
GoGreenLight,
BitcoinIndiaCamel, // duplicate-ish entry preserved with slight name change
EkanemBTC,
CanoeUppercase,
TigerLowercase,
OneM1X,
EkanemBtc,
Canoe,
Tiger,
OneM1x,
Zulupool,
SECPOOL,
OCEAN,
SecPool,
Ocean,
WhitePool,
Wiz,
Mononaut,
Rijndael,
Wk057,
FutureBitApolloSolo,
Emzy,
Knorrium,
CarbonNegative,
PortlandHODL,
PortlandHodl,
Phoenix,
Neopool,
MaxiPool,
DrDetroit,
BitFuFuPool,
LuckyPool,
MiningDutch,
PublicPool,
MiningSquared,
InnopolisTech,
Nymkappa,
BTCLab,
BtcLab,
Parasite,
Dummy158,
Dummy159,
Dummy160,
Dummy161,
Dummy162,
Dummy163,
Dummy164,
Dummy165,
Dummy166,
Dummy167,
Dummy168,
Dummy169,
Dummy170,
Dummy171,
Dummy172,
Dummy173,
Dummy174,
Dummy175,
Dummy176,
Dummy177,
Dummy178,
Dummy179,
Dummy180,
Dummy181,
Dummy182,
Dummy183,
Dummy184,
Dummy185,
Dummy186,
Dummy187,
Dummy188,
Dummy189,
Dummy190,
Dummy191,
Dummy192,
Dummy193,
Dummy194,
Dummy195,
Dummy196,
Dummy197,
Dummy198,
Dummy199,
Dummy200,
Dummy201,
Dummy202,
Dummy203,
Dummy204,
Dummy205,
Dummy206,
Dummy207,
Dummy208,
Dummy209,
Dummy210,
Dummy211,
Dummy212,
Dummy213,
Dummy214,
Dummy215,
Dummy216,
Dummy217,
Dummy218,
Dummy219,
Dummy220,
Dummy221,
Dummy222,
Dummy223,
Dummy224,
Dummy225,
Dummy226,
Dummy227,
Dummy228,
Dummy229,
Dummy230,
Dummy231,
Dummy232,
Dummy233,
Dummy234,
Dummy235,
Dummy236,
Dummy237,
Dummy238,
Dummy239,
Dummy240,
Dummy241,
Dummy242,
Dummy243,
Dummy244,
Dummy245,
Dummy246,
Dummy247,
Dummy248,
Dummy249,
Dummy250,
Dummy251,
Dummy252,
Dummy253,
Dummy254,
Dummy255,
}
crates/brk_computer/src/pools/mod.rs
+233
View File
@@ -1,7 +1,240 @@
use std::{collections::BTreeMap, path::Path};
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_store::AnyStore;
use brk_structs::{AddressBytes, Height, OutputIndex, OutputType};
use rayon::prelude::*;
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec,
PAGE_SIZE, RawVec, StoredIndex, VecIterator, Version,
};
mod id;
mod pool;
#[allow(clippy::module_inception)]
mod pools;
mod vecs;
pub use id::*;
pub use pool::*;
pub use pools::*;
use crate::{
chain,
indexes::{self, Indexes},
price,
};
#[derive(Clone, Allocative)]
pub struct Vecs {
db: Database,
pools: &'static Pools,
height_to_pool: RawVec<Height, PoolId>,
vecs: BTreeMap<PoolId, vecs::Vecs>,
}
impl Vecs {
pub fn forced_import(
parent_path: &Path,
parent_version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let db = Database::open(&parent_path.join("pools"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let pools = pools();
let version = parent_version + Version::new(3) + Version::new(pools.len() as u64);
let this = Self {
height_to_pool: RawVec::forced_import(&db, "pool", version + Version::ZERO)?,
vecs: pools
.iter()
.map(|pool| {
vecs::Vecs::forced_import(
&db,
pool.id,
pools,
version + Version::ZERO,
indexes,
price,
)
.map(|vecs| (pool.id, vecs))
})
.collect::<Result<BTreeMap<_, _>>>()?,
pools,
db,
};
this.db.retain_regions(
this.vecs()
.into_iter()
.flat_map(|v| v.region_names())
.collect(),
)?;
Ok(this)
}
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
chain: &chain::Vecs,
price: Option<&price::Vecs>,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, chain, price, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
chain: &chain::Vecs,
price: Option<&price::Vecs>,
exit: &Exit,
) -> Result<()> {
self.compute_height_to_pool(indexer, indexes, starting_indexes, exit)?;
self.vecs.par_iter_mut().try_for_each(|(_, vecs)| {
vecs.compute(
indexer,
indexes,
starting_indexes,
&self.height_to_pool,
chain,
price,
exit,
)
})?;
Ok(())
}
fn compute_height_to_pool(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.height_to_pool.validate_computed_version_or_reset(
self.height_to_pool.version() + indexer.stores.height_to_coinbase_tag.version(),
)?;
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
let mut txindex_to_first_outputindex_iter =
indexer.vecs.txindex_to_first_outputindex.iter();
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut outputindex_to_outputtype_iter = indexer.vecs.outputindex_to_outputtype.iter();
let mut outputindex_to_typeindex_iter = indexer.vecs.outputindex_to_typeindex.iter();
let mut p2pk65addressindex_to_p2pk65bytes_iter =
indexer.vecs.p2pk65addressindex_to_p2pk65bytes.iter();
let mut p2pk33addressindex_to_p2pk33bytes_iter =
indexer.vecs.p2pk33addressindex_to_p2pk33bytes.iter();
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.iter();
let mut p2shaddressindex_to_p2shbytes_iter =
indexer.vecs.p2shaddressindex_to_p2shbytes.iter();
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter();
let mut p2wshaddressindex_to_p2wshbytes_iter =
indexer.vecs.p2wshaddressindex_to_p2wshbytes.iter();
let mut p2traddressindex_to_p2trbytes_iter =
indexer.vecs.p2traddressindex_to_p2trbytes.iter();
let mut p2aaddressindex_to_p2abytes_iter = indexer.vecs.p2aaddressindex_to_p2abytes.iter();
let unknown = self.pools.get_unknown();
let min = starting_indexes
.height
.unwrap_to_usize()
.min(self.height_to_pool.len());
indexer
.stores
.height_to_coinbase_tag
.iter()
.skip(min)
.try_for_each(|(height, coinbase_tag)| -> Result<()> {
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
let outputindex = txindex_to_first_outputindex_iter.unwrap_get_inner(txindex);
let outputcount = txindex_to_output_count_iter.unwrap_get_inner(txindex);
let pool = (*outputindex..(*outputindex + *outputcount))
.map(OutputIndex::from)
.find_map(|outputindex| {
let outputtype =
outputindex_to_outputtype_iter.unwrap_get_inner(outputindex);
let typeindex = outputindex_to_typeindex_iter.unwrap_get_inner(outputindex);
let address = match outputtype {
OutputType::P2PK65 => Some(AddressBytes::from(
p2pk65addressindex_to_p2pk65bytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2PK33 => Some(AddressBytes::from(
p2pk33addressindex_to_p2pk33bytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2PKH => Some(AddressBytes::from(
p2pkhaddressindex_to_p2pkhbytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2SH => Some(AddressBytes::from(
p2shaddressindex_to_p2shbytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2WPKH => Some(AddressBytes::from(
p2wpkhaddressindex_to_p2wpkhbytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2WSH => Some(AddressBytes::from(
p2wshaddressindex_to_p2wshbytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2TR => Some(AddressBytes::from(
p2traddressindex_to_p2trbytes_iter
.unwrap_get_inner(typeindex.into()),
)),
OutputType::P2A => Some(AddressBytes::from(
p2aaddressindex_to_p2abytes_iter.unwrap_get_inner(typeindex.into()),
)),
_ => None,
};
address
.and_then(|address| self.pools.find_from_address(&address.to_string()))
})
.or_else(|| self.pools.find_from_coinbase_tag(&coinbase_tag))
.unwrap_or(unknown);
self.height_to_pool.push_if_needed(height, pool.id)?;
Ok(())
})?;
self.height_to_pool.safe_flush(exit)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.vecs
.iter()
.flat_map(|(_, vecs)| vecs.vecs())
.collect::<Vec<_>>(),
vec![&self.height_to_pool],
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
crates/brk_computer/src/pools/pool.rs
+36 -2
View File
@@ -1,10 +1,44 @@
use serde::{Deserialize, Serialize};
use allocative::Allocative;
use crate::pools::PoolId;
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Allocative)]
pub struct Pool {
pub id: PoolId,
pub name: &'static str,
pub addresses: Box<[&'static str]>,
pub tags: Box<[&'static str]>,
pub tags_lowercase: Box<[String]>,
pub link: &'static str,
}
impl Pool {
pub fn serialized_id(&self) -> String {
let value = serde_json::to_value(self.id).unwrap();
value.as_str().unwrap().to_string()
}
}
impl From<(usize, JSONPool)> for Pool {
fn from((index, pool): (usize, JSONPool)) -> Self {
Self {
id: (index as u8).into(),
name: pool.name,
addresses: pool.addresses,
tags_lowercase: pool
.tags
.iter()
.map(|t| t.to_lowercase())
.collect::<Vec<_>>()
.into_boxed_slice(),
tags: pool.tags,
link: pool.link,
}
}
}
#[derive(Debug)]
pub struct JSONPool {
pub name: &'static str,
pub addresses: Box<[&'static str]>,
pub tags: Box<[&'static str]>,
crates/brk_computer/src/pools/pools.rs
+203 -377
View File
File diff suppressed because it is too large Load Diff
crates/brk_computer/src/pools/vecs.rs
+467
View File
@@ -0,0 +1,467 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Height, Sats, StoredF32, StoredU16, StoredU32};
use vecdb::{AnyCollectableVec, AnyIterableVec, Database, Exit, StoredIndex, VecIterator, Version};
use crate::{
PoolId, Pools, chain,
grouped::{
ComputedValueVecsFromHeight, ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source,
VecBuilderOptions,
},
indexes::{self, Indexes},
price,
};
#[derive(Clone, Allocative)]
pub struct Vecs {
id: PoolId,
indexes_to_blocks_mined: ComputedVecsFromHeight<StoredU32>,
indexes_to_1w_blocks_mined: ComputedVecsFromDateIndex<StoredU32>,
indexes_to_1m_blocks_mined: ComputedVecsFromDateIndex<StoredU32>,
indexes_to_1y_blocks_mined: ComputedVecsFromDateIndex<StoredU32>,
indexes_to_subsidy: ComputedValueVecsFromHeight,
indexes_to_fee: ComputedValueVecsFromHeight,
indexes_to_coinbase: ComputedValueVecsFromHeight,
indexes_to_dominance: ComputedVecsFromDateIndex<StoredF32>,
indexes_to_1d_dominance: ComputedVecsFromDateIndex<StoredF32>,
indexes_to_1w_dominance: ComputedVecsFromDateIndex<StoredF32>,
indexes_to_1m_dominance: ComputedVecsFromDateIndex<StoredF32>,
indexes_to_1y_dominance: ComputedVecsFromDateIndex<StoredF32>,
indexes_to_days_since_block: ComputedVecsFromDateIndex<StoredU16>,
}
impl Vecs {
pub fn forced_import(
db: &Database,
id: PoolId,
pools: &Pools,
parent_version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let pool = pools.get(id);
let name = pool.serialized_id();
let suffix = |s: &str| format!("{name}_{s}");
let compute_dollars = price.is_some();
let version = parent_version + Version::ZERO;
Ok(Self {
id,
indexes_to_blocks_mined: ComputedVecsFromHeight::forced_import(
db,
&suffix("blocks_mined"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_1w_blocks_mined: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1w_blocks_mined"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1m_blocks_mined: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1m_blocks_mined"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1y_blocks_mined: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1y_blocks_mined"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
db,
&suffix("subsidy"),
Source::Compute,
version + Version::ZERO,
VecBuilderOptions::default().add_sum().add_cumulative(),
compute_dollars,
indexes,
)?,
indexes_to_fee: ComputedValueVecsFromHeight::forced_import(
db,
&suffix("fee"),
Source::Compute,
version + Version::ZERO,
VecBuilderOptions::default().add_sum().add_cumulative(),
compute_dollars,
indexes,
)?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
db,
&suffix("coinbase"),
Source::Compute,
version + Version::ZERO,
VecBuilderOptions::default().add_sum().add_cumulative(),
compute_dollars,
indexes,
)?,
indexes_to_dominance: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("dominance"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1d_dominance: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1d_dominance"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1w_dominance: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1w_dominance"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1m_dominance: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1m_dominance"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_1y_dominance: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("1y_dominance"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
indexes_to_days_since_block: ComputedVecsFromDateIndex::forced_import(
db,
&suffix("days_since_block"),
Source::Compute,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
})
}
#[allow(clippy::too_many_arguments)]
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
height_to_pool: &impl AnyIterableVec<Height, PoolId>,
chain: &chain::Vecs,
price: Option<&price::Vecs>,
exit: &Exit,
) -> Result<()> {
self.indexes_to_blocks_mined.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_transform(
starting_indexes.height,
height_to_pool,
|(h, id, ..)| {
(
h,
if id == self.id {
StoredU32::ONE
} else {
StoredU32::ZERO
},
)
},
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1w_blocks_mined.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_sum(),
7,
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1m_blocks_mined.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_sum(),
30,
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1y_blocks_mined.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
v.compute_sum(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_sum(),
365,
exit,
)?;
Ok(())
},
)?;
let height_to_blocks_mined = self.indexes_to_blocks_mined.height.as_ref().unwrap();
self.indexes_to_subsidy.compute_all(
indexer,
indexes,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_transform2(
starting_indexes.height,
height_to_blocks_mined,
chain.indexes_to_subsidy.sats.height.as_ref().unwrap(),
|(h, mined, sats, ..)| {
(
h,
if mined == StoredU32::ONE {
sats
} else {
Sats::ZERO
},
)
},
exit,
)?;
Ok(())
},
)?;
self.indexes_to_fee.compute_all(
indexer,
indexes,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_transform2(
starting_indexes.height,
height_to_blocks_mined,
chain.indexes_to_fee.sats.height.unwrap_sum(),
|(h, mined, sats, ..)| {
(
h,
if mined == StoredU32::ONE {
sats
} else {
Sats::ZERO
},
)
},
exit,
)?;
Ok(())
},
)?;
self.indexes_to_coinbase.compute_all(
indexer,
indexes,
price,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_transform2(
starting_indexes.height,
height_to_blocks_mined,
chain.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|(h, mined, sats, ..)| {
(
h,
if mined == StoredU32::ONE {
sats
} else {
Sats::ZERO
},
)
},
exit,
)?;
Ok(())
},
)?;
self.indexes_to_dominance.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_cumulative(),
chain.indexes_to_block_count.dateindex.unwrap_cumulative(),
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1d_dominance.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_sum(),
chain.indexes_to_block_count.dateindex.unwrap_sum(),
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1w_dominance.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_1w_blocks_mined.dateindex.as_ref().unwrap(),
chain.indexes_to_1w_block_count.dateindex.as_ref().unwrap(),
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1m_dominance.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_1m_blocks_mined.dateindex.as_ref().unwrap(),
chain.indexes_to_1m_block_count.dateindex.as_ref().unwrap(),
exit,
)?;
Ok(())
},
)?;
self.indexes_to_1y_dominance.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_percentage(
starting_indexes.dateindex,
self.indexes_to_1y_blocks_mined.dateindex.as_ref().unwrap(),
chain.indexes_to_1y_block_count.dateindex.as_ref().unwrap(),
exit,
)?;
Ok(())
},
)?;
self.indexes_to_days_since_block.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
let mut prev = None;
v.compute_transform2(
starting_indexes.dateindex,
self.indexes_to_blocks_mined.dateindex.unwrap_sum(),
self.indexes_to_blocks_mined.dateindex.unwrap_cumulative(),
|(i, sum, cumulative, slf)| {
if prev.is_none() {
let i = i.unwrap_to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
} else {
StoredU16::ZERO
});
}
let days = if !cumulative.is_zero() && sum.is_zero() {
prev.unwrap() + StoredU16::ONE
} else {
StoredU16::ZERO
};
prev.replace(days);
(i, days)
},
exit,
)?;
Ok(())
},
)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.indexes_to_blocks_mined.vecs(),
self.indexes_to_1w_blocks_mined.vecs(),
self.indexes_to_1m_blocks_mined.vecs(),
self.indexes_to_1y_blocks_mined.vecs(),
self.indexes_to_subsidy.vecs(),
self.indexes_to_fee.vecs(),
self.indexes_to_coinbase.vecs(),
self.indexes_to_dominance.vecs(),
self.indexes_to_1d_dominance.vecs(),
self.indexes_to_1w_dominance.vecs(),
self.indexes_to_1m_dominance.vecs(),
self.indexes_to_1y_dominance.vecs(),
self.indexes_to_days_since_block.vecs(),
]
.into_iter()
.flatten()
.collect()
}
}