global: MASSIVE snapshot

This commit is contained in:
nym21
2026-02-23 17:22:12 +01:00
parent be0d749f9c
commit 3b7aa8242a
703 changed files with 29130 additions and 30779 deletions

View File

@@ -7,7 +7,7 @@ use brk_types::{
AddressIndexTxIndex, AddressStats, AnyAddressDataIndexEnum, OutputType, Sats, TxIndex,
TxStatus, Txid, TypeIndex, Unit, Utxo, Vout,
};
use vecdb::TypedVecIterator;
use vecdb::{ReadableVec, VecIndex};
use crate::Query;
@@ -62,14 +62,14 @@ impl Query {
.distribution
.addresses_data
.funded
.iter()?
.get_unwrap(index),
.reader()
.get(usize::from(index)),
AnyAddressDataIndexEnum::Empty(index) => computer
.distribution
.addresses_data
.empty
.iter()?
.get_unwrap(index)
.reader()
.get(usize::from(index))
.into(),
};
@@ -138,10 +138,10 @@ impl Query {
.map(|(key, _)| key.txindex())
.collect();
let mut txindex_to_txid_iter = indexer.vecs.transactions.txid.iter()?;
let txid_reader = indexer.vecs.transactions.txid.reader();
let txids: Vec<Txid> = txindices
.into_iter()
.map(|txindex| txindex_to_txid_iter.get_unwrap(txindex))
.map(|txindex| txid_reader.get(txindex.to_usize()))
.collect();
Ok(txids)
@@ -166,23 +166,21 @@ impl Query {
.map(|(key, _): (AddressIndexOutPoint, Unit)| (key.txindex(), key.vout()))
.collect();
let mut txindex_to_txid_iter = vecs.transactions.txid.iter()?;
let mut txindex_to_height_iter = vecs.transactions.height.iter()?;
let mut txindex_to_first_txoutindex_iter = vecs.transactions.first_txoutindex.iter()?;
let mut txoutindex_to_value_iter = vecs.outputs.value.iter()?;
let mut height_to_blockhash_iter = vecs.blocks.blockhash.iter()?;
let mut height_to_timestamp_iter = vecs.blocks.timestamp.iter()?;
let txid_reader = vecs.transactions.txid.reader();
let first_txoutindex_reader = vecs.transactions.first_txoutindex.reader();
let value_reader = vecs.outputs.value.reader();
let blockhash_reader = vecs.blocks.blockhash.reader();
let utxos: Vec<Utxo> = outpoints
.into_iter()
.map(|(txindex, vout)| {
let txid: Txid = txindex_to_txid_iter.get_unwrap(txindex);
let height = txindex_to_height_iter.get_unwrap(txindex);
let first_txoutindex = txindex_to_first_txoutindex_iter.get_unwrap(txindex);
let txid: Txid = txid_reader.get(txindex.to_usize());
let height = vecs.transactions.height.collect_one_at(txindex.to_usize()).unwrap();
let first_txoutindex = first_txoutindex_reader.get(txindex.to_usize());
let txoutindex = first_txoutindex + vout;
let value: Sats = txoutindex_to_value_iter.get_unwrap(txoutindex);
let block_hash = height_to_blockhash_iter.get_unwrap(height);
let block_time = height_to_timestamp_iter.get_unwrap(height);
let value: Sats = value_reader.get(usize::from(txoutindex));
let block_hash = blockhash_reader.get(usize::from(height));
let block_time = vecs.blocks.timestamp.collect_one_at(usize::from(height)).unwrap();
Utxo {
txid,

View File

@@ -1,6 +1,6 @@
use brk_error::{Error, Result};
use brk_types::{BlockHash, BlockHashPrefix, BlockInfo, Height, TxIndex};
use vecdb::{AnyVec, GenericStoredVec, VecIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
use crate::Query;
@@ -21,10 +21,10 @@ impl Query {
}
let blockhash = indexer.vecs.blocks.blockhash.read_once(height)?;
let difficulty = indexer.vecs.blocks.difficulty.read_once(height)?;
let timestamp = indexer.vecs.blocks.timestamp.read_once(height)?;
let size = indexer.vecs.blocks.total_size.read_once(height)?;
let weight = indexer.vecs.blocks.weight.read_once(height)?;
let difficulty = indexer.vecs.blocks.difficulty.collect_one(height).unwrap();
let timestamp = indexer.vecs.blocks.timestamp.collect_one(height).unwrap();
let size = indexer.vecs.blocks.total_size.collect_one(height).unwrap();
let weight = indexer.vecs.blocks.weight.collect_one(height).unwrap();
let tx_count = self.tx_count_at_height(height, max_height)?;
Ok(BlockInfo {
@@ -45,12 +45,58 @@ impl Query {
let start = start.min(max_height);
let start_u32: u32 = start.into();
let count = DEFAULT_BLOCK_COUNT.min(start_u32 + 1);
let count = DEFAULT_BLOCK_COUNT.min(start_u32 + 1) as usize;
let mut blocks = Vec::with_capacity(count as usize);
for i in 0..count {
let height = Height::from(start_u32 - i);
blocks.push(self.block_by_height(height)?);
if count == 0 {
return Ok(Vec::new());
}
let indexer = self.indexer();
let computer = self.computer();
// Batch-read all PcoVec data for the contiguous range (avoids
// per-block page decompression — 4 reads instead of 4*count).
let end = start_u32 as usize + 1;
let begin = end - count;
let difficulties = indexer.vecs.blocks.difficulty.collect_range_at(begin, end);
let timestamps = indexer.vecs.blocks.timestamp.collect_range_at(begin, end);
let sizes = indexer.vecs.blocks.total_size.collect_range_at(begin, end);
let weights = indexer.vecs.blocks.weight.collect_range_at(begin, end);
// Batch-read first_txindex for tx_count computation (need one extra for next boundary)
let txindex_end = if end <= max_height.to_usize() {
end + 1
} else {
end
};
let first_txindexes: Vec<TxIndex> = indexer
.vecs
.transactions
.first_txindex
.collect_range_at(begin, txindex_end);
let total_txs = computer.indexes.txindex.identity.len();
let mut blocks = Vec::with_capacity(count);
for i in (0..count).rev() {
let height = Height::from(begin + i);
let blockhash = indexer.vecs.blocks.blockhash.read_once(height)?;
let tx_count = if i + 1 < first_txindexes.len() {
first_txindexes[i + 1].to_usize() - first_txindexes[i].to_usize()
} else {
total_txs - first_txindexes[i].to_usize()
};
blocks.push(BlockInfo {
id: blockhash,
height,
tx_count: tx_count as u32,
size: *sizes[i],
weight: weights[i],
timestamp: timestamps[i],
difficulty: *difficulties[i],
});
}
Ok(blocks)
@@ -72,27 +118,26 @@ impl Query {
}
fn max_height(&self) -> Height {
Height::from(
self.indexer()
.vecs
.blocks
.blockhash
.len()
.saturating_sub(1),
)
Height::from(self.indexer().vecs.blocks.blockhash.len().saturating_sub(1))
}
fn tx_count_at_height(&self, height: Height, max_height: Height) -> Result<u32> {
let indexer = self.indexer();
let computer = self.computer();
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
let first_txindex = indexer
.vecs
.transactions
.first_txindex
.collect_one(height)
.unwrap();
let next_first_txindex = if height < max_height {
indexer
.vecs
.transactions
.first_txindex
.read_once(height.incremented())?
.collect_one(height.incremented())
.unwrap()
} else {
TxIndex::from(computer.indexes.txindex.identity.len())
};

View File

@@ -1,6 +1,6 @@
use brk_error::{Error, Result};
use brk_types::{BlockHash, Height};
use vecdb::{AnyVec, GenericStoredVec};
use vecdb::{AnyVec, ReadableVec};
use crate::Query;
@@ -20,8 +20,8 @@ impl Query {
return Err(Error::OutOfRange("Block height out of range".into()));
}
let position = computer.positions.block_position.read_once(height)?;
let size = indexer.vecs.blocks.total_size.read_once(height)?;
let position = computer.positions.block_position.collect_one(height).unwrap();
let size = indexer.vecs.blocks.total_size.collect_one(height).unwrap();
reader.read_raw_bytes(position, *size as usize)
}

View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_types::{BlockHash, BlockStatus, Height};
use vecdb::{AnyVec, GenericStoredVec};
use vecdb::AnyVec;
use crate::Query;

View File

@@ -1,7 +1,7 @@
use brk_error::{Error, Result};
use brk_types::{BlockTimestamp, Date, DateIndex, Height, Timestamp};
use brk_types::{BlockTimestamp, Date, Day1, Height, Timestamp};
use jiff::Timestamp as JiffTimestamp;
use vecdb::{GenericStoredVec, TypedVecIterator};
use vecdb::ReadableVec;
use crate::Query;
@@ -19,28 +19,26 @@ impl Query {
let target = timestamp;
let date = Date::from(target);
let dateindex = DateIndex::try_from(date).unwrap_or_default();
let day1 = Day1::try_from(date).unwrap_or_default();
// Get first height of the target date
let first_height_of_day = computer
.indexes
.dateindex
.day1
.first_height
.read_once(dateindex)
.collect_one(day1)
.unwrap_or(Height::from(0usize));
let start: usize = usize::from(first_height_of_day).min(max_height_usize);
// Use iterator for efficient sequential access
let mut timestamp_iter = indexer.vecs.blocks.timestamp.iter()?;
let timestamps = &indexer.vecs.blocks.timestamp;
// Search forward from start to find the last block <= target timestamp
let mut best_height = start;
let mut best_ts = timestamp_iter.get_unwrap(Height::from(start));
let mut best_ts = timestamps.collect_one_at(start).unwrap();
for h in (start + 1)..=max_height_usize {
let height = Height::from(h);
let block_ts = timestamp_iter.get_unwrap(height);
let block_ts = timestamps.collect_one_at(h).unwrap();
if block_ts <= target {
best_height = h;
best_ts = block_ts;
@@ -51,8 +49,7 @@ impl Query {
// Check one block before start in case we need to go backward
if start > 0 && best_ts > target {
let prev_height = Height::from(start - 1);
let prev_ts = timestamp_iter.get_unwrap(prev_height);
let prev_ts = timestamps.collect_one_at(start - 1).unwrap();
if prev_ts <= target {
best_height = start - 1;
best_ts = prev_ts;
@@ -60,12 +57,7 @@ impl Query {
}
let height = Height::from(best_height);
let blockhash = indexer
.vecs
.blocks
.blockhash
.iter()?
.get_unwrap(height);
let blockhash = indexer.vecs.blocks.blockhash.reader().get(usize::from(height));
// Convert timestamp to ISO 8601 format
let ts_secs: i64 = (*best_ts).into();

View File

@@ -1,6 +1,6 @@
use brk_error::{Error, Result};
use brk_types::{BlockHash, Height, Transaction, TxIndex, Txid};
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator};
use vecdb::{AnyVec, ReadableVec};
use super::BLOCK_TXS_PAGE_SIZE;
use crate::Query;
@@ -31,26 +31,22 @@ impl Query {
return Err(Error::OutOfRange("Block height out of range".into()));
}
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
let first_txindex = indexer.vecs.transactions.first_txindex.collect_one(height).unwrap();
let next_first_txindex = indexer
.vecs
.transactions
.first_txindex
.read_once(height.incremented())
.unwrap_or_else(|_| TxIndex::from(indexer.vecs.transactions.txid.len()));
.collect_one(height.incremented())
.unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len()));
let first: usize = first_txindex.into();
let next: usize = next_first_txindex.into();
let count = next - first;
let txids: Vec<Txid> = indexer
.vecs
.transactions
.txid
.iter()?
.skip(first)
.take(count)
.collect();
.collect_range_at(first, next);
Ok(txids)
}
@@ -67,13 +63,13 @@ impl Query {
return Err(Error::OutOfRange("Block height out of range".into()));
}
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
let first_txindex = indexer.vecs.transactions.first_txindex.collect_one(height).unwrap();
let next_first_txindex = indexer
.vecs
.transactions
.first_txindex
.read_once(height.incremented())
.unwrap_or_else(|_| TxIndex::from(indexer.vecs.transactions.txid.len()));
.collect_one(height.incremented())
.unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len()));
let first: usize = first_txindex.into();
let next: usize = next_first_txindex.into();
@@ -104,13 +100,13 @@ impl Query {
return Err(Error::OutOfRange("Block height out of range".into()));
}
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
let first_txindex = indexer.vecs.transactions.first_txindex.collect_one(height).unwrap();
let next_first_txindex = indexer
.vecs
.transactions
.first_txindex
.read_once(height.incremented())
.unwrap_or_else(|_| TxIndex::from(indexer.vecs.transactions.txid.len()));
.collect_one(height.incremented())
.unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len()));
let first: usize = first_txindex.into();
let next: usize = next_first_txindex.into();
@@ -120,8 +116,8 @@ impl Query {
return Err(Error::OutOfRange("Transaction index out of range".into()));
}
let txindex = TxIndex::from(first + index);
let txid = indexer.vecs.transactions.txid.iter()?.get_unwrap(txindex);
let txindex = first + index;
let txid = indexer.vecs.transactions.txid.reader().get(txindex);
Ok(txid)
}

View File

@@ -2,9 +2,9 @@ use std::{fs, path::PathBuf};
use brk_error::{Error, Result};
use brk_types::{
CostBasisBucket, CostBasisDistribution, CostBasisFormatted, CostBasisValue, Date, DateIndex,
CostBasisBucket, CostBasisDistribution, CostBasisFormatted, CostBasisValue, Date, Day1,
};
use vecdb::IterableVec;
use vecdb::ReadableVec;
use crate::Query;
@@ -81,20 +81,14 @@ impl Query {
value: CostBasisValue,
) -> Result<CostBasisFormatted> {
let distribution = self.cost_basis_distribution(cohort, date)?;
let dateindex =
DateIndex::try_from(date).map_err(|e| Error::Parse(e.to_string()))?;
let price = self
.computer()
.price
.as_ref()
.ok_or_else(|| Error::NotFound("Price data not available".to_string()))?;
let spot = *price
let day1 = Day1::try_from(date).map_err(|e| Error::Parse(e.to_string()))?;
let price = &self.computer().prices;
let spot = price
.cents
.split
.dateindex
.close
.iter()
.get(dateindex)
.day1
.collect_one(day1)
.ok_or_else(|| Error::NotFound(format!("No price data for {date}")))?;
Ok(distribution.format(bucket, value, spot))
}

View File

@@ -3,13 +3,14 @@ use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_traversable::TreeNode;
use brk_types::{
DetailedMetricCount, Format, Index, IndexInfo, Limit, Metric, MetricData, MetricOutput,
MetricSelection, Output, PaginatedMetrics, Pagination, PaginationIndex,
DetailedMetricCount, Etag, Format, Index, IndexInfo, LegacyValue, Limit, Metric, MetricData,
MetricOutput, MetricOutputLegacy, MetricSelection, Output, OutputLegacy, PaginatedMetrics,
Pagination, PaginationIndex, Version,
};
use vecdb::AnyExportableVec;
use crate::{
Query, ResolvedQuery,
Query,
vecs::{IndexToVec, MetricToVec},
};
@@ -120,15 +121,11 @@ impl Query {
/// Resolve query metadata without formatting (cheap).
/// Use with `format` for lazy formatting after ETag check.
pub fn resolve(
&self,
params: MetricSelection,
max_weight: usize,
) -> Result<ResolvedQuery> {
pub fn resolve(&self, params: MetricSelection, max_weight: usize) -> Result<ResolvedQuery> {
let vecs = self.search(&params)?;
let total = vecs.iter().map(|v| v.len()).min().unwrap_or(0);
let version: u64 = vecs.iter().map(|v| u64::from(v.version())).sum();
let version: Version = vecs.iter().map(|v| v.version()).sum();
let start = params
.start()
@@ -182,12 +179,13 @@ impl Query {
let output = match format {
Format::CSV => Output::CSV(Self::columns_to_csv(&vecs, start, end)?),
Format::JSON => {
let count = end.saturating_sub(start);
if vecs.len() == 1 {
let mut buf = Vec::new();
let mut buf = Vec::with_capacity(count * 12 + 256);
MetricData::serialize(vecs[0], index, start, end, &mut buf)?;
Output::Json(buf)
} else {
let mut buf = Vec::new();
let mut buf = Vec::with_capacity(count * 12 * vecs.len() + 256);
buf.push(b'[');
for (i, vec) in vecs.iter().enumerate() {
if i > 0 {
@@ -244,4 +242,87 @@ impl Query {
pub fn metric_to_indexes(&self, metric: Metric) -> Option<&Vec<Index>> {
self.vecs().metric_to_indexes(metric)
}
/// Deprecated - format a resolved query as legacy output (expensive).
pub fn format_legacy(&self, resolved: ResolvedQuery) -> Result<MetricOutputLegacy> {
let ResolvedQuery {
vecs,
format,
version,
total,
start,
end,
..
} = resolved;
if vecs.is_empty() {
return Ok(MetricOutputLegacy {
output: OutputLegacy::default(format),
version: Version::ZERO,
total: 0,
start: 0,
end: 0,
});
}
let from = Some(start as i64);
let to = Some(end as i64);
let output = match format {
Format::CSV => OutputLegacy::CSV(Self::columns_to_csv(&vecs, start, end)?),
Format::JSON => {
if vecs.len() == 1 {
let metric = vecs[0];
let count = metric.range_count(from, to);
let mut buf = Vec::new();
if count == 1 {
metric.write_json_value(Some(start), &mut buf)?;
OutputLegacy::Json(LegacyValue::Value(buf))
} else {
metric.write_json(Some(start), Some(end), &mut buf)?;
OutputLegacy::Json(LegacyValue::List(buf))
}
} else {
let mut values = Vec::with_capacity(vecs.len());
for vec in &vecs {
let mut buf = Vec::new();
vec.write_json(Some(start), Some(end), &mut buf)?;
values.push(buf);
}
OutputLegacy::Json(LegacyValue::Matrix(values))
}
}
};
Ok(MetricOutputLegacy {
output,
version,
total,
start,
end,
})
}
}
/// A resolved metric query ready for formatting.
/// Contains the vecs and metadata needed to build an ETag or format the output.
pub struct ResolvedQuery {
pub vecs: Vec<&'static dyn AnyExportableVec>,
pub format: Format,
pub index: Index,
pub version: Version,
pub total: usize,
pub start: usize,
pub end: usize,
pub height: u32,
}
impl ResolvedQuery {
pub fn etag(&self) -> Etag {
Etag::from_metric(self.version, self.total, self.start, self.end, self.height)
}
pub fn format(&self) -> Format {
self.format
}
}

View File

@@ -1,66 +0,0 @@
use brk_error::Result;
use brk_types::{Format, LegacyValue, MetricOutputLegacy, OutputLegacy};
use crate::{Query, ResolvedQuery};
impl Query {
/// Deprecated - format a resolved query as legacy output (expensive).
pub fn format_legacy(&self, resolved: ResolvedQuery) -> Result<MetricOutputLegacy> {
let ResolvedQuery {
vecs,
format,
version,
total,
start,
end,
..
} = resolved;
if vecs.is_empty() {
return Ok(MetricOutputLegacy {
output: OutputLegacy::default(format),
version: 0,
total: 0,
start: 0,
end: 0,
});
}
let from = Some(start as i64);
let to = Some(end as i64);
let output = match format {
Format::CSV => OutputLegacy::CSV(Self::columns_to_csv(&vecs, start, end)?),
Format::JSON => {
if vecs.len() == 1 {
let metric = vecs[0];
let count = metric.range_count(from, to);
let mut buf = Vec::new();
if count == 1 {
metric.write_json_value(Some(start), &mut buf)?;
OutputLegacy::Json(LegacyValue::Value(buf))
} else {
metric.write_json(Some(start), Some(end), &mut buf)?;
OutputLegacy::Json(LegacyValue::List(buf))
}
} else {
let mut values = Vec::with_capacity(vecs.len());
for vec in &vecs {
let mut buf = Vec::new();
vec.write_json(Some(start), Some(end), &mut buf)?;
values.push(buf);
}
OutputLegacy::Json(LegacyValue::Matrix(values))
}
}
};
Ok(MetricOutputLegacy {
output,
version,
total,
start,
end,
})
}
}

View File

@@ -1,9 +1,9 @@
// TODO: INCOMPLETE - indexes_to_fee_rate.dateindex doesn't have percentile fields
// because from_txindex.rs calls remove_percentiles() before creating dateindex.
// TODO: INCOMPLETE - indexes_to_fee_rate.day1 doesn't have percentile fields
// because from_txindex.rs calls remove_percentiles() before creating day1.
// Need to either:
// 1. Use .height instead and convert height to dateindex for iteration
// 2. Fix from_txindex.rs to preserve percentiles for dateindex
// 3. Create a separate dateindex computation path with percentiles
// 1. Use .height instead and convert height to day1 for iteration
// 2. Fix from_txindex.rs to preserve percentiles for day1
// 3. Create a separate day1 computation path with percentiles
#![allow(dead_code)]
@@ -15,12 +15,11 @@ use brk_types::{
};
// use vecdb::{IterableVec, VecIndex};
// use super::dateindex_iter::DateIndexIter;
use crate::Query;
impl Query {
pub fn block_fee_rates(&self, _time_period: TimePeriod) -> Result<Vec<BlockFeeRatesEntry>> {
// Disabled until percentile data is available at dateindex level
// Disabled until percentile data is available at day1 level
Ok(Vec::new())
// Original implementation:
@@ -30,9 +29,9 @@ impl Query {
// .to_usize()
// .saturating_sub(time_period.block_count());
//
// let iter = DateIndexIter::new(computer, start, current_height.to_usize());
// let iter = Day1Iter::new(computer, start, current_height.to_usize());
//
// let vecs = &computer.transactions.transaction.indexes_to_fee_rate.dateindex;
// let vecs = &computer.transactions.transaction.indexes_to_fee_rate.day1;
// let mut min = vecs.unwrap_min().iter();
// let mut pct10 = vecs.unwrap_pct10().iter();
// let mut pct25 = vecs.unwrap_pct25().iter();

View File

@@ -1,8 +1,8 @@
use brk_error::Result;
use brk_types::{BlockFeesEntry, TimePeriod};
use vecdb::{IterableVec, VecIndex};
use vecdb::{ReadableVec, VecIndex};
use super::dateindex_iter::DateIndexIter;
use super::day1_iter::Day1Iter;
use crate::Query;
impl Query {
@@ -13,19 +13,18 @@ impl Query {
.to_usize()
.saturating_sub(time_period.block_count());
let iter = DateIndexIter::new(computer, start, current_height.to_usize());
let iter = Day1Iter::new(computer, start, current_height.to_usize());
let mut fees = computer
let fees_vec = &computer
.transactions
.fees
.fee
.sats
.dateindex
.average()
.iter();
.day1
.average;
Ok(iter.collect(|di, ts, h| {
fees.get(di).map(|fee| BlockFeesEntry {
fees_vec.collect_one(di).map(|fee| BlockFeesEntry {
avg_height: h,
timestamp: ts,
avg_fees: fee,

View File

@@ -1,8 +1,8 @@
use brk_error::Result;
use brk_types::{BlockRewardsEntry, TimePeriod};
use vecdb::{IterableVec, VecIndex};
use vecdb::{ReadableVec, VecIndex};
use super::dateindex_iter::DateIndexIter;
use super::day1_iter::Day1Iter;
use crate::Query;
impl Query {
@@ -13,20 +13,18 @@ impl Query {
.to_usize()
.saturating_sub(time_period.block_count());
let iter = DateIndexIter::new(computer, start, current_height.to_usize());
let iter = Day1Iter::new(computer, start, current_height.to_usize());
let mut rewards = computer
.blocks
let rewards_vec = &computer
.mining
.rewards
.coinbase
.sats
.dateindex
.distribution
.average()
.iter();
.day1
.average;
Ok(iter.collect(|di, ts, h| {
rewards.get(di).map(|reward| BlockRewardsEntry {
rewards_vec.collect_one(di).map(|reward| BlockRewardsEntry {
avg_height: h.into(),
timestamp: *ts,
avg_rewards: *reward,

View File

@@ -1,8 +1,8 @@
use brk_error::Result;
use brk_types::{BlockSizeEntry, BlockSizesWeights, BlockWeightEntry, TimePeriod};
use vecdb::{IterableVec, VecIndex};
use vecdb::{ReadableVec, VecIndex};
use super::dateindex_iter::DateIndexIter;
use super::day1_iter::Day1Iter;
use crate::Query;
impl Query {
@@ -13,34 +13,30 @@ impl Query {
.to_usize()
.saturating_sub(time_period.block_count());
let iter = DateIndexIter::new(computer, start, current_height.to_usize());
let iter = Day1Iter::new(computer, start, current_height.to_usize());
let mut sizes_vec = computer
let sizes_vec = &computer
.blocks
.size
.size
.dateindex
.distribution
.average()
.iter();
let mut weights_vec = computer
.day1
.average;
let weights_vec = &computer
.blocks
.weight
.weight
.dateindex
.distribution
.average()
.iter();
.day1
.average;
let entries: Vec<_> = iter.collect(|di, ts, h| {
let size = sizes_vec.get(di).map(|s| *s);
let weight = weights_vec.get(di).map(|w| *w);
let size = sizes_vec.collect_one(di).map(|s| *s);
let weight = weights_vec.collect_one(di).map(|w| *w);
Some((h.into(), (*ts), size, weight))
});
let sizes = entries
.iter()
.filter_map(|(h, ts, size, _)| {
.filter_map(|(h, ts, size, _): &(u32, _, _, _)| {
size.map(|s| BlockSizeEntry {
avg_height: *h,
timestamp: *ts,
@@ -51,7 +47,7 @@ impl Query {
let weights = entries
.iter()
.filter_map(|(h, ts, _, weight)| {
.filter_map(|(h, ts, _, weight): &(u32, _, _, _)| {
weight.map(|w| BlockWeightEntry {
avg_height: *h,
timestamp: *ts,

View File

@@ -1,28 +1,28 @@
use brk_computer::Computer;
use brk_types::{DateIndex, Height, Timestamp};
use vecdb::{GenericStoredVec, IterableVec, VecIndex};
use brk_types::{Day1, Height, Timestamp};
use vecdb::{ReadableVec, Ro, VecIndex};
/// Helper for iterating over dateindex ranges with sampling.
pub struct DateIndexIter<'a> {
computer: &'a Computer,
start_di: DateIndex,
end_di: DateIndex,
/// Helper for iterating over day1 ranges with sampling.
pub struct Day1Iter<'a> {
computer: &'a Computer<Ro>,
start_di: Day1,
end_di: Day1,
step: usize,
}
impl<'a> DateIndexIter<'a> {
pub fn new(computer: &'a Computer, start_height: usize, end_height: usize) -> Self {
impl<'a> Day1Iter<'a> {
pub fn new(computer: &'a Computer<Ro>, start_height: usize, end_height: usize) -> Self {
let start_di = computer
.indexes
.height
.dateindex
.read_once(Height::from(start_height))
.day1
.collect_one(Height::from(start_height))
.unwrap_or_default();
let end_di = computer
.indexes
.height
.dateindex
.read_once(Height::from(end_height))
.day1
.collect_one(Height::from(end_height))
.unwrap_or_default();
let total = end_di.to_usize().saturating_sub(start_di.to_usize()) + 1;
@@ -39,22 +39,22 @@ impl<'a> DateIndexIter<'a> {
/// Iterate and collect entries using the provided transform function.
pub fn collect<T, F>(&self, mut transform: F) -> Vec<T>
where
F: FnMut(DateIndex, Timestamp, Height) -> Option<T>,
F: FnMut(Day1, Timestamp, Height) -> Option<T>,
{
let total = self
.end_di
.to_usize()
.saturating_sub(self.start_di.to_usize())
+ 1;
let mut timestamps = self.computer.blocks.time.timestamp.dateindex.iter();
let mut heights = self.computer.indexes.dateindex.first_height.iter();
let timestamps = &self.computer.blocks.time.timestamp.day1;
let heights = &self.computer.indexes.day1.first_height;
let mut entries = Vec::with_capacity(total / self.step + 1);
let mut i = self.start_di.to_usize();
while i <= self.end_di.to_usize() {
let di = DateIndex::from(i);
if let (Some(ts), Some(h)) = (timestamps.get(di), heights.get(di))
let di = Day1::from(i);
if let (Some(ts), Some(h)) = (timestamps.collect_one(di), heights.collect_one(di))
&& let Some(entry) = transform(di, ts, h)
{
entries.push(entry);

View File

@@ -2,7 +2,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use brk_error::Result;
use brk_types::{DifficultyAdjustment, DifficultyEpoch, Height};
use vecdb::GenericStoredVec;
use vecdb::ReadableVec;
use crate::Query;
@@ -24,7 +24,8 @@ impl Query {
.indexes
.height
.difficultyepoch
.read_once(current_height)?;
.collect_one(current_height)
.unwrap();
let current_epoch_usize: usize = current_epoch.into();
// Get epoch start height
@@ -32,7 +33,8 @@ impl Query {
.indexes
.difficultyepoch
.first_height
.read_once(current_epoch)?;
.collect_one(current_epoch)
.unwrap();
let epoch_start_u32: u32 = epoch_start_height.into();
// Calculate epoch progress
@@ -47,12 +49,14 @@ impl Query {
.time
.timestamp
.difficultyepoch
.read_once(current_epoch)?;
.collect_one(current_epoch)
.unwrap();
let current_timestamp = indexer
.vecs
.blocks
.timestamp
.read_once(current_height)?;
.collect_one(current_height)
.unwrap();
// Calculate average block time in current epoch
let elapsed_time = (*current_timestamp - *epoch_start_timestamp) as u64;
@@ -88,18 +92,21 @@ impl Query {
.indexes
.difficultyepoch
.first_height
.read_once(prev_epoch)?;
.collect_one(prev_epoch)
.unwrap();
let prev_difficulty = indexer
.vecs
.blocks
.difficulty
.read_once(prev_epoch_start)?;
.collect_one(prev_epoch_start)
.unwrap();
let curr_difficulty = indexer
.vecs
.blocks
.difficulty
.read_once(epoch_start_height)?;
.collect_one(epoch_start_height)
.unwrap();
if *prev_difficulty > 0.0 {
((*curr_difficulty / *prev_difficulty) - 1.0) * 100.0

View File

@@ -1,10 +1,10 @@
use brk_computer::Computer;
use brk_types::{DifficultyAdjustmentEntry, DifficultyEpoch, Height};
use vecdb::{GenericStoredVec, IterableVec, VecIndex};
use vecdb::{ReadableVec, Ro, VecIndex};
/// Iterate over difficulty epochs within a height range.
pub fn iter_difficulty_epochs(
computer: &Computer,
computer: &Computer<Ro>,
start_height: usize,
end_height: usize,
) -> Vec<DifficultyAdjustmentEntry> {
@@ -12,38 +12,34 @@ pub fn iter_difficulty_epochs(
.indexes
.height
.difficultyepoch
.read_once(Height::from(start_height))
.collect_one(Height::from(start_height))
.unwrap_or_default();
let end_epoch = computer
.indexes
.height
.difficultyepoch
.read_once(Height::from(end_height))
.collect_one(Height::from(end_height))
.unwrap_or_default();
let mut epoch_to_height_iter = computer
.indexes
.difficultyepoch
.first_height
.iter();
let mut epoch_to_timestamp_iter = computer.blocks.time.timestamp.difficultyepoch.iter();
let mut epoch_to_difficulty_iter = computer.blocks.difficulty.raw.difficultyepoch.iter();
let epoch_to_height = &computer.indexes.difficultyepoch.first_height;
let epoch_to_timestamp = &computer.blocks.time.timestamp.difficultyepoch;
let epoch_to_difficulty = &computer.blocks.difficulty.raw.difficultyepoch;
let mut results = Vec::with_capacity(end_epoch.to_usize() - start_epoch.to_usize() + 1);
let mut prev_difficulty: Option<f64> = None;
for epoch_usize in start_epoch.to_usize()..=end_epoch.to_usize() {
let epoch = DifficultyEpoch::from(epoch_usize);
let epoch_height = epoch_to_height_iter.get(epoch).unwrap_or_default();
let epoch_height = epoch_to_height.collect_one(epoch).unwrap_or_default();
// Skip epochs before our start height but track difficulty
if epoch_height.to_usize() < start_height {
prev_difficulty = epoch_to_difficulty_iter.get(epoch).map(|d| *d);
prev_difficulty = epoch_to_difficulty.collect_one(epoch).map(|d| *d);
continue;
}
let epoch_timestamp = epoch_to_timestamp_iter.get(epoch).unwrap_or_default();
let epoch_difficulty = *epoch_to_difficulty_iter.get(epoch).unwrap_or_default();
let epoch_timestamp = epoch_to_timestamp.collect_one(epoch).unwrap_or_default();
let epoch_difficulty = *epoch_to_difficulty.collect_one(epoch).unwrap_or_default();
let change_percent = match prev_difficulty {
Some(prev) if prev > 0.0 => ((epoch_difficulty / prev) - 1.0) * 100.0,

View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_types::{DateIndex, DifficultyEntry, HashrateEntry, HashrateSummary, Height, TimePeriod};
use vecdb::{GenericStoredVec, IterableVec, VecIndex};
use brk_types::{Day1, DifficultyEntry, HashrateEntry, HashrateSummary, Height, TimePeriod};
use vecdb::{ReadableVec, VecIndex};
use super::epochs::iter_difficulty_epochs;
use crate::Query;
@@ -12,21 +12,23 @@ impl Query {
let current_height = self.height();
// Get current difficulty
let current_difficulty = *indexer.vecs.blocks.difficulty.read_once(current_height)?;
let current_difficulty = *indexer.vecs.blocks.difficulty.collect_one(current_height).unwrap();
// Get current hashrate
let current_dateindex = computer
let current_day1 = computer
.indexes
.height
.dateindex
.read_once(current_height)?;
.day1
.collect_one(current_height)
.unwrap();
let current_hashrate = *computer
.blocks
.mining
.hashrate
.hash_rate
.dateindex
.read_once(current_dateindex)? as u128;
.day1
.collect_one(current_day1)
.unwrap() as u128;
// Calculate start height based on time period
let end = current_height.to_usize();
@@ -36,31 +38,30 @@ impl Query {
};
// Get hashrate entries using iterators for efficiency
let start_dateindex = computer
let start_day1 = computer
.indexes
.height
.dateindex
.read_once(Height::from(start))?;
let end_dateindex = current_dateindex;
.day1
.collect_one(Height::from(start))
.unwrap();
let end_day1 = current_day1;
// Sample at regular intervals to avoid too many data points
let total_days = end_dateindex
let total_days = end_day1
.to_usize()
.saturating_sub(start_dateindex.to_usize())
.saturating_sub(start_day1.to_usize())
+ 1;
let step = (total_days / 200).max(1); // Max ~200 data points
// Create iterators for the loop
let mut hashrate_iter = computer.blocks.mining.hash_rate.dateindex.iter();
let mut timestamp_iter = computer.blocks.time.timestamp.dateindex.iter();
let hashrate_vec = &computer.mining.hashrate.hash_rate.day1;
let timestamp_vec = &computer.blocks.time.timestamp.day1;
let mut hashrates = Vec::with_capacity(total_days / step + 1);
let mut di = start_dateindex.to_usize();
while di <= end_dateindex.to_usize() {
let dateindex = DateIndex::from(di);
let mut di = start_day1.to_usize();
while di <= end_day1.to_usize() {
let day1 = Day1::from(di);
if let (Some(hr), Some(timestamp)) =
(hashrate_iter.get(dateindex), timestamp_iter.get(dateindex))
(hashrate_vec.collect_one(day1), timestamp_vec.collect_one(day1))
{
hashrates.push(HashrateEntry {
timestamp,

View File

@@ -2,7 +2,7 @@ mod block_fee_rates;
mod block_fees;
mod block_rewards;
mod block_sizes;
mod dateindex_iter;
mod day1_iter;
mod difficulty;
mod difficulty_adjustments;
mod epochs;

View File

@@ -3,7 +3,7 @@ use brk_types::{
Height, PoolBlockCounts, PoolBlockShares, PoolDetail, PoolDetailInfo, PoolInfo, PoolSlug,
PoolStats, PoolsSummary, TimePeriod, pools,
};
use vecdb::{AnyVec, IterableVec, VecIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
use crate::Query;
@@ -30,18 +30,16 @@ impl Query {
// For each pool, get cumulative count at end and start, subtract to get range count
for (pool_id, pool_vecs) in &computer.pools.vecs {
let mut cumulative = pool_vecs
let cumulative = &pool_vecs
.blocks_mined
.height_cumulative
.inner()
.iter();
.height_cumulative;
let count_at_end: u32 = *cumulative.get(current_height).unwrap_or_default();
let count_at_end: u32 = *cumulative.collect_one(current_height).unwrap_or_default();
let count_at_start: u32 = if start == 0 {
0
} else {
*cumulative.get(Height::from(start - 1)).unwrap_or_default()
*cumulative.collect_one(Height::from(start - 1)).unwrap_or_default()
};
let block_count = count_at_end.saturating_sub(count_at_start);
@@ -100,14 +98,12 @@ impl Query {
.get(&slug)
.ok_or_else(|| Error::NotFound("Pool data not found".into()))?;
let mut cumulative = pool_vecs
let cumulative = &pool_vecs
.blocks_mined
.height_cumulative
.inner()
.iter();
.height_cumulative;
// Get total blocks (all time)
let total_all: u32 = *cumulative.get(current_height).unwrap_or_default();
let total_all: u32 = *cumulative.collect_one(current_height).unwrap_or_default();
// Get blocks for 24h (144 blocks)
let start_24h = end.saturating_sub(144);
@@ -115,7 +111,7 @@ impl Query {
0
} else {
*cumulative
.get(Height::from(start_24h - 1))
.collect_one(Height::from(start_24h - 1))
.unwrap_or_default()
};
let total_24h = total_all.saturating_sub(count_before_24h);
@@ -126,7 +122,7 @@ impl Query {
0
} else {
*cumulative
.get(Height::from(start_1w - 1))
.collect_one(Height::from(start_1w - 1))
.unwrap_or_default()
};
let total_1w = total_all.saturating_sub(count_before_1w);

View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_types::{Height, RewardStats, Sats};
use vecdb::{IterableVec, VecIndex};
use vecdb::{ReadableVec, VecIndex};
use crate::Query;
@@ -12,39 +12,16 @@ impl Query {
let end_block = current_height;
let start_block = Height::from(current_height.to_usize().saturating_sub(block_count - 1));
let mut coinbase_iter = computer.blocks.rewards.coinbase.sats.height.iter();
let coinbase_vec = &computer.mining.rewards.coinbase.sats.height;
let fee_vec = &computer.transactions.fees.fee.sats.height.sum_cum.sum.0;
let tx_count_vec = &computer.transactions.count.tx_count.height;
let mut fee_iter = computer
.transactions
.fees
.fee
.sats
.height
.sum_cum
.sum
.0
.iter();
let mut tx_count_iter = computer.transactions.count.tx_count.height.iter();
let start = start_block.to_usize();
let end = end_block.to_usize() + 1;
let mut total_reward = Sats::ZERO;
let mut total_fee = Sats::ZERO;
let mut total_tx: u64 = 0;
for height in start_block.to_usize()..=end_block.to_usize() {
let h = Height::from(height);
if let Some(coinbase) = coinbase_iter.get(h) {
total_reward += coinbase;
}
if let Some(fee) = fee_iter.get(h) {
total_fee += fee;
}
if let Some(tx_count) = tx_count_iter.get(h) {
total_tx += *tx_count;
}
}
let total_reward = coinbase_vec.fold_range_at(start, end, Sats::ZERO, |acc, v| acc + v);
let total_fee = fee_vec.fold_range_at(start, end, Sats::ZERO, |acc, v| acc + v);
let total_tx = tx_count_vec.fold_range_at(start, end, 0u64, |acc, v| acc + *v);
Ok(RewardStats {
start_block,

View File

@@ -3,7 +3,6 @@ mod block;
mod cost_basis;
mod mempool;
mod metrics;
mod metrics_legacy;
mod mining;
mod price;
mod transaction;

View File

@@ -1,18 +1,11 @@
use brk_error::{Error, Result};
use brk_error::Result;
use brk_types::Dollars;
use crate::Query;
impl Query {
pub fn live_price(&self) -> Result<Dollars> {
let oracle_vecs = &self
.computer()
.price
.as_ref()
.ok_or_else(|| Error::OutOfRange("Oracle prices not computed yet".into()))?
.oracle;
let mut oracle = oracle_vecs.live_oracle(self.indexer())?;
let mut oracle = self.computer().prices.cents.live_oracle(self.indexer())?;
if let Some(mempool) = self.mempool() {
let txs = mempool.get_txs();

View File

@@ -6,7 +6,7 @@ use brk_types::{
Sats, Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxOutspend, TxStatus, Txid, TxidParam,
TxidPrefix, Vin, Vout, Weight,
};
use vecdb::{GenericStoredVec, TypedVecIterator};
use vecdb::{ReadableVec, VecIndex};
use crate::Query;
@@ -55,9 +55,9 @@ impl Query {
};
// Get block info for status
let height = indexer.vecs.transactions.height.read_once(txindex)?;
let height = indexer.vecs.transactions.height.collect_one(txindex).unwrap();
let block_hash = indexer.vecs.blocks.blockhash.read_once(height)?;
let block_time = indexer.vecs.blocks.timestamp.read_once(height)?;
let block_time = indexer.vecs.blocks.timestamp.collect_one(height).unwrap();
Ok(TxStatus {
confirmed: true,
@@ -120,11 +120,7 @@ impl Query {
// Look up spend status
let computer = self.computer();
let txinindex = computer
.outputs
.spent
.txinindex
.read_once(txoutindex)?;
let txinindex = computer.outputs.spent.txinindex.read_once(txoutindex)?;
if txinindex == TxInIndex::UNSPENT {
return Ok(TxOutspend::UNSPENT);
@@ -169,13 +165,12 @@ impl Query {
// Get spend status for each output
let computer = self.computer();
let mut txoutindex_to_txinindex_iter =
computer.outputs.spent.txinindex.iter()?;
let txinindex_reader = computer.outputs.spent.txinindex.reader();
let mut outspends = Vec::with_capacity(output_count);
for i in 0..output_count {
let txoutindex = first_txoutindex + Vout::from(i);
let txinindex = txoutindex_to_txinindex_iter.get_unwrap(txoutindex);
let txinindex = txinindex_reader.get(usize::from(txoutindex));
if txinindex == TxInIndex::UNSPENT {
outspends.push(TxOutspend::UNSPENT);
@@ -194,22 +189,23 @@ impl Query {
let reader = self.reader();
let computer = self.computer();
// Get tx metadata using read_once for single lookups
// Get tx metadata using collect_one for PcoVec, read_once for BytesVec
let txid = indexer.vecs.transactions.txid.read_once(txindex)?;
let height = indexer.vecs.transactions.height.read_once(txindex)?;
let version = indexer.vecs.transactions.txversion.read_once(txindex)?;
let lock_time = indexer.vecs.transactions.rawlocktime.read_once(txindex)?;
let total_size = indexer.vecs.transactions.total_size.read_once(txindex)?;
let height = indexer.vecs.transactions.height.collect_one(txindex).unwrap();
let version = indexer.vecs.transactions.txversion.collect_one(txindex).unwrap();
let lock_time = indexer.vecs.transactions.rawlocktime.collect_one(txindex).unwrap();
let total_size = indexer.vecs.transactions.total_size.collect_one(txindex).unwrap();
let first_txinindex = indexer
.vecs
.transactions
.first_txinindex
.read_once(txindex)?;
let position = computer.positions.tx_position.read_once(txindex)?;
.collect_one(txindex)
.unwrap();
let position = computer.positions.tx_position.collect_one(txindex).unwrap();
// Get block info for status
let block_hash = indexer.vecs.blocks.blockhash.read_once(height)?;
let block_time = indexer.vecs.blocks.timestamp.read_once(height)?;
let block_time = indexer.vecs.blocks.timestamp.collect_one(height).unwrap();
// Read and decode the raw transaction from blk file
let buffer = reader.read_raw_bytes(position, *total_size as usize)?;
@@ -217,12 +213,16 @@ impl Query {
let tx = bitcoin::Transaction::consensus_decode(&mut cursor)
.map_err(|_| Error::Parse("Failed to decode transaction".into()))?;
// For iterating through inputs, we need iterators (multiple lookups)
let mut txindex_to_txid_iter = indexer.vecs.transactions.txid.iter()?;
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.transactions.first_txoutindex.iter()?;
let mut txinindex_to_outpoint_iter = indexer.vecs.inputs.outpoint.iter()?;
let mut txoutindex_to_value_iter = indexer.vecs.outputs.value.iter()?;
// Create readers for random access lookups
let txid_reader = indexer.vecs.transactions.txid.reader();
let first_txoutindex_reader = indexer.vecs.transactions.first_txoutindex.reader();
let value_reader = indexer.vecs.outputs.value.reader();
// Batch-read outpoints for all inputs (avoids per-input PcoVec page decompression)
let outpoints: Vec<_> = indexer.vecs.inputs.outpoint.collect_range_at(
usize::from(first_txinindex),
usize::from(first_txinindex) + tx.input.len(),
);
// Build inputs with prevout information
let input: Vec<TxIn> = tx
@@ -230,8 +230,7 @@ impl Query {
.iter()
.enumerate()
.map(|(i, txin)| {
let txinindex = first_txinindex + i;
let outpoint = txinindex_to_outpoint_iter.get_unwrap(txinindex);
let outpoint = outpoints[i];
let is_coinbase = outpoint.is_coinbase();
@@ -241,15 +240,14 @@ impl Query {
} else {
let prev_txindex = outpoint.txindex();
let prev_vout = outpoint.vout();
let prev_txid = txindex_to_txid_iter.get_unwrap(prev_txindex);
let prev_txid = txid_reader.get(prev_txindex.to_usize());
// Calculate the txoutindex for the prevout
let prev_first_txoutindex =
txindex_to_first_txoutindex_iter.get_unwrap(prev_txindex);
let prev_first_txoutindex = first_txoutindex_reader.get(prev_txindex.to_usize());
let prev_txoutindex = prev_first_txoutindex + prev_vout;
// Get the value of the prevout
let prev_value = txoutindex_to_value_iter.get_unwrap(prev_txoutindex);
let prev_value = value_reader.get(usize::from(prev_txoutindex));
let prevout = Some(TxOut::from((
bitcoin::ScriptBuf::new(), // Placeholder - would need to reconstruct
@@ -314,8 +312,8 @@ impl Query {
let reader = self.reader();
let computer = self.computer();
let total_size = indexer.vecs.transactions.total_size.read_once(txindex)?;
let position = computer.positions.tx_position.read_once(txindex)?;
let total_size = indexer.vecs.transactions.total_size.collect_one(txindex).unwrap();
let position = computer.positions.tx_position.collect_one(txindex).unwrap();
let buffer = reader.read_raw_bytes(position, *total_size as usize)?;
@@ -326,14 +324,15 @@ impl Query {
let indexer = self.indexer();
// Look up spending txindex directly
let spending_txindex = indexer.vecs.inputs.txindex.read_once(txinindex)?;
let spending_txindex = indexer.vecs.inputs.txindex.collect_one(txinindex).unwrap();
// Calculate vin
let spending_first_txinindex = indexer
.vecs
.transactions
.first_txinindex
.read_once(spending_txindex)?;
.collect_one(spending_txindex)
.unwrap();
let vin = Vin::from(usize::from(txinindex) - usize::from(spending_first_txinindex));
// Get spending tx details
@@ -342,9 +341,10 @@ impl Query {
.vecs
.transactions
.height
.read_once(spending_txindex)?;
.collect_one(spending_txindex)
.unwrap();
let block_hash = indexer.vecs.blocks.blockhash.read_once(spending_height)?;
let block_time = indexer.vecs.blocks.timestamp.read_once(spending_height)?;
let block_time = indexer.vecs.blocks.timestamp.collect_one(spending_height).unwrap();
Ok(TxOutspend {
spent: true,