mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-04-27 08:09:58 -07:00
global: MASSIVE snapshot
This commit is contained in:
@@ -7,7 +7,7 @@ use brk_types::{
|
||||
AddressIndexTxIndex, AddressStats, AnyAddressDataIndexEnum, OutputType, Sats, TxIndex,
|
||||
TxStatus, Txid, TypeIndex, Unit, Utxo, Vout,
|
||||
};
|
||||
use vecdb::TypedVecIterator;
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -62,14 +62,14 @@ impl Query {
|
||||
.distribution
|
||||
.addresses_data
|
||||
.funded
|
||||
.iter()?
|
||||
.get_unwrap(index),
|
||||
.reader()
|
||||
.get(usize::from(index)),
|
||||
AnyAddressDataIndexEnum::Empty(index) => computer
|
||||
.distribution
|
||||
.addresses_data
|
||||
.empty
|
||||
.iter()?
|
||||
.get_unwrap(index)
|
||||
.reader()
|
||||
.get(usize::from(index))
|
||||
.into(),
|
||||
};
|
||||
|
||||
@@ -138,10 +138,10 @@ impl Query {
|
||||
.map(|(key, _)| key.txindex())
|
||||
.collect();
|
||||
|
||||
let mut txindex_to_txid_iter = indexer.vecs.transactions.txid.iter()?;
|
||||
let txid_reader = indexer.vecs.transactions.txid.reader();
|
||||
let txids: Vec<Txid> = txindices
|
||||
.into_iter()
|
||||
.map(|txindex| txindex_to_txid_iter.get_unwrap(txindex))
|
||||
.map(|txindex| txid_reader.get(txindex.to_usize()))
|
||||
.collect();
|
||||
|
||||
Ok(txids)
|
||||
@@ -166,23 +166,21 @@ impl Query {
|
||||
.map(|(key, _): (AddressIndexOutPoint, Unit)| (key.txindex(), key.vout()))
|
||||
.collect();
|
||||
|
||||
let mut txindex_to_txid_iter = vecs.transactions.txid.iter()?;
|
||||
let mut txindex_to_height_iter = vecs.transactions.height.iter()?;
|
||||
let mut txindex_to_first_txoutindex_iter = vecs.transactions.first_txoutindex.iter()?;
|
||||
let mut txoutindex_to_value_iter = vecs.outputs.value.iter()?;
|
||||
let mut height_to_blockhash_iter = vecs.blocks.blockhash.iter()?;
|
||||
let mut height_to_timestamp_iter = vecs.blocks.timestamp.iter()?;
|
||||
let txid_reader = vecs.transactions.txid.reader();
|
||||
let first_txoutindex_reader = vecs.transactions.first_txoutindex.reader();
|
||||
let value_reader = vecs.outputs.value.reader();
|
||||
let blockhash_reader = vecs.blocks.blockhash.reader();
|
||||
|
||||
let utxos: Vec<Utxo> = outpoints
|
||||
.into_iter()
|
||||
.map(|(txindex, vout)| {
|
||||
let txid: Txid = txindex_to_txid_iter.get_unwrap(txindex);
|
||||
let height = txindex_to_height_iter.get_unwrap(txindex);
|
||||
let first_txoutindex = txindex_to_first_txoutindex_iter.get_unwrap(txindex);
|
||||
let txid: Txid = txid_reader.get(txindex.to_usize());
|
||||
let height = vecs.transactions.height.collect_one_at(txindex.to_usize()).unwrap();
|
||||
let first_txoutindex = first_txoutindex_reader.get(txindex.to_usize());
|
||||
let txoutindex = first_txoutindex + vout;
|
||||
let value: Sats = txoutindex_to_value_iter.get_unwrap(txoutindex);
|
||||
let block_hash = height_to_blockhash_iter.get_unwrap(height);
|
||||
let block_time = height_to_timestamp_iter.get_unwrap(height);
|
||||
let value: Sats = value_reader.get(usize::from(txoutindex));
|
||||
let block_hash = blockhash_reader.get(usize::from(height));
|
||||
let block_time = vecs.blocks.timestamp.collect_one_at(usize::from(height)).unwrap();
|
||||
|
||||
Utxo {
|
||||
txid,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{BlockHash, BlockHashPrefix, BlockInfo, Height, TxIndex};
|
||||
use vecdb::{AnyVec, GenericStoredVec, VecIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -21,10 +21,10 @@ impl Query {
|
||||
}
|
||||
|
||||
let blockhash = indexer.vecs.blocks.blockhash.read_once(height)?;
|
||||
let difficulty = indexer.vecs.blocks.difficulty.read_once(height)?;
|
||||
let timestamp = indexer.vecs.blocks.timestamp.read_once(height)?;
|
||||
let size = indexer.vecs.blocks.total_size.read_once(height)?;
|
||||
let weight = indexer.vecs.blocks.weight.read_once(height)?;
|
||||
let difficulty = indexer.vecs.blocks.difficulty.collect_one(height).unwrap();
|
||||
let timestamp = indexer.vecs.blocks.timestamp.collect_one(height).unwrap();
|
||||
let size = indexer.vecs.blocks.total_size.collect_one(height).unwrap();
|
||||
let weight = indexer.vecs.blocks.weight.collect_one(height).unwrap();
|
||||
let tx_count = self.tx_count_at_height(height, max_height)?;
|
||||
|
||||
Ok(BlockInfo {
|
||||
@@ -45,12 +45,58 @@ impl Query {
|
||||
let start = start.min(max_height);
|
||||
|
||||
let start_u32: u32 = start.into();
|
||||
let count = DEFAULT_BLOCK_COUNT.min(start_u32 + 1);
|
||||
let count = DEFAULT_BLOCK_COUNT.min(start_u32 + 1) as usize;
|
||||
|
||||
let mut blocks = Vec::with_capacity(count as usize);
|
||||
for i in 0..count {
|
||||
let height = Height::from(start_u32 - i);
|
||||
blocks.push(self.block_by_height(height)?);
|
||||
if count == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let indexer = self.indexer();
|
||||
let computer = self.computer();
|
||||
|
||||
// Batch-read all PcoVec data for the contiguous range (avoids
|
||||
// per-block page decompression — 4 reads instead of 4*count).
|
||||
let end = start_u32 as usize + 1;
|
||||
let begin = end - count;
|
||||
|
||||
let difficulties = indexer.vecs.blocks.difficulty.collect_range_at(begin, end);
|
||||
let timestamps = indexer.vecs.blocks.timestamp.collect_range_at(begin, end);
|
||||
let sizes = indexer.vecs.blocks.total_size.collect_range_at(begin, end);
|
||||
let weights = indexer.vecs.blocks.weight.collect_range_at(begin, end);
|
||||
|
||||
// Batch-read first_txindex for tx_count computation (need one extra for next boundary)
|
||||
let txindex_end = if end <= max_height.to_usize() {
|
||||
end + 1
|
||||
} else {
|
||||
end
|
||||
};
|
||||
let first_txindexes: Vec<TxIndex> = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txindex
|
||||
.collect_range_at(begin, txindex_end);
|
||||
let total_txs = computer.indexes.txindex.identity.len();
|
||||
|
||||
let mut blocks = Vec::with_capacity(count);
|
||||
for i in (0..count).rev() {
|
||||
let height = Height::from(begin + i);
|
||||
let blockhash = indexer.vecs.blocks.blockhash.read_once(height)?;
|
||||
|
||||
let tx_count = if i + 1 < first_txindexes.len() {
|
||||
first_txindexes[i + 1].to_usize() - first_txindexes[i].to_usize()
|
||||
} else {
|
||||
total_txs - first_txindexes[i].to_usize()
|
||||
};
|
||||
|
||||
blocks.push(BlockInfo {
|
||||
id: blockhash,
|
||||
height,
|
||||
tx_count: tx_count as u32,
|
||||
size: *sizes[i],
|
||||
weight: weights[i],
|
||||
timestamp: timestamps[i],
|
||||
difficulty: *difficulties[i],
|
||||
});
|
||||
}
|
||||
|
||||
Ok(blocks)
|
||||
@@ -72,27 +118,26 @@ impl Query {
|
||||
}
|
||||
|
||||
fn max_height(&self) -> Height {
|
||||
Height::from(
|
||||
self.indexer()
|
||||
.vecs
|
||||
.blocks
|
||||
.blockhash
|
||||
.len()
|
||||
.saturating_sub(1),
|
||||
)
|
||||
Height::from(self.indexer().vecs.blocks.blockhash.len().saturating_sub(1))
|
||||
}
|
||||
|
||||
fn tx_count_at_height(&self, height: Height, max_height: Height) -> Result<u32> {
|
||||
let indexer = self.indexer();
|
||||
let computer = self.computer();
|
||||
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
|
||||
let first_txindex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txindex
|
||||
.collect_one(height)
|
||||
.unwrap();
|
||||
let next_first_txindex = if height < max_height {
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txindex
|
||||
.read_once(height.incremented())?
|
||||
.collect_one(height.incremented())
|
||||
.unwrap()
|
||||
} else {
|
||||
TxIndex::from(computer.indexes.txindex.identity.len())
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{BlockHash, Height};
|
||||
use vecdb::{AnyVec, GenericStoredVec};
|
||||
use vecdb::{AnyVec, ReadableVec};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -20,8 +20,8 @@ impl Query {
|
||||
return Err(Error::OutOfRange("Block height out of range".into()));
|
||||
}
|
||||
|
||||
let position = computer.positions.block_position.read_once(height)?;
|
||||
let size = indexer.vecs.blocks.total_size.read_once(height)?;
|
||||
let position = computer.positions.block_position.collect_one(height).unwrap();
|
||||
let size = indexer.vecs.blocks.total_size.collect_one(height).unwrap();
|
||||
|
||||
reader.read_raw_bytes(position, *size as usize)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{BlockHash, BlockStatus, Height};
|
||||
use vecdb::{AnyVec, GenericStoredVec};
|
||||
use vecdb::AnyVec;
|
||||
|
||||
use crate::Query;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{BlockTimestamp, Date, DateIndex, Height, Timestamp};
|
||||
use brk_types::{BlockTimestamp, Date, Day1, Height, Timestamp};
|
||||
use jiff::Timestamp as JiffTimestamp;
|
||||
use vecdb::{GenericStoredVec, TypedVecIterator};
|
||||
use vecdb::ReadableVec;
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -19,28 +19,26 @@ impl Query {
|
||||
|
||||
let target = timestamp;
|
||||
let date = Date::from(target);
|
||||
let dateindex = DateIndex::try_from(date).unwrap_or_default();
|
||||
let day1 = Day1::try_from(date).unwrap_or_default();
|
||||
|
||||
// Get first height of the target date
|
||||
let first_height_of_day = computer
|
||||
.indexes
|
||||
.dateindex
|
||||
.day1
|
||||
.first_height
|
||||
.read_once(dateindex)
|
||||
.collect_one(day1)
|
||||
.unwrap_or(Height::from(0usize));
|
||||
|
||||
let start: usize = usize::from(first_height_of_day).min(max_height_usize);
|
||||
|
||||
// Use iterator for efficient sequential access
|
||||
let mut timestamp_iter = indexer.vecs.blocks.timestamp.iter()?;
|
||||
let timestamps = &indexer.vecs.blocks.timestamp;
|
||||
|
||||
// Search forward from start to find the last block <= target timestamp
|
||||
let mut best_height = start;
|
||||
let mut best_ts = timestamp_iter.get_unwrap(Height::from(start));
|
||||
let mut best_ts = timestamps.collect_one_at(start).unwrap();
|
||||
|
||||
for h in (start + 1)..=max_height_usize {
|
||||
let height = Height::from(h);
|
||||
let block_ts = timestamp_iter.get_unwrap(height);
|
||||
let block_ts = timestamps.collect_one_at(h).unwrap();
|
||||
if block_ts <= target {
|
||||
best_height = h;
|
||||
best_ts = block_ts;
|
||||
@@ -51,8 +49,7 @@ impl Query {
|
||||
|
||||
// Check one block before start in case we need to go backward
|
||||
if start > 0 && best_ts > target {
|
||||
let prev_height = Height::from(start - 1);
|
||||
let prev_ts = timestamp_iter.get_unwrap(prev_height);
|
||||
let prev_ts = timestamps.collect_one_at(start - 1).unwrap();
|
||||
if prev_ts <= target {
|
||||
best_height = start - 1;
|
||||
best_ts = prev_ts;
|
||||
@@ -60,12 +57,7 @@ impl Query {
|
||||
}
|
||||
|
||||
let height = Height::from(best_height);
|
||||
let blockhash = indexer
|
||||
.vecs
|
||||
.blocks
|
||||
.blockhash
|
||||
.iter()?
|
||||
.get_unwrap(height);
|
||||
let blockhash = indexer.vecs.blocks.blockhash.reader().get(usize::from(height));
|
||||
|
||||
// Convert timestamp to ISO 8601 format
|
||||
let ts_secs: i64 = (*best_ts).into();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{BlockHash, Height, Transaction, TxIndex, Txid};
|
||||
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator};
|
||||
use vecdb::{AnyVec, ReadableVec};
|
||||
|
||||
use super::BLOCK_TXS_PAGE_SIZE;
|
||||
use crate::Query;
|
||||
@@ -31,26 +31,22 @@ impl Query {
|
||||
return Err(Error::OutOfRange("Block height out of range".into()));
|
||||
}
|
||||
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.collect_one(height).unwrap();
|
||||
let next_first_txindex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txindex
|
||||
.read_once(height.incremented())
|
||||
.unwrap_or_else(|_| TxIndex::from(indexer.vecs.transactions.txid.len()));
|
||||
.collect_one(height.incremented())
|
||||
.unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len()));
|
||||
|
||||
let first: usize = first_txindex.into();
|
||||
let next: usize = next_first_txindex.into();
|
||||
let count = next - first;
|
||||
|
||||
let txids: Vec<Txid> = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.txid
|
||||
.iter()?
|
||||
.skip(first)
|
||||
.take(count)
|
||||
.collect();
|
||||
.collect_range_at(first, next);
|
||||
|
||||
Ok(txids)
|
||||
}
|
||||
@@ -67,13 +63,13 @@ impl Query {
|
||||
return Err(Error::OutOfRange("Block height out of range".into()));
|
||||
}
|
||||
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.collect_one(height).unwrap();
|
||||
let next_first_txindex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txindex
|
||||
.read_once(height.incremented())
|
||||
.unwrap_or_else(|_| TxIndex::from(indexer.vecs.transactions.txid.len()));
|
||||
.collect_one(height.incremented())
|
||||
.unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len()));
|
||||
|
||||
let first: usize = first_txindex.into();
|
||||
let next: usize = next_first_txindex.into();
|
||||
@@ -104,13 +100,13 @@ impl Query {
|
||||
return Err(Error::OutOfRange("Block height out of range".into()));
|
||||
}
|
||||
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.read_once(height)?;
|
||||
let first_txindex = indexer.vecs.transactions.first_txindex.collect_one(height).unwrap();
|
||||
let next_first_txindex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txindex
|
||||
.read_once(height.incremented())
|
||||
.unwrap_or_else(|_| TxIndex::from(indexer.vecs.transactions.txid.len()));
|
||||
.collect_one(height.incremented())
|
||||
.unwrap_or_else(|| TxIndex::from(indexer.vecs.transactions.txid.len()));
|
||||
|
||||
let first: usize = first_txindex.into();
|
||||
let next: usize = next_first_txindex.into();
|
||||
@@ -120,8 +116,8 @@ impl Query {
|
||||
return Err(Error::OutOfRange("Transaction index out of range".into()));
|
||||
}
|
||||
|
||||
let txindex = TxIndex::from(first + index);
|
||||
let txid = indexer.vecs.transactions.txid.iter()?.get_unwrap(txindex);
|
||||
let txindex = first + index;
|
||||
let txid = indexer.vecs.transactions.txid.reader().get(txindex);
|
||||
|
||||
Ok(txid)
|
||||
}
|
||||
|
||||
@@ -2,9 +2,9 @@ use std::{fs, path::PathBuf};
|
||||
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{
|
||||
CostBasisBucket, CostBasisDistribution, CostBasisFormatted, CostBasisValue, Date, DateIndex,
|
||||
CostBasisBucket, CostBasisDistribution, CostBasisFormatted, CostBasisValue, Date, Day1,
|
||||
};
|
||||
use vecdb::IterableVec;
|
||||
use vecdb::ReadableVec;
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -81,20 +81,14 @@ impl Query {
|
||||
value: CostBasisValue,
|
||||
) -> Result<CostBasisFormatted> {
|
||||
let distribution = self.cost_basis_distribution(cohort, date)?;
|
||||
let dateindex =
|
||||
DateIndex::try_from(date).map_err(|e| Error::Parse(e.to_string()))?;
|
||||
let price = self
|
||||
.computer()
|
||||
.price
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::NotFound("Price data not available".to_string()))?;
|
||||
let spot = *price
|
||||
let day1 = Day1::try_from(date).map_err(|e| Error::Parse(e.to_string()))?;
|
||||
let price = &self.computer().prices;
|
||||
let spot = price
|
||||
.cents
|
||||
.split
|
||||
.dateindex
|
||||
.close
|
||||
.iter()
|
||||
.get(dateindex)
|
||||
.day1
|
||||
.collect_one(day1)
|
||||
.ok_or_else(|| Error::NotFound(format!("No price data for {date}")))?;
|
||||
Ok(distribution.format(bucket, value, spot))
|
||||
}
|
||||
|
||||
@@ -3,13 +3,14 @@ use std::collections::BTreeMap;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_traversable::TreeNode;
|
||||
use brk_types::{
|
||||
DetailedMetricCount, Format, Index, IndexInfo, Limit, Metric, MetricData, MetricOutput,
|
||||
MetricSelection, Output, PaginatedMetrics, Pagination, PaginationIndex,
|
||||
DetailedMetricCount, Etag, Format, Index, IndexInfo, LegacyValue, Limit, Metric, MetricData,
|
||||
MetricOutput, MetricOutputLegacy, MetricSelection, Output, OutputLegacy, PaginatedMetrics,
|
||||
Pagination, PaginationIndex, Version,
|
||||
};
|
||||
use vecdb::AnyExportableVec;
|
||||
|
||||
use crate::{
|
||||
Query, ResolvedQuery,
|
||||
Query,
|
||||
vecs::{IndexToVec, MetricToVec},
|
||||
};
|
||||
|
||||
@@ -120,15 +121,11 @@ impl Query {
|
||||
|
||||
/// Resolve query metadata without formatting (cheap).
|
||||
/// Use with `format` for lazy formatting after ETag check.
|
||||
pub fn resolve(
|
||||
&self,
|
||||
params: MetricSelection,
|
||||
max_weight: usize,
|
||||
) -> Result<ResolvedQuery> {
|
||||
pub fn resolve(&self, params: MetricSelection, max_weight: usize) -> Result<ResolvedQuery> {
|
||||
let vecs = self.search(¶ms)?;
|
||||
|
||||
let total = vecs.iter().map(|v| v.len()).min().unwrap_or(0);
|
||||
let version: u64 = vecs.iter().map(|v| u64::from(v.version())).sum();
|
||||
let version: Version = vecs.iter().map(|v| v.version()).sum();
|
||||
|
||||
let start = params
|
||||
.start()
|
||||
@@ -182,12 +179,13 @@ impl Query {
|
||||
let output = match format {
|
||||
Format::CSV => Output::CSV(Self::columns_to_csv(&vecs, start, end)?),
|
||||
Format::JSON => {
|
||||
let count = end.saturating_sub(start);
|
||||
if vecs.len() == 1 {
|
||||
let mut buf = Vec::new();
|
||||
let mut buf = Vec::with_capacity(count * 12 + 256);
|
||||
MetricData::serialize(vecs[0], index, start, end, &mut buf)?;
|
||||
Output::Json(buf)
|
||||
} else {
|
||||
let mut buf = Vec::new();
|
||||
let mut buf = Vec::with_capacity(count * 12 * vecs.len() + 256);
|
||||
buf.push(b'[');
|
||||
for (i, vec) in vecs.iter().enumerate() {
|
||||
if i > 0 {
|
||||
@@ -244,4 +242,87 @@ impl Query {
|
||||
pub fn metric_to_indexes(&self, metric: Metric) -> Option<&Vec<Index>> {
|
||||
self.vecs().metric_to_indexes(metric)
|
||||
}
|
||||
|
||||
/// Deprecated - format a resolved query as legacy output (expensive).
|
||||
pub fn format_legacy(&self, resolved: ResolvedQuery) -> Result<MetricOutputLegacy> {
|
||||
let ResolvedQuery {
|
||||
vecs,
|
||||
format,
|
||||
version,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
..
|
||||
} = resolved;
|
||||
|
||||
if vecs.is_empty() {
|
||||
return Ok(MetricOutputLegacy {
|
||||
output: OutputLegacy::default(format),
|
||||
version: Version::ZERO,
|
||||
total: 0,
|
||||
start: 0,
|
||||
end: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let from = Some(start as i64);
|
||||
let to = Some(end as i64);
|
||||
|
||||
let output = match format {
|
||||
Format::CSV => OutputLegacy::CSV(Self::columns_to_csv(&vecs, start, end)?),
|
||||
Format::JSON => {
|
||||
if vecs.len() == 1 {
|
||||
let metric = vecs[0];
|
||||
let count = metric.range_count(from, to);
|
||||
let mut buf = Vec::new();
|
||||
if count == 1 {
|
||||
metric.write_json_value(Some(start), &mut buf)?;
|
||||
OutputLegacy::Json(LegacyValue::Value(buf))
|
||||
} else {
|
||||
metric.write_json(Some(start), Some(end), &mut buf)?;
|
||||
OutputLegacy::Json(LegacyValue::List(buf))
|
||||
}
|
||||
} else {
|
||||
let mut values = Vec::with_capacity(vecs.len());
|
||||
for vec in &vecs {
|
||||
let mut buf = Vec::new();
|
||||
vec.write_json(Some(start), Some(end), &mut buf)?;
|
||||
values.push(buf);
|
||||
}
|
||||
OutputLegacy::Json(LegacyValue::Matrix(values))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(MetricOutputLegacy {
|
||||
output,
|
||||
version,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A resolved metric query ready for formatting.
|
||||
/// Contains the vecs and metadata needed to build an ETag or format the output.
|
||||
pub struct ResolvedQuery {
|
||||
pub vecs: Vec<&'static dyn AnyExportableVec>,
|
||||
pub format: Format,
|
||||
pub index: Index,
|
||||
pub version: Version,
|
||||
pub total: usize,
|
||||
pub start: usize,
|
||||
pub end: usize,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
impl ResolvedQuery {
|
||||
pub fn etag(&self) -> Etag {
|
||||
Etag::from_metric(self.version, self.total, self.start, self.end, self.height)
|
||||
}
|
||||
|
||||
pub fn format(&self) -> Format {
|
||||
self.format
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{Format, LegacyValue, MetricOutputLegacy, OutputLegacy};
|
||||
|
||||
use crate::{Query, ResolvedQuery};
|
||||
|
||||
impl Query {
|
||||
/// Deprecated - format a resolved query as legacy output (expensive).
|
||||
pub fn format_legacy(&self, resolved: ResolvedQuery) -> Result<MetricOutputLegacy> {
|
||||
let ResolvedQuery {
|
||||
vecs,
|
||||
format,
|
||||
version,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
..
|
||||
} = resolved;
|
||||
|
||||
if vecs.is_empty() {
|
||||
return Ok(MetricOutputLegacy {
|
||||
output: OutputLegacy::default(format),
|
||||
version: 0,
|
||||
total: 0,
|
||||
start: 0,
|
||||
end: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let from = Some(start as i64);
|
||||
let to = Some(end as i64);
|
||||
|
||||
let output = match format {
|
||||
Format::CSV => OutputLegacy::CSV(Self::columns_to_csv(&vecs, start, end)?),
|
||||
Format::JSON => {
|
||||
if vecs.len() == 1 {
|
||||
let metric = vecs[0];
|
||||
let count = metric.range_count(from, to);
|
||||
let mut buf = Vec::new();
|
||||
if count == 1 {
|
||||
metric.write_json_value(Some(start), &mut buf)?;
|
||||
OutputLegacy::Json(LegacyValue::Value(buf))
|
||||
} else {
|
||||
metric.write_json(Some(start), Some(end), &mut buf)?;
|
||||
OutputLegacy::Json(LegacyValue::List(buf))
|
||||
}
|
||||
} else {
|
||||
let mut values = Vec::with_capacity(vecs.len());
|
||||
for vec in &vecs {
|
||||
let mut buf = Vec::new();
|
||||
vec.write_json(Some(start), Some(end), &mut buf)?;
|
||||
values.push(buf);
|
||||
}
|
||||
OutputLegacy::Json(LegacyValue::Matrix(values))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(MetricOutputLegacy {
|
||||
output,
|
||||
version,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
// TODO: INCOMPLETE - indexes_to_fee_rate.dateindex doesn't have percentile fields
|
||||
// because from_txindex.rs calls remove_percentiles() before creating dateindex.
|
||||
// TODO: INCOMPLETE - indexes_to_fee_rate.day1 doesn't have percentile fields
|
||||
// because from_txindex.rs calls remove_percentiles() before creating day1.
|
||||
// Need to either:
|
||||
// 1. Use .height instead and convert height to dateindex for iteration
|
||||
// 2. Fix from_txindex.rs to preserve percentiles for dateindex
|
||||
// 3. Create a separate dateindex computation path with percentiles
|
||||
// 1. Use .height instead and convert height to day1 for iteration
|
||||
// 2. Fix from_txindex.rs to preserve percentiles for day1
|
||||
// 3. Create a separate day1 computation path with percentiles
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
@@ -15,12 +15,11 @@ use brk_types::{
|
||||
};
|
||||
// use vecdb::{IterableVec, VecIndex};
|
||||
|
||||
// use super::dateindex_iter::DateIndexIter;
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
pub fn block_fee_rates(&self, _time_period: TimePeriod) -> Result<Vec<BlockFeeRatesEntry>> {
|
||||
// Disabled until percentile data is available at dateindex level
|
||||
// Disabled until percentile data is available at day1 level
|
||||
Ok(Vec::new())
|
||||
|
||||
// Original implementation:
|
||||
@@ -30,9 +29,9 @@ impl Query {
|
||||
// .to_usize()
|
||||
// .saturating_sub(time_period.block_count());
|
||||
//
|
||||
// let iter = DateIndexIter::new(computer, start, current_height.to_usize());
|
||||
// let iter = Day1Iter::new(computer, start, current_height.to_usize());
|
||||
//
|
||||
// let vecs = &computer.transactions.transaction.indexes_to_fee_rate.dateindex;
|
||||
// let vecs = &computer.transactions.transaction.indexes_to_fee_rate.day1;
|
||||
// let mut min = vecs.unwrap_min().iter();
|
||||
// let mut pct10 = vecs.unwrap_pct10().iter();
|
||||
// let mut pct25 = vecs.unwrap_pct25().iter();
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{BlockFeesEntry, TimePeriod};
|
||||
use vecdb::{IterableVec, VecIndex};
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use super::dateindex_iter::DateIndexIter;
|
||||
use super::day1_iter::Day1Iter;
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
@@ -13,19 +13,18 @@ impl Query {
|
||||
.to_usize()
|
||||
.saturating_sub(time_period.block_count());
|
||||
|
||||
let iter = DateIndexIter::new(computer, start, current_height.to_usize());
|
||||
let iter = Day1Iter::new(computer, start, current_height.to_usize());
|
||||
|
||||
let mut fees = computer
|
||||
let fees_vec = &computer
|
||||
.transactions
|
||||
.fees
|
||||
.fee
|
||||
.sats
|
||||
.dateindex
|
||||
.average()
|
||||
.iter();
|
||||
.day1
|
||||
.average;
|
||||
|
||||
Ok(iter.collect(|di, ts, h| {
|
||||
fees.get(di).map(|fee| BlockFeesEntry {
|
||||
fees_vec.collect_one(di).map(|fee| BlockFeesEntry {
|
||||
avg_height: h,
|
||||
timestamp: ts,
|
||||
avg_fees: fee,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{BlockRewardsEntry, TimePeriod};
|
||||
use vecdb::{IterableVec, VecIndex};
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use super::dateindex_iter::DateIndexIter;
|
||||
use super::day1_iter::Day1Iter;
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
@@ -13,20 +13,18 @@ impl Query {
|
||||
.to_usize()
|
||||
.saturating_sub(time_period.block_count());
|
||||
|
||||
let iter = DateIndexIter::new(computer, start, current_height.to_usize());
|
||||
let iter = Day1Iter::new(computer, start, current_height.to_usize());
|
||||
|
||||
let mut rewards = computer
|
||||
.blocks
|
||||
let rewards_vec = &computer
|
||||
.mining
|
||||
.rewards
|
||||
.coinbase
|
||||
.sats
|
||||
.dateindex
|
||||
.distribution
|
||||
.average()
|
||||
.iter();
|
||||
.day1
|
||||
.average;
|
||||
|
||||
Ok(iter.collect(|di, ts, h| {
|
||||
rewards.get(di).map(|reward| BlockRewardsEntry {
|
||||
rewards_vec.collect_one(di).map(|reward| BlockRewardsEntry {
|
||||
avg_height: h.into(),
|
||||
timestamp: *ts,
|
||||
avg_rewards: *reward,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{BlockSizeEntry, BlockSizesWeights, BlockWeightEntry, TimePeriod};
|
||||
use vecdb::{IterableVec, VecIndex};
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use super::dateindex_iter::DateIndexIter;
|
||||
use super::day1_iter::Day1Iter;
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
@@ -13,34 +13,30 @@ impl Query {
|
||||
.to_usize()
|
||||
.saturating_sub(time_period.block_count());
|
||||
|
||||
let iter = DateIndexIter::new(computer, start, current_height.to_usize());
|
||||
let iter = Day1Iter::new(computer, start, current_height.to_usize());
|
||||
|
||||
let mut sizes_vec = computer
|
||||
let sizes_vec = &computer
|
||||
.blocks
|
||||
.size
|
||||
.size
|
||||
.dateindex
|
||||
.distribution
|
||||
.average()
|
||||
.iter();
|
||||
let mut weights_vec = computer
|
||||
.day1
|
||||
.average;
|
||||
let weights_vec = &computer
|
||||
.blocks
|
||||
.weight
|
||||
.weight
|
||||
.dateindex
|
||||
.distribution
|
||||
.average()
|
||||
.iter();
|
||||
.day1
|
||||
.average;
|
||||
|
||||
let entries: Vec<_> = iter.collect(|di, ts, h| {
|
||||
let size = sizes_vec.get(di).map(|s| *s);
|
||||
let weight = weights_vec.get(di).map(|w| *w);
|
||||
let size = sizes_vec.collect_one(di).map(|s| *s);
|
||||
let weight = weights_vec.collect_one(di).map(|w| *w);
|
||||
Some((h.into(), (*ts), size, weight))
|
||||
});
|
||||
|
||||
let sizes = entries
|
||||
.iter()
|
||||
.filter_map(|(h, ts, size, _)| {
|
||||
.filter_map(|(h, ts, size, _): &(u32, _, _, _)| {
|
||||
size.map(|s| BlockSizeEntry {
|
||||
avg_height: *h,
|
||||
timestamp: *ts,
|
||||
@@ -51,7 +47,7 @@ impl Query {
|
||||
|
||||
let weights = entries
|
||||
.iter()
|
||||
.filter_map(|(h, ts, _, weight)| {
|
||||
.filter_map(|(h, ts, _, weight): &(u32, _, _, _)| {
|
||||
weight.map(|w| BlockWeightEntry {
|
||||
avg_height: *h,
|
||||
timestamp: *ts,
|
||||
|
||||
@@ -1,28 +1,28 @@
|
||||
use brk_computer::Computer;
|
||||
use brk_types::{DateIndex, Height, Timestamp};
|
||||
use vecdb::{GenericStoredVec, IterableVec, VecIndex};
|
||||
use brk_types::{Day1, Height, Timestamp};
|
||||
use vecdb::{ReadableVec, Ro, VecIndex};
|
||||
|
||||
/// Helper for iterating over dateindex ranges with sampling.
|
||||
pub struct DateIndexIter<'a> {
|
||||
computer: &'a Computer,
|
||||
start_di: DateIndex,
|
||||
end_di: DateIndex,
|
||||
/// Helper for iterating over day1 ranges with sampling.
|
||||
pub struct Day1Iter<'a> {
|
||||
computer: &'a Computer<Ro>,
|
||||
start_di: Day1,
|
||||
end_di: Day1,
|
||||
step: usize,
|
||||
}
|
||||
|
||||
impl<'a> DateIndexIter<'a> {
|
||||
pub fn new(computer: &'a Computer, start_height: usize, end_height: usize) -> Self {
|
||||
impl<'a> Day1Iter<'a> {
|
||||
pub fn new(computer: &'a Computer<Ro>, start_height: usize, end_height: usize) -> Self {
|
||||
let start_di = computer
|
||||
.indexes
|
||||
.height
|
||||
.dateindex
|
||||
.read_once(Height::from(start_height))
|
||||
.day1
|
||||
.collect_one(Height::from(start_height))
|
||||
.unwrap_or_default();
|
||||
let end_di = computer
|
||||
.indexes
|
||||
.height
|
||||
.dateindex
|
||||
.read_once(Height::from(end_height))
|
||||
.day1
|
||||
.collect_one(Height::from(end_height))
|
||||
.unwrap_or_default();
|
||||
|
||||
let total = end_di.to_usize().saturating_sub(start_di.to_usize()) + 1;
|
||||
@@ -39,22 +39,22 @@ impl<'a> DateIndexIter<'a> {
|
||||
/// Iterate and collect entries using the provided transform function.
|
||||
pub fn collect<T, F>(&self, mut transform: F) -> Vec<T>
|
||||
where
|
||||
F: FnMut(DateIndex, Timestamp, Height) -> Option<T>,
|
||||
F: FnMut(Day1, Timestamp, Height) -> Option<T>,
|
||||
{
|
||||
let total = self
|
||||
.end_di
|
||||
.to_usize()
|
||||
.saturating_sub(self.start_di.to_usize())
|
||||
+ 1;
|
||||
let mut timestamps = self.computer.blocks.time.timestamp.dateindex.iter();
|
||||
let mut heights = self.computer.indexes.dateindex.first_height.iter();
|
||||
let timestamps = &self.computer.blocks.time.timestamp.day1;
|
||||
let heights = &self.computer.indexes.day1.first_height;
|
||||
|
||||
let mut entries = Vec::with_capacity(total / self.step + 1);
|
||||
let mut i = self.start_di.to_usize();
|
||||
|
||||
while i <= self.end_di.to_usize() {
|
||||
let di = DateIndex::from(i);
|
||||
if let (Some(ts), Some(h)) = (timestamps.get(di), heights.get(di))
|
||||
let di = Day1::from(i);
|
||||
if let (Some(ts), Some(h)) = (timestamps.collect_one(di), heights.collect_one(di))
|
||||
&& let Some(entry) = transform(di, ts, h)
|
||||
{
|
||||
entries.push(entry);
|
||||
@@ -2,7 +2,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::{DifficultyAdjustment, DifficultyEpoch, Height};
|
||||
use vecdb::GenericStoredVec;
|
||||
use vecdb::ReadableVec;
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -24,7 +24,8 @@ impl Query {
|
||||
.indexes
|
||||
.height
|
||||
.difficultyepoch
|
||||
.read_once(current_height)?;
|
||||
.collect_one(current_height)
|
||||
.unwrap();
|
||||
let current_epoch_usize: usize = current_epoch.into();
|
||||
|
||||
// Get epoch start height
|
||||
@@ -32,7 +33,8 @@ impl Query {
|
||||
.indexes
|
||||
.difficultyepoch
|
||||
.first_height
|
||||
.read_once(current_epoch)?;
|
||||
.collect_one(current_epoch)
|
||||
.unwrap();
|
||||
let epoch_start_u32: u32 = epoch_start_height.into();
|
||||
|
||||
// Calculate epoch progress
|
||||
@@ -47,12 +49,14 @@ impl Query {
|
||||
.time
|
||||
.timestamp
|
||||
.difficultyepoch
|
||||
.read_once(current_epoch)?;
|
||||
.collect_one(current_epoch)
|
||||
.unwrap();
|
||||
let current_timestamp = indexer
|
||||
.vecs
|
||||
.blocks
|
||||
.timestamp
|
||||
.read_once(current_height)?;
|
||||
.collect_one(current_height)
|
||||
.unwrap();
|
||||
|
||||
// Calculate average block time in current epoch
|
||||
let elapsed_time = (*current_timestamp - *epoch_start_timestamp) as u64;
|
||||
@@ -88,18 +92,21 @@ impl Query {
|
||||
.indexes
|
||||
.difficultyepoch
|
||||
.first_height
|
||||
.read_once(prev_epoch)?;
|
||||
.collect_one(prev_epoch)
|
||||
.unwrap();
|
||||
|
||||
let prev_difficulty = indexer
|
||||
.vecs
|
||||
.blocks
|
||||
.difficulty
|
||||
.read_once(prev_epoch_start)?;
|
||||
.collect_one(prev_epoch_start)
|
||||
.unwrap();
|
||||
let curr_difficulty = indexer
|
||||
.vecs
|
||||
.blocks
|
||||
.difficulty
|
||||
.read_once(epoch_start_height)?;
|
||||
.collect_one(epoch_start_height)
|
||||
.unwrap();
|
||||
|
||||
if *prev_difficulty > 0.0 {
|
||||
((*curr_difficulty / *prev_difficulty) - 1.0) * 100.0
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use brk_computer::Computer;
|
||||
use brk_types::{DifficultyAdjustmentEntry, DifficultyEpoch, Height};
|
||||
use vecdb::{GenericStoredVec, IterableVec, VecIndex};
|
||||
use vecdb::{ReadableVec, Ro, VecIndex};
|
||||
|
||||
/// Iterate over difficulty epochs within a height range.
|
||||
pub fn iter_difficulty_epochs(
|
||||
computer: &Computer,
|
||||
computer: &Computer<Ro>,
|
||||
start_height: usize,
|
||||
end_height: usize,
|
||||
) -> Vec<DifficultyAdjustmentEntry> {
|
||||
@@ -12,38 +12,34 @@ pub fn iter_difficulty_epochs(
|
||||
.indexes
|
||||
.height
|
||||
.difficultyepoch
|
||||
.read_once(Height::from(start_height))
|
||||
.collect_one(Height::from(start_height))
|
||||
.unwrap_or_default();
|
||||
let end_epoch = computer
|
||||
.indexes
|
||||
.height
|
||||
.difficultyepoch
|
||||
.read_once(Height::from(end_height))
|
||||
.collect_one(Height::from(end_height))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut epoch_to_height_iter = computer
|
||||
.indexes
|
||||
.difficultyepoch
|
||||
.first_height
|
||||
.iter();
|
||||
let mut epoch_to_timestamp_iter = computer.blocks.time.timestamp.difficultyepoch.iter();
|
||||
let mut epoch_to_difficulty_iter = computer.blocks.difficulty.raw.difficultyepoch.iter();
|
||||
let epoch_to_height = &computer.indexes.difficultyepoch.first_height;
|
||||
let epoch_to_timestamp = &computer.blocks.time.timestamp.difficultyepoch;
|
||||
let epoch_to_difficulty = &computer.blocks.difficulty.raw.difficultyepoch;
|
||||
|
||||
let mut results = Vec::with_capacity(end_epoch.to_usize() - start_epoch.to_usize() + 1);
|
||||
let mut prev_difficulty: Option<f64> = None;
|
||||
|
||||
for epoch_usize in start_epoch.to_usize()..=end_epoch.to_usize() {
|
||||
let epoch = DifficultyEpoch::from(epoch_usize);
|
||||
let epoch_height = epoch_to_height_iter.get(epoch).unwrap_or_default();
|
||||
let epoch_height = epoch_to_height.collect_one(epoch).unwrap_or_default();
|
||||
|
||||
// Skip epochs before our start height but track difficulty
|
||||
if epoch_height.to_usize() < start_height {
|
||||
prev_difficulty = epoch_to_difficulty_iter.get(epoch).map(|d| *d);
|
||||
prev_difficulty = epoch_to_difficulty.collect_one(epoch).map(|d| *d);
|
||||
continue;
|
||||
}
|
||||
|
||||
let epoch_timestamp = epoch_to_timestamp_iter.get(epoch).unwrap_or_default();
|
||||
let epoch_difficulty = *epoch_to_difficulty_iter.get(epoch).unwrap_or_default();
|
||||
let epoch_timestamp = epoch_to_timestamp.collect_one(epoch).unwrap_or_default();
|
||||
let epoch_difficulty = *epoch_to_difficulty.collect_one(epoch).unwrap_or_default();
|
||||
|
||||
let change_percent = match prev_difficulty {
|
||||
Some(prev) if prev > 0.0 => ((epoch_difficulty / prev) - 1.0) * 100.0,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{DateIndex, DifficultyEntry, HashrateEntry, HashrateSummary, Height, TimePeriod};
|
||||
use vecdb::{GenericStoredVec, IterableVec, VecIndex};
|
||||
use brk_types::{Day1, DifficultyEntry, HashrateEntry, HashrateSummary, Height, TimePeriod};
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use super::epochs::iter_difficulty_epochs;
|
||||
use crate::Query;
|
||||
@@ -12,21 +12,23 @@ impl Query {
|
||||
let current_height = self.height();
|
||||
|
||||
// Get current difficulty
|
||||
let current_difficulty = *indexer.vecs.blocks.difficulty.read_once(current_height)?;
|
||||
let current_difficulty = *indexer.vecs.blocks.difficulty.collect_one(current_height).unwrap();
|
||||
|
||||
// Get current hashrate
|
||||
let current_dateindex = computer
|
||||
let current_day1 = computer
|
||||
.indexes
|
||||
.height
|
||||
.dateindex
|
||||
.read_once(current_height)?;
|
||||
.day1
|
||||
.collect_one(current_height)
|
||||
.unwrap();
|
||||
|
||||
let current_hashrate = *computer
|
||||
.blocks
|
||||
.mining
|
||||
.hashrate
|
||||
.hash_rate
|
||||
.dateindex
|
||||
.read_once(current_dateindex)? as u128;
|
||||
.day1
|
||||
.collect_one(current_day1)
|
||||
.unwrap() as u128;
|
||||
|
||||
// Calculate start height based on time period
|
||||
let end = current_height.to_usize();
|
||||
@@ -36,31 +38,30 @@ impl Query {
|
||||
};
|
||||
|
||||
// Get hashrate entries using iterators for efficiency
|
||||
let start_dateindex = computer
|
||||
let start_day1 = computer
|
||||
.indexes
|
||||
.height
|
||||
.dateindex
|
||||
.read_once(Height::from(start))?;
|
||||
let end_dateindex = current_dateindex;
|
||||
.day1
|
||||
.collect_one(Height::from(start))
|
||||
.unwrap();
|
||||
let end_day1 = current_day1;
|
||||
|
||||
// Sample at regular intervals to avoid too many data points
|
||||
let total_days = end_dateindex
|
||||
let total_days = end_day1
|
||||
.to_usize()
|
||||
.saturating_sub(start_dateindex.to_usize())
|
||||
.saturating_sub(start_day1.to_usize())
|
||||
+ 1;
|
||||
let step = (total_days / 200).max(1); // Max ~200 data points
|
||||
|
||||
// Create iterators for the loop
|
||||
let mut hashrate_iter = computer.blocks.mining.hash_rate.dateindex.iter();
|
||||
|
||||
let mut timestamp_iter = computer.blocks.time.timestamp.dateindex.iter();
|
||||
let hashrate_vec = &computer.mining.hashrate.hash_rate.day1;
|
||||
let timestamp_vec = &computer.blocks.time.timestamp.day1;
|
||||
|
||||
let mut hashrates = Vec::with_capacity(total_days / step + 1);
|
||||
let mut di = start_dateindex.to_usize();
|
||||
while di <= end_dateindex.to_usize() {
|
||||
let dateindex = DateIndex::from(di);
|
||||
let mut di = start_day1.to_usize();
|
||||
while di <= end_day1.to_usize() {
|
||||
let day1 = Day1::from(di);
|
||||
if let (Some(hr), Some(timestamp)) =
|
||||
(hashrate_iter.get(dateindex), timestamp_iter.get(dateindex))
|
||||
(hashrate_vec.collect_one(day1), timestamp_vec.collect_one(day1))
|
||||
{
|
||||
hashrates.push(HashrateEntry {
|
||||
timestamp,
|
||||
|
||||
@@ -2,7 +2,7 @@ mod block_fee_rates;
|
||||
mod block_fees;
|
||||
mod block_rewards;
|
||||
mod block_sizes;
|
||||
mod dateindex_iter;
|
||||
mod day1_iter;
|
||||
mod difficulty;
|
||||
mod difficulty_adjustments;
|
||||
mod epochs;
|
||||
|
||||
@@ -3,7 +3,7 @@ use brk_types::{
|
||||
Height, PoolBlockCounts, PoolBlockShares, PoolDetail, PoolDetailInfo, PoolInfo, PoolSlug,
|
||||
PoolStats, PoolsSummary, TimePeriod, pools,
|
||||
};
|
||||
use vecdb::{AnyVec, IterableVec, VecIndex};
|
||||
use vecdb::{AnyVec, ReadableVec, VecIndex};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -30,18 +30,16 @@ impl Query {
|
||||
|
||||
// For each pool, get cumulative count at end and start, subtract to get range count
|
||||
for (pool_id, pool_vecs) in &computer.pools.vecs {
|
||||
let mut cumulative = pool_vecs
|
||||
let cumulative = &pool_vecs
|
||||
.blocks_mined
|
||||
.height_cumulative
|
||||
.inner()
|
||||
.iter();
|
||||
.height_cumulative;
|
||||
|
||||
let count_at_end: u32 = *cumulative.get(current_height).unwrap_or_default();
|
||||
let count_at_end: u32 = *cumulative.collect_one(current_height).unwrap_or_default();
|
||||
|
||||
let count_at_start: u32 = if start == 0 {
|
||||
0
|
||||
} else {
|
||||
*cumulative.get(Height::from(start - 1)).unwrap_or_default()
|
||||
*cumulative.collect_one(Height::from(start - 1)).unwrap_or_default()
|
||||
};
|
||||
|
||||
let block_count = count_at_end.saturating_sub(count_at_start);
|
||||
@@ -100,14 +98,12 @@ impl Query {
|
||||
.get(&slug)
|
||||
.ok_or_else(|| Error::NotFound("Pool data not found".into()))?;
|
||||
|
||||
let mut cumulative = pool_vecs
|
||||
let cumulative = &pool_vecs
|
||||
.blocks_mined
|
||||
.height_cumulative
|
||||
.inner()
|
||||
.iter();
|
||||
.height_cumulative;
|
||||
|
||||
// Get total blocks (all time)
|
||||
let total_all: u32 = *cumulative.get(current_height).unwrap_or_default();
|
||||
let total_all: u32 = *cumulative.collect_one(current_height).unwrap_or_default();
|
||||
|
||||
// Get blocks for 24h (144 blocks)
|
||||
let start_24h = end.saturating_sub(144);
|
||||
@@ -115,7 +111,7 @@ impl Query {
|
||||
0
|
||||
} else {
|
||||
*cumulative
|
||||
.get(Height::from(start_24h - 1))
|
||||
.collect_one(Height::from(start_24h - 1))
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let total_24h = total_all.saturating_sub(count_before_24h);
|
||||
@@ -126,7 +122,7 @@ impl Query {
|
||||
0
|
||||
} else {
|
||||
*cumulative
|
||||
.get(Height::from(start_1w - 1))
|
||||
.collect_one(Height::from(start_1w - 1))
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let total_1w = total_all.saturating_sub(count_before_1w);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::{Height, RewardStats, Sats};
|
||||
use vecdb::{IterableVec, VecIndex};
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -12,39 +12,16 @@ impl Query {
|
||||
let end_block = current_height;
|
||||
let start_block = Height::from(current_height.to_usize().saturating_sub(block_count - 1));
|
||||
|
||||
let mut coinbase_iter = computer.blocks.rewards.coinbase.sats.height.iter();
|
||||
let coinbase_vec = &computer.mining.rewards.coinbase.sats.height;
|
||||
let fee_vec = &computer.transactions.fees.fee.sats.height.sum_cum.sum.0;
|
||||
let tx_count_vec = &computer.transactions.count.tx_count.height;
|
||||
|
||||
let mut fee_iter = computer
|
||||
.transactions
|
||||
.fees
|
||||
.fee
|
||||
.sats
|
||||
.height
|
||||
.sum_cum
|
||||
.sum
|
||||
.0
|
||||
.iter();
|
||||
let mut tx_count_iter = computer.transactions.count.tx_count.height.iter();
|
||||
let start = start_block.to_usize();
|
||||
let end = end_block.to_usize() + 1;
|
||||
|
||||
let mut total_reward = Sats::ZERO;
|
||||
let mut total_fee = Sats::ZERO;
|
||||
let mut total_tx: u64 = 0;
|
||||
|
||||
for height in start_block.to_usize()..=end_block.to_usize() {
|
||||
let h = Height::from(height);
|
||||
|
||||
if let Some(coinbase) = coinbase_iter.get(h) {
|
||||
total_reward += coinbase;
|
||||
}
|
||||
|
||||
if let Some(fee) = fee_iter.get(h) {
|
||||
total_fee += fee;
|
||||
}
|
||||
|
||||
if let Some(tx_count) = tx_count_iter.get(h) {
|
||||
total_tx += *tx_count;
|
||||
}
|
||||
}
|
||||
let total_reward = coinbase_vec.fold_range_at(start, end, Sats::ZERO, |acc, v| acc + v);
|
||||
let total_fee = fee_vec.fold_range_at(start, end, Sats::ZERO, |acc, v| acc + v);
|
||||
let total_tx = tx_count_vec.fold_range_at(start, end, 0u64, |acc, v| acc + *v);
|
||||
|
||||
Ok(RewardStats {
|
||||
start_block,
|
||||
|
||||
@@ -3,7 +3,6 @@ mod block;
|
||||
mod cost_basis;
|
||||
mod mempool;
|
||||
mod metrics;
|
||||
mod metrics_legacy;
|
||||
mod mining;
|
||||
mod price;
|
||||
mod transaction;
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_error::Result;
|
||||
use brk_types::Dollars;
|
||||
|
||||
use crate::Query;
|
||||
|
||||
impl Query {
|
||||
pub fn live_price(&self) -> Result<Dollars> {
|
||||
let oracle_vecs = &self
|
||||
.computer()
|
||||
.price
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::OutOfRange("Oracle prices not computed yet".into()))?
|
||||
.oracle;
|
||||
|
||||
let mut oracle = oracle_vecs.live_oracle(self.indexer())?;
|
||||
let mut oracle = self.computer().prices.cents.live_oracle(self.indexer())?;
|
||||
|
||||
if let Some(mempool) = self.mempool() {
|
||||
let txs = mempool.get_txs();
|
||||
|
||||
@@ -6,7 +6,7 @@ use brk_types::{
|
||||
Sats, Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxOutspend, TxStatus, Txid, TxidParam,
|
||||
TxidPrefix, Vin, Vout, Weight,
|
||||
};
|
||||
use vecdb::{GenericStoredVec, TypedVecIterator};
|
||||
use vecdb::{ReadableVec, VecIndex};
|
||||
|
||||
use crate::Query;
|
||||
|
||||
@@ -55,9 +55,9 @@ impl Query {
|
||||
};
|
||||
|
||||
// Get block info for status
|
||||
let height = indexer.vecs.transactions.height.read_once(txindex)?;
|
||||
let height = indexer.vecs.transactions.height.collect_one(txindex).unwrap();
|
||||
let block_hash = indexer.vecs.blocks.blockhash.read_once(height)?;
|
||||
let block_time = indexer.vecs.blocks.timestamp.read_once(height)?;
|
||||
let block_time = indexer.vecs.blocks.timestamp.collect_one(height).unwrap();
|
||||
|
||||
Ok(TxStatus {
|
||||
confirmed: true,
|
||||
@@ -120,11 +120,7 @@ impl Query {
|
||||
|
||||
// Look up spend status
|
||||
let computer = self.computer();
|
||||
let txinindex = computer
|
||||
.outputs
|
||||
.spent
|
||||
.txinindex
|
||||
.read_once(txoutindex)?;
|
||||
let txinindex = computer.outputs.spent.txinindex.read_once(txoutindex)?;
|
||||
|
||||
if txinindex == TxInIndex::UNSPENT {
|
||||
return Ok(TxOutspend::UNSPENT);
|
||||
@@ -169,13 +165,12 @@ impl Query {
|
||||
|
||||
// Get spend status for each output
|
||||
let computer = self.computer();
|
||||
let mut txoutindex_to_txinindex_iter =
|
||||
computer.outputs.spent.txinindex.iter()?;
|
||||
let txinindex_reader = computer.outputs.spent.txinindex.reader();
|
||||
|
||||
let mut outspends = Vec::with_capacity(output_count);
|
||||
for i in 0..output_count {
|
||||
let txoutindex = first_txoutindex + Vout::from(i);
|
||||
let txinindex = txoutindex_to_txinindex_iter.get_unwrap(txoutindex);
|
||||
let txinindex = txinindex_reader.get(usize::from(txoutindex));
|
||||
|
||||
if txinindex == TxInIndex::UNSPENT {
|
||||
outspends.push(TxOutspend::UNSPENT);
|
||||
@@ -194,22 +189,23 @@ impl Query {
|
||||
let reader = self.reader();
|
||||
let computer = self.computer();
|
||||
|
||||
// Get tx metadata using read_once for single lookups
|
||||
// Get tx metadata using collect_one for PcoVec, read_once for BytesVec
|
||||
let txid = indexer.vecs.transactions.txid.read_once(txindex)?;
|
||||
let height = indexer.vecs.transactions.height.read_once(txindex)?;
|
||||
let version = indexer.vecs.transactions.txversion.read_once(txindex)?;
|
||||
let lock_time = indexer.vecs.transactions.rawlocktime.read_once(txindex)?;
|
||||
let total_size = indexer.vecs.transactions.total_size.read_once(txindex)?;
|
||||
let height = indexer.vecs.transactions.height.collect_one(txindex).unwrap();
|
||||
let version = indexer.vecs.transactions.txversion.collect_one(txindex).unwrap();
|
||||
let lock_time = indexer.vecs.transactions.rawlocktime.collect_one(txindex).unwrap();
|
||||
let total_size = indexer.vecs.transactions.total_size.collect_one(txindex).unwrap();
|
||||
let first_txinindex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txinindex
|
||||
.read_once(txindex)?;
|
||||
let position = computer.positions.tx_position.read_once(txindex)?;
|
||||
.collect_one(txindex)
|
||||
.unwrap();
|
||||
let position = computer.positions.tx_position.collect_one(txindex).unwrap();
|
||||
|
||||
// Get block info for status
|
||||
let block_hash = indexer.vecs.blocks.blockhash.read_once(height)?;
|
||||
let block_time = indexer.vecs.blocks.timestamp.read_once(height)?;
|
||||
let block_time = indexer.vecs.blocks.timestamp.collect_one(height).unwrap();
|
||||
|
||||
// Read and decode the raw transaction from blk file
|
||||
let buffer = reader.read_raw_bytes(position, *total_size as usize)?;
|
||||
@@ -217,12 +213,16 @@ impl Query {
|
||||
let tx = bitcoin::Transaction::consensus_decode(&mut cursor)
|
||||
.map_err(|_| Error::Parse("Failed to decode transaction".into()))?;
|
||||
|
||||
// For iterating through inputs, we need iterators (multiple lookups)
|
||||
let mut txindex_to_txid_iter = indexer.vecs.transactions.txid.iter()?;
|
||||
let mut txindex_to_first_txoutindex_iter =
|
||||
indexer.vecs.transactions.first_txoutindex.iter()?;
|
||||
let mut txinindex_to_outpoint_iter = indexer.vecs.inputs.outpoint.iter()?;
|
||||
let mut txoutindex_to_value_iter = indexer.vecs.outputs.value.iter()?;
|
||||
// Create readers for random access lookups
|
||||
let txid_reader = indexer.vecs.transactions.txid.reader();
|
||||
let first_txoutindex_reader = indexer.vecs.transactions.first_txoutindex.reader();
|
||||
let value_reader = indexer.vecs.outputs.value.reader();
|
||||
|
||||
// Batch-read outpoints for all inputs (avoids per-input PcoVec page decompression)
|
||||
let outpoints: Vec<_> = indexer.vecs.inputs.outpoint.collect_range_at(
|
||||
usize::from(first_txinindex),
|
||||
usize::from(first_txinindex) + tx.input.len(),
|
||||
);
|
||||
|
||||
// Build inputs with prevout information
|
||||
let input: Vec<TxIn> = tx
|
||||
@@ -230,8 +230,7 @@ impl Query {
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, txin)| {
|
||||
let txinindex = first_txinindex + i;
|
||||
let outpoint = txinindex_to_outpoint_iter.get_unwrap(txinindex);
|
||||
let outpoint = outpoints[i];
|
||||
|
||||
let is_coinbase = outpoint.is_coinbase();
|
||||
|
||||
@@ -241,15 +240,14 @@ impl Query {
|
||||
} else {
|
||||
let prev_txindex = outpoint.txindex();
|
||||
let prev_vout = outpoint.vout();
|
||||
let prev_txid = txindex_to_txid_iter.get_unwrap(prev_txindex);
|
||||
let prev_txid = txid_reader.get(prev_txindex.to_usize());
|
||||
|
||||
// Calculate the txoutindex for the prevout
|
||||
let prev_first_txoutindex =
|
||||
txindex_to_first_txoutindex_iter.get_unwrap(prev_txindex);
|
||||
let prev_first_txoutindex = first_txoutindex_reader.get(prev_txindex.to_usize());
|
||||
let prev_txoutindex = prev_first_txoutindex + prev_vout;
|
||||
|
||||
// Get the value of the prevout
|
||||
let prev_value = txoutindex_to_value_iter.get_unwrap(prev_txoutindex);
|
||||
let prev_value = value_reader.get(usize::from(prev_txoutindex));
|
||||
|
||||
let prevout = Some(TxOut::from((
|
||||
bitcoin::ScriptBuf::new(), // Placeholder - would need to reconstruct
|
||||
@@ -314,8 +312,8 @@ impl Query {
|
||||
let reader = self.reader();
|
||||
let computer = self.computer();
|
||||
|
||||
let total_size = indexer.vecs.transactions.total_size.read_once(txindex)?;
|
||||
let position = computer.positions.tx_position.read_once(txindex)?;
|
||||
let total_size = indexer.vecs.transactions.total_size.collect_one(txindex).unwrap();
|
||||
let position = computer.positions.tx_position.collect_one(txindex).unwrap();
|
||||
|
||||
let buffer = reader.read_raw_bytes(position, *total_size as usize)?;
|
||||
|
||||
@@ -326,14 +324,15 @@ impl Query {
|
||||
let indexer = self.indexer();
|
||||
|
||||
// Look up spending txindex directly
|
||||
let spending_txindex = indexer.vecs.inputs.txindex.read_once(txinindex)?;
|
||||
let spending_txindex = indexer.vecs.inputs.txindex.collect_one(txinindex).unwrap();
|
||||
|
||||
// Calculate vin
|
||||
let spending_first_txinindex = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txinindex
|
||||
.read_once(spending_txindex)?;
|
||||
.collect_one(spending_txindex)
|
||||
.unwrap();
|
||||
let vin = Vin::from(usize::from(txinindex) - usize::from(spending_first_txinindex));
|
||||
|
||||
// Get spending tx details
|
||||
@@ -342,9 +341,10 @@ impl Query {
|
||||
.vecs
|
||||
.transactions
|
||||
.height
|
||||
.read_once(spending_txindex)?;
|
||||
.collect_one(spending_txindex)
|
||||
.unwrap();
|
||||
let block_hash = indexer.vecs.blocks.blockhash.read_once(spending_height)?;
|
||||
let block_time = indexer.vecs.blocks.timestamp.read_once(spending_height)?;
|
||||
let block_time = indexer.vecs.blocks.timestamp.collect_one(spending_height).unwrap();
|
||||
|
||||
Ok(TxOutspend {
|
||||
spent: true,
|
||||
|
||||
Reference in New Issue
Block a user