From 5a3e1b4e6e6f2a6a6a9b2497abf819428d97a34b Mon Sep 17 00:00:00 2001 From: nym21 Date: Thu, 9 Apr 2026 14:02:26 +0200 Subject: [PATCH] global: speed improvement part2 --- crates/brk_client/src/lib.rs | 4 - crates/brk_computer/src/prices/compute.rs | 2 +- .../src/transactions/count/import.rs | 19 +-- .../src/transactions/count/vecs.rs | 5 +- .../src/transactions/fees/compute.rs | 10 +- .../brk_computer/src/transactions/import.rs | 2 +- crates/brk_indexer/src/processor/mod.rs | 3 +- crates/brk_indexer/src/processor/tx.rs | 4 +- crates/brk_indexer/src/vecs/blocks.rs | 12 +- crates/brk_indexer/src/vecs/transactions.rs | 8 -- crates/brk_oracle/examples/compare_digits.rs | 2 +- crates/brk_oracle/examples/determinism.rs | 2 +- crates/brk_oracle/examples/noise.rs | 2 +- crates/brk_oracle/examples/report.rs | 2 +- crates/brk_oracle/examples/sweep_digits.rs | 2 +- crates/brk_oracle/examples/sweep_tolerance.rs | 2 +- crates/brk_oracle/examples/validate.rs | 2 +- crates/brk_query/src/impl/addr.rs | 26 ++-- crates/brk_query/src/impl/block/timestamp.rs | 6 +- crates/brk_query/src/impl/block/txs.rs | 119 +++++++++--------- crates/brk_query/src/impl/tx.rs | 37 +++--- crates/brk_server/src/state.rs | 20 ++- modules/brk-client/index.js | 4 - packages/brk_client/brk_client/__init__.py | 2 - 24 files changed, 137 insertions(+), 160 deletions(-) diff --git a/crates/brk_client/src/lib.rs b/crates/brk_client/src/lib.rs index 676896395..47f4f275c 100644 --- a/crates/brk_client/src/lib.rs +++ b/crates/brk_client/src/lib.rs @@ -3464,7 +3464,6 @@ impl SeriesTree_Transactions { /// Series tree node. pub struct SeriesTree_Transactions_Raw { pub first_tx_index: SeriesPattern18, - pub height: SeriesPattern19, pub txid: SeriesPattern19, pub tx_version: SeriesPattern19, pub raw_locktime: SeriesPattern19, @@ -3479,7 +3478,6 @@ impl SeriesTree_Transactions_Raw { pub fn new(client: Arc, base_path: String) -> Self { Self { first_tx_index: SeriesPattern18::new(client.clone(), "first_tx_index".to_string()), - height: SeriesPattern19::new(client.clone(), "height".to_string()), txid: SeriesPattern19::new(client.clone(), "txid".to_string()), tx_version: SeriesPattern19::new(client.clone(), "tx_version".to_string()), raw_locktime: SeriesPattern19::new(client.clone(), "raw_locktime".to_string()), @@ -3495,14 +3493,12 @@ impl SeriesTree_Transactions_Raw { /// Series tree node. pub struct SeriesTree_Transactions_Count { pub total: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern, - pub is_coinbase: SeriesPattern19, } impl SeriesTree_Transactions_Count { pub fn new(client: Arc, base_path: String) -> Self { Self { total: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new(client.clone(), "tx_count".to_string()), - is_coinbase: SeriesPattern19::new(client.clone(), "is_coinbase".to_string()), } } } diff --git a/crates/brk_computer/src/prices/compute.rs b/crates/brk_computer/src/prices/compute.rs index 746583cf5..e983317a9 100644 --- a/crates/brk_computer/src/prices/compute.rs +++ b/crates/brk_computer/src/prices/compute.rs @@ -157,7 +157,7 @@ impl Vecs { indexer: &Indexer, range: Range, ) -> Vec { - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); // Pre-collect height-indexed data for the range (plus one extra for next-block lookups) diff --git a/crates/brk_computer/src/transactions/count/import.rs b/crates/brk_computer/src/transactions/count/import.rs index 37c56ba5a..8648dbb85 100644 --- a/crates/brk_computer/src/transactions/count/import.rs +++ b/crates/brk_computer/src/transactions/count/import.rs @@ -1,7 +1,6 @@ use brk_error::Result; -use brk_indexer::Indexer; -use brk_types::{StoredBool, TxIndex, Version}; -use vecdb::{Database, LazyVecFrom2, ReadableCloneableVec}; +use brk_types::Version; +use vecdb::Database; use super::Vecs; use crate::{ @@ -13,25 +12,11 @@ impl Vecs { pub(crate) fn forced_import( db: &Database, version: Version, - indexer: &Indexer, indexes: &indexes::Vecs, cached_starts: &CachedWindowStarts, ) -> Result { - let tx_index_to_is_coinbase = LazyVecFrom2::init( - "is_coinbase", - version, - indexer.vecs.transactions.height.read_only_boxed_clone(), - indexer - .vecs - .transactions - .first_tx_index - .read_only_boxed_clone(), - |index: TxIndex, _height, first_tx_index| StoredBool::from(index == first_tx_index), - ); - Ok(Self { total: PerBlockFull::forced_import(db, "tx_count", version, indexes, cached_starts)?, - is_coinbase: tx_index_to_is_coinbase, }) } } diff --git a/crates/brk_computer/src/transactions/count/vecs.rs b/crates/brk_computer/src/transactions/count/vecs.rs index 7719959f6..3b9c1e438 100644 --- a/crates/brk_computer/src/transactions/count/vecs.rs +++ b/crates/brk_computer/src/transactions/count/vecs.rs @@ -1,11 +1,10 @@ use brk_traversable::Traversable; -use brk_types::{Height, StoredBool, StoredU64, TxIndex}; -use vecdb::{LazyVecFrom2, Rw, StorageMode}; +use brk_types::StoredU64; +use vecdb::{Rw, StorageMode}; use crate::internal::PerBlockFull; #[derive(Traversable)] pub struct Vecs { pub total: PerBlockFull, - pub is_coinbase: LazyVecFrom2, } diff --git a/crates/brk_computer/src/transactions/fees/compute.rs b/crates/brk_computer/src/transactions/fees/compute.rs index 0ec200689..bb0b2a2f1 100644 --- a/crates/brk_computer/src/transactions/fees/compute.rs +++ b/crates/brk_computer/src/transactions/fees/compute.rs @@ -1,6 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{FeeRate, Indexes, OutPoint, Sats, TxInIndex, VSize}; +use brk_types::{FeeRate, Indexes, OutPoint, Sats, TxInIndex, TxIndex, VSize}; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely}; use super::super::size; @@ -125,11 +125,9 @@ impl Vecs { let start_height = if start_tx == 0 { 0 } else { - indexer - .vecs - .transactions - .height - .collect_one_at(start_tx) + indexes + .tx_heights + .get_shared(TxIndex::from(start_tx)) .unwrap() .to_usize() }; diff --git a/crates/brk_computer/src/transactions/import.rs b/crates/brk_computer/src/transactions/import.rs index de1753b39..81c9c275a 100644 --- a/crates/brk_computer/src/transactions/import.rs +++ b/crates/brk_computer/src/transactions/import.rs @@ -25,7 +25,7 @@ impl Vecs { let db = open_db(parent_path, super::DB_NAME, 10_000_000)?; let version = parent_version; - let count = CountVecs::forced_import(&db, version, indexer, indexes, cached_starts)?; + let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?; let size = SizeVecs::forced_import(&db, version, indexer, indexes)?; let fees = FeesVecs::forced_import(&db, version, indexes)?; let versions = VersionsVecs::forced_import(&db, version, indexes, cached_starts)?; diff --git a/crates/brk_indexer/src/processor/mod.rs b/crates/brk_indexer/src/processor/mod.rs index 7f3981175..d57393b22 100644 --- a/crates/brk_indexer/src/processor/mod.rs +++ b/crates/brk_indexer/src/processor/mod.rs @@ -43,7 +43,6 @@ impl BlockProcessor<'_> { already_added: &mut ByAddrType>, same_block_info: &mut FxHashMap, ) -> Result<()> { - let height = self.height; let indexes = &mut *self.indexes; // Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest @@ -85,7 +84,7 @@ impl BlockProcessor<'_> { same_block_info, ) }, - || tx::store_tx_metadata(height, txs, txid_prefix_store, &mut tx_metadata), + || tx::store_tx_metadata(txs, txid_prefix_store, &mut tx_metadata), ); finalize_result?; diff --git a/crates/brk_indexer/src/processor/tx.rs b/crates/brk_indexer/src/processor/tx.rs index 2cd7a06d0..1d5e44293 100644 --- a/crates/brk_indexer/src/processor/tx.rs +++ b/crates/brk_indexer/src/processor/tx.rs @@ -1,6 +1,6 @@ use brk_error::{Error, Result}; use brk_store::Store; -use brk_types::{Height, StoredBool, TxIndex, Txid, TxidPrefix}; +use brk_types::{StoredBool, TxIndex, Txid, TxidPrefix}; use rayon::prelude::*; use tracing::error; use vecdb::{AnyVec, WritableVec, likely}; @@ -89,7 +89,6 @@ impl<'a> BlockProcessor<'a> { } pub(super) fn store_tx_metadata( - height: Height, txs: Vec, store: &mut Store, md: &mut TxMetadataVecs<'_>, @@ -98,7 +97,6 @@ pub(super) fn store_tx_metadata( if ct.prev_tx_index_opt.is_none() { store.insert(ct.txid_prefix, ct.tx_index); } - md.height.checked_push(ct.tx_index, height)?; md.tx_version .checked_push(ct.tx_index, ct.tx.version.into())?; md.txid.checked_push(ct.tx_index, ct.txid)?; diff --git a/crates/brk_indexer/src/vecs/blocks.rs b/crates/brk_indexer/src/vecs/blocks.rs index acce76f83..c6274ed4e 100644 --- a/crates/brk_indexer/src/vecs/blocks.rs +++ b/crates/brk_indexer/src/vecs/blocks.rs @@ -6,7 +6,8 @@ use brk_types::{ }; use rayon::prelude::*; use vecdb::{ - AnyStoredVec, BytesVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec, + AnyStoredVec, BytesVec, CachedVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, + WritableVec, }; use crate::parallel_import; @@ -14,12 +15,16 @@ use crate::parallel_import; #[derive(Traversable)] pub struct BlocksVecs { pub blockhash: M::Stored>, + #[traversable(skip)] + pub cached_blockhash: CachedVec, pub coinbase_tag: M::Stored>, #[traversable(wrap = "difficulty", rename = "value")] pub difficulty: M::Stored>, /// Doesn't guarantee continuity due to possible reorgs and more generally the nature of mining #[traversable(wrap = "time")] pub timestamp: M::Stored>, + #[traversable(skip)] + pub cached_timestamp: CachedVec, #[traversable(wrap = "size", rename = "base")] pub total: M::Stored>, #[traversable(wrap = "weight", rename = "base")] @@ -56,11 +61,16 @@ impl BlocksVecs { segwit_size = PcoVec::forced_import(db, "segwit_size", version), segwit_weight = PcoVec::forced_import(db, "segwit_weight", version), }; + let cached_blockhash = CachedVec::new(&blockhash); + let cached_timestamp = CachedVec::new(×tamp); + Ok(Self { blockhash, + cached_blockhash, coinbase_tag, difficulty, timestamp, + cached_timestamp, total, weight, position, diff --git a/crates/brk_indexer/src/vecs/transactions.rs b/crates/brk_indexer/src/vecs/transactions.rs index d97055ac2..2b10b3ba4 100644 --- a/crates/brk_indexer/src/vecs/transactions.rs +++ b/crates/brk_indexer/src/vecs/transactions.rs @@ -14,7 +14,6 @@ use crate::parallel_import; #[derive(Traversable)] pub struct TransactionsVecs { pub first_tx_index: M::Stored>, - pub height: M::Stored>, pub txid: M::Stored>, pub tx_version: M::Stored>, pub raw_locktime: M::Stored>, @@ -28,7 +27,6 @@ pub struct TransactionsVecs { } pub struct TxMetadataVecs<'a> { - pub height: &'a mut PcoVec, pub tx_version: &'a mut PcoVec, pub txid: &'a mut BytesVec, pub raw_locktime: &'a mut PcoVec, @@ -49,7 +47,6 @@ impl TransactionsVecs { &mut self.first_txout_index, &mut self.first_txin_index, TxMetadataVecs { - height: &mut self.height, tx_version: &mut self.tx_version, txid: &mut self.txid, raw_locktime: &mut self.raw_locktime, @@ -63,7 +60,6 @@ impl TransactionsVecs { pub fn forced_import(db: &Database, version: Version) -> Result { let ( first_tx_index, - height, txid, tx_version, raw_locktime, @@ -75,7 +71,6 @@ impl TransactionsVecs { position, ) = parallel_import! { first_tx_index = PcoVec::forced_import(db, "first_tx_index", version), - height = PcoVec::forced_import(db, "height", version), txid = BytesVec::forced_import(db, "txid", version), tx_version = PcoVec::forced_import(db, "tx_version", version), raw_locktime = PcoVec::forced_import(db, "raw_locktime", version), @@ -88,7 +83,6 @@ impl TransactionsVecs { }; Ok(Self { first_tx_index, - height, txid, tx_version, raw_locktime, @@ -104,7 +98,6 @@ impl TransactionsVecs { pub fn truncate(&mut self, height: Height, tx_index: TxIndex, stamp: Stamp) -> Result<()> { self.first_tx_index .truncate_if_needed_with_stamp(height, stamp)?; - self.height.truncate_if_needed_with_stamp(tx_index, stamp)?; self.txid.truncate_if_needed_with_stamp(tx_index, stamp)?; self.tx_version .truncate_if_needed_with_stamp(tx_index, stamp)?; @@ -128,7 +121,6 @@ impl TransactionsVecs { pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator { [ &mut self.first_tx_index as &mut dyn AnyStoredVec, - &mut self.height, &mut self.txid, &mut self.tx_version, &mut self.raw_locktime, diff --git a/crates/brk_oracle/examples/compare_digits.rs b/crates/brk_oracle/examples/compare_digits.rs index 6d3431ff2..71ca15596 100644 --- a/crates/brk_oracle/examples/compare_digits.rs +++ b/crates/brk_oracle/examples/compare_digits.rs @@ -150,7 +150,7 @@ fn main() { let idx = |m: usize, s: usize| -> usize { m * start_heights.len() + s }; - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); diff --git a/crates/brk_oracle/examples/determinism.rs b/crates/brk_oracle/examples/determinism.rs index 0a6a1f0fa..57a54e1a8 100644 --- a/crates/brk_oracle/examples/determinism.rs +++ b/crates/brk_oracle/examples/determinism.rs @@ -46,7 +46,7 @@ fn main() { let config = Config::default(); let window_size = config.window_size; - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); diff --git a/crates/brk_oracle/examples/noise.rs b/crates/brk_oracle/examples/noise.rs index 0ca9e7dff..aae9d09c3 100644 --- a/crates/brk_oracle/examples/noise.rs +++ b/crates/brk_oracle/examples/noise.rs @@ -91,7 +91,7 @@ fn main() { // Build per-block RAW histograms from the lowest start height. eprintln!("Building histograms from height {}...", lowest); - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader(); diff --git a/crates/brk_oracle/examples/report.rs b/crates/brk_oracle/examples/report.rs index 227435a74..375e67f42 100644 --- a/crates/brk_oracle/examples/report.rs +++ b/crates/brk_oracle/examples/report.rs @@ -181,7 +181,7 @@ fn main() { let config = Config::default(); let mut oracle = Oracle::new(cents_to_bin(start_price * 100.0), config); - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); // Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too large. diff --git a/crates/brk_oracle/examples/sweep_digits.rs b/crates/brk_oracle/examples/sweep_digits.rs index 81ac05602..a022cfff7 100644 --- a/crates/brk_oracle/examples/sweep_digits.rs +++ b/crates/brk_oracle/examples/sweep_digits.rs @@ -163,7 +163,7 @@ fn main() { // Phase 1: precompute per-block data in a single pass over the indexer. eprintln!("Phase 1: precomputing block data..."); - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); diff --git a/crates/brk_oracle/examples/sweep_tolerance.rs b/crates/brk_oracle/examples/sweep_tolerance.rs index 71059b0f0..d33a57fdc 100644 --- a/crates/brk_oracle/examples/sweep_tolerance.rs +++ b/crates/brk_oracle/examples/sweep_tolerance.rs @@ -161,7 +161,7 @@ fn main() { // filter at different tolerance thresholds in Phase 2. eprintln!("Phase 1: precomputing block data..."); - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); diff --git a/crates/brk_oracle/examples/validate.rs b/crates/brk_oracle/examples/validate.rs index ae9bfad51..07941c731 100644 --- a/crates/brk_oracle/examples/validate.rs +++ b/crates/brk_oracle/examples/validate.rs @@ -148,7 +148,7 @@ fn main() { ]; // Build per-block filtered histograms from the indexer, feeding all oracles in one pass. - let total_txs = indexer.vecs.transactions.height.len(); + let total_txs = indexer.vecs.transactions.txid.len(); let total_outputs = indexer.vecs.outputs.value.len(); // Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too large. diff --git a/crates/brk_query/src/impl/addr.rs b/crates/brk_query/src/impl/addr.rs index bc5c660e9..8d9ef4916 100644 --- a/crates/brk_query/src/impl/addr.rs +++ b/crates/brk_query/src/impl/addr.rs @@ -4,7 +4,7 @@ use bitcoin::{Network, PublicKey, ScriptBuf}; use brk_error::{Error, OptionData, Result}; use brk_types::{ Addr, AddrBytes, AddrChainStats, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, AddrStats, - AnyAddrDataIndexEnum, BlockHash, Dollars, Height, OutputType, Timestamp, Transaction, TxIndex, + AnyAddrDataIndexEnum, Dollars, Height, OutputType, Transaction, TxIndex, TxStatus, Txid, TypeIndex, Unit, Utxo, Vout, }; use vecdb::{ReadableVec, VecIndex}; @@ -189,11 +189,8 @@ impl Query { let txid_reader = vecs.transactions.txid.reader(); let first_txout_index_reader = vecs.transactions.first_txout_index.reader(); let value_reader = vecs.outputs.value.reader(); - let blockhash_reader = vecs.blocks.blockhash.reader(); let tx_heights = &self.computer().indexes.tx_heights; - let mut block_ts_cursor = vecs.blocks.timestamp.cursor(); - let mut cached_block: Option<(Height, BlockHash, Timestamp)> = None; let mut utxos = Vec::with_capacity(outpoints.len()); for (tx_index, vout) in outpoints { @@ -202,16 +199,8 @@ impl Query { let first_txout_index = first_txout_index_reader.get(tx_index.to_usize()); let value = value_reader.get(usize::from(first_txout_index + vout)); - let (block_hash, block_time) = if let Some((h, ref bh, bt)) = cached_block - && h == height - { - (bh.clone(), bt) - } else { - let bh = blockhash_reader.get(height.to_usize()); - let bt = block_ts_cursor.get(height.to_usize()).data()?; - cached_block = Some((height, bh.clone(), bt)); - (bh, bt) - }; + let block_hash = vecs.blocks.cached_blockhash.collect_one(height).data()?; + let block_time = vecs.blocks.cached_timestamp.collect_one(height).data()?; utxos.push(Utxo { txid, @@ -268,11 +257,10 @@ impl Query { .next_back() .map(|(key, _): (AddrIndexTxIndex, Unit)| key.tx_index()) .ok_or(Error::UnknownAddr)?; - self.indexer() - .vecs - .transactions - .height - .collect_one(last_tx_index) + self.computer() + .indexes + .tx_heights + .get_shared(last_tx_index) .ok_or(Error::UnknownAddr) } diff --git a/crates/brk_query/src/impl/block/timestamp.rs b/crates/brk_query/src/impl/block/timestamp.rs index f5d1090a6..dd77f4595 100644 --- a/crates/brk_query/src/impl/block/timestamp.rs +++ b/crates/brk_query/src/impl/block/timestamp.rs @@ -60,9 +60,9 @@ impl Query { let blockhash = indexer .vecs .blocks - .blockhash - .reader() - .get(usize::from(height)); + .cached_blockhash + .collect_one(height) + .data()?; // Convert timestamp to ISO 8601 format let ts_secs: i64 = (*best_ts).into(); diff --git a/crates/brk_query/src/impl/block/txs.rs b/crates/brk_query/src/impl/block/txs.rs index 95235b253..bb18dc6d5 100644 --- a/crates/brk_query/src/impl/block/txs.rs +++ b/crates/brk_query/src/impl/block/txs.rs @@ -4,7 +4,7 @@ use bitcoin::{consensus::Decodable, hex::DisplayHex}; use brk_error::{Error, OptionData, Result}; use brk_types::{ BlkPosition, BlockHash, Height, OutPoint, OutputType, RawLockTime, Sats, StoredU32, Timestamp, - Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxOutIndex, TxStatus, Txid, Vout, Weight, + Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxStatus, Txid, TypeIndex, Vout, Weight, }; use rustc_hash::FxHashMap; use vecdb::{AnyVec, ReadableVec, VecIndex}; @@ -75,6 +75,7 @@ impl Query { return Ok(Vec::new()); } + let t0 = std::time::Instant::now(); let len = indices.len(); // Sort positions ascending for sequential I/O (O(n) when already sorted) @@ -92,9 +93,6 @@ impl Query { let mut total_size_cursor = indexer.vecs.transactions.total_size.cursor(); let mut first_txin_cursor = indexer.vecs.transactions.first_txin_index.cursor(); let mut position_cursor = indexer.vecs.transactions.position.cursor(); - let blockhash_reader = indexer.vecs.blocks.blockhash.reader(); - let mut block_ts_cursor = indexer.vecs.blocks.timestamp.cursor(); - struct DecodedTx { pos: usize, tx_index: TxIndex, @@ -109,7 +107,6 @@ impl Query { outpoints: Vec, } - let mut cached_block: Option<(Height, BlockHash, Timestamp)> = None; let mut decoded_txs: Vec = Vec::with_capacity(len); let mut total_inputs: usize = 0; @@ -125,16 +122,18 @@ impl Query { let first_txin_index: TxInIndex = first_txin_cursor.get(idx).data()?; let position: BlkPosition = position_cursor.get(idx).data()?; - let (block_hash, block_time) = if let Some((h, ref bh, bt)) = cached_block - && h == height - { - (bh.clone(), bt) - } else { - let bh = blockhash_reader.get(height.to_usize()); - let bt = block_ts_cursor.get(height.to_usize()).data()?; - cached_block = Some((height, bh.clone(), bt)); - (bh, bt) - }; + let block_hash = indexer + .vecs + .blocks + .cached_blockhash + .collect_one(height) + .data()?; + let block_time = indexer + .vecs + .blocks + .cached_timestamp + .collect_one(height) + .data()?; let buffer = reader.read_raw_bytes(position, *total_size as usize)?; let decoded = bitcoin::Transaction::consensus_decode(&mut Cursor::new(buffer)) @@ -157,19 +156,39 @@ impl Query { }); } - // Phase 1b: Batch-read outpoints via cursor (PcoVec — sequential - // cursor avoids re-decompressing the same pages) + let t_phase1a = t0.elapsed(); + + // Phase 1b: Batch-read outpoints + prevout data via cursors (PcoVec — + // sequential cursor avoids re-decompressing the same pages). + // Reading output_type/type_index/value HERE from inputs vecs (sequential) + // avoids random-reading them from outputs vecs in Phase 2. let mut outpoint_cursor = indexer.vecs.inputs.outpoint.cursor(); + let mut input_output_type_cursor = indexer.vecs.inputs.output_type.cursor(); + let mut input_type_index_cursor = indexer.vecs.inputs.type_index.cursor(); + let mut input_value_cursor = self.computer().inputs.spent.value.cursor(); + + let mut prevout_input_data: FxHashMap = + FxHashMap::with_capacity_and_hasher(total_inputs, Default::default()); + for dtx in &mut decoded_txs { let start = usize::from(dtx.first_txin_index); let count = dtx.decoded.input.len(); let mut outpoints = Vec::with_capacity(count); for i in 0..count { - outpoints.push(outpoint_cursor.get(start + i).data()?); + let op: OutPoint = outpoint_cursor.get(start + i).data()?; + if op.is_not_coinbase() { + let ot: OutputType = input_output_type_cursor.get(start + i).data()?; + let ti: TypeIndex = input_type_index_cursor.get(start + i).data()?; + let val: Sats = input_value_cursor.get(start + i).data()?; + prevout_input_data.insert(op, (ot, ti, val)); + } + outpoints.push(op); } dtx.outpoints = outpoints; } + let t_phase1b = t0.elapsed(); + // ── Phase 2: Batch-read prevout data in sorted order ──────────── // Collect all non-coinbase outpoints, deduplicate, sort by tx_index @@ -184,53 +203,25 @@ impl Query { prevout_keys.sort_unstable(); prevout_keys.dedup(); - // Batch-read txid + first_txout_index sorted by prev_tx_index + // Batch-read txid sorted by prev_tx_index (only remaining random read) let txid_reader = indexer.vecs.transactions.txid.reader(); - let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader(); - struct PrevoutIntermediate { - outpoint: OutPoint, - txid: Txid, - txout_index: TxOutIndex, - } - - let mut intermediates: Vec = Vec::with_capacity(prevout_keys.len()); - - for &op in &prevout_keys { - let prev_tx_idx = op.tx_index().to_usize(); - let txid = txid_reader.get(prev_tx_idx); - let first_txout = first_txout_index_reader.get(prev_tx_idx); - let txout_index = first_txout + op.vout(); - intermediates.push(PrevoutIntermediate { - outpoint: op, - txid, - txout_index, - }); - } - - // Re-sort by txout_index for sequential output data reads - intermediates.sort_unstable_by_key(|i| i.txout_index); - - let value_reader = indexer.vecs.outputs.value.reader(); - let output_type_reader = indexer.vecs.outputs.output_type.reader(); - let type_index_reader = indexer.vecs.outputs.type_index.reader(); let addr_readers = indexer.vecs.addrs.addr_readers(); let mut prevout_map: FxHashMap = - FxHashMap::with_capacity_and_hasher(intermediates.len(), Default::default()); + FxHashMap::with_capacity_and_hasher(prevout_keys.len(), Default::default()); - for inter in &intermediates { - let txout_idx = usize::from(inter.txout_index); - let value: Sats = value_reader.get(txout_idx); - let output_type: OutputType = output_type_reader.get(txout_idx); - let type_index = type_index_reader.get(txout_idx); + for &op in &prevout_keys { + let txid = txid_reader.get(op.tx_index().to_usize()); + // output_type, type_index, value pre-read from inputs vecs (sequential) + let &(output_type, type_index, value) = + prevout_input_data.get(&op).unwrap(); let script_pubkey = addr_readers.script_pubkey(output_type, type_index); - prevout_map.insert( - inter.outpoint, - (inter.txid.clone(), TxOut::from((script_pubkey, value))), - ); + prevout_map.insert(op, (txid, TxOut::from((script_pubkey, value)))); } + let t_phase2 = t0.elapsed(); + // ── Phase 3: Assemble Transaction objects ─────────────────────── let mut txs: Vec> = (0..len).map(|_| None).collect(); @@ -320,6 +311,22 @@ impl Query { txs[dtx.pos] = Some(transaction); } + let t_phase3 = t0.elapsed(); + + if t_phase3.as_millis() > 50 { + eprintln!( + "[perf:txs] n={} vin={} prevouts={} | 1a={:.1?} 1b={:.1?} | 2={:.1?} | 3={:.1?} | total={:.1?}", + len, + total_inputs, + prevout_keys.len(), + t_phase1a, + t_phase1b - t_phase1a, + t_phase2 - t_phase1b, + t_phase3 - t_phase2, + t_phase3, + ); + } + Ok(txs.into_iter().map(Option::unwrap).collect()) } diff --git a/crates/brk_query/src/impl/tx.rs b/crates/brk_query/src/impl/tx.rs index 3e6e07b19..f04722ab7 100644 --- a/crates/brk_query/src/impl/tx.rs +++ b/crates/brk_query/src/impl/tx.rs @@ -1,7 +1,7 @@ use bitcoin::hex::{DisplayHex, FromHex}; use brk_error::{Error, OptionData, Result}; use brk_types::{ - BlockHash, Height, MerkleProof, Timestamp, Transaction, TxInIndex, TxIndex, TxOutIndex, + Height, MerkleProof, Transaction, TxInIndex, TxIndex, TxOutIndex, TxOutspend, TxStatus, Txid, TxidPrefix, Vin, Vout, }; use vecdb::{ReadableVec, VecIndex}; @@ -59,13 +59,8 @@ impl Query { .tx_heights .get_shared(tx_index) .data()?; - let block_hash = indexer - .vecs - .blocks - .blockhash - .reader() - .get(height.to_usize()); - let block_time = indexer.vecs.blocks.timestamp.collect_one(height).data()?; + let block_hash = indexer.vecs.blocks.cached_blockhash.collect_one(height).data()?; + let block_time = indexer.vecs.blocks.cached_timestamp.collect_one(height).data()?; Ok(TxStatus { confirmed: true, @@ -144,14 +139,10 @@ impl Query { let indexer = self.indexer(); let txin_index_reader = self.computer().outputs.spent.txin_index.reader(); let txid_reader = indexer.vecs.transactions.txid.reader(); - let blockhash_reader = indexer.vecs.blocks.blockhash.reader(); let tx_heights = &self.computer().indexes.tx_heights; let mut input_tx_cursor = indexer.vecs.inputs.tx_index.cursor(); let mut first_txin_cursor = indexer.vecs.transactions.first_txin_index.cursor(); - let mut block_ts_cursor = indexer.vecs.blocks.timestamp.cursor(); - - let mut cached_block: Option<(Height, BlockHash, Timestamp)> = None; let mut outspends = Vec::with_capacity(output_count); for i in 0..output_count { @@ -168,16 +159,18 @@ impl Query { let spending_txid = txid_reader.get(spending_tx_index.to_usize()); let spending_height = tx_heights.get_shared(spending_tx_index).data()?; - let (block_hash, block_time) = if let Some((h, ref bh, bt)) = cached_block - && h == spending_height - { - (bh.clone(), bt) - } else { - let bh = blockhash_reader.get(spending_height.to_usize()); - let bt = block_ts_cursor.get(spending_height.to_usize()).data()?; - cached_block = Some((spending_height, bh.clone(), bt)); - (bh, bt) - }; + let block_hash = indexer + .vecs + .blocks + .cached_blockhash + .collect_one(spending_height) + .data()?; + let block_time = indexer + .vecs + .blocks + .cached_timestamp + .collect_one(spending_height) + .data()?; outspends.push(TxOutspend { spent: true, diff --git a/crates/brk_server/src/state.rs b/crates/brk_server/src/state.rs index 047366a65..c67b91e1d 100644 --- a/crates/brk_server/src/state.rs +++ b/crates/brk_server/src/state.rs @@ -189,8 +189,26 @@ impl AppState { F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, { self.cached(headers, strategy, uri, "application/json", move |q, enc| { + let t0 = std::time::Instant::now(); let value = f(q)?; - Ok(enc.compress(Bytes::from(serde_json::to_vec(&value).unwrap()))) + let t_query = t0.elapsed(); + let json = serde_json::to_vec(&value).unwrap(); + let t_json = t0.elapsed(); + let json_len = json.len(); + let compressed = enc.compress(Bytes::from(json)); + let t_total = t0.elapsed(); + if t_total.as_millis() > 100 { + eprintln!( + "[perf] query={:.1?} json={:.1?}({:.1}MB) compress={:.1?}({}) total={:.1?}", + t_query, + t_json - t_query, + json_len as f64 / 1_048_576.0, + t_total - t_json, + enc.as_str(), + t_total, + ); + } + Ok(compressed) }) .await } diff --git a/modules/brk-client/index.js b/modules/brk-client/index.js index ae7acecba..df0121344 100644 --- a/modules/brk-client/index.js +++ b/modules/brk-client/index.js @@ -4539,7 +4539,6 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Transactions_Raw * @property {SeriesPattern18} firstTxIndex - * @property {SeriesPattern19} height * @property {SeriesPattern19} txid * @property {SeriesPattern19} txVersion * @property {SeriesPattern19} rawLocktime @@ -4553,7 +4552,6 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Transactions_Count * @property {AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern} total - * @property {SeriesPattern19} isCoinbase */ /** @@ -7921,7 +7919,6 @@ class BrkClient extends BrkClientBase { transactions: { raw: { firstTxIndex: createSeriesPattern18(this, 'first_tx_index'), - height: createSeriesPattern19(this, 'height'), txid: createSeriesPattern19(this, 'txid'), txVersion: createSeriesPattern19(this, 'tx_version'), rawLocktime: createSeriesPattern19(this, 'raw_locktime'), @@ -7933,7 +7930,6 @@ class BrkClient extends BrkClientBase { }, count: { total: createAverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern(this, 'tx_count'), - isCoinbase: createSeriesPattern19(this, 'is_coinbase'), }, size: { vsize: create_6bBlockTxPattern(this, 'tx_vsize'), diff --git a/packages/brk_client/brk_client/__init__.py b/packages/brk_client/brk_client/__init__.py index 9edff8754..45ebac05e 100644 --- a/packages/brk_client/brk_client/__init__.py +++ b/packages/brk_client/brk_client/__init__.py @@ -3776,7 +3776,6 @@ class SeriesTree_Transactions_Raw: def __init__(self, client: BrkClientBase, base_path: str = ''): self.first_tx_index: SeriesPattern18[TxIndex] = SeriesPattern18(client, 'first_tx_index') - self.height: SeriesPattern19[Height] = SeriesPattern19(client, 'height') self.txid: SeriesPattern19[Txid] = SeriesPattern19(client, 'txid') self.tx_version: SeriesPattern19[TxVersion] = SeriesPattern19(client, 'tx_version') self.raw_locktime: SeriesPattern19[RawLockTime] = SeriesPattern19(client, 'raw_locktime') @@ -3791,7 +3790,6 @@ class SeriesTree_Transactions_Count: def __init__(self, client: BrkClientBase, base_path: str = ''): self.total: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern = AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern(client, 'tx_count') - self.is_coinbase: SeriesPattern19[StoredBool] = SeriesPattern19(client, 'is_coinbase') class SeriesTree_Transactions_Size_Weight: """Series tree node."""