Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 15:19:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: blk metadata fixes

Browse Source
This commit is contained in:
nym21
2025-09-19 16:45:57 +02:00
parent 43117825d7
commit 23f6397a97
18 changed files with 197 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
crates/brk_computer/src/positions.rs → crates/brk_computer/src/blks.rs
View File

@@ -3,8 +3,7 @@ use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::{BlkPosition, Height, TxIndex, Version};
use log::info;
use brk_structs::{BlkPosition, Height, StoredU32, TxIndex, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
GenericStoredVec, PAGE_SIZE, VecIterator,
@@ -17,12 +16,14 @@ pub struct Vecs {
db: Database,
pub height_to_position: CompressedVec<Height, BlkPosition>,
pub height_to_len: CompressedVec<Height, StoredU32>,
pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
pub txindex_to_len: CompressedVec<TxIndex, StoredU32>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("positions"))?;
let db = Database::open(&parent_path.join("blks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let version = parent_version + Version::ZERO;
@@ -31,13 +32,15 @@ impl Vecs {
height_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
version + Version::TWO,
)?,
height_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
txindex_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
version + Version::TWO,
)?,
txindex_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
db,
};
@@ -75,48 +78,66 @@ impl Vecs {
let min_txindex =
TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
let min_height = indexes
let Some(min_height) = indexes
.txindex_to_height
.iter()
.unwrap_get_inner(min_txindex)
.min(starting_indexes.height);
.get_inner(min_txindex)
.map(|h| h.min(starting_indexes.height))
else {
return Ok(());
};
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
parser
.parse(Some(min_height), None)
.parse(
Some(min_height),
Some((indexer.vecs.height_to_first_txindex.len() - 1).into()),
)
.iter()
.try_for_each(|block| -> Result<()> {
let height = block.height();
info!("{height}");
self.height_to_position.forced_push_at(
height,
block.metadata().position(),
exit,
)?;
self.height_to_position
.forced_push_at(height, *block.position(), exit)?;
self.height_to_len
.forced_push_at(height, block.metadata().len().into(), exit)?;
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
block.tx_positions().iter().enumerate().try_for_each(
|(index, position)| -> Result<()> {
block.tx_metadata().iter().enumerate().try_for_each(
|(index, metadata)| -> Result<()> {
let txindex = txindex + index;
self.txindex_to_position
.forced_push_at(txindex, *position, exit)?;
self.txindex_to_position.forced_push_at(
txindex,
metadata.position(),
exit,
)?;
self.txindex_to_len
.forced_push_at(txindex, metadata.len().into(), exit)?;
Ok(())
},
)?;
// Stuck, why ??
if *height % 1_000 == 0 {
let _lock = exit.lock();
self.height_to_position.flush()?;
self.height_to_len.flush()?;
self.txindex_to_position.flush()?;
self.txindex_to_len.flush()?;
}
Ok(())
})?;
let _lock = exit.lock();
self.height_to_position.flush()?;
self.height_to_len.flush()?;
self.txindex_to_position.flush()?;
self.txindex_to_len.flush()?;
Ok(())
}
@@ -125,7 +146,9 @@ impl Vecs {
Box::new(
[
&self.height_to_position as &dyn AnyCollectableVec,
&self.height_to_len,
&self.txindex_to_position,
&self.txindex_to_len,
]
.into_iter(),
)

20
crates/brk_computer/src/lib.rs
View File

@@ -10,6 +10,7 @@ use brk_structs::Version;
use log::info;
use vecdb::{AnyCollectableVec, Exit, Format};
mod blks;
mod chain;
mod cointime;
mod constants;
@@ -18,7 +19,6 @@ mod grouped;
mod indexes;
mod market;
mod pools;
mod positions;
mod price;
mod stateful;
mod states;
@@ -40,7 +40,7 @@ pub struct Computer {
pub indexes: indexes::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub positions: positions::Vecs,
pub blks: blks::Vecs,
pub price: Option<price::Vecs>,
pub stateful: stateful::Vecs,
}
@@ -90,7 +90,7 @@ impl Computer {
&indexes,
price.as_ref(),
)?,
positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
blks: blks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
@@ -132,8 +132,8 @@ impl Computer {
)?;
}
info!("Computing positions...");
self.positions
info!("Computing BLKs metadata...");
self.blks
.compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
@@ -144,9 +144,9 @@ impl Computer {
Ok(())
});
// let positions = scope.spawn(|| -> Result<()> {
// info!("Computing positions...");
// self.positions
// let blks = scope.spawn(|| -> Result<()> {
// info!("Computing blks...");
// self.blks
// .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
// Ok(())
// });
@@ -169,7 +169,7 @@ impl Computer {
}
constants.join().unwrap()?;
// positions.join().unwrap()?;
// blks.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;
@@ -216,7 +216,7 @@ impl Computer {
iter = Box::new(iter.chain(self.indexes.iter_any_collectable()));
iter = Box::new(iter.chain(self.market.iter_any_collectable()));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.positions.iter_any_collectable()));
iter = Box::new(iter.chain(self.blks.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.stateful.iter_any_collectable()));