Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-13 08:23:32 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: blk metadata fixes

Browse Source
This commit is contained in:
nym21
2025-09-19 16:45:57 +02:00
parent 43117825d7
commit 23f6397a97
18 changed files with 197 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_cli/Cargo.toml
+2 -3
View File
@@ -20,14 +20,13 @@ brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_server = { workspace = true }
vecdb = { workspace = true }
clap = { version = "4.5.47", features = ["string"] }
clap_derive = "4.5.47"
clap = { version = "4.5.47", features = ["derive", "string"] }
color-eyre = "0.6.5"
log = { workspace = true }
minreq = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.6"
toml = "0.9.7"
zip = { version = "5.1.1", default-features = false, features = ["deflate"] }
[[bin]]
crates/brk_cli/src/website.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use clap_derive::ValueEnum;
use clap::ValueEnum;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, ValueEnum)]
crates/brk_computer/Cargo.toml
-3
View File
@@ -11,7 +11,6 @@ build = "build.rs"
[dependencies]
allocative = { workspace = true }
allocative_derive = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_structs = { workspace = true }
@@ -23,8 +22,6 @@ brk_store = { workspace = true }
brk_parser = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
inferno = "0.12.3"
jiff = { workspace = true }
log = { workspace = true }
num_enum = "0.7.4"
pco = "0.4.6"
crates/brk_computer/src/positions.rs → crates/brk_computer/src/blks.rs
+40 -17
View File
@@ -3,8 +3,7 @@ use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::{BlkPosition, Height, TxIndex, Version};
use log::info;
use brk_structs::{BlkPosition, Height, StoredU32, TxIndex, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
GenericStoredVec, PAGE_SIZE, VecIterator,
@@ -17,12 +16,14 @@ pub struct Vecs {
db: Database,
pub height_to_position: CompressedVec<Height, BlkPosition>,
pub height_to_len: CompressedVec<Height, StoredU32>,
pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
pub txindex_to_len: CompressedVec<TxIndex, StoredU32>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("positions"))?;
let db = Database::open(&parent_path.join("blks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let version = parent_version + Version::ZERO;
@@ -31,13 +32,15 @@ impl Vecs {
height_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
version + Version::TWO,
)?,
height_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
txindex_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
version + Version::TWO,
)?,
txindex_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
db,
};
@@ -75,48 +78,66 @@ impl Vecs {
let min_txindex =
TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
let min_height = indexes
let Some(min_height) = indexes
.txindex_to_height
.iter()
.unwrap_get_inner(min_txindex)
.min(starting_indexes.height);
.get_inner(min_txindex)
.map(|h| h.min(starting_indexes.height))
else {
return Ok(());
};
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
parser
.parse(Some(min_height), None)
.parse(
Some(min_height),
Some((indexer.vecs.height_to_first_txindex.len() - 1).into()),
)
.iter()
.try_for_each(|block| -> Result<()> {
let height = block.height();
info!("{height}");
self.height_to_position.forced_push_at(
height,
block.metadata().position(),
exit,
)?;
self.height_to_position
.forced_push_at(height, *block.position(), exit)?;
self.height_to_len
.forced_push_at(height, block.metadata().len().into(), exit)?;
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
block.tx_positions().iter().enumerate().try_for_each(
|(index, position)| -> Result<()> {
block.tx_metadata().iter().enumerate().try_for_each(
|(index, metadata)| -> Result<()> {
let txindex = txindex + index;
self.txindex_to_position
.forced_push_at(txindex, *position, exit)?;
self.txindex_to_position.forced_push_at(
txindex,
metadata.position(),
exit,
)?;
self.txindex_to_len
.forced_push_at(txindex, metadata.len().into(), exit)?;
Ok(())
},
)?;
// Stuck, why ??
if *height % 1_000 == 0 {
let _lock = exit.lock();
self.height_to_position.flush()?;
self.height_to_len.flush()?;
self.txindex_to_position.flush()?;
self.txindex_to_len.flush()?;
}
Ok(())
})?;
let _lock = exit.lock();
self.height_to_position.flush()?;
self.height_to_len.flush()?;
self.txindex_to_position.flush()?;
self.txindex_to_len.flush()?;
Ok(())
}
@@ -125,7 +146,9 @@ impl Vecs {
Box::new(
[
&self.height_to_position as &dyn AnyCollectableVec,
&self.height_to_len,
&self.txindex_to_position,
&self.txindex_to_len,
]
.into_iter(),
)
crates/brk_computer/src/lib.rs
+10 -10
View File
@@ -10,6 +10,7 @@ use brk_structs::Version;
use log::info;
use vecdb::{AnyCollectableVec, Exit, Format};
mod blks;
mod chain;
mod cointime;
mod constants;
@@ -18,7 +19,6 @@ mod grouped;
mod indexes;
mod market;
mod pools;
mod positions;
mod price;
mod stateful;
mod states;
@@ -40,7 +40,7 @@ pub struct Computer {
pub indexes: indexes::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub positions: positions::Vecs,
pub blks: blks::Vecs,
pub price: Option<price::Vecs>,
pub stateful: stateful::Vecs,
}
@@ -90,7 +90,7 @@ impl Computer {
&indexes,
price.as_ref(),
)?,
positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
blks: blks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
@@ -132,8 +132,8 @@ impl Computer {
)?;
}
info!("Computing positions...");
self.positions
info!("Computing BLKs metadata...");
self.blks
.compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
@@ -144,9 +144,9 @@ impl Computer {
Ok(())
});
// let positions = scope.spawn(|| -> Result<()> {
// info!("Computing positions...");
// self.positions
// let blks = scope.spawn(|| -> Result<()> {
// info!("Computing blks...");
// self.blks
// .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
// Ok(())
// });
@@ -169,7 +169,7 @@ impl Computer {
}
constants.join().unwrap()?;
// positions.join().unwrap()?;
// blks.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;
@@ -216,7 +216,7 @@ impl Computer {
iter = Box::new(iter.chain(self.indexes.iter_any_collectable()));
iter = Box::new(iter.chain(self.market.iter_any_collectable()));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.positions.iter_any_collectable()));
iter = Box::new(iter.chain(self.blks.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.stateful.iter_any_collectable()));
crates/brk_parser/Cargo.toml
-3
View File
@@ -20,6 +20,3 @@ crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
derive_deref = { workspace = true }
parking_lot = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }
crates/brk_parser/examples/main.rs
+1 -1
View File
@@ -26,7 +26,7 @@ fn main() -> Result<()> {
let mut diff = BTreeMap::new();
parser.parse(start, end).iter().for_each(|block| {
println!("{}: {}", block.height(), block.hash());
let new_blk_index = block.position().blk_index();
let new_blk_index = block.metadata().blk_index();
if new_blk_index < blk_index {
diff.insert(blk_index - new_blk_index, block.height());
}
crates/brk_parser/src/any_block.rs
+8 -6
View File
@@ -1,7 +1,7 @@
use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor};
use bitcoincore_rpc::RpcApi;
use brk_error::Result;
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use brk_structs::{BlkMetadata, Block, Height, ParsedBlock};
use crate::{XORBytes, XORIndex};
@@ -14,7 +14,7 @@ pub enum AnyBlock {
impl AnyBlock {
pub fn decode(
self,
position: BlkPosition,
metadata: BlkMetadata,
rpc: &'static bitcoincore_rpc::Client,
mut xor_i: XORIndex,
xor_bytes: XORBytes,
@@ -55,17 +55,19 @@ impl AnyBlock {
}
let mut txdata = Vec::with_capacity(tx_count as usize);
let mut tx_positions = Vec::with_capacity(tx_count as usize);
let mut tx_metadata = Vec::with_capacity(tx_count as usize);
for _ in 0..tx_count {
let tx_position = BlkPosition::new(position.blk_index(), cursor.position() as u32);
tx_positions.push(tx_position);
let offset = cursor.position() as u32;
let position = metadata.position() + offset;
let tx = Transaction::consensus_decode(&mut cursor)?;
txdata.push(tx);
let len = cursor.position() as u32 - offset;
tx_metadata.push(BlkMetadata::new(position, len));
}
let block = bitcoin::Block { header, txdata };
let block = Block::from((height, hash, block));
let block = ParsedBlock::from((block, position, tx_positions));
let block = ParsedBlock::from((block, metadata, tx_metadata));
Ok(Self::Decoded(block))
}
crates/brk_parser/src/lib.rs
+8 -7
View File
@@ -15,7 +15,7 @@ use bitcoin::{block::Header, consensus::Decodable};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use brk_error::Result;
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use brk_structs::{BlkMetadata, BlkPosition, Block, Height, ParsedBlock};
use crossbeam::channel::{Receiver, bounded};
use parking_lot::{RwLock, RwLockReadGuard};
use rayon::prelude::*;
@@ -123,8 +123,6 @@ impl Parser {
}
}
let position = BlkPosition::new(blk_index, i as u32);
let len = u32::from_le_bytes(
xor_i
.bytes(&mut blk_bytes[i..(i + 4)], xor_bytes)
@@ -133,10 +131,13 @@ impl Parser {
) as usize;
i += 4;
let position = BlkPosition::new(blk_index, i as u32);
let metadata = BlkMetadata::new(position, len as u32);
let block_bytes = (blk_bytes[i..(i + len)]).to_vec();
if send_bytes
.send((position, AnyBlock::Raw(block_bytes), xor_i))
.send((metadata, AnyBlock::Raw(block_bytes), xor_i))
.is_err()
{
return ControlFlow::Break(());
@@ -156,13 +157,13 @@ impl Parser {
let mut bulk = vec![];
let drain_and_send = |bulk: &mut Vec<(BlkPosition, AnyBlock, XORIndex)>| {
let drain_and_send = |bulk: &mut Vec<(BlkMetadata, AnyBlock, XORIndex)>| {
// Using a vec and sending after to not end up with stuck threads in par iter
mem::take(bulk)
.into_par_iter()
.try_for_each(|(position, any_block, xor_i)| {
.try_for_each(|(metdata, any_block, xor_i)| {
if let Ok(AnyBlock::Decoded(block)) =
any_block.decode(position, rpc, xor_i, xor_bytes, start, end)
any_block.decode(metdata, rpc, xor_i, xor_bytes, start, end)
&& send_block.send(block).is_err()
{
return ControlFlow::Break(());
crates/brk_server/src/api/mod.rs
+18 -5
View File
@@ -1,6 +1,6 @@
use std::{
fs::File,
io::{BufReader, Seek, SeekFrom},
io::{Cursor, Read, Seek, SeekFrom},
str::FromStr,
};
@@ -162,6 +162,11 @@ impl ApiRoutes for Router<AppState> {
.map(|opt| opt.map(|cow| cow.into_owned())) else {
return "Unknown transaction".into_response();
};
let txid = indexer
.vecs
.txindex_to_txid
.iter()
.unwrap_get_inner(txindex);
let version = indexer
.vecs
.txindex_to_txversion
@@ -177,7 +182,8 @@ impl ApiRoutes for Router<AppState> {
let parser = interface.parser();
let computer = interface.computer();
let position = computer.positions.txindex_to_position.iter().unwrap_get_inner(txindex);
let position = computer.blks.txindex_to_position.iter().unwrap_get_inner(txindex);
let len = computer.blks.txindex_to_len.iter().unwrap_get_inner(txindex);
let blk_index_to_blk_path = parser.blk_index_to_blk_path();
@@ -188,14 +194,21 @@ impl ApiRoutes for Router<AppState> {
let mut xori = XORIndex::default();
xori.add_assign(position.offset() as usize);
let Ok(file) = File::open(blk_path) else {
let Ok(mut file) = File::open(blk_path) else {
return "Error opening blk file".into_response();
};
let mut reader = BufReader::new(file);
if reader.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
if file.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
return "Error seeking position in blk file".into_response();
}
let mut buffer = vec![0u8; *len as usize];
if file.read_exact(&mut buffer).is_err() {
return "File fail read exact".into_response();
}
xori.bytes(&mut buffer, parser.xor_bytes());
let mut reader = Cursor::new(buffer);
let Ok(tx) = Transaction::consensus_decode(&mut reader) else {
return "Error decoding transaction".into_response();
};
crates/brk_store/src/lib.rs
+16 -22
View File
@@ -25,12 +25,13 @@ use log::info;
use meta::*;
use parking_lot::RwLock;
#[derive(Clone)]
pub struct Store<Key, Value> {
meta: StoreMeta,
name: &'static str,
keyspace: TransactionalKeyspace,
partition: Arc<RwLock<Option<TransactionalPartitionHandle>>>,
rtx: ReadTransaction,
rtx: Arc<RwLock<Option<ReadTransaction>>>,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
bloom_filters: Option<bool>,
@@ -79,7 +80,7 @@ where
name: Box::leak(Box::new(name.to_string())),
keyspace: keyspace.clone(),
partition: Arc::new(RwLock::new(Some(partition))),
rtx,
rtx: Arc::new(RwLock::new(Some(rtx))),
puts: BTreeMap::new(),
dels: BTreeSet::new(),
bloom_filters,
@@ -91,6 +92,9 @@ where
Ok(Some(Cow::Borrowed(v)))
} else if let Some(slice) = self
.rtx
.read()
.as_ref()
.unwrap()
.get(self.partition.read().as_ref().unwrap(), ByteView::from(key))?
{
Ok(Some(Cow::Owned(V::from(ByteView::from(slice)))))
@@ -101,6 +105,9 @@ where
pub fn is_empty(&self) -> Result<bool> {
self.rtx
.read()
.as_ref()
.unwrap()
.is_empty(self.partition.read().as_ref().unwrap())
.map_err(|e| e.into())
}
@@ -129,6 +136,9 @@ where
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
self.rtx
.read()
.as_ref()
.unwrap()
.iter(self.partition.read().as_ref().unwrap())
.map(|res| res.unwrap())
.map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v))))
@@ -202,6 +212,9 @@ where
self.meta.export(height)?;
let mut rtx = self.rtx.write();
let _ = rtx.take();
let mut wtx = self.keyspace.write_tx();
let partition = self.partition.read();
@@ -228,7 +241,7 @@ where
wtx.commit()?;
self.rtx = self.keyspace.read_tx();
rtx.replace(self.keyspace.read_tx());
Ok(())
}
@@ -295,22 +308,3 @@ where
self.meta.version()
}
}
impl<Key, Value> Clone for Store<Key, Value>
where
Key: Clone,
Value: Clone,
{
fn clone(&self) -> Self {
Self {
meta: self.meta.clone(),
name: self.name,
keyspace: self.keyspace.clone(),
partition: self.partition.clone(),
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),
bloom_filters: self.bloom_filters,
}
}
}
crates/brk_structs/Cargo.toml
-1
View File
@@ -11,7 +11,6 @@ build = "build.rs"
[dependencies]
allocative = { workspace = true }
allocative_derive = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_error = {workspace = true}
crates/brk_structs/src/structs/blkmetadata.rs
+29
View File
@@ -0,0 +1,29 @@
use crate::BlkPosition;
#[derive(Debug)]
pub struct BlkMetadata {
position: BlkPosition,
len: u32,
}
impl BlkMetadata {
pub fn new(position: BlkPosition, len: u32) -> Self {
Self { position, len }
}
pub fn position(&self) -> BlkPosition {
self.position
}
pub fn blk_index(&self) -> u16 {
self.position.blk_index()
}
pub fn offset(&self) -> u32 {
self.position.offset()
}
#[allow(clippy::len_without_is_empty)]
pub fn len(&self) -> u32 {
self.len
}
}
crates/brk_structs/src/structs/blkposition.rs
+10 -1
View File
@@ -1,3 +1,5 @@
use std::ops::Add;
use serde::Serialize;
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -13,10 +15,17 @@ impl BlkPosition {
}
pub fn blk_index(&self) -> u16 {
(self.0 >> 31) as u16
(self.0 >> 32) as u16
}
pub fn offset(&self) -> u32 {
self.0 as u32
}
}
impl Add<u32> for BlkPosition {
type Output = Self;
fn add(self, rhs: u32) -> Self::Output {
Self(self.0 + rhs as u64)
}
}
crates/brk_structs/src/structs/block.rs
+11 -11
View File
@@ -1,6 +1,6 @@
use std::{borrow::Cow, ops::Deref};
use crate::BlkPosition;
use crate::BlkMetadata;
use super::{BlockHash, Height};
@@ -58,27 +58,27 @@ impl Deref for Block {
#[derive(Debug)]
pub struct ParsedBlock {
block: Block,
position: BlkPosition,
tx_positions: Vec<BlkPosition>,
metadata: BlkMetadata,
tx_metadata: Vec<BlkMetadata>,
}
impl From<(Block, BlkPosition, Vec<BlkPosition>)> for ParsedBlock {
fn from((block, position, tx_positions): (Block, BlkPosition, Vec<BlkPosition>)) -> Self {
impl From<(Block, BlkMetadata, Vec<BlkMetadata>)> for ParsedBlock {
fn from((block, metadata, tx_metadata): (Block, BlkMetadata, Vec<BlkMetadata>)) -> Self {
Self {
block,
position,
tx_positions,
metadata,
tx_metadata,
}
}
}
impl ParsedBlock {
pub fn position(&self) -> &BlkPosition {
&self.position
pub fn metadata(&self) -> &BlkMetadata {
&self.metadata
}
pub fn tx_positions(&self) -> &Vec<BlkPosition> {
&self.tx_positions
pub fn tx_metadata(&self) -> &Vec<BlkMetadata> {
&self.tx_metadata
}
}
crates/brk_structs/src/structs/mod.rs
+2
View File
@@ -4,6 +4,7 @@ mod addressbytes;
mod addressbyteshash;
mod anyaddressindex;
mod bitcoin;
mod blkmetadata;
mod blkposition;
mod block;
mod blockhash;
@@ -69,6 +70,7 @@ pub use addressbytes::*;
pub use addressbyteshash::*;
pub use anyaddressindex::*;
pub use bitcoin::*;
pub use blkmetadata::*;
pub use blkposition::*;
pub use block::*;
pub use blockhash::*;