Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: blk metadata fixes

Browse Source
This commit is contained in:
nym21
2025-09-19 16:45:57 +02:00
parent 43117825d7
commit 23f6397a97
18 changed files with 197 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
Cargo.lock generated
View File

@@ -540,7 +540,6 @@ dependencies = [
"brk_parser",
"brk_server",
"clap",
"clap_derive",
"color-eyre",
"log",
"minreq",
@@ -556,7 +555,6 @@ name = "brk_computer"
version = "0.0.107"
dependencies = [
"allocative",
"allocative_derive",
"bitcoin",
"bitcoincore-rpc",
"brk_error",
@@ -567,8 +565,6 @@ dependencies = [
"brk_store",
"brk_structs",
"derive_deref",
"inferno",
"jiff",
"log",
"num_enum",
"pco",
@@ -676,9 +672,6 @@ dependencies = [
"derive_deref",
"parking_lot 0.12.4",
"rayon",
"serde",
"serde_json",
"zerocopy",
]
[[package]]
@@ -755,7 +748,7 @@ dependencies = [
"css-module-lexer",
"dunce",
"futures",
"indexmap 2.11.3",
"indexmap 2.11.4",
"itertools",
"itoa",
"memchr",
@@ -1025,7 +1018,7 @@ dependencies = [
"fast-glob",
"form_urlencoded",
"futures",
"indexmap 2.11.3",
"indexmap 2.11.4",
"infer",
"itoa",
"memchr",
@@ -1112,7 +1105,6 @@ name = "brk_structs"
version = "0.0.107"
dependencies = [
"allocative",
"allocative_derive",
"bitcoin",
"bitcoincore-rpc",
"brk_error",
@@ -1163,12 +1155,6 @@ version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
[[package]]
name = "bytemuck"
version = "1.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677"
[[package]]
name = "byteorder"
version = "1.5.0"
@@ -1198,9 +1184,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.37"
version = "1.2.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44"
checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1815,9 +1801,9 @@ dependencies = [
[[package]]
name = "fancy-regex"
version = "0.16.1"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf04c5ec15464ace8355a7b440a33aece288993475556d461154d7a62ad9947c"
checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f"
dependencies = [
"bit-set",
"regex-automata",
@@ -1862,9 +1848,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d"
checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959"
[[package]]
name = "fixedbitset"
@@ -2372,12 +2358,12 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.11.3"
version = "2.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3"
checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
dependencies = [
"equivalent",
"hashbrown 0.15.5",
"hashbrown 0.16.0",
"serde",
"serde_core",
]
@@ -2391,28 +2377,6 @@ dependencies = [
"cfb",
]
[[package]]
name = "inferno"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e96d2465363ed2d81857759fc864cf6bb7997f79327aec028d65bd7989393685"
dependencies = [
"ahash",
"clap",
"crossbeam-channel",
"crossbeam-utils",
"dashmap",
"env_logger",
"indexmap 2.11.3",
"itoa",
"log",
"num-format",
"once_cell",
"quick-xml",
"rgb",
"str_stack",
]
[[package]]
name = "inotify"
version = "0.11.0"
@@ -2844,16 +2808,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-format"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
dependencies = [
"arrayvec",
"itoa",
]
[[package]]
name = "num-integer"
version = "0.1.46"
@@ -3249,7 +3203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49c4a4d746f42bac28538163952aa66da2f0ea781a0708772d8ad3f6fc066963"
dependencies = [
"cfg-if",
"indexmap 2.11.3",
"indexmap 2.11.4",
"json-strip-comments",
"libc",
"once_cell",
@@ -3344,7 +3298,7 @@ dependencies = [
"base64 0.22.1",
"compact_str",
"cow-utils",
"indexmap 2.11.3",
"indexmap 2.11.4",
"itoa",
"memchr",
"oxc-browserslist",
@@ -3531,7 +3485,7 @@ checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca"
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
"indexmap 2.11.3",
"indexmap 2.11.4",
"serde",
]
@@ -3714,15 +3668,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quick-xml"
version = "0.37.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb"
dependencies = [
"memchr",
]
[[package]]
name = "quick_cache"
version = "0.6.16"
@@ -3945,15 +3890,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "rgb"
version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce"
dependencies = [
"bytemuck",
]
[[package]]
name = "ring"
version = "0.17.14"
@@ -4155,12 +4091,11 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "seqdb"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0068ee8855b29d0251d0277ffb2c2d8872afe0a7124fc9f33a29102af1096f90"
checksum = "b31ad41314db97a84d637ce25b5323f46572f0a0e356a789fd1fa3e617c04bd0"
dependencies = [
"allocative",
"allocative_derive",
"libc",
"log",
"memmap2",
@@ -4227,7 +4162,7 @@ version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"indexmap 2.11.3",
"indexmap 2.11.4",
"itoa",
"memchr",
"ryu",
@@ -4248,9 +4183,9 @@ dependencies = [
[[package]]
name = "serde_spanned"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2789234a13a53fc4be1b51ea1bab45a3c338bdb884862a257d10e5a74ae009e6"
checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee"
dependencies = [
"serde_core",
]
@@ -4277,7 +4212,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.11.3",
"indexmap 2.11.4",
"schemars 0.9.0",
"schemars 1.0.4",
"serde",
@@ -4408,12 +4343,6 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6"
[[package]]
name = "str_stack"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -4582,11 +4511,12 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.43"
version = "0.3.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031"
checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d"
dependencies = [
"deranged",
"itoa",
"num-conv",
"powerfmt",
"serde",
@@ -4675,11 +4605,11 @@ dependencies = [
[[package]]
name = "toml"
version = "0.9.6"
version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae2a4cf385da23d1d53bc15cdfa5c2109e93d8d362393c801e87da2f72f0e201"
checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0"
dependencies = [
"indexmap 2.11.3",
"indexmap 2.11.4",
"serde_core",
"serde_spanned",
"toml_datetime",
@@ -4690,20 +4620,20 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.7.1"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a197c0ec7d131bfc6f7e82c8442ba1595aeab35da7adbf05b6b73cd06a16b6be"
checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1"
dependencies = [
"serde_core",
]
[[package]]
name = "toml_edit"
version = "0.23.5"
version = "0.23.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2ad0b7ae9cfeef5605163839cb9221f453399f15cfb5c10be9885fcf56611f9"
checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b"
dependencies = [
"indexmap 2.11.3",
"indexmap 2.11.4",
"toml_datetime",
"toml_parser",
"winnow",
@@ -4711,18 +4641,18 @@ dependencies = [
[[package]]
name = "toml_parser"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10"
checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627"
dependencies = [
"winnow",
]
[[package]]
name = "toml_writer"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64"
checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109"
[[package]]
name = "tower"
@@ -5021,17 +4951,15 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vecdb"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e834cf5275445308e2ab867847d059ac7603de71c35dbe6c197726d1db7c74b"
checksum = "fe63b28536d63d008e824ff8fd5a66147b97d9cd893d392e19d8450488cfc8b0"
dependencies = [
"allocative",
"allocative_derive",
"ctrlc",
"log",
"parking_lot 0.12.4",
"pco",
"rayon",
"seqdb",
"serde",
"serde_derive",
@@ -5043,9 +4971,9 @@ dependencies = [
[[package]]
name = "vecdb_derive"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9acc9a0b85d010ddcd2a4825d2a804520d9c52759bb9b0bc5ef7c4a1e7fce1"
checksum = "19d38e2229203d0a3eda93507f0bf3d1ceb67c85832508d48b0268281cda3f9c"
dependencies = [
"quote",
"syn 2.0.106",
@@ -5630,7 +5558,7 @@ dependencies = [
"arbitrary",
"crc32fast",
"flate2",
"indexmap 2.11.3",
"indexmap 2.11.4",
"memchr",
"zopfli",
]

3
Cargo.toml
View File

@@ -42,7 +42,6 @@ debug-assertions = false
[workspace.dependencies]
allocative = { version = "0.3.4", features = ["parking_lot"] }
allocative_derive = "0.3.3"
axum = "0.8.4"
bitcoin = { version = "0.32.7", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
@@ -74,7 +73,7 @@ serde_derive = "1.0.225"
serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
vecdb = { version = "0.2.14", features = ["derive"]}
vecdb = { version = "0.2.15", features = ["derive"]}
zerocopy = "0.8.27"
zerocopy-derive = "0.8.27"

5
crates/brk_cli/Cargo.toml
View File

@@ -20,14 +20,13 @@ brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_server = { workspace = true }
vecdb = { workspace = true }
clap = { version = "4.5.47", features = ["string"] }
clap_derive = "4.5.47"
clap = { version = "4.5.47", features = ["derive", "string"] }
color-eyre = "0.6.5"
log = { workspace = true }
minreq = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.6"
toml = "0.9.7"
zip = { version = "5.1.1", default-features = false, features = ["deflate"] }
[[bin]]

2
crates/brk_cli/src/website.rs
View File

@@ -1,4 +1,4 @@
use clap_derive::ValueEnum;
use clap::ValueEnum;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, ValueEnum)]

3
crates/brk_computer/Cargo.toml
View File

@@ -11,7 +11,6 @@ build = "build.rs"
[dependencies]
allocative = { workspace = true }
allocative_derive = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_structs = { workspace = true }
@@ -23,8 +22,6 @@ brk_store = { workspace = true }
brk_parser = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
inferno = "0.12.3"
jiff = { workspace = true }
log = { workspace = true }
num_enum = "0.7.4"
pco = "0.4.6"

57
crates/brk_computer/src/positions.rs → crates/brk_computer/src/blks.rs
View File

@@ -3,8 +3,7 @@ use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::{BlkPosition, Height, TxIndex, Version};
use log::info;
use brk_structs::{BlkPosition, Height, StoredU32, TxIndex, Version};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
GenericStoredVec, PAGE_SIZE, VecIterator,
@@ -17,12 +16,14 @@ pub struct Vecs {
db: Database,
pub height_to_position: CompressedVec<Height, BlkPosition>,
pub height_to_len: CompressedVec<Height, StoredU32>,
pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
pub txindex_to_len: CompressedVec<TxIndex, StoredU32>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("positions"))?;
let db = Database::open(&parent_path.join("blks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let version = parent_version + Version::ZERO;
@@ -31,13 +32,15 @@ impl Vecs {
height_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
version + Version::TWO,
)?,
height_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
txindex_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
version + Version::TWO,
)?,
txindex_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
db,
};
@@ -75,48 +78,66 @@ impl Vecs {
let min_txindex =
TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
let min_height = indexes
let Some(min_height) = indexes
.txindex_to_height
.iter()
.unwrap_get_inner(min_txindex)
.min(starting_indexes.height);
.get_inner(min_txindex)
.map(|h| h.min(starting_indexes.height))
else {
return Ok(());
};
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
parser
.parse(Some(min_height), None)
.parse(
Some(min_height),
Some((indexer.vecs.height_to_first_txindex.len() - 1).into()),
)
.iter()
.try_for_each(|block| -> Result<()> {
let height = block.height();
info!("{height}");
self.height_to_position.forced_push_at(
height,
block.metadata().position(),
exit,
)?;
self.height_to_position
.forced_push_at(height, *block.position(), exit)?;
self.height_to_len
.forced_push_at(height, block.metadata().len().into(), exit)?;
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
block.tx_positions().iter().enumerate().try_for_each(
|(index, position)| -> Result<()> {
block.tx_metadata().iter().enumerate().try_for_each(
|(index, metadata)| -> Result<()> {
let txindex = txindex + index;
self.txindex_to_position
.forced_push_at(txindex, *position, exit)?;
self.txindex_to_position.forced_push_at(
txindex,
metadata.position(),
exit,
)?;
self.txindex_to_len
.forced_push_at(txindex, metadata.len().into(), exit)?;
Ok(())
},
)?;
// Stuck, why ??
if *height % 1_000 == 0 {
let _lock = exit.lock();
self.height_to_position.flush()?;
self.height_to_len.flush()?;
self.txindex_to_position.flush()?;
self.txindex_to_len.flush()?;
}
Ok(())
})?;
let _lock = exit.lock();
self.height_to_position.flush()?;
self.height_to_len.flush()?;
self.txindex_to_position.flush()?;
self.txindex_to_len.flush()?;
Ok(())
}
@@ -125,7 +146,9 @@ impl Vecs {
Box::new(
[
&self.height_to_position as &dyn AnyCollectableVec,
&self.height_to_len,
&self.txindex_to_position,
&self.txindex_to_len,
]
.into_iter(),
)

20
crates/brk_computer/src/lib.rs
View File

@@ -10,6 +10,7 @@ use brk_structs::Version;
use log::info;
use vecdb::{AnyCollectableVec, Exit, Format};
mod blks;
mod chain;
mod cointime;
mod constants;
@@ -18,7 +19,6 @@ mod grouped;
mod indexes;
mod market;
mod pools;
mod positions;
mod price;
mod stateful;
mod states;
@@ -40,7 +40,7 @@ pub struct Computer {
pub indexes: indexes::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub positions: positions::Vecs,
pub blks: blks::Vecs,
pub price: Option<price::Vecs>,
pub stateful: stateful::Vecs,
}
@@ -90,7 +90,7 @@ impl Computer {
&indexes,
price.as_ref(),
)?,
positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
blks: blks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
@@ -132,8 +132,8 @@ impl Computer {
)?;
}
info!("Computing positions...");
self.positions
info!("Computing BLKs metadata...");
self.blks
.compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
@@ -144,9 +144,9 @@ impl Computer {
Ok(())
});
// let positions = scope.spawn(|| -> Result<()> {
// info!("Computing positions...");
// self.positions
// let blks = scope.spawn(|| -> Result<()> {
// info!("Computing blks...");
// self.blks
// .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
// Ok(())
// });
@@ -169,7 +169,7 @@ impl Computer {
}
constants.join().unwrap()?;
// positions.join().unwrap()?;
// blks.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;
@@ -216,7 +216,7 @@ impl Computer {
iter = Box::new(iter.chain(self.indexes.iter_any_collectable()));
iter = Box::new(iter.chain(self.market.iter_any_collectable()));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.positions.iter_any_collectable()));
iter = Box::new(iter.chain(self.blks.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.stateful.iter_any_collectable()));

3
crates/brk_parser/Cargo.toml
View File

@@ -20,6 +20,3 @@ crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
derive_deref = { workspace = true }
parking_lot = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }

2
crates/brk_parser/examples/main.rs
View File

@@ -26,7 +26,7 @@ fn main() -> Result<()> {
let mut diff = BTreeMap::new();
parser.parse(start, end).iter().for_each(|block| {
println!("{}: {}", block.height(), block.hash());
let new_blk_index = block.position().blk_index();
let new_blk_index = block.metadata().blk_index();
if new_blk_index < blk_index {
diff.insert(blk_index - new_blk_index, block.height());
}

14
crates/brk_parser/src/any_block.rs
View File

@@ -1,7 +1,7 @@
use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor};
use bitcoincore_rpc::RpcApi;
use brk_error::Result;
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use brk_structs::{BlkMetadata, Block, Height, ParsedBlock};
use crate::{XORBytes, XORIndex};
@@ -14,7 +14,7 @@ pub enum AnyBlock {
impl AnyBlock {
pub fn decode(
self,
position: BlkPosition,
metadata: BlkMetadata,
rpc: &'static bitcoincore_rpc::Client,
mut xor_i: XORIndex,
xor_bytes: XORBytes,
@@ -55,17 +55,19 @@ impl AnyBlock {
}
let mut txdata = Vec::with_capacity(tx_count as usize);
let mut tx_positions = Vec::with_capacity(tx_count as usize);
let mut tx_metadata = Vec::with_capacity(tx_count as usize);
for _ in 0..tx_count {
let tx_position = BlkPosition::new(position.blk_index(), cursor.position() as u32);
tx_positions.push(tx_position);
let offset = cursor.position() as u32;
let position = metadata.position() + offset;
let tx = Transaction::consensus_decode(&mut cursor)?;
txdata.push(tx);
let len = cursor.position() as u32 - offset;
tx_metadata.push(BlkMetadata::new(position, len));
}
let block = bitcoin::Block { header, txdata };
let block = Block::from((height, hash, block));
let block = ParsedBlock::from((block, position, tx_positions));
let block = ParsedBlock::from((block, metadata, tx_metadata));
Ok(Self::Decoded(block))
}

15
crates/brk_parser/src/lib.rs
View File

@@ -15,7 +15,7 @@ use bitcoin::{block::Header, consensus::Decodable};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use brk_error::Result;
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use brk_structs::{BlkMetadata, BlkPosition, Block, Height, ParsedBlock};
use crossbeam::channel::{Receiver, bounded};
use parking_lot::{RwLock, RwLockReadGuard};
use rayon::prelude::*;
@@ -123,8 +123,6 @@ impl Parser {
}
}
let position = BlkPosition::new(blk_index, i as u32);
let len = u32::from_le_bytes(
xor_i
.bytes(&mut blk_bytes[i..(i + 4)], xor_bytes)
@@ -133,10 +131,13 @@ impl Parser {
) as usize;
i += 4;
let position = BlkPosition::new(blk_index, i as u32);
let metadata = BlkMetadata::new(position, len as u32);
let block_bytes = (blk_bytes[i..(i + len)]).to_vec();
if send_bytes
.send((position, AnyBlock::Raw(block_bytes), xor_i))
.send((metadata, AnyBlock::Raw(block_bytes), xor_i))
.is_err()
{
return ControlFlow::Break(());
@@ -156,13 +157,13 @@ impl Parser {
let mut bulk = vec![];
let drain_and_send = |bulk: &mut Vec<(BlkPosition, AnyBlock, XORIndex)>| {
let drain_and_send = |bulk: &mut Vec<(BlkMetadata, AnyBlock, XORIndex)>| {
// Using a vec and sending after to not end up with stuck threads in par iter
mem::take(bulk)
.into_par_iter()
.try_for_each(|(position, any_block, xor_i)| {
.try_for_each(|(metdata, any_block, xor_i)| {
if let Ok(AnyBlock::Decoded(block)) =
any_block.decode(position, rpc, xor_i, xor_bytes, start, end)
any_block.decode(metdata, rpc, xor_i, xor_bytes, start, end)
&& send_block.send(block).is_err()
{
return ControlFlow::Break(());

23
crates/brk_server/src/api/mod.rs
View File

@@ -1,6 +1,6 @@
use std::{
fs::File,
io::{BufReader, Seek, SeekFrom},
io::{Cursor, Read, Seek, SeekFrom},
str::FromStr,
};
@@ -162,6 +162,11 @@ impl ApiRoutes for Router<AppState> {
.map(|opt| opt.map(|cow| cow.into_owned())) else {
return "Unknown transaction".into_response();
};
let txid = indexer
.vecs
.txindex_to_txid
.iter()
.unwrap_get_inner(txindex);
let version = indexer
.vecs
.txindex_to_txversion
@@ -177,7 +182,8 @@ impl ApiRoutes for Router<AppState> {
let parser = interface.parser();
let computer = interface.computer();
let position = computer.positions.txindex_to_position.iter().unwrap_get_inner(txindex);
let position = computer.blks.txindex_to_position.iter().unwrap_get_inner(txindex);
let len = computer.blks.txindex_to_len.iter().unwrap_get_inner(txindex);
let blk_index_to_blk_path = parser.blk_index_to_blk_path();
@@ -188,14 +194,21 @@ impl ApiRoutes for Router<AppState> {
let mut xori = XORIndex::default();
xori.add_assign(position.offset() as usize);
let Ok(file) = File::open(blk_path) else {
let Ok(mut file) = File::open(blk_path) else {
return "Error opening blk file".into_response();
};
let mut reader = BufReader::new(file);
if reader.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
if file.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
return "Error seeking position in blk file".into_response();
}
let mut buffer = vec![0u8; *len as usize];
if file.read_exact(&mut buffer).is_err() {
return "File fail read exact".into_response();
}
xori.bytes(&mut buffer, parser.xor_bytes());
let mut reader = Cursor::new(buffer);
let Ok(tx) = Transaction::consensus_decode(&mut reader) else {
return "Error decoding transaction".into_response();
};

38
crates/brk_store/src/lib.rs
View File

@@ -25,12 +25,13 @@ use log::info;
use meta::*;
use parking_lot::RwLock;
#[derive(Clone)]
pub struct Store<Key, Value> {
meta: StoreMeta,
name: &'static str,
keyspace: TransactionalKeyspace,
partition: Arc<RwLock<Option<TransactionalPartitionHandle>>>,
rtx: ReadTransaction,
rtx: Arc<RwLock<Option<ReadTransaction>>>,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
bloom_filters: Option<bool>,
@@ -79,7 +80,7 @@ where
name: Box::leak(Box::new(name.to_string())),
keyspace: keyspace.clone(),
partition: Arc::new(RwLock::new(Some(partition))),
rtx,
rtx: Arc::new(RwLock::new(Some(rtx))),
puts: BTreeMap::new(),
dels: BTreeSet::new(),
bloom_filters,
@@ -91,6 +92,9 @@ where
Ok(Some(Cow::Borrowed(v)))
} else if let Some(slice) = self
.rtx
.read()
.as_ref()
.unwrap()
.get(self.partition.read().as_ref().unwrap(), ByteView::from(key))?
{
Ok(Some(Cow::Owned(V::from(ByteView::from(slice)))))
@@ -101,6 +105,9 @@ where
pub fn is_empty(&self) -> Result<bool> {
self.rtx
.read()
.as_ref()
.unwrap()
.is_empty(self.partition.read().as_ref().unwrap())
.map_err(|e| e.into())
}
@@ -129,6 +136,9 @@ where
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
self.rtx
.read()
.as_ref()
.unwrap()
.iter(self.partition.read().as_ref().unwrap())
.map(|res| res.unwrap())
.map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v))))
@@ -202,6 +212,9 @@ where
self.meta.export(height)?;
let mut rtx = self.rtx.write();
let _ = rtx.take();
let mut wtx = self.keyspace.write_tx();
let partition = self.partition.read();
@@ -228,7 +241,7 @@ where
wtx.commit()?;
self.rtx = self.keyspace.read_tx();
rtx.replace(self.keyspace.read_tx());
Ok(())
}
@@ -295,22 +308,3 @@ where
self.meta.version()
}
}
impl<Key, Value> Clone for Store<Key, Value>
where
Key: Clone,
Value: Clone,
{
fn clone(&self) -> Self {
Self {
meta: self.meta.clone(),
name: self.name,
keyspace: self.keyspace.clone(),
partition: self.partition.clone(),
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),
bloom_filters: self.bloom_filters,
}
}
}

1
crates/brk_structs/Cargo.toml
View File

@@ -11,7 +11,6 @@ build = "build.rs"
[dependencies]
allocative = { workspace = true }
allocative_derive = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_error = {workspace = true}

29
crates/brk_structs/src/structs/blkmetadata.rs Normal file
View File

@@ -0,0 +1,29 @@
use crate::BlkPosition;
#[derive(Debug)]
pub struct BlkMetadata {
position: BlkPosition,
len: u32,
}
impl BlkMetadata {
pub fn new(position: BlkPosition, len: u32) -> Self {
Self { position, len }
}
pub fn position(&self) -> BlkPosition {
self.position
}
pub fn blk_index(&self) -> u16 {
self.position.blk_index()
}
pub fn offset(&self) -> u32 {
self.position.offset()
}
#[allow(clippy::len_without_is_empty)]
pub fn len(&self) -> u32 {
self.len
}
}

11
crates/brk_structs/src/structs/blkposition.rs
View File

@@ -1,3 +1,5 @@
use std::ops::Add;
use serde::Serialize;
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -13,10 +15,17 @@ impl BlkPosition {
}
pub fn blk_index(&self) -> u16 {
(self.0 >> 31) as u16
(self.0 >> 32) as u16
}
pub fn offset(&self) -> u32 {
self.0 as u32
}
}
impl Add<u32> for BlkPosition {
type Output = Self;
fn add(self, rhs: u32) -> Self::Output {
Self(self.0 + rhs as u64)
}
}

22
crates/brk_structs/src/structs/block.rs
View File

@@ -1,6 +1,6 @@
use std::{borrow::Cow, ops::Deref};
use crate::BlkPosition;
use crate::BlkMetadata;
use super::{BlockHash, Height};
@@ -58,27 +58,27 @@ impl Deref for Block {
#[derive(Debug)]
pub struct ParsedBlock {
block: Block,
position: BlkPosition,
tx_positions: Vec<BlkPosition>,
metadata: BlkMetadata,
tx_metadata: Vec<BlkMetadata>,
}
impl From<(Block, BlkPosition, Vec<BlkPosition>)> for ParsedBlock {
fn from((block, position, tx_positions): (Block, BlkPosition, Vec<BlkPosition>)) -> Self {
impl From<(Block, BlkMetadata, Vec<BlkMetadata>)> for ParsedBlock {
fn from((block, metadata, tx_metadata): (Block, BlkMetadata, Vec<BlkMetadata>)) -> Self {
Self {
block,
position,
tx_positions,
metadata,
tx_metadata,
}
}
}
impl ParsedBlock {
pub fn position(&self) -> &BlkPosition {
&self.position
pub fn metadata(&self) -> &BlkMetadata {
&self.metadata
}
pub fn tx_positions(&self) -> &Vec<BlkPosition> {
&self.tx_positions
pub fn tx_metadata(&self) -> &Vec<BlkMetadata> {
&self.tx_metadata
}
}

2
crates/brk_structs/src/structs/mod.rs
View File

@@ -4,6 +4,7 @@ mod addressbytes;
mod addressbyteshash;
mod anyaddressindex;
mod bitcoin;
mod blkmetadata;
mod blkposition;
mod block;
mod blockhash;
@@ -69,6 +70,7 @@ pub use addressbytes::*;
pub use addressbyteshash::*;
pub use anyaddressindex::*;
pub use bitcoin::*;
pub use blkmetadata::*;
pub use blkposition::*;
pub use block::*;
pub use blockhash::*;