diff --git a/Cargo.lock b/Cargo.lock index 75c5eaf22..83286cafe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,7 +540,6 @@ dependencies = [ "brk_parser", "brk_server", "clap", - "clap_derive", "color-eyre", "log", "minreq", @@ -556,7 +555,6 @@ name = "brk_computer" version = "0.0.107" dependencies = [ "allocative", - "allocative_derive", "bitcoin", "bitcoincore-rpc", "brk_error", @@ -567,8 +565,6 @@ dependencies = [ "brk_store", "brk_structs", "derive_deref", - "inferno", - "jiff", "log", "num_enum", "pco", @@ -676,9 +672,6 @@ dependencies = [ "derive_deref", "parking_lot 0.12.4", "rayon", - "serde", - "serde_json", - "zerocopy", ] [[package]] @@ -755,7 +748,7 @@ dependencies = [ "css-module-lexer", "dunce", "futures", - "indexmap 2.11.3", + "indexmap 2.11.4", "itertools", "itoa", "memchr", @@ -1025,7 +1018,7 @@ dependencies = [ "fast-glob", "form_urlencoded", "futures", - "indexmap 2.11.3", + "indexmap 2.11.4", "infer", "itoa", "memchr", @@ -1112,7 +1105,6 @@ name = "brk_structs" version = "0.0.107" dependencies = [ "allocative", - "allocative_derive", "bitcoin", "bitcoincore-rpc", "brk_error", @@ -1163,12 +1155,6 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" -[[package]] -name = "bytemuck" -version = "1.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" - [[package]] name = "byteorder" version = "1.5.0" @@ -1198,9 +1184,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.37" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "jobserver", @@ -1815,9 +1801,9 @@ dependencies = [ [[package]] name = "fancy-regex" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf04c5ec15464ace8355a7b440a33aece288993475556d461154d7a62ad9947c" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" dependencies = [ "bit-set", "regex-automata", @@ -1862,9 +1848,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "fixedbitset" @@ -2372,12 +2358,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.3" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "serde", "serde_core", ] @@ -2391,28 +2377,6 @@ dependencies = [ "cfb", ] -[[package]] -name = "inferno" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96d2465363ed2d81857759fc864cf6bb7997f79327aec028d65bd7989393685" -dependencies = [ - "ahash", - "clap", - "crossbeam-channel", - "crossbeam-utils", - "dashmap", - "env_logger", - "indexmap 2.11.3", - "itoa", - "log", - "num-format", - "once_cell", - "quick-xml", - "rgb", - "str_stack", -] - [[package]] name = "inotify" version = "0.11.0" @@ -2844,16 +2808,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - [[package]] name = "num-integer" version = "0.1.46" @@ -3249,7 +3203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49c4a4d746f42bac28538163952aa66da2f0ea781a0708772d8ad3f6fc066963" dependencies = [ "cfg-if", - "indexmap 2.11.3", + "indexmap 2.11.4", "json-strip-comments", "libc", "once_cell", @@ -3344,7 +3298,7 @@ dependencies = [ "base64 0.22.1", "compact_str", "cow-utils", - "indexmap 2.11.3", + "indexmap 2.11.4", "itoa", "memchr", "oxc-browserslist", @@ -3531,7 +3485,7 @@ checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.11.3", + "indexmap 2.11.4", "serde", ] @@ -3714,15 +3668,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", -] - [[package]] name = "quick_cache" version = "0.6.16" @@ -3945,15 +3890,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "rgb" -version = "0.8.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" -dependencies = [ - "bytemuck", -] - [[package]] name = "ring" version = "0.17.14" @@ -4155,12 +4091,11 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "seqdb" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0068ee8855b29d0251d0277ffb2c2d8872afe0a7124fc9f33a29102af1096f90" +checksum = "b31ad41314db97a84d637ce25b5323f46572f0a0e356a789fd1fa3e617c04bd0" dependencies = [ "allocative", - "allocative_derive", "libc", "log", "memmap2", @@ -4227,7 +4162,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.11.3", + "indexmap 2.11.4", "itoa", "memchr", "ryu", @@ -4248,9 +4183,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2789234a13a53fc4be1b51ea1bab45a3c338bdb884862a257d10e5a74ae009e6" +checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee" dependencies = [ "serde_core", ] @@ -4277,7 +4212,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.3", + "indexmap 2.11.4", "schemars 0.9.0", "schemars 1.0.4", "serde", @@ -4408,12 +4343,6 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" -[[package]] -name = "str_stack" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" - [[package]] name = "strsim" version = "0.11.1" @@ -4582,11 +4511,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -4675,11 +4605,11 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae2a4cf385da23d1d53bc15cdfa5c2109e93d8d362393c801e87da2f72f0e201" +checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0" dependencies = [ - "indexmap 2.11.3", + "indexmap 2.11.4", "serde_core", "serde_spanned", "toml_datetime", @@ -4690,20 +4620,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a197c0ec7d131bfc6f7e82c8442ba1595aeab35da7adbf05b6b73cd06a16b6be" +checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.5" +version = "0.23.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ad0b7ae9cfeef5605163839cb9221f453399f15cfb5c10be9885fcf56611f9" +checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b" dependencies = [ - "indexmap 2.11.3", + "indexmap 2.11.4", "toml_datetime", "toml_parser", "winnow", @@ -4711,18 +4641,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" +checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109" [[package]] name = "tower" @@ -5021,17 +4951,15 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e834cf5275445308e2ab867847d059ac7603de71c35dbe6c197726d1db7c74b" +checksum = "fe63b28536d63d008e824ff8fd5a66147b97d9cd893d392e19d8450488cfc8b0" dependencies = [ "allocative", - "allocative_derive", "ctrlc", "log", "parking_lot 0.12.4", "pco", - "rayon", "seqdb", "serde", "serde_derive", @@ -5043,9 +4971,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9acc9a0b85d010ddcd2a4825d2a804520d9c52759bb9b0bc5ef7c4a1e7fce1" +checksum = "19d38e2229203d0a3eda93507f0bf3d1ceb67c85832508d48b0268281cda3f9c" dependencies = [ "quote", "syn 2.0.106", @@ -5630,7 +5558,7 @@ dependencies = [ "arbitrary", "crc32fast", "flate2", - "indexmap 2.11.3", + "indexmap 2.11.4", "memchr", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index e9b956cb5..b22998231 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,6 @@ debug-assertions = false [workspace.dependencies] allocative = { version = "0.3.4", features = ["parking_lot"] } -allocative_derive = "0.3.3" axum = "0.8.4" bitcoin = { version = "0.32.7", features = ["serde"] } bitcoincore-rpc = "0.19.0" @@ -74,7 +73,7 @@ serde_derive = "1.0.225" serde_json = { version = "1.0.145", features = ["float_roundtrip"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} -vecdb = { version = "0.2.14", features = ["derive"]} +vecdb = { version = "0.2.15", features = ["derive"]} zerocopy = "0.8.27" zerocopy-derive = "0.8.27" diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index f707625e6..b792e0ae7 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -20,14 +20,13 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_server = { workspace = true } vecdb = { workspace = true } -clap = { version = "4.5.47", features = ["string"] } -clap_derive = "4.5.47" +clap = { version = "4.5.47", features = ["derive", "string"] } color-eyre = "0.6.5" log = { workspace = true } minreq = { workspace = true } serde = { workspace = true } tokio = { workspace = true } -toml = "0.9.6" +toml = "0.9.7" zip = { version = "5.1.1", default-features = false, features = ["deflate"] } [[bin]] diff --git a/crates/brk_cli/src/website.rs b/crates/brk_cli/src/website.rs index 8a71c4e9b..d526ae6f5 100644 --- a/crates/brk_cli/src/website.rs +++ b/crates/brk_cli/src/website.rs @@ -1,4 +1,4 @@ -use clap_derive::ValueEnum; +use clap::ValueEnum; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, ValueEnum)] diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index b356a9213..56ea4e00e 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -11,7 +11,6 @@ build = "build.rs" [dependencies] allocative = { workspace = true } -allocative_derive = { workspace = true } bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_structs = { workspace = true } @@ -23,8 +22,6 @@ brk_store = { workspace = true } brk_parser = { workspace = true } vecdb = { workspace = true } derive_deref = { workspace = true } -inferno = "0.12.3" -jiff = { workspace = true } log = { workspace = true } num_enum = "0.7.4" pco = "0.4.6" diff --git a/crates/brk_computer/src/positions.rs b/crates/brk_computer/src/blks.rs similarity index 61% rename from crates/brk_computer/src/positions.rs rename to crates/brk_computer/src/blks.rs index 8efeaab45..cc60872ab 100644 --- a/crates/brk_computer/src/positions.rs +++ b/crates/brk_computer/src/blks.rs @@ -3,8 +3,7 @@ use std::path::Path; use brk_error::Result; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_structs::{BlkPosition, Height, TxIndex, Version}; -use log::info; +use brk_structs::{BlkPosition, Height, StoredU32, TxIndex, Version}; use vecdb::{ AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit, GenericStoredVec, PAGE_SIZE, VecIterator, @@ -17,12 +16,14 @@ pub struct Vecs { db: Database, pub height_to_position: CompressedVec, + pub height_to_len: CompressedVec, pub txindex_to_position: CompressedVec, + pub txindex_to_len: CompressedVec, } impl Vecs { pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result { - let db = Database::open(&parent_path.join("positions"))?; + let db = Database::open(&parent_path.join("blks"))?; db.set_min_len(PAGE_SIZE * 1_000_000)?; let version = parent_version + Version::ZERO; @@ -31,13 +32,15 @@ impl Vecs { height_to_position: CompressedVec::forced_import( &db, "position", - version + Version::ZERO, + version + Version::TWO, )?, + height_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?, txindex_to_position: CompressedVec::forced_import( &db, "position", - version + Version::ZERO, + version + Version::TWO, )?, + txindex_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?, db, }; @@ -75,48 +78,66 @@ impl Vecs { let min_txindex = TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex); - let min_height = indexes + let Some(min_height) = indexes .txindex_to_height .iter() - .unwrap_get_inner(min_txindex) - .min(starting_indexes.height); + .get_inner(min_txindex) + .map(|h| h.min(starting_indexes.height)) + else { + return Ok(()); + }; let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter(); parser - .parse(Some(min_height), None) + .parse( + Some(min_height), + Some((indexer.vecs.height_to_first_txindex.len() - 1).into()), + ) .iter() .try_for_each(|block| -> Result<()> { let height = block.height(); - info!("{height}"); + self.height_to_position.forced_push_at( + height, + block.metadata().position(), + exit, + )?; - self.height_to_position - .forced_push_at(height, *block.position(), exit)?; + self.height_to_len + .forced_push_at(height, block.metadata().len().into(), exit)?; let txindex = height_to_first_txindex_iter.unwrap_get_inner(height); - block.tx_positions().iter().enumerate().try_for_each( - |(index, position)| -> Result<()> { + block.tx_metadata().iter().enumerate().try_for_each( + |(index, metadata)| -> Result<()> { let txindex = txindex + index; - self.txindex_to_position - .forced_push_at(txindex, *position, exit)?; + self.txindex_to_position.forced_push_at( + txindex, + metadata.position(), + exit, + )?; + self.txindex_to_len + .forced_push_at(txindex, metadata.len().into(), exit)?; Ok(()) }, )?; - // Stuck, why ?? if *height % 1_000 == 0 { let _lock = exit.lock(); self.height_to_position.flush()?; + self.height_to_len.flush()?; self.txindex_to_position.flush()?; + self.txindex_to_len.flush()?; } Ok(()) })?; let _lock = exit.lock(); self.height_to_position.flush()?; + self.height_to_len.flush()?; self.txindex_to_position.flush()?; + self.txindex_to_len.flush()?; Ok(()) } @@ -125,7 +146,9 @@ impl Vecs { Box::new( [ &self.height_to_position as &dyn AnyCollectableVec, + &self.height_to_len, &self.txindex_to_position, + &self.txindex_to_len, ] .into_iter(), ) diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index d9a9109d7..73a5f8ee2 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -10,6 +10,7 @@ use brk_structs::Version; use log::info; use vecdb::{AnyCollectableVec, Exit, Format}; +mod blks; mod chain; mod cointime; mod constants; @@ -18,7 +19,6 @@ mod grouped; mod indexes; mod market; mod pools; -mod positions; mod price; mod stateful; mod states; @@ -40,7 +40,7 @@ pub struct Computer { pub indexes: indexes::Vecs, pub market: market::Vecs, pub pools: pools::Vecs, - pub positions: positions::Vecs, + pub blks: blks::Vecs, pub price: Option, pub stateful: stateful::Vecs, } @@ -90,7 +90,7 @@ impl Computer { &indexes, price.as_ref(), )?, - positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?, + blks: blks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?, pools: pools::Vecs::forced_import( &computed_path, VERSION + Version::ZERO, @@ -132,8 +132,8 @@ impl Computer { )?; } - info!("Computing positions..."); - self.positions + info!("Computing BLKs metadata..."); + self.blks .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?; std::thread::scope(|scope| -> Result<()> { @@ -144,9 +144,9 @@ impl Computer { Ok(()) }); - // let positions = scope.spawn(|| -> Result<()> { - // info!("Computing positions..."); - // self.positions + // let blks = scope.spawn(|| -> Result<()> { + // info!("Computing blks..."); + // self.blks // .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?; // Ok(()) // }); @@ -169,7 +169,7 @@ impl Computer { } constants.join().unwrap()?; - // positions.join().unwrap()?; + // blks.join().unwrap()?; chain.join().unwrap()?; Ok(()) })?; @@ -216,7 +216,7 @@ impl Computer { iter = Box::new(iter.chain(self.indexes.iter_any_collectable())); iter = Box::new(iter.chain(self.market.iter_any_collectable())); iter = Box::new(iter.chain(self.pools.iter_any_collectable())); - iter = Box::new(iter.chain(self.positions.iter_any_collectable())); + iter = Box::new(iter.chain(self.blks.iter_any_collectable())); iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable()))); iter = Box::new(iter.chain(self.stateful.iter_any_collectable())); diff --git a/crates/brk_parser/Cargo.toml b/crates/brk_parser/Cargo.toml index 6d8562b82..67f62c810 100644 --- a/crates/brk_parser/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -20,6 +20,3 @@ crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_deref = { workspace = true } parking_lot = { workspace = true } rayon = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -zerocopy = { workspace = true } diff --git a/crates/brk_parser/examples/main.rs b/crates/brk_parser/examples/main.rs index 6b61a3280..ef00e12ca 100644 --- a/crates/brk_parser/examples/main.rs +++ b/crates/brk_parser/examples/main.rs @@ -26,7 +26,7 @@ fn main() -> Result<()> { let mut diff = BTreeMap::new(); parser.parse(start, end).iter().for_each(|block| { println!("{}: {}", block.height(), block.hash()); - let new_blk_index = block.position().blk_index(); + let new_blk_index = block.metadata().blk_index(); if new_blk_index < blk_index { diff.insert(blk_index - new_blk_index, block.height()); } diff --git a/crates/brk_parser/src/any_block.rs b/crates/brk_parser/src/any_block.rs index 350ac613d..760fdc054 100644 --- a/crates/brk_parser/src/any_block.rs +++ b/crates/brk_parser/src/any_block.rs @@ -1,7 +1,7 @@ use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor}; use bitcoincore_rpc::RpcApi; use brk_error::Result; -use brk_structs::{BlkPosition, Block, Height, ParsedBlock}; +use brk_structs::{BlkMetadata, Block, Height, ParsedBlock}; use crate::{XORBytes, XORIndex}; @@ -14,7 +14,7 @@ pub enum AnyBlock { impl AnyBlock { pub fn decode( self, - position: BlkPosition, + metadata: BlkMetadata, rpc: &'static bitcoincore_rpc::Client, mut xor_i: XORIndex, xor_bytes: XORBytes, @@ -55,17 +55,19 @@ impl AnyBlock { } let mut txdata = Vec::with_capacity(tx_count as usize); - let mut tx_positions = Vec::with_capacity(tx_count as usize); + let mut tx_metadata = Vec::with_capacity(tx_count as usize); for _ in 0..tx_count { - let tx_position = BlkPosition::new(position.blk_index(), cursor.position() as u32); - tx_positions.push(tx_position); + let offset = cursor.position() as u32; + let position = metadata.position() + offset; let tx = Transaction::consensus_decode(&mut cursor)?; txdata.push(tx); + let len = cursor.position() as u32 - offset; + tx_metadata.push(BlkMetadata::new(position, len)); } let block = bitcoin::Block { header, txdata }; let block = Block::from((height, hash, block)); - let block = ParsedBlock::from((block, position, tx_positions)); + let block = ParsedBlock::from((block, metadata, tx_metadata)); Ok(Self::Decoded(block)) } diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index 4efb6130a..f36d8102d 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -15,7 +15,7 @@ use bitcoin::{block::Header, consensus::Decodable}; use bitcoincore_rpc::RpcApi; use blk_index_to_blk_path::*; use brk_error::Result; -use brk_structs::{BlkPosition, Block, Height, ParsedBlock}; +use brk_structs::{BlkMetadata, BlkPosition, Block, Height, ParsedBlock}; use crossbeam::channel::{Receiver, bounded}; use parking_lot::{RwLock, RwLockReadGuard}; use rayon::prelude::*; @@ -123,8 +123,6 @@ impl Parser { } } - let position = BlkPosition::new(blk_index, i as u32); - let len = u32::from_le_bytes( xor_i .bytes(&mut blk_bytes[i..(i + 4)], xor_bytes) @@ -133,10 +131,13 @@ impl Parser { ) as usize; i += 4; + let position = BlkPosition::new(blk_index, i as u32); + let metadata = BlkMetadata::new(position, len as u32); + let block_bytes = (blk_bytes[i..(i + len)]).to_vec(); if send_bytes - .send((position, AnyBlock::Raw(block_bytes), xor_i)) + .send((metadata, AnyBlock::Raw(block_bytes), xor_i)) .is_err() { return ControlFlow::Break(()); @@ -156,13 +157,13 @@ impl Parser { let mut bulk = vec![]; - let drain_and_send = |bulk: &mut Vec<(BlkPosition, AnyBlock, XORIndex)>| { + let drain_and_send = |bulk: &mut Vec<(BlkMetadata, AnyBlock, XORIndex)>| { // Using a vec and sending after to not end up with stuck threads in par iter mem::take(bulk) .into_par_iter() - .try_for_each(|(position, any_block, xor_i)| { + .try_for_each(|(metdata, any_block, xor_i)| { if let Ok(AnyBlock::Decoded(block)) = - any_block.decode(position, rpc, xor_i, xor_bytes, start, end) + any_block.decode(metdata, rpc, xor_i, xor_bytes, start, end) && send_block.send(block).is_err() { return ControlFlow::Break(()); diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index 5ffaea620..c6338a900 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -1,6 +1,6 @@ use std::{ fs::File, - io::{BufReader, Seek, SeekFrom}, + io::{Cursor, Read, Seek, SeekFrom}, str::FromStr, }; @@ -162,6 +162,11 @@ impl ApiRoutes for Router { .map(|opt| opt.map(|cow| cow.into_owned())) else { return "Unknown transaction".into_response(); }; + let txid = indexer + .vecs + .txindex_to_txid + .iter() + .unwrap_get_inner(txindex); let version = indexer .vecs .txindex_to_txversion @@ -177,7 +182,8 @@ impl ApiRoutes for Router { let parser = interface.parser(); let computer = interface.computer(); - let position = computer.positions.txindex_to_position.iter().unwrap_get_inner(txindex); + let position = computer.blks.txindex_to_position.iter().unwrap_get_inner(txindex); + let len = computer.blks.txindex_to_len.iter().unwrap_get_inner(txindex); let blk_index_to_blk_path = parser.blk_index_to_blk_path(); @@ -188,14 +194,21 @@ impl ApiRoutes for Router { let mut xori = XORIndex::default(); xori.add_assign(position.offset() as usize); - let Ok(file) = File::open(blk_path) else { + let Ok(mut file) = File::open(blk_path) else { return "Error opening blk file".into_response(); }; - let mut reader = BufReader::new(file); - if reader.seek(SeekFrom::Start(position.offset() as u64)).is_err() { + + if file.seek(SeekFrom::Start(position.offset() as u64)).is_err() { return "Error seeking position in blk file".into_response(); } + let mut buffer = vec![0u8; *len as usize]; + if file.read_exact(&mut buffer).is_err() { + return "File fail read exact".into_response(); + } + xori.bytes(&mut buffer, parser.xor_bytes()); + + let mut reader = Cursor::new(buffer); let Ok(tx) = Transaction::consensus_decode(&mut reader) else { return "Error decoding transaction".into_response(); }; diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 63ea6a9d9..bbaa32d88 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -25,12 +25,13 @@ use log::info; use meta::*; use parking_lot::RwLock; +#[derive(Clone)] pub struct Store { meta: StoreMeta, name: &'static str, keyspace: TransactionalKeyspace, partition: Arc>>, - rtx: ReadTransaction, + rtx: Arc>>, puts: BTreeMap, dels: BTreeSet, bloom_filters: Option, @@ -79,7 +80,7 @@ where name: Box::leak(Box::new(name.to_string())), keyspace: keyspace.clone(), partition: Arc::new(RwLock::new(Some(partition))), - rtx, + rtx: Arc::new(RwLock::new(Some(rtx))), puts: BTreeMap::new(), dels: BTreeSet::new(), bloom_filters, @@ -91,6 +92,9 @@ where Ok(Some(Cow::Borrowed(v))) } else if let Some(slice) = self .rtx + .read() + .as_ref() + .unwrap() .get(self.partition.read().as_ref().unwrap(), ByteView::from(key))? { Ok(Some(Cow::Owned(V::from(ByteView::from(slice))))) @@ -101,6 +105,9 @@ where pub fn is_empty(&self) -> Result { self.rtx + .read() + .as_ref() + .unwrap() .is_empty(self.partition.read().as_ref().unwrap()) .map_err(|e| e.into()) } @@ -129,6 +136,9 @@ where pub fn iter(&self) -> impl Iterator { self.rtx + .read() + .as_ref() + .unwrap() .iter(self.partition.read().as_ref().unwrap()) .map(|res| res.unwrap()) .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))) @@ -202,6 +212,9 @@ where self.meta.export(height)?; + let mut rtx = self.rtx.write(); + let _ = rtx.take(); + let mut wtx = self.keyspace.write_tx(); let partition = self.partition.read(); @@ -228,7 +241,7 @@ where wtx.commit()?; - self.rtx = self.keyspace.read_tx(); + rtx.replace(self.keyspace.read_tx()); Ok(()) } @@ -295,22 +308,3 @@ where self.meta.version() } } - -impl Clone for Store -where - Key: Clone, - Value: Clone, -{ - fn clone(&self) -> Self { - Self { - meta: self.meta.clone(), - name: self.name, - keyspace: self.keyspace.clone(), - partition: self.partition.clone(), - rtx: self.keyspace.read_tx(), - puts: self.puts.clone(), - dels: self.dels.clone(), - bloom_filters: self.bloom_filters, - } - } -} diff --git a/crates/brk_structs/Cargo.toml b/crates/brk_structs/Cargo.toml index a9831b469..bb503507d 100644 --- a/crates/brk_structs/Cargo.toml +++ b/crates/brk_structs/Cargo.toml @@ -11,7 +11,6 @@ build = "build.rs" [dependencies] allocative = { workspace = true } -allocative_derive = { workspace = true } bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_error = {workspace = true} diff --git a/crates/brk_structs/src/structs/blkmetadata.rs b/crates/brk_structs/src/structs/blkmetadata.rs new file mode 100644 index 000000000..70d0b1a58 --- /dev/null +++ b/crates/brk_structs/src/structs/blkmetadata.rs @@ -0,0 +1,29 @@ +use crate::BlkPosition; + +#[derive(Debug)] +pub struct BlkMetadata { + position: BlkPosition, + len: u32, +} + +impl BlkMetadata { + pub fn new(position: BlkPosition, len: u32) -> Self { + Self { position, len } + } + pub fn position(&self) -> BlkPosition { + self.position + } + + pub fn blk_index(&self) -> u16 { + self.position.blk_index() + } + + pub fn offset(&self) -> u32 { + self.position.offset() + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> u32 { + self.len + } +} diff --git a/crates/brk_structs/src/structs/blkposition.rs b/crates/brk_structs/src/structs/blkposition.rs index 8415922c6..3416716e6 100644 --- a/crates/brk_structs/src/structs/blkposition.rs +++ b/crates/brk_structs/src/structs/blkposition.rs @@ -1,3 +1,5 @@ +use std::ops::Add; + use serde::Serialize; use vecdb::StoredCompressed; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -13,10 +15,17 @@ impl BlkPosition { } pub fn blk_index(&self) -> u16 { - (self.0 >> 31) as u16 + (self.0 >> 32) as u16 } pub fn offset(&self) -> u32 { self.0 as u32 } } + +impl Add for BlkPosition { + type Output = Self; + fn add(self, rhs: u32) -> Self::Output { + Self(self.0 + rhs as u64) + } +} diff --git a/crates/brk_structs/src/structs/block.rs b/crates/brk_structs/src/structs/block.rs index b9cff9a8e..856e1b877 100644 --- a/crates/brk_structs/src/structs/block.rs +++ b/crates/brk_structs/src/structs/block.rs @@ -1,6 +1,6 @@ use std::{borrow::Cow, ops::Deref}; -use crate::BlkPosition; +use crate::BlkMetadata; use super::{BlockHash, Height}; @@ -58,27 +58,27 @@ impl Deref for Block { #[derive(Debug)] pub struct ParsedBlock { block: Block, - position: BlkPosition, - tx_positions: Vec, + metadata: BlkMetadata, + tx_metadata: Vec, } -impl From<(Block, BlkPosition, Vec)> for ParsedBlock { - fn from((block, position, tx_positions): (Block, BlkPosition, Vec)) -> Self { +impl From<(Block, BlkMetadata, Vec)> for ParsedBlock { + fn from((block, metadata, tx_metadata): (Block, BlkMetadata, Vec)) -> Self { Self { block, - position, - tx_positions, + metadata, + tx_metadata, } } } impl ParsedBlock { - pub fn position(&self) -> &BlkPosition { - &self.position + pub fn metadata(&self) -> &BlkMetadata { + &self.metadata } - pub fn tx_positions(&self) -> &Vec { - &self.tx_positions + pub fn tx_metadata(&self) -> &Vec { + &self.tx_metadata } } diff --git a/crates/brk_structs/src/structs/mod.rs b/crates/brk_structs/src/structs/mod.rs index e525ec202..11c51e5a8 100644 --- a/crates/brk_structs/src/structs/mod.rs +++ b/crates/brk_structs/src/structs/mod.rs @@ -4,6 +4,7 @@ mod addressbytes; mod addressbyteshash; mod anyaddressindex; mod bitcoin; +mod blkmetadata; mod blkposition; mod block; mod blockhash; @@ -69,6 +70,7 @@ pub use addressbytes::*; pub use addressbyteshash::*; pub use anyaddressindex::*; pub use bitcoin::*; +pub use blkmetadata::*; pub use blkposition::*; pub use block::*; pub use blockhash::*;