diff --git a/Cargo.lock b/Cargo.lock index 3bc67b457..75c5eaf22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,11 @@ dependencies = [ name = "brk_interface" version = "0.0.107" dependencies = [ + "bitcoincore-rpc", "brk_computer", "brk_error", "brk_indexer", + "brk_parser", "brk_structs", "derive_deref", "nucleo-matcher", @@ -672,6 +674,7 @@ dependencies = [ "brk_structs", "crossbeam", "derive_deref", + "parking_lot 0.12.4", "rayon", "serde", "serde_json", @@ -752,7 +755,7 @@ dependencies = [ "css-module-lexer", "dunce", "futures", - "indexmap 2.11.1", + "indexmap 2.11.3", "itertools", "itoa", "memchr", @@ -1022,7 +1025,7 @@ dependencies = [ "fast-glob", "form_urlencoded", "futures", - "indexmap 2.11.1", + "indexmap 2.11.3", "infer", "itoa", "memchr", @@ -2046,7 +2049,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi 0.14.5+wasi-0.2.4", + "wasi 0.14.7+wasi-0.2.4", ] [[package]] @@ -2199,9 +2202,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "bytes", "futures-core", @@ -2369,13 +2372,14 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.1" +version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" +checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3" dependencies = [ "equivalent", "hashbrown 0.15.5", "serde", + "serde_core", ] [[package]] @@ -2399,7 +2403,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.11.1", + "indexmap 2.11.3", "itoa", "log", "num-format", @@ -2532,9 +2536,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.78" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" +checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e" dependencies = [ "once_cell", "wasm-bindgen", @@ -2542,9 +2546,9 @@ dependencies = [ [[package]] name = "json-strip-comments" -version = "1.0.4" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b271732a960335e715b6b2ae66a086f115c74eb97360e996d2bd809bfc063bba" +checksum = "c4135b29c84322dbc3327272084360785665452213a576a991b3ac2f63148e82" dependencies = [ "memchr", ] @@ -2955,9 +2959,9 @@ dependencies = [ [[package]] name = "oxc-browserslist" -version = "2.0.16" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd843fdc95833f2faf4251b701a812d488ef35958144af91b9ff540d9fe8ceaa" +checksum = "4ca83f99170f095592784131ed47b6782974c98165ec4da64e90df64bdc2b3cf" dependencies = [ "bincode", "flate2", @@ -2971,23 +2975,24 @@ dependencies = [ [[package]] name = "oxc-miette" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31cfb121c9d3e0f9082856927f5cff9594279c91b544f4436e4bc971563caa60" +checksum = "2d5495f6099fa0b25fa25755c1d59ed79ffa64dda80f5366a4cdfc8fc20f5932" dependencies = [ "cfg-if", "owo-colors", "oxc-miette-derive", "textwrap", "thiserror 2.0.16", + "unicode-segmentation", "unicode-width", ] [[package]] name = "oxc-miette-derive" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6eabb57f935b454fbe0552ea0abaaf9eb0019b5fa05a7bbe7efd5bd8c765085" +checksum = "9dbbc96af6e37c35f2303b2bedbf8ce9cc563e4fbbf7776be6f0803cb0095652" dependencies = [ "proc-macro2", "quote", @@ -3239,13 +3244,14 @@ dependencies = [ [[package]] name = "oxc_resolver" -version = "11.7.2" +version = "11.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7faa92a909b5e1a2018babf171a1973e3be4c0a2e29e3337c0cfa3c0ba04ebf" +checksum = "49c4a4d746f42bac28538163952aa66da2f0ea781a0708772d8ad3f6fc066963" dependencies = [ "cfg-if", - "indexmap 2.11.1", + "indexmap 2.11.3", "json-strip-comments", + "libc", "once_cell", "papaya", "pnp", @@ -3283,9 +3289,9 @@ dependencies = [ [[package]] name = "oxc_sourcemap" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60410c89a9da2ce5b10946012b8a8605ecc41129cb7976bbfb73b85902071208" +checksum = "87a91923e9269b32c91cd956596f9c3bd24b60a89df5c815abb2812a0ffa8b7b" dependencies = [ "base64-simd", "rustc-hash", @@ -3338,7 +3344,7 @@ dependencies = [ "base64 0.22.1", "compact_str", "cow-utils", - "indexmap 2.11.1", + "indexmap 2.11.3", "itoa", "memchr", "oxc-browserslist", @@ -3525,7 +3531,7 @@ checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.11.1", + "indexmap 2.11.3", "serde", ] @@ -3670,9 +3676,9 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ "toml_edit", ] @@ -4166,9 +4172,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.223" +version = "1.0.225" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a505d71960adde88e293da5cb5eda57093379f64e61cf77bf0e6a63af07a7bac" +checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d" dependencies = [ "serde_core", "serde_derive", @@ -4176,9 +4182,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe07b5d88710e3b807c16a06ccbc9dfecd5fff6a4d2745c59e3e26774f10de6a" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", "serde_core", @@ -4186,18 +4192,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.223" +version = "1.0.225" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20f57cbd357666aa7b3ac84a90b4ea328f1d4ddb6772b430caa5d9e1309bb9e9" +checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.223" +version = "1.0.225" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d428d07faf17e306e699ec1e91996e5a165ba5d6bce5b5155173e91a8a01a56" +checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516" dependencies = [ "proc-macro2", "quote", @@ -4221,7 +4227,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.11.1", + "indexmap 2.11.3", "itoa", "memchr", "ryu", @@ -4231,9 +4237,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a30a8abed938137c7183c173848e3c9b3517f5e038226849a4ecc9b21a4b4e2a" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" dependencies = [ "itoa", "serde", @@ -4242,11 +4248,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" +checksum = "2789234a13a53fc4be1b51ea1bab45a3c338bdb884862a257d10e5a74ae009e6" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -4271,7 +4277,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.1", + "indexmap 2.11.3", "schemars 0.9.0", "schemars 1.0.4", "serde", @@ -4669,14 +4675,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" +checksum = "ae2a4cf385da23d1d53bc15cdfa5c2109e93d8d362393c801e87da2f72f0e201" dependencies = [ - "indexmap 2.11.1", - "serde", + "indexmap 2.11.3", + "serde_core", "serde_spanned", - "toml_datetime 0.7.0", + "toml_datetime", "toml_parser", "toml_writer", "winnow", @@ -4684,27 +4690,22 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.11" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" - -[[package]] -name = "toml_datetime" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +checksum = "a197c0ec7d131bfc6f7e82c8442ba1595aeab35da7adbf05b6b73cd06a16b6be" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "toml_edit" -version = "0.22.27" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "c2ad0b7ae9cfeef5605163839cb9221f453399f15cfb5c10be9885fcf56611f9" dependencies = [ - "indexmap 2.11.1", - "toml_datetime 0.6.11", + "indexmap 2.11.3", + "toml_datetime", + "toml_parser", "winnow", ] @@ -5095,27 +5096,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.5+wasi-0.2.4" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" dependencies = [ "wasip2", ] [[package]] name = "wasip2" -version = "1.0.0+wasi-0.2.4" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" +checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" dependencies = [ "cfg-if", "once_cell", @@ -5126,9 +5127,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" +checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" dependencies = [ "bumpalo", "log", @@ -5140,9 +5141,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" +checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5150,9 +5151,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" +checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", @@ -5163,9 +5164,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" +checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" dependencies = [ "unicode-ident", ] @@ -5500,9 +5501,9 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.45.1" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" @@ -5629,7 +5630,7 @@ dependencies = [ "arbitrary", "crc32fast", "flate2", - "indexmap 2.11.1", + "indexmap 2.11.3", "memchr", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 82bf059a0..e9b956cb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,9 +68,9 @@ minreq = { version = "2.14.1", features = ["https", "serde_json"] } parking_lot = "0.12.4" quick_cache = "0.6.16" rayon = "1.11.0" -serde = "1.0.223" -serde_bytes = "0.11.18" -serde_derive = "1.0.223" +serde = "1.0.225" +serde_bytes = "0.11.19" +serde_derive = "1.0.225" serde_json = { version = "1.0.145", features = ["float_roundtrip"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 09419650c..f707625e6 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -27,7 +27,7 @@ log = { workspace = true } minreq = { workspace = true } serde = { workspace = true } tokio = { workspace = true } -toml = "0.9.5" +toml = "0.9.6" zip = { version = "5.1.1", default-features = false, features = ["deflate"] } [[bin]] diff --git a/crates/brk_cli/src/lib.rs b/crates/brk_cli/src/lib.rs index ee075167d..d9f22088b 100644 --- a/crates/brk_cli/src/lib.rs +++ b/crates/brk_cli/src/lib.rs @@ -73,7 +73,7 @@ pub fn run() -> color_eyre::Result<()> { .enable_all() .build()? .block_on(async { - let interface = Interface::build(&indexer, &computer); + let interface = Interface::build(&parser, &indexer, &computer); let website = config.website(); @@ -135,9 +135,7 @@ pub fn run() -> color_eyre::Result<()> { let starting_indexes = indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap(); - // dbg!(&starting_indexes); - - computer.compute(&indexer, starting_indexes, &exit).unwrap(); + computer.compute(&indexer, starting_indexes, &parser, &exit).unwrap(); info!("Waiting for new blocks..."); diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 46b78b276..0e3886210 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -45,7 +45,7 @@ pub fn main() -> Result<()> { loop { let i = Instant::now(); let starting_indexes = indexer.index(&parser, rpc, &exit, true)?; - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, starting_indexes, &parser, &exit)?; dbg!(i.elapsed()); sleep(Duration::from_secs(10)); } diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index db1d9cd66..2cf1d5879 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -12,8 +12,6 @@ use super::{ indexes, }; -const VERSION: Version = Version::ZERO; - #[derive(Clone)] pub struct Vecs { db: Database, @@ -35,15 +33,21 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result { - let db = Database::open(&parent.join("constants"))?; + pub fn forced_import( + parent_path: &Path, + parent_version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let db = Database::open(&parent_path.join("constants"))?; + + let version = parent_version + Version::ZERO; let this = Self { constant_0: ComputedVecsFromHeight::forced_import( &db, "constant_0", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -51,7 +55,7 @@ impl Vecs { &db, "constant_1", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -59,7 +63,7 @@ impl Vecs { &db, "constant_2", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -67,7 +71,7 @@ impl Vecs { &db, "constant_3", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -75,7 +79,7 @@ impl Vecs { &db, "constant_4", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -83,7 +87,7 @@ impl Vecs { &db, "constant_38_2", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -91,7 +95,7 @@ impl Vecs { &db, "constant_50", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -99,7 +103,7 @@ impl Vecs { &db, "constant_61_8", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -107,7 +111,7 @@ impl Vecs { &db, "constant_100", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -115,7 +119,7 @@ impl Vecs { &db, "constant_600", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -123,7 +127,7 @@ impl Vecs { &db, "constant_minus_1", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -131,7 +135,7 @@ impl Vecs { &db, "constant_minus_2", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -139,7 +143,7 @@ impl Vecs { &db, "constant_minus_3", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, @@ -147,7 +151,7 @@ impl Vecs { &db, "constant_minus_4", Source::Compute, - version + VERSION + Version::ZERO, + version + Version::ZERO, indexes, VecBuilderOptions::default().add_last(), )?, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 30cfdd514..d9a9109d7 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -5,6 +5,7 @@ use std::path::Path; use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; +use brk_parser::Parser; use brk_structs::Version; use log::info; use vecdb::{AnyCollectableVec, Exit, Format}; @@ -17,6 +18,7 @@ mod grouped; mod indexes; mod market; mod pools; +mod positions; mod price; mod stateful; mod states; @@ -31,15 +33,16 @@ use states::*; #[derive(Clone)] pub struct Computer { - pub indexes: indexes::Vecs, + pub chain: chain::Vecs, + pub cointime: cointime::Vecs, pub constants: constants::Vecs, + pub fetched: Option, + pub indexes: indexes::Vecs, pub market: market::Vecs, pub pools: pools::Vecs, + pub positions: positions::Vecs, pub price: Option, - pub chain: chain::Vecs, pub stateful: stateful::Vecs, - pub fetched: Option, - pub cointime: cointime::Vecs, } const VERSION: Version = Version::new(4); @@ -87,6 +90,7 @@ impl Computer { &indexes, price.as_ref(), )?, + positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?, pools: pools::Vecs::forced_import( &computed_path, VERSION + Version::ZERO, @@ -109,15 +113,12 @@ impl Computer { &mut self, indexer: &Indexer, starting_indexes: brk_indexer::Indexes, + parser: &Parser, exit: &Exit, ) -> Result<()> { info!("Computing indexes..."); let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; - info!("Computing constants..."); - self.constants - .compute(&self.indexes, &starting_indexes, exit)?; - if let Some(fetched) = self.fetched.as_mut() { info!("Computing fetched..."); fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?; @@ -131,7 +132,25 @@ impl Computer { )?; } + info!("Computing positions..."); + self.positions + .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?; + std::thread::scope(|scope| -> Result<()> { + let constants = scope.spawn(|| -> Result<()> { + info!("Computing constants..."); + self.constants + .compute(&self.indexes, &starting_indexes, exit)?; + Ok(()) + }); + + // let positions = scope.spawn(|| -> Result<()> { + // info!("Computing positions..."); + // self.positions + // .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?; + // Ok(()) + // }); + let chain = scope.spawn(|| -> Result<()> { info!("Computing chain..."); self.chain.compute( @@ -149,6 +168,8 @@ impl Computer { self.market.compute(price, &starting_indexes, exit)?; } + constants.join().unwrap()?; + // positions.join().unwrap()?; chain.join().unwrap()?; Ok(()) })?; @@ -162,8 +183,6 @@ impl Computer { exit, )?; - // return Ok(()); - info!("Computing stateful..."); self.stateful.compute( indexer, @@ -191,14 +210,15 @@ impl Computer { let mut iter: Box> = Box::new(self.fetched.iter().flat_map(|v| v.iter_any_collectable())); - iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable()))); - iter = Box::new(iter.chain(self.pools.iter_any_collectable())); + iter = Box::new(iter.chain(self.chain.iter_any_collectable())); + iter = Box::new(iter.chain(self.cointime.iter_any_collectable())); iter = Box::new(iter.chain(self.constants.iter_any_collectable())); iter = Box::new(iter.chain(self.indexes.iter_any_collectable())); iter = Box::new(iter.chain(self.market.iter_any_collectable())); - iter = Box::new(iter.chain(self.chain.iter_any_collectable())); + iter = Box::new(iter.chain(self.pools.iter_any_collectable())); + iter = Box::new(iter.chain(self.positions.iter_any_collectable())); + iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable()))); iter = Box::new(iter.chain(self.stateful.iter_any_collectable())); - iter = Box::new(iter.chain(self.cointime.iter_any_collectable())); iter } diff --git a/crates/brk_computer/src/positions.rs b/crates/brk_computer/src/positions.rs new file mode 100644 index 000000000..8efeaab45 --- /dev/null +++ b/crates/brk_computer/src/positions.rs @@ -0,0 +1,133 @@ +use std::path::Path; + +use brk_error::Result; +use brk_indexer::Indexer; +use brk_parser::Parser; +use brk_structs::{BlkPosition, Height, TxIndex, Version}; +use log::info; +use vecdb::{ + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit, + GenericStoredVec, PAGE_SIZE, VecIterator, +}; + +use super::{Indexes, indexes}; + +#[derive(Clone)] +pub struct Vecs { + db: Database, + + pub height_to_position: CompressedVec, + pub txindex_to_position: CompressedVec, +} + +impl Vecs { + pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result { + let db = Database::open(&parent_path.join("positions"))?; + db.set_min_len(PAGE_SIZE * 1_000_000)?; + + let version = parent_version + Version::ZERO; + + let this = Self { + height_to_position: CompressedVec::forced_import( + &db, + "position", + version + Version::ZERO, + )?, + txindex_to_position: CompressedVec::forced_import( + &db, + "position", + version + Version::ZERO, + )?, + + db, + }; + + this.db.retain_regions( + this.iter_any_collectable() + .flat_map(|v| v.region_names()) + .collect(), + )?; + + Ok(this) + } + + pub fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + parser: &Parser, + exit: &Exit, + ) -> Result<()> { + self.compute_(indexer, indexes, starting_indexes, parser, exit)?; + self.db.flush_then_punch()?; + Ok(()) + } + + fn compute_( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + parser: &Parser, + exit: &Exit, + ) -> Result<()> { + let min_txindex = + TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex); + + let min_height = indexes + .txindex_to_height + .iter() + .unwrap_get_inner(min_txindex) + .min(starting_indexes.height); + + let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter(); + + parser + .parse(Some(min_height), None) + .iter() + .try_for_each(|block| -> Result<()> { + let height = block.height(); + + info!("{height}"); + + self.height_to_position + .forced_push_at(height, *block.position(), exit)?; + + let txindex = height_to_first_txindex_iter.unwrap_get_inner(height); + + block.tx_positions().iter().enumerate().try_for_each( + |(index, position)| -> Result<()> { + let txindex = txindex + index; + self.txindex_to_position + .forced_push_at(txindex, *position, exit)?; + Ok(()) + }, + )?; + + // Stuck, why ?? + if *height % 1_000 == 0 { + let _lock = exit.lock(); + self.height_to_position.flush()?; + self.txindex_to_position.flush()?; + } + Ok(()) + })?; + + let _lock = exit.lock(); + self.height_to_position.flush()?; + self.txindex_to_position.flush()?; + + Ok(()) + } + + pub fn iter_any_collectable(&self) -> impl Iterator { + Box::new( + [ + &self.height_to_position as &dyn AnyCollectableVec, + &self.txindex_to_position, + ] + .into_iter(), + ) + } +} diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index f217ab935..20022df88 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -20,7 +20,6 @@ use crate::Indexes; #[derive(Clone)] pub struct Vecs { db: Database, - pub emptyoutputindex_to_txindex: CompressedVec, pub height_to_blockhash: RawVec, pub height_to_difficulty: CompressedVec, @@ -39,7 +38,7 @@ pub struct Vecs { pub height_to_first_p2wshaddressindex: CompressedVec, pub height_to_first_txindex: CompressedVec, pub height_to_first_unknownoutputindex: CompressedVec, - /// Doesn't guarantee continuity due to possible reorgs + /// Doesn't guarantee continuity due to possible reorgs and more generally the nature of mining pub height_to_timestamp: CompressedVec, pub height_to_total_size: CompressedVec, pub height_to_weight: CompressedVec, @@ -72,9 +71,11 @@ pub struct Vecs { impl Vecs { pub fn forced_import(parent: &Path, version: Version) -> Result { let db = Database::open(&parent.join("vecs"))?; - db.set_min_len(PAGE_SIZE * 50_000_000)?; + let db_positions = Database::open(&parent.join("vecs/positions"))?; + db_positions.set_min_len(PAGE_SIZE * 1_000_000)?; + let this = Self { emptyoutputindex_to_txindex: CompressedVec::forced_import(&db, "txindex", version)?, height_to_blockhash: RawVec::forced_import(&db, "blockhash", version)?, @@ -192,12 +193,8 @@ impl Vecs { db, }; - // self.db.retain_regions( - // this.vecs() - // .into_iter() - // .flat_map(|v| v.region_names()) - // .collect(), - // )?; + this.db + .retain_regions(this.iter().flat_map(|v| v.region_names()).collect())?; Ok(this) } @@ -349,16 +346,15 @@ impl Vecs { } pub fn flush(&mut self, height: Height) -> Result<()> { - self.mut_vecs() - .into_par_iter() + self.iter_mut() + .par_bridge() .try_for_each(|vec| vec.stamped_flush(Stamp::from(height)))?; self.db.flush()?; Ok(()) } pub fn starting_height(&mut self) -> Height { - self.mut_vecs() - .into_iter() + self.iter_mut() .map(|vec| { let h = Height::from(vec.stamp()); if h > Height::ZERO { h.incremented() } else { h } @@ -372,9 +368,9 @@ impl Vecs { Ok(()) } - pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { - vec![ - &self.emptyoutputindex_to_txindex, + pub fn iter(&self) -> impl Iterator { + [ + &self.emptyoutputindex_to_txindex as &dyn AnyCollectableVec, &self.height_to_blockhash, &self.height_to_difficulty, &self.height_to_first_emptyoutputindex, @@ -419,11 +415,12 @@ impl Vecs { &self.txindex_to_txversion, &self.unknownoutputindex_to_txindex, ] + .into_iter() } - fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStoredVec> { - vec![ - &mut self.emptyoutputindex_to_txindex, + fn iter_mut(&mut self) -> impl Iterator { + [ + &mut self.emptyoutputindex_to_txindex as &mut dyn AnyStoredVec, &mut self.height_to_blockhash, &mut self.height_to_difficulty, &mut self.height_to_first_emptyoutputindex, @@ -468,5 +465,6 @@ impl Vecs { &mut self.txindex_to_txversion, &mut self.unknownoutputindex_to_txindex, ] + .into_iter() } } diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index e36b1f7ba..151c7347a 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -10,9 +10,11 @@ rust-version.workspace = true build = "build.rs" [dependencies] +bitcoincore-rpc = { workspace = true } brk_computer = { workspace = true } brk_error = { workspace = true } brk_indexer = { workspace = true } +brk_parser = { workspace = true } brk_structs = { workspace = true } vecdb = { workspace = true } derive_deref = { workspace = true } diff --git a/crates/brk_interface/examples/main.rs b/crates/brk_interface/examples/main.rs index f55bd2703..a449b8cb0 100644 --- a/crates/brk_interface/examples/main.rs +++ b/crates/brk_interface/examples/main.rs @@ -1,18 +1,40 @@ -use std::path::Path; +use std::{fs, path::Path}; use brk_computer::Computer; use brk_error::Result; use brk_indexer::Indexer; use brk_interface::{Index, Interface, Params, ParamsOpt}; +use brk_parser::Parser; +use vecdb::Exit; pub fn main() -> Result<()> { - let outputs_dir = Path::new("../../_outputs"); + let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap()) + .join("Library") + .join("Application Support") + .join("Bitcoin"); + // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); - let indexer = Indexer::forced_import(outputs_dir)?; + let blocks_dir = bitcoin_dir.join("blocks"); - let computer = Computer::forced_import(outputs_dir, &indexer, None)?; + let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk"); + fs::create_dir_all(&outputs_dir)?; + // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); - let interface = Interface::build(&indexer, &computer); + let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( + "http://localhost:8332", + bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), + )?)); + + let exit = Exit::new(); + exit.set_ctrlc_handler(); + + let parser = Parser::new(blocks_dir, rpc); + + let indexer = Indexer::forced_import(&outputs_dir)?; + + let computer = Computer::forced_import(&outputs_dir, &indexer, None)?; + + let interface = Interface::build(&parser, &indexer, &computer); dbg!(interface.search_and_format(Params { index: Index::Height, diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 86b48e731..5aafb6a74 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -5,6 +5,7 @@ use std::{collections::BTreeMap, sync::OnceLock}; use brk_computer::Computer; use brk_error::{Error, Result}; use brk_indexer::Indexer; +use brk_parser::Parser; use brk_structs::Height; use nucleo_matcher::{ Config, Matcher, @@ -42,18 +43,21 @@ pub fn cached_errors() -> &'static Cache { #[allow(dead_code)] pub struct Interface<'a> { vecs: Vecs<'a>, + parser: &'a Parser, indexer: &'a Indexer, computer: &'a Computer, } impl<'a> Interface<'a> { - pub fn build(indexer: &Indexer, computer: &Computer) -> Self { + pub fn build(parser: &Parser, indexer: &Indexer, computer: &Computer) -> Self { + let parser = parser.static_clone(); let indexer = indexer.static_clone(); let computer = computer.static_clone(); let vecs = Vecs::build(indexer, computer); Self { vecs, + parser, indexer, computer, } @@ -262,6 +266,10 @@ impl<'a> Interface<'a> { self.vecs.id_to_indexes(id) } + pub fn parser(&self) -> &Parser { + self.parser + } + pub fn indexer(&self) -> &Indexer { self.indexer } diff --git a/crates/brk_interface/src/vecs.rs b/crates/brk_interface/src/vecs.rs index 58359a907..aa7f5ca25 100644 --- a/crates/brk_interface/src/vecs.rs +++ b/crates/brk_interface/src/vecs.rs @@ -27,11 +27,7 @@ impl<'a> Vecs<'a> { pub fn build(indexer: &'a Indexer, computer: &'a Computer) -> Self { let mut this = Vecs::default(); - indexer - .vecs - .vecs() - .into_iter() - .for_each(|vec| this.insert(vec)); + indexer.vecs.iter().for_each(|vec| this.insert(vec)); computer .iter_any_collectable() diff --git a/crates/brk_parser/Cargo.toml b/crates/brk_parser/Cargo.toml index 929e4d3da..6d8562b82 100644 --- a/crates/brk_parser/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -18,6 +18,7 @@ brk_error = { workspace = true } brk_structs = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_deref = { workspace = true } +parking_lot = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/brk_parser/examples/main.rs b/crates/brk_parser/examples/main.rs index df23d5434..6b61a3280 100644 --- a/crates/brk_parser/examples/main.rs +++ b/crates/brk_parser/examples/main.rs @@ -3,7 +3,6 @@ use std::{collections::BTreeMap, path::Path}; use bitcoincore_rpc::{Auth, Client}; use brk_error::Result; use brk_parser::Parser; -use brk_structs::Height; #[allow(clippy::needless_doctest_main)] fn main() -> Result<()> { @@ -21,13 +20,13 @@ fn main() -> Result<()> { let parser = Parser::new(bitcoin_dir.join("blocks"), rpc); - let start = Some(915_155_u32.into()); + let start = Some(915_138_u32.into()); let end = None; let mut blk_index = 0; let mut diff = BTreeMap::new(); parser.parse(start, end).iter().for_each(|block| { println!("{}: {}", block.height(), block.hash()); - let new_blk_index = block.position().blk_index; + let new_blk_index = block.position().blk_index(); if new_blk_index < blk_index { diff.insert(blk_index - new_blk_index, block.height()); } diff --git a/crates/brk_parser/src/any_block.rs b/crates/brk_parser/src/any_block.rs index ba1562ce9..350ac613d 100644 --- a/crates/brk_parser/src/any_block.rs +++ b/crates/brk_parser/src/any_block.rs @@ -1,7 +1,7 @@ use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor}; use bitcoincore_rpc::RpcApi; use brk_error::Result; -use brk_structs::{Block, BlockPosition, Height, ParsedBlock}; +use brk_structs::{BlkPosition, Block, Height, ParsedBlock}; use crate::{XORBytes, XORIndex}; @@ -14,10 +14,10 @@ pub enum AnyBlock { impl AnyBlock { pub fn decode( self, - position: BlockPosition, + position: BlkPosition, rpc: &'static bitcoincore_rpc::Client, mut xor_i: XORIndex, - xor_bytes: &XORBytes, + xor_bytes: XORBytes, start: Option, end: Option, ) -> Result { @@ -55,14 +55,17 @@ impl AnyBlock { } let mut txdata = Vec::with_capacity(tx_count as usize); + let mut tx_positions = Vec::with_capacity(tx_count as usize); for _ in 0..tx_count { + let tx_position = BlkPosition::new(position.blk_index(), cursor.position() as u32); + tx_positions.push(tx_position); let tx = Transaction::consensus_decode(&mut cursor)?; txdata.push(tx); } let block = bitcoin::Block { header, txdata }; let block = Block::from((height, hash, block)); - let block = ParsedBlock::from((block, position)); + let block = ParsedBlock::from((block, position, tx_positions)); Ok(Self::Decoded(block)) } diff --git a/crates/brk_parser/src/blk_index_to_blk_path.rs b/crates/brk_parser/src/blk_index_to_blk_path.rs index e098cd36f..c94619c91 100644 --- a/crates/brk_parser/src/blk_index_to_blk_path.rs +++ b/crates/brk_parser/src/blk_index_to_blk_path.rs @@ -9,7 +9,7 @@ use derive_deref::{Deref, DerefMut}; const BLK: &str = "blk"; const DOT_DAT: &str = ".dat"; -#[derive(Debug, Deref, DerefMut)] +#[derive(Debug, Clone, Deref, DerefMut)] pub struct BlkIndexToBlkPath(BTreeMap); impl BlkIndexToBlkPath { diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index 8a4bc38bd..4efb6130a 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -7,6 +7,7 @@ use std::{ mem, ops::ControlFlow, path::PathBuf, + sync::Arc, thread, }; @@ -14,8 +15,9 @@ use bitcoin::{block::Header, consensus::Decodable}; use bitcoincore_rpc::RpcApi; use blk_index_to_blk_path::*; use brk_error::Result; -use brk_structs::{Block, BlockPosition, Height, ParsedBlock}; +use brk_structs::{BlkPosition, Block, Height, ParsedBlock}; use crossbeam::channel::{Receiver, bounded}; +use parking_lot::{RwLock, RwLockReadGuard}; use rayon::prelude::*; mod any_block; @@ -24,22 +26,40 @@ mod xor_bytes; mod xor_index; use any_block::*; -use xor_bytes::*; -use xor_index::*; +pub use xor_bytes::*; +pub use xor_index::*; pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100; const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217]; const BOUND_CAP: usize = 50; +#[derive(Debug, Clone)] pub struct Parser { + blk_index_to_blk_path: Arc>, + xor_bytes: XORBytes, blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client, } impl Parser { pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self { - Self { blocks_dir, rpc } + Self { + xor_bytes: XORBytes::from(blocks_dir.as_path()), + blk_index_to_blk_path: Arc::new(RwLock::new(BlkIndexToBlkPath::scan( + blocks_dir.as_path(), + ))), + blocks_dir, + rpc, + } + } + + pub fn blk_index_to_blk_path(&self) -> RwLockReadGuard<'_, BlkIndexToBlkPath> { + self.blk_index_to_blk_path.read() + } + + pub fn xor_bytes(&self) -> XORBytes { + self.xor_bytes } pub fn get(&self, height: Height) -> Result { @@ -57,19 +77,19 @@ impl Parser { /// For an example checkout `./main.rs` /// pub fn parse(&self, start: Option, end: Option) -> Receiver { - let blocks_dir = self.blocks_dir.as_path(); let rpc = self.rpc; let (send_bytes, recv_bytes) = bounded(BOUND_CAP); let (send_block, recv_block) = bounded(BOUND_CAP); let (send_ordered, recv_ordered) = bounded(BOUND_CAP); - let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir); + let blk_index_to_blk_path = BlkIndexToBlkPath::scan(&self.blocks_dir); + *self.blk_index_to_blk_path.write() = blk_index_to_blk_path.clone(); - let xor_bytes = XORBytes::from(blocks_dir); + let xor_bytes = self.xor_bytes; let first_blk_index = self - .find_start_blk_index(start, &blk_index_to_blk_path, &xor_bytes) + .find_start_blk_index(start, &blk_index_to_blk_path, xor_bytes) .unwrap_or_default(); thread::spawn(move || { @@ -95,7 +115,7 @@ impl Parser { current_4bytes.rotate_left(1); - current_4bytes[3] = xor_i.byte(blk_bytes[i], &xor_bytes); + current_4bytes[3] = xor_i.byte(blk_bytes[i], xor_bytes); i += 1; if current_4bytes == MAGIC_BYTES { @@ -103,27 +123,20 @@ impl Parser { } } - let deser_len = u32::from_le_bytes( + let position = BlkPosition::new(blk_index, i as u32); + + let len = u32::from_le_bytes( xor_i - .bytes(&mut blk_bytes[i..(i + 4)], &xor_bytes) + .bytes(&mut blk_bytes[i..(i + 4)], xor_bytes) .try_into() .unwrap(), - ); - let len = deser_len as usize; + ) as usize; i += 4; let block_bytes = (blk_bytes[i..(i + len)]).to_vec(); if send_bytes - .send(( - BlockPosition { - blk_index, - offset: i, - len: deser_len, - }, - AnyBlock::Raw(block_bytes), - xor_i, - )) + .send((position, AnyBlock::Raw(block_bytes), xor_i)) .is_err() { return ControlFlow::Break(()); @@ -143,13 +156,13 @@ impl Parser { let mut bulk = vec![]; - let drain_and_send = |bulk: &mut Vec<(BlockPosition, AnyBlock, XORIndex)>| { + let drain_and_send = |bulk: &mut Vec<(BlkPosition, AnyBlock, XORIndex)>| { // Using a vec and sending after to not end up with stuck threads in par iter mem::take(bulk) .into_par_iter() .try_for_each(|(position, any_block, xor_i)| { if let Ok(AnyBlock::Decoded(block)) = - any_block.decode(position, rpc, xor_i, &xor_bytes, start, end) + any_block.decode(position, rpc, xor_i, xor_bytes, start, end) && send_block.send(block).is_err() { return ControlFlow::Break(()); @@ -211,7 +224,7 @@ impl Parser { &self, target_start: Option, blk_index_to_blk_path: &BlkIndexToBlkPath, - xor_bytes: &XORBytes, + xor_bytes: XORBytes, ) -> Result { let Some(target_start) = target_start else { return Ok(0); @@ -280,7 +293,7 @@ impl Parser { Ok(blk_indices.get(final_idx).copied().unwrap_or(0)) } - fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: &XORBytes) -> Result { + fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: XORBytes) -> Result { let mut file = File::open(blk_path)?; let mut xor_i = XORIndex::default(); let mut current_4bytes = [0; 4]; @@ -314,4 +327,8 @@ impl Parser { Ok(Height::new(height)) } + + pub fn static_clone(&self) -> &'static Self { + Box::leak(Box::new(self.clone())) + } } diff --git a/crates/brk_parser/src/xor_index.rs b/crates/brk_parser/src/xor_index.rs index 0ce4cf682..05d314b8f 100644 --- a/crates/brk_parser/src/xor_index.rs +++ b/crates/brk_parser/src/xor_index.rs @@ -4,7 +4,7 @@ use crate::xor_bytes::{XOR_LEN, XORBytes}; pub struct XORIndex(usize); impl XORIndex { - pub fn bytes<'a>(&mut self, bytes: &'a mut [u8], xor_bytes: &XORBytes) -> &'a mut [u8] { + pub fn bytes<'a>(&mut self, bytes: &'a mut [u8], xor_bytes: XORBytes) -> &'a mut [u8] { let len = bytes.len(); let mut bytes_index = 0; @@ -18,7 +18,7 @@ impl XORIndex { } #[inline] - pub fn byte(&mut self, mut byte: u8, xor_bytes: &XORBytes) -> u8 { + pub fn byte(&mut self, mut byte: u8, xor_bytes: XORBytes) -> u8 { byte ^= xor_bytes[self.0]; self.increment(); byte diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 8054f07cf..9b3f9e48a 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -39,7 +39,7 @@ pub fn main() -> Result<()> { .enable_all() .build()? .block_on(async { - let interface = Interface::build(&indexer, &computer); + let interface = Interface::build(&parser, &indexer, &computer); let server = Server::new(interface, None); @@ -53,7 +53,7 @@ pub fn main() -> Result<()> { let starting_indexes = indexer.index(&parser, rpc, &exit, true)?; - computer.compute(&indexer, starting_indexes, &exit)?; + computer.compute(&indexer, starting_indexes, &parser, &exit)?; while block_count == rpc.get_block_count()? { sleep(Duration::from_secs(1)) diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index 7475cd8ce..5ffaea620 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -1,4 +1,8 @@ -use std::str::FromStr; +use std::{ + fs::File, + io::{BufReader, Seek, SeekFrom}, + str::FromStr, +}; use axum::{ Json, Router, @@ -7,9 +11,10 @@ use axum::{ response::{IntoResponse, Redirect, Response}, routing::get, }; -use bitcoin::{Address, Network, absolute::LockTime}; +use bitcoin::{Address, Network, Transaction, absolute::LockTime, consensus::Decodable}; use bitcoincore_rpc::bitcoin; use brk_interface::{IdParam, Index, PaginatedIndexParam, PaginationParam, Params, ParamsOpt}; +use brk_parser::XORIndex; use brk_structs::{ AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, Txid, TxidPrefix, }; @@ -45,13 +50,6 @@ impl ApiRoutes for Router { let computer = interface.computer(); let stores = &indexer.stores; let hash = AddressBytesHash::from(&address); - dbg!(&hash); - dbg!( - &address, - address.address_type(), - address.script_pubkey(), - OutputType::from(&address) - ); let Ok(Some(addri)) = stores .addressbyteshash_to_typeindex @@ -60,8 +58,6 @@ impl ApiRoutes for Router { return "Unknown address".into_response(); }; - println!("Script pubkey: {}", address.script_pubkey()); - println!("Address type: {:?}", address.address_type()); let output_type = OutputType::from(&address); let stateful = &computer.stateful; let price = computer.price.as_ref().map(|v| { @@ -178,11 +174,38 @@ impl ApiRoutes for Router { .unwrap_get_inner(txindex); let locktime = LockTime::from(rawlocktime); + let parser = interface.parser(); + let computer = interface.computer(); + + let position = computer.positions.txindex_to_position.iter().unwrap_get_inner(txindex); + + let blk_index_to_blk_path = parser.blk_index_to_blk_path(); + + let Some(blk_path) = blk_index_to_blk_path.get(&position.blk_index()) else { + return "Unknown blk index".into_response(); + }; + + let mut xori = XORIndex::default(); + xori.add_assign(position.offset() as usize); + + let Ok(file) = File::open(blk_path) else { + return "Error opening blk file".into_response(); + }; + let mut reader = BufReader::new(file); + if reader.seek(SeekFrom::Start(position.offset() as u64)).is_err() { + return "Error seeking position in blk file".into_response(); + } + + let Ok(tx) = Transaction::consensus_decode(&mut reader) else { + return "Error decoding transaction".into_response(); + }; + Json(serde_json::json!({ "txid": txid, "index": txindex, "version": version, - "locktime": locktime + "locktime": locktime, + "tx": tx, })) .into_response() }, diff --git a/crates/brk_structs/src/structs/blkposition.rs b/crates/brk_structs/src/structs/blkposition.rs new file mode 100644 index 000000000..8415922c6 --- /dev/null +++ b/crates/brk_structs/src/structs/blkposition.rs @@ -0,0 +1,22 @@ +use serde::Serialize; +use vecdb::StoredCompressed; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive( + Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout, StoredCompressed, +)] +pub struct BlkPosition(u64); + +impl BlkPosition { + pub fn new(blk_index: u16, offset: u32) -> Self { + Self(((blk_index as u64) << 32) | offset as u64) + } + + pub fn blk_index(&self) -> u16 { + (self.0 >> 31) as u16 + } + + pub fn offset(&self) -> u32 { + self.0 as u32 + } +} diff --git a/crates/brk_structs/src/structs/block.rs b/crates/brk_structs/src/structs/block.rs index 10926b642..b9cff9a8e 100644 --- a/crates/brk_structs/src/structs/block.rs +++ b/crates/brk_structs/src/structs/block.rs @@ -1,14 +1,6 @@ -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - io::Cursor, - ops::Deref, -}; +use std::{borrow::Cow, ops::Deref}; -use bitcoin::{block::Header, consensus::Decodable}; -use brk_error::Result; - -use crate::BlockPosition; +use crate::BlkPosition; use super::{BlockHash, Height}; @@ -28,40 +20,6 @@ impl Block { &self.hash } - pub fn parse(block_data: &[u8]) -> Result { - let mut cursor = Cursor::new(block_data); - - let header = Header::consensus_decode(&mut cursor)?; - - // Parse transactions with positions - let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize; - let mut transactions = Vec::with_capacity(tx_count); - let mut tx_positions = HashMap::with_capacity(tx_count); - - for _ in 0..tx_count { - let start = cursor.position() as usize; - let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?; - - tx_positions.insert(tx.compute_txid(), start); - transactions.push(tx); - } - - let block = bitcoin::Block { - header, - txdata: transactions, - }; - - // block.bip34_block_height() - - let hash = block.block_hash(); - - Ok(Block { - block, - hash: hash.into(), - height: Height::ZERO, - }) - } - pub fn coinbase_tag(&self) -> Cow<'_, str> { String::from_utf8_lossy( self.txdata @@ -100,70 +58,28 @@ impl Deref for Block { #[derive(Debug)] pub struct ParsedBlock { block: Block, - position: BlockPosition, - tx_positions: BTreeMap, // txid -> offset + position: BlkPosition, + tx_positions: Vec, } -impl From<(Block, BlockPosition)> for ParsedBlock { - fn from((block, position): (Block, BlockPosition)) -> Self { +impl From<(Block, BlkPosition, Vec)> for ParsedBlock { + fn from((block, position, tx_positions): (Block, BlkPosition, Vec)) -> Self { Self { block, position, - tx_positions: BTreeMap::default(), + tx_positions, } } } impl ParsedBlock { - pub fn position(&self) -> &BlockPosition { + pub fn position(&self) -> &BlkPosition { &self.position } - pub fn tx_positions(&self) -> &BTreeMap { + pub fn tx_positions(&self) -> &Vec { &self.tx_positions } - - // pub fn parse(block_data: &[u8], file_offset: usize) -> Result { - // let mut cursor = std::io::Cursor::new(block_data); - - // let header = Header::consensus_decode(&mut cursor)?; - - // // Parse transactions with positions - // let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize; - // let mut transactions = Vec::with_capacity(tx_count); - // let mut tx_positions = HashMap::with_capacity(tx_count); - - // for _ in 0..tx_count { - // let start = cursor.position() as usize; - // let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?; - - // tx_positions.insert(tx.compute_txid(), start); - // transactions.push(tx); - // } - - // let block = bitcoin::Block { - // header, - // txdata: transactions, - // }; - - // // block.bip34_block_height() - - // let hash = block.block_hash(); - - // Ok(Block { - // block, - // hash: hash.into(), - // height: Height::ZERO, - // tx_positions, - // block_start_offset: file_offset, - // }) - // } - - // pub fn get_absolute_position(&self, txid: &Txid) -> Option { - // self.tx_positions - // .get(txid) - // .map(|offset| self.block_start_offset + offset) - // } } impl Deref for ParsedBlock { diff --git a/crates/brk_structs/src/structs/blockmeta.rs b/crates/brk_structs/src/structs/blockmeta.rs deleted file mode 100644 index 801d78879..000000000 --- a/crates/brk_structs/src/structs/blockmeta.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[derive(Debug, Clone, Copy)] -pub struct BlockPosition { - pub blk_index: u16, - pub offset: usize, - pub len: u32, -} diff --git a/crates/brk_structs/src/structs/mod.rs b/crates/brk_structs/src/structs/mod.rs index 477dabea8..e525ec202 100644 --- a/crates/brk_structs/src/structs/mod.rs +++ b/crates/brk_structs/src/structs/mod.rs @@ -4,10 +4,10 @@ mod addressbytes; mod addressbyteshash; mod anyaddressindex; mod bitcoin; +mod blkposition; mod block; mod blockhash; mod blockhashprefix; -mod blockmeta; mod cents; mod date; mod dateindex; @@ -69,10 +69,10 @@ pub use addressbytes::*; pub use addressbyteshash::*; pub use anyaddressindex::*; pub use bitcoin::*; +pub use blkposition::*; pub use block::*; pub use blockhash::*; pub use blockhashprefix::*; -pub use blockmeta::*; pub use cents::*; pub use date::*; pub use dateindex::*; diff --git a/docs/TODO.md b/docs/TODO.md index e77f828b9..3437d397b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -132,6 +132,7 @@ - https://fonts.google.com/specimen/Space+Mono - keep as many files as possible [under 14kb](https://endtimes.dev/why-your-website-should-be-under-14kb-in-size/) + - No classes: https://news.ycombinator.com/item?id=45287155 - __GLOBAL__ - check `TODO`s in codebase - rename `output` to `txout` or `vout`, `input` to `txin` or `vin` diff --git a/scripts/update.sh b/scripts/update.sh index 66347cd1e..2fbdb2286 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -4,4 +4,4 @@ cargo clean rustup update cargo upgrade --incompatible cargo update -cargo build --package brk -r +cargo build --package brk