Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 14:11:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: add positions

Browse Source
This commit is contained in:
nym21
2025-09-18 19:45:16 +02:00
parent cc5701ea62
commit 43117825d7
27 changed files with 458 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+78 -77
View File
@@ -627,9 +627,11 @@ dependencies = [
name = "brk_interface"
version = "0.0.107"
dependencies = [
"bitcoincore-rpc",
"brk_computer",
"brk_error",
"brk_indexer",
"brk_parser",
"brk_structs",
"derive_deref",
"nucleo-matcher",
@@ -672,6 +674,7 @@ dependencies = [
"brk_structs",
"crossbeam",
"derive_deref",
"parking_lot 0.12.4",
"rayon",
"serde",
"serde_json",
@@ -752,7 +755,7 @@ dependencies = [
"css-module-lexer",
"dunce",
"futures",
"indexmap 2.11.1",
"indexmap 2.11.3",
"itertools",
"itoa",
"memchr",
@@ -1022,7 +1025,7 @@ dependencies = [
"fast-glob",
"form_urlencoded",
"futures",
"indexmap 2.11.1",
"indexmap 2.11.3",
"infer",
"itoa",
"memchr",
@@ -2046,7 +2049,7 @@ dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi 0.14.5+wasi-0.2.4",
"wasi 0.14.7+wasi-0.2.4",
]
[[package]]
@@ -2199,9 +2202,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.16"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8"
dependencies = [
"bytes",
"futures-core",
@@ -2369,13 +2372,14 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.11.1"
version = "2.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921"
checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3"
dependencies = [
"equivalent",
"hashbrown 0.15.5",
"serde",
"serde_core",
]
[[package]]
@@ -2399,7 +2403,7 @@ dependencies = [
"crossbeam-utils",
"dashmap",
"env_logger",
"indexmap 2.11.1",
"indexmap 2.11.3",
"itoa",
"log",
"num-format",
@@ -2532,9 +2536,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.78"
version = "0.3.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738"
checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e"
dependencies = [
"once_cell",
"wasm-bindgen",
@@ -2542,9 +2546,9 @@ dependencies = [
[[package]]
name = "json-strip-comments"
version = "1.0.4"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b271732a960335e715b6b2ae66a086f115c74eb97360e996d2bd809bfc063bba"
checksum = "c4135b29c84322dbc3327272084360785665452213a576a991b3ac2f63148e82"
dependencies = [
"memchr",
]
@@ -2955,9 +2959,9 @@ dependencies = [
[[package]]
name = "oxc-browserslist"
version = "2.0.16"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd843fdc95833f2faf4251b701a812d488ef35958144af91b9ff540d9fe8ceaa"
checksum = "4ca83f99170f095592784131ed47b6782974c98165ec4da64e90df64bdc2b3cf"
dependencies = [
"bincode",
"flate2",
@@ -2971,23 +2975,24 @@ dependencies = [
[[package]]
name = "oxc-miette"
version = "2.4.0"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31cfb121c9d3e0f9082856927f5cff9594279c91b544f4436e4bc971563caa60"
checksum = "2d5495f6099fa0b25fa25755c1d59ed79ffa64dda80f5366a4cdfc8fc20f5932"
dependencies = [
"cfg-if",
"owo-colors",
"oxc-miette-derive",
"textwrap",
"thiserror 2.0.16",
"unicode-segmentation",
"unicode-width",
]
[[package]]
name = "oxc-miette-derive"
version = "2.4.0"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6eabb57f935b454fbe0552ea0abaaf9eb0019b5fa05a7bbe7efd5bd8c765085"
checksum = "9dbbc96af6e37c35f2303b2bedbf8ce9cc563e4fbbf7776be6f0803cb0095652"
dependencies = [
"proc-macro2",
"quote",
@@ -3239,13 +3244,14 @@ dependencies = [
[[package]]
name = "oxc_resolver"
version = "11.7.2"
version = "11.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7faa92a909b5e1a2018babf171a1973e3be4c0a2e29e3337c0cfa3c0ba04ebf"
checksum = "49c4a4d746f42bac28538163952aa66da2f0ea781a0708772d8ad3f6fc066963"
dependencies = [
"cfg-if",
"indexmap 2.11.1",
"indexmap 2.11.3",
"json-strip-comments",
"libc",
"once_cell",
"papaya",
"pnp",
@@ -3283,9 +3289,9 @@ dependencies = [
[[package]]
name = "oxc_sourcemap"
version = "4.1.1"
version = "4.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60410c89a9da2ce5b10946012b8a8605ecc41129cb7976bbfb73b85902071208"
checksum = "87a91923e9269b32c91cd956596f9c3bd24b60a89df5c815abb2812a0ffa8b7b"
dependencies = [
"base64-simd",
"rustc-hash",
@@ -3338,7 +3344,7 @@ dependencies = [
"base64 0.22.1",
"compact_str",
"cow-utils",
"indexmap 2.11.1",
"indexmap 2.11.3",
"itoa",
"memchr",
"oxc-browserslist",
@@ -3525,7 +3531,7 @@ checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca"
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
"indexmap 2.11.1",
"indexmap 2.11.3",
"serde",
]
@@ -3670,9 +3676,9 @@ dependencies = [
[[package]]
name = "proc-macro-crate"
version = "3.3.0"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35"
checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983"
dependencies = [
"toml_edit",
]
@@ -4166,9 +4172,9 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.223"
version = "1.0.225"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a505d71960adde88e293da5cb5eda57093379f64e61cf77bf0e6a63af07a7bac"
checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d"
dependencies = [
"serde_core",
"serde_derive",
@@ -4176,9 +4182,9 @@ dependencies = [
[[package]]
name = "serde_bytes"
version = "0.11.18"
version = "0.11.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe07b5d88710e3b807c16a06ccbc9dfecd5fff6a4d2745c59e3e26774f10de6a"
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
dependencies = [
"serde",
"serde_core",
@@ -4186,18 +4192,18 @@ dependencies = [
[[package]]
name = "serde_core"
version = "1.0.223"
version = "1.0.225"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20f57cbd357666aa7b3ac84a90b4ea328f1d4ddb6772b430caa5d9e1309bb9e9"
checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.223"
version = "1.0.225"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d428d07faf17e306e699ec1e91996e5a165ba5d6bce5b5155173e91a8a01a56"
checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516"
dependencies = [
"proc-macro2",
"quote",
@@ -4221,7 +4227,7 @@ version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"indexmap 2.11.1",
"indexmap 2.11.3",
"itoa",
"memchr",
"ryu",
@@ -4231,9 +4237,9 @@ dependencies = [
[[package]]
name = "serde_path_to_error"
version = "0.1.19"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a30a8abed938137c7183c173848e3c9b3517f5e038226849a4ecc9b21a4b4e2a"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
@@ -4242,11 +4248,11 @@ dependencies = [
[[package]]
name = "serde_spanned"
version = "1.0.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83"
checksum = "2789234a13a53fc4be1b51ea1bab45a3c338bdb884862a257d10e5a74ae009e6"
dependencies = [
"serde",
"serde_core",
]
[[package]]
@@ -4271,7 +4277,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.11.1",
"indexmap 2.11.3",
"schemars 0.9.0",
"schemars 1.0.4",
"serde",
@@ -4669,14 +4675,14 @@ dependencies = [
[[package]]
name = "toml"
version = "0.9.5"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8"
checksum = "ae2a4cf385da23d1d53bc15cdfa5c2109e93d8d362393c801e87da2f72f0e201"
dependencies = [
"indexmap 2.11.1",
"serde",
"indexmap 2.11.3",
"serde_core",
"serde_spanned",
"toml_datetime 0.7.0",
"toml_datetime",
"toml_parser",
"toml_writer",
"winnow",
@@ -4684,27 +4690,22 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.11"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
[[package]]
name = "toml_datetime"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3"
checksum = "a197c0ec7d131bfc6f7e82c8442ba1595aeab35da7adbf05b6b73cd06a16b6be"
dependencies = [
"serde",
"serde_core",
]
[[package]]
name = "toml_edit"
version = "0.22.27"
version = "0.23.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
checksum = "c2ad0b7ae9cfeef5605163839cb9221f453399f15cfb5c10be9885fcf56611f9"
dependencies = [
"indexmap 2.11.1",
"toml_datetime 0.6.11",
"indexmap 2.11.3",
"toml_datetime",
"toml_parser",
"winnow",
]
@@ -5095,27 +5096,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasi"
version = "0.14.5+wasi-0.2.4"
version = "0.14.7+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4"
checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
dependencies = [
"wasip2",
]
[[package]]
name = "wasip2"
version = "1.0.0+wasi-0.2.4"
version = "1.0.1+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24"
checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.101"
version = "0.2.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b"
checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819"
dependencies = [
"cfg-if",
"once_cell",
@@ -5126,9 +5127,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.101"
version = "0.2.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb"
checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c"
dependencies = [
"bumpalo",
"log",
@@ -5140,9 +5141,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.101"
version = "0.2.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d"
checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -5150,9 +5151,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.101"
version = "0.2.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa"
checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32"
dependencies = [
"proc-macro2",
"quote",
@@ -5163,9 +5164,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.101"
version = "0.2.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1"
checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf"
dependencies = [
"unicode-ident",
]
@@ -5500,9 +5501,9 @@ dependencies = [
[[package]]
name = "wit-bindgen"
version = "0.45.1"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36"
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
[[package]]
name = "writeable"
@@ -5629,7 +5630,7 @@ dependencies = [
"arbitrary",
"crc32fast",
"flate2",
"indexmap 2.11.1",
"indexmap 2.11.3",
"memchr",
"zopfli",
]
Cargo.toml
+3 -3
View File
@@ -68,9 +68,9 @@ minreq = { version = "2.14.1", features = ["https", "serde_json"] }
parking_lot = "0.12.4"
quick_cache = "0.6.16"
rayon = "1.11.0"
serde = "1.0.223"
serde_bytes = "0.11.18"
serde_derive = "1.0.223"
serde = "1.0.225"
serde_bytes = "0.11.19"
serde_derive = "1.0.225"
serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
crates/brk_cli/Cargo.toml
+1 -1
View File
@@ -27,7 +27,7 @@ log = { workspace = true }
minreq = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.5"
toml = "0.9.6"
zip = { version = "5.1.1", default-features = false, features = ["deflate"] }
[[bin]]
crates/brk_cli/src/lib.rs
+2 -4
View File
@@ -73,7 +73,7 @@ pub fn run() -> color_eyre::Result<()> {
.enable_all()
.build()?
.block_on(async {
let interface = Interface::build(&indexer, &computer);
let interface = Interface::build(&parser, &indexer, &computer);
let website = config.website();
@@ -135,9 +135,7 @@ pub fn run() -> color_eyre::Result<()> {
let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap();
// dbg!(&starting_indexes);
computer.compute(&indexer, starting_indexes, &exit).unwrap();
computer.compute(&indexer, starting_indexes, &parser, &exit).unwrap();
info!("Waiting for new blocks...");
crates/brk_computer/examples/computer.rs
+1 -1
View File
@@ -45,7 +45,7 @@ pub fn main() -> Result<()> {
loop {
let i = Instant::now();
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &parser, &exit)?;
dbg!(i.elapsed());
sleep(Duration::from_secs(10));
}
crates/brk_computer/src/constants.rs
+22 -18
View File
@@ -12,8 +12,6 @@ use super::{
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
db: Database,
@@ -35,15 +33,21 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
let db = Database::open(&parent.join("constants"))?;
pub fn forced_import(
parent_path: &Path,
parent_version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let db = Database::open(&parent_path.join("constants"))?;
let version = parent_version + Version::ZERO;
let this = Self {
constant_0: ComputedVecsFromHeight::forced_import(
&db,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -51,7 +55,7 @@ impl Vecs {
&db,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -59,7 +63,7 @@ impl Vecs {
&db,
"constant_2",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -67,7 +71,7 @@ impl Vecs {
&db,
"constant_3",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -75,7 +79,7 @@ impl Vecs {
&db,
"constant_4",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -83,7 +87,7 @@ impl Vecs {
&db,
"constant_38_2",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -91,7 +95,7 @@ impl Vecs {
&db,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -99,7 +103,7 @@ impl Vecs {
&db,
"constant_61_8",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -107,7 +111,7 @@ impl Vecs {
&db,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -115,7 +119,7 @@ impl Vecs {
&db,
"constant_600",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -123,7 +127,7 @@ impl Vecs {
&db,
"constant_minus_1",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -131,7 +135,7 @@ impl Vecs {
&db,
"constant_minus_2",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -139,7 +143,7 @@ impl Vecs {
&db,
"constant_minus_3",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
@@ -147,7 +151,7 @@ impl Vecs {
&db,
"constant_minus_4",
Source::Compute,
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexes,
VecBuilderOptions::default().add_last(),
)?,
crates/brk_computer/src/lib.rs
+34 -14
View File
@@ -5,6 +5,7 @@ use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::Version;
use log::info;
use vecdb::{AnyCollectableVec, Exit, Format};
@@ -17,6 +18,7 @@ mod grouped;
mod indexes;
mod market;
mod pools;
mod positions;
mod price;
mod stateful;
mod states;
@@ -31,15 +33,16 @@ use states::*;
#[derive(Clone)]
pub struct Computer {
pub indexes: indexes::Vecs,
pub chain: chain::Vecs,
pub cointime: cointime::Vecs,
pub constants: constants::Vecs,
pub fetched: Option<fetched::Vecs>,
pub indexes: indexes::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub positions: positions::Vecs,
pub price: Option<price::Vecs>,
pub chain: chain::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
pub cointime: cointime::Vecs,
}
const VERSION: Version = Version::new(4);
@@ -87,6 +90,7 @@ impl Computer {
&indexes,
price.as_ref(),
)?,
positions: positions::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
@@ -109,15 +113,12 @@ impl Computer {
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
parser: &Parser,
exit: &Exit,
) -> Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(&self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
@@ -131,7 +132,25 @@ impl Computer {
)?;
}
info!("Computing positions...");
self.positions
.compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
let constants = scope.spawn(|| -> Result<()> {
info!("Computing constants...");
self.constants
.compute(&self.indexes, &starting_indexes, exit)?;
Ok(())
});
// let positions = scope.spawn(|| -> Result<()> {
// info!("Computing positions...");
// self.positions
// .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
// Ok(())
// });
let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
self.chain.compute(
@@ -149,6 +168,8 @@ impl Computer {
self.market.compute(price, &starting_indexes, exit)?;
}
constants.join().unwrap()?;
// positions.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;
@@ -162,8 +183,6 @@ impl Computer {
exit,
)?;
// return Ok(());
info!("Computing stateful...");
self.stateful.compute(
indexer,
@@ -191,14 +210,15 @@ impl Computer {
let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec>> =
Box::new(self.fetched.iter().flat_map(|v| v.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.chain.iter_any_collectable()));
iter = Box::new(iter.chain(self.cointime.iter_any_collectable()));
iter = Box::new(iter.chain(self.constants.iter_any_collectable()));
iter = Box::new(iter.chain(self.indexes.iter_any_collectable()));
iter = Box::new(iter.chain(self.market.iter_any_collectable()));
iter = Box::new(iter.chain(self.chain.iter_any_collectable()));
iter = Box::new(iter.chain(self.pools.iter_any_collectable()));
iter = Box::new(iter.chain(self.positions.iter_any_collectable()));
iter = Box::new(iter.chain(self.price.iter().flat_map(|v| v.iter_any_collectable())));
iter = Box::new(iter.chain(self.stateful.iter_any_collectable()));
iter = Box::new(iter.chain(self.cointime.iter_any_collectable()));
iter
}
crates/brk_computer/src/positions.rs
+133
View File
@@ -0,0 +1,133 @@
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::{BlkPosition, Height, TxIndex, Version};
use log::info;
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
GenericStoredVec, PAGE_SIZE, VecIterator,
};
use super::{Indexes, indexes};
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub height_to_position: CompressedVec<Height, BlkPosition>,
pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
}
impl Vecs {
pub fn forced_import(parent_path: &Path, parent_version: Version) -> Result<Self> {
let db = Database::open(&parent_path.join("positions"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let version = parent_version + Version::ZERO;
let this = Self {
height_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
)?,
txindex_to_position: CompressedVec::forced_import(
&db,
"position",
version + Version::ZERO,
)?,
db,
};
this.db.retain_regions(
this.iter_any_collectable()
.flat_map(|v| v.region_names())
.collect(),
)?;
Ok(this)
}
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
parser: &Parser,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, parser, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
parser: &Parser,
exit: &Exit,
) -> Result<()> {
let min_txindex =
TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
let min_height = indexes
.txindex_to_height
.iter()
.unwrap_get_inner(min_txindex)
.min(starting_indexes.height);
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
parser
.parse(Some(min_height), None)
.iter()
.try_for_each(|block| -> Result<()> {
let height = block.height();
info!("{height}");
self.height_to_position
.forced_push_at(height, *block.position(), exit)?;
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
block.tx_positions().iter().enumerate().try_for_each(
|(index, position)| -> Result<()> {
let txindex = txindex + index;
self.txindex_to_position
.forced_push_at(txindex, *position, exit)?;
Ok(())
},
)?;
// Stuck, why ??
if *height % 1_000 == 0 {
let _lock = exit.lock();
self.height_to_position.flush()?;
self.txindex_to_position.flush()?;
}
Ok(())
})?;
let _lock = exit.lock();
self.height_to_position.flush()?;
self.txindex_to_position.flush()?;
Ok(())
}
pub fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
Box::new(
[
&self.height_to_position as &dyn AnyCollectableVec,
&self.txindex_to_position,
]
.into_iter(),
)
}
}
crates/brk_indexer/src/vecs.rs
+17 -19
View File
@@ -20,7 +20,6 @@ use crate::Indexes;
#[derive(Clone)]
pub struct Vecs {
db: Database,
pub emptyoutputindex_to_txindex: CompressedVec<EmptyOutputIndex, TxIndex>,
pub height_to_blockhash: RawVec<Height, BlockHash>,
pub height_to_difficulty: CompressedVec<Height, StoredF64>,
@@ -39,7 +38,7 @@ pub struct Vecs {
pub height_to_first_p2wshaddressindex: CompressedVec<Height, P2WSHAddressIndex>,
pub height_to_first_txindex: CompressedVec<Height, TxIndex>,
pub height_to_first_unknownoutputindex: CompressedVec<Height, UnknownOutputIndex>,
/// Doesn't guarantee continuity due to possible reorgs
/// Doesn't guarantee continuity due to possible reorgs and more generally the nature of mining
pub height_to_timestamp: CompressedVec<Height, Timestamp>,
pub height_to_total_size: CompressedVec<Height, StoredU64>,
pub height_to_weight: CompressedVec<Height, Weight>,
@@ -72,9 +71,11 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(parent: &Path, version: Version) -> Result<Self> {
let db = Database::open(&parent.join("vecs"))?;
db.set_min_len(PAGE_SIZE * 50_000_000)?;
let db_positions = Database::open(&parent.join("vecs/positions"))?;
db_positions.set_min_len(PAGE_SIZE * 1_000_000)?;
let this = Self {
emptyoutputindex_to_txindex: CompressedVec::forced_import(&db, "txindex", version)?,
height_to_blockhash: RawVec::forced_import(&db, "blockhash", version)?,
@@ -192,12 +193,8 @@ impl Vecs {
db,
};
// self.db.retain_regions(
// this.vecs()
// .into_iter()
// .flat_map(|v| v.region_names())
// .collect(),
// )?;
this.db
.retain_regions(this.iter().flat_map(|v| v.region_names()).collect())?;
Ok(this)
}
@@ -349,16 +346,15 @@ impl Vecs {
}
pub fn flush(&mut self, height: Height) -> Result<()> {
self.mut_vecs()
.into_par_iter()
self.iter_mut()
.par_bridge()
.try_for_each(|vec| vec.stamped_flush(Stamp::from(height)))?;
self.db.flush()?;
Ok(())
}
pub fn starting_height(&mut self) -> Height {
self.mut_vecs()
.into_iter()
self.iter_mut()
.map(|vec| {
let h = Height::from(vec.stamp());
if h > Height::ZERO { h.incremented() } else { h }
@@ -372,9 +368,9 @@ impl Vecs {
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
vec![
&self.emptyoutputindex_to_txindex,
pub fn iter(&self) -> impl Iterator<Item = &dyn AnyCollectableVec> {
[
&self.emptyoutputindex_to_txindex as &dyn AnyCollectableVec,
&self.height_to_blockhash,
&self.height_to_difficulty,
&self.height_to_first_emptyoutputindex,
@@ -419,11 +415,12 @@ impl Vecs {
&self.txindex_to_txversion,
&self.unknownoutputindex_to_txindex,
]
.into_iter()
}
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStoredVec> {
vec![
&mut self.emptyoutputindex_to_txindex,
fn iter_mut(&mut self) -> impl Iterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.emptyoutputindex_to_txindex as &mut dyn AnyStoredVec,
&mut self.height_to_blockhash,
&mut self.height_to_difficulty,
&mut self.height_to_first_emptyoutputindex,
@@ -468,5 +465,6 @@ impl Vecs {
&mut self.txindex_to_txversion,
&mut self.unknownoutputindex_to_txindex,
]
.into_iter()
}
}
crates/brk_interface/Cargo.toml
+2
View File
@@ -10,9 +10,11 @@ rust-version.workspace = true
build = "build.rs"
[dependencies]
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_error = { workspace = true }
brk_indexer = { workspace = true }
brk_parser = { workspace = true }
brk_structs = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
crates/brk_interface/examples/main.rs
+27 -5
View File
@@ -1,18 +1,40 @@
use std::path::Path;
use std::{fs, path::Path};
use brk_computer::Computer;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_interface::{Index, Interface, Params, ParamsOpt};
use brk_parser::Parser;
use vecdb::Exit;
pub fn main() -> Result<()> {
let outputs_dir = Path::new("../../_outputs");
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let indexer = Indexer::forced_import(outputs_dir)?;
let blocks_dir = bitcoin_dir.join("blocks");
let computer = Computer::forced_import(outputs_dir, &indexer, None)?;
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
fs::create_dir_all(&outputs_dir)?;
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let interface = Interface::build(&indexer, &computer);
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(blocks_dir, rpc);
let indexer = Indexer::forced_import(&outputs_dir)?;
let computer = Computer::forced_import(&outputs_dir, &indexer, None)?;
let interface = Interface::build(&parser, &indexer, &computer);
dbg!(interface.search_and_format(Params {
index: Index::Height,
crates/brk_interface/src/lib.rs
+9 -1
View File
@@ -5,6 +5,7 @@ use std::{collections::BTreeMap, sync::OnceLock};
use brk_computer::Computer;
use brk_error::{Error, Result};
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_structs::Height;
use nucleo_matcher::{
Config, Matcher,
@@ -42,18 +43,21 @@ pub fn cached_errors() -> &'static Cache<String, String> {
#[allow(dead_code)]
pub struct Interface<'a> {
vecs: Vecs<'a>,
parser: &'a Parser,
indexer: &'a Indexer,
computer: &'a Computer,
}
impl<'a> Interface<'a> {
pub fn build(indexer: &Indexer, computer: &Computer) -> Self {
pub fn build(parser: &Parser, indexer: &Indexer, computer: &Computer) -> Self {
let parser = parser.static_clone();
let indexer = indexer.static_clone();
let computer = computer.static_clone();
let vecs = Vecs::build(indexer, computer);
Self {
vecs,
parser,
indexer,
computer,
}
@@ -262,6 +266,10 @@ impl<'a> Interface<'a> {
self.vecs.id_to_indexes(id)
}
pub fn parser(&self) -> &Parser {
self.parser
}
pub fn indexer(&self) -> &Indexer {
self.indexer
}
crates/brk_interface/src/vecs.rs
+1 -5
View File
@@ -27,11 +27,7 @@ impl<'a> Vecs<'a> {
pub fn build(indexer: &'a Indexer, computer: &'a Computer) -> Self {
let mut this = Vecs::default();
indexer
.vecs
.vecs()
.into_iter()
.for_each(|vec| this.insert(vec));
indexer.vecs.iter().for_each(|vec| this.insert(vec));
computer
.iter_any_collectable()
crates/brk_parser/Cargo.toml
+1
View File
@@ -18,6 +18,7 @@ brk_error = { workspace = true }
brk_structs = { workspace = true }
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
derive_deref = { workspace = true }
parking_lot = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
crates/brk_parser/examples/main.rs
+2 -3
View File
@@ -3,7 +3,6 @@ use std::{collections::BTreeMap, path::Path};
use bitcoincore_rpc::{Auth, Client};
use brk_error::Result;
use brk_parser::Parser;
use brk_structs::Height;
#[allow(clippy::needless_doctest_main)]
fn main() -> Result<()> {
@@ -21,13 +20,13 @@ fn main() -> Result<()> {
let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
let start = Some(915_155_u32.into());
let start = Some(915_138_u32.into());
let end = None;
let mut blk_index = 0;
let mut diff = BTreeMap::new();
parser.parse(start, end).iter().for_each(|block| {
println!("{}: {}", block.height(), block.hash());
let new_blk_index = block.position().blk_index;
let new_blk_index = block.position().blk_index();
if new_blk_index < blk_index {
diff.insert(blk_index - new_blk_index, block.height());
}
crates/brk_parser/src/any_block.rs
+7 -4
View File
@@ -1,7 +1,7 @@
use bitcoin::{Transaction, VarInt, block::Header, consensus::Decodable, io::Cursor};
use bitcoincore_rpc::RpcApi;
use brk_error::Result;
use brk_structs::{Block, BlockPosition, Height, ParsedBlock};
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use crate::{XORBytes, XORIndex};
@@ -14,10 +14,10 @@ pub enum AnyBlock {
impl AnyBlock {
pub fn decode(
self,
position: BlockPosition,
position: BlkPosition,
rpc: &'static bitcoincore_rpc::Client,
mut xor_i: XORIndex,
xor_bytes: &XORBytes,
xor_bytes: XORBytes,
start: Option<Height>,
end: Option<Height>,
) -> Result<Self> {
@@ -55,14 +55,17 @@ impl AnyBlock {
}
let mut txdata = Vec::with_capacity(tx_count as usize);
let mut tx_positions = Vec::with_capacity(tx_count as usize);
for _ in 0..tx_count {
let tx_position = BlkPosition::new(position.blk_index(), cursor.position() as u32);
tx_positions.push(tx_position);
let tx = Transaction::consensus_decode(&mut cursor)?;
txdata.push(tx);
}
let block = bitcoin::Block { header, txdata };
let block = Block::from((height, hash, block));
let block = ParsedBlock::from((block, position));
let block = ParsedBlock::from((block, position, tx_positions));
Ok(Self::Decoded(block))
}
crates/brk_parser/src/blk_index_to_blk_path.rs
+1 -1
View File
@@ -9,7 +9,7 @@ use derive_deref::{Deref, DerefMut};
const BLK: &str = "blk";
const DOT_DAT: &str = ".dat";
#[derive(Debug, Deref, DerefMut)]
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct BlkIndexToBlkPath(BTreeMap<u16, PathBuf>);
impl BlkIndexToBlkPath {
crates/brk_parser/src/lib.rs
+43 -26
View File
@@ -7,6 +7,7 @@ use std::{
mem,
ops::ControlFlow,
path::PathBuf,
sync::Arc,
thread,
};
@@ -14,8 +15,9 @@ use bitcoin::{block::Header, consensus::Decodable};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use brk_error::Result;
use brk_structs::{Block, BlockPosition, Height, ParsedBlock};
use brk_structs::{BlkPosition, Block, Height, ParsedBlock};
use crossbeam::channel::{Receiver, bounded};
use parking_lot::{RwLock, RwLockReadGuard};
use rayon::prelude::*;
mod any_block;
@@ -24,22 +26,40 @@ mod xor_bytes;
mod xor_index;
use any_block::*;
use xor_bytes::*;
use xor_index::*;
pub use xor_bytes::*;
pub use xor_index::*;
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 50;
#[derive(Debug, Clone)]
pub struct Parser {
blk_index_to_blk_path: Arc<RwLock<BlkIndexToBlkPath>>,
xor_bytes: XORBytes,
blocks_dir: PathBuf,
rpc: &'static bitcoincore_rpc::Client,
}
impl Parser {
pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
Self { blocks_dir, rpc }
Self {
xor_bytes: XORBytes::from(blocks_dir.as_path()),
blk_index_to_blk_path: Arc::new(RwLock::new(BlkIndexToBlkPath::scan(
blocks_dir.as_path(),
))),
blocks_dir,
rpc,
}
}
pub fn blk_index_to_blk_path(&self) -> RwLockReadGuard<'_, BlkIndexToBlkPath> {
self.blk_index_to_blk_path.read()
}
pub fn xor_bytes(&self) -> XORBytes {
self.xor_bytes
}
pub fn get(&self, height: Height) -> Result<Block> {
@@ -57,19 +77,19 @@ impl Parser {
/// For an example checkout `./main.rs`
///
pub fn parse(&self, start: Option<Height>, end: Option<Height>) -> Receiver<ParsedBlock> {
let blocks_dir = self.blocks_dir.as_path();
let rpc = self.rpc;
let (send_bytes, recv_bytes) = bounded(BOUND_CAP);
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_ordered, recv_ordered) = bounded(BOUND_CAP);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(&self.blocks_dir);
*self.blk_index_to_blk_path.write() = blk_index_to_blk_path.clone();
let xor_bytes = XORBytes::from(blocks_dir);
let xor_bytes = self.xor_bytes;
let first_blk_index = self
.find_start_blk_index(start, &blk_index_to_blk_path, &xor_bytes)
.find_start_blk_index(start, &blk_index_to_blk_path, xor_bytes)
.unwrap_or_default();
thread::spawn(move || {
@@ -95,7 +115,7 @@ impl Parser {
current_4bytes.rotate_left(1);
current_4bytes[3] = xor_i.byte(blk_bytes[i], &xor_bytes);
current_4bytes[3] = xor_i.byte(blk_bytes[i], xor_bytes);
i += 1;
if current_4bytes == MAGIC_BYTES {
@@ -103,27 +123,20 @@ impl Parser {
}
}
let deser_len = u32::from_le_bytes(
let position = BlkPosition::new(blk_index, i as u32);
let len = u32::from_le_bytes(
xor_i
.bytes(&mut blk_bytes[i..(i + 4)], &xor_bytes)
.bytes(&mut blk_bytes[i..(i + 4)], xor_bytes)
.try_into()
.unwrap(),
);
let len = deser_len as usize;
) as usize;
i += 4;
let block_bytes = (blk_bytes[i..(i + len)]).to_vec();
if send_bytes
.send((
BlockPosition {
blk_index,
offset: i,
len: deser_len,
},
AnyBlock::Raw(block_bytes),
xor_i,
))
.send((position, AnyBlock::Raw(block_bytes), xor_i))
.is_err()
{
return ControlFlow::Break(());
@@ -143,13 +156,13 @@ impl Parser {
let mut bulk = vec![];
let drain_and_send = |bulk: &mut Vec<(BlockPosition, AnyBlock, XORIndex)>| {
let drain_and_send = |bulk: &mut Vec<(BlkPosition, AnyBlock, XORIndex)>| {
// Using a vec and sending after to not end up with stuck threads in par iter
mem::take(bulk)
.into_par_iter()
.try_for_each(|(position, any_block, xor_i)| {
if let Ok(AnyBlock::Decoded(block)) =
any_block.decode(position, rpc, xor_i, &xor_bytes, start, end)
any_block.decode(position, rpc, xor_i, xor_bytes, start, end)
&& send_block.send(block).is_err()
{
return ControlFlow::Break(());
@@ -211,7 +224,7 @@ impl Parser {
&self,
target_start: Option<Height>,
blk_index_to_blk_path: &BlkIndexToBlkPath,
xor_bytes: &XORBytes,
xor_bytes: XORBytes,
) -> Result<u16> {
let Some(target_start) = target_start else {
return Ok(0);
@@ -280,7 +293,7 @@ impl Parser {
Ok(blk_indices.get(final_idx).copied().unwrap_or(0))
}
fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: &XORBytes) -> Result<Height> {
fn get_first_block_height(&self, blk_path: &PathBuf, xor_bytes: XORBytes) -> Result<Height> {
let mut file = File::open(blk_path)?;
let mut xor_i = XORIndex::default();
let mut current_4bytes = [0; 4];
@@ -314,4 +327,8 @@ impl Parser {
Ok(Height::new(height))
}
pub fn static_clone(&self) -> &'static Self {
Box::leak(Box::new(self.clone()))
}
}
crates/brk_parser/src/xor_index.rs
+2 -2
View File
@@ -4,7 +4,7 @@ use crate::xor_bytes::{XOR_LEN, XORBytes};
pub struct XORIndex(usize);
impl XORIndex {
pub fn bytes<'a>(&mut self, bytes: &'a mut [u8], xor_bytes: &XORBytes) -> &'a mut [u8] {
pub fn bytes<'a>(&mut self, bytes: &'a mut [u8], xor_bytes: XORBytes) -> &'a mut [u8] {
let len = bytes.len();
let mut bytes_index = 0;
@@ -18,7 +18,7 @@ impl XORIndex {
}
#[inline]
pub fn byte(&mut self, mut byte: u8, xor_bytes: &XORBytes) -> u8 {
pub fn byte(&mut self, mut byte: u8, xor_bytes: XORBytes) -> u8 {
byte ^= xor_bytes[self.0];
self.increment();
byte
crates/brk_server/examples/main.rs
+2 -2
View File
@@ -39,7 +39,7 @@ pub fn main() -> Result<()> {
.enable_all()
.build()?
.block_on(async {
let interface = Interface::build(&indexer, &computer);
let interface = Interface::build(&parser, &indexer, &computer);
let server = Server::new(interface, None);
@@ -53,7 +53,7 @@ pub fn main() -> Result<()> {
let starting_indexes = indexer.index(&parser, rpc, &exit, true)?;
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &parser, &exit)?;
while block_count == rpc.get_block_count()? {
sleep(Duration::from_secs(1))
crates/brk_server/src/api/mod.rs
+35 -12
View File
@@ -1,4 +1,8 @@
use std::str::FromStr;
use std::{
fs::File,
io::{BufReader, Seek, SeekFrom},
str::FromStr,
};
use axum::{
Json, Router,
@@ -7,9 +11,10 @@ use axum::{
response::{IntoResponse, Redirect, Response},
routing::get,
};
use bitcoin::{Address, Network, absolute::LockTime};
use bitcoin::{Address, Network, Transaction, absolute::LockTime, consensus::Decodable};
use bitcoincore_rpc::bitcoin;
use brk_interface::{IdParam, Index, PaginatedIndexParam, PaginationParam, Params, ParamsOpt};
use brk_parser::XORIndex;
use brk_structs::{
AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, Txid, TxidPrefix,
};
@@ -45,13 +50,6 @@ impl ApiRoutes for Router<AppState> {
let computer = interface.computer();
let stores = &indexer.stores;
let hash = AddressBytesHash::from(&address);
dbg!(&hash);
dbg!(
&address,
address.address_type(),
address.script_pubkey(),
OutputType::from(&address)
);
let Ok(Some(addri)) = stores
.addressbyteshash_to_typeindex
@@ -60,8 +58,6 @@ impl ApiRoutes for Router<AppState> {
return "Unknown address".into_response();
};
println!("Script pubkey: {}", address.script_pubkey());
println!("Address type: {:?}", address.address_type());
let output_type = OutputType::from(&address);
let stateful = &computer.stateful;
let price = computer.price.as_ref().map(|v| {
@@ -178,11 +174,38 @@ impl ApiRoutes for Router<AppState> {
.unwrap_get_inner(txindex);
let locktime = LockTime::from(rawlocktime);
let parser = interface.parser();
let computer = interface.computer();
let position = computer.positions.txindex_to_position.iter().unwrap_get_inner(txindex);
let blk_index_to_blk_path = parser.blk_index_to_blk_path();
let Some(blk_path) = blk_index_to_blk_path.get(&position.blk_index()) else {
return "Unknown blk index".into_response();
};
let mut xori = XORIndex::default();
xori.add_assign(position.offset() as usize);
let Ok(file) = File::open(blk_path) else {
return "Error opening blk file".into_response();
};
let mut reader = BufReader::new(file);
if reader.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
return "Error seeking position in blk file".into_response();
}
let Ok(tx) = Transaction::consensus_decode(&mut reader) else {
return "Error decoding transaction".into_response();
};
Json(serde_json::json!({
"txid": txid,
"index": txindex,
"version": version,
"locktime": locktime
"locktime": locktime,
"tx": tx,
}))
.into_response()
},
crates/brk_structs/src/structs/blkposition.rs
+22
View File
@@ -0,0 +1,22 @@
use serde::Serialize;
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout, StoredCompressed,
)]
pub struct BlkPosition(u64);
impl BlkPosition {
pub fn new(blk_index: u16, offset: u32) -> Self {
Self(((blk_index as u64) << 32) | offset as u64)
}
pub fn blk_index(&self) -> u16 {
(self.0 >> 31) as u16
}
pub fn offset(&self) -> u32 {
self.0 as u32
}
}
crates/brk_structs/src/structs/block.rs
+9 -93
View File
@@ -1,14 +1,6 @@
use std::{
borrow::Cow,
collections::{BTreeMap, HashMap},
io::Cursor,
ops::Deref,
};
use std::{borrow::Cow, ops::Deref};
use bitcoin::{block::Header, consensus::Decodable};
use brk_error::Result;
use crate::BlockPosition;
use crate::BlkPosition;
use super::{BlockHash, Height};
@@ -28,40 +20,6 @@ impl Block {
&self.hash
}
pub fn parse(block_data: &[u8]) -> Result<Self> {
let mut cursor = Cursor::new(block_data);
let header = Header::consensus_decode(&mut cursor)?;
// Parse transactions with positions
let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize;
let mut transactions = Vec::with_capacity(tx_count);
let mut tx_positions = HashMap::with_capacity(tx_count);
for _ in 0..tx_count {
let start = cursor.position() as usize;
let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?;
tx_positions.insert(tx.compute_txid(), start);
transactions.push(tx);
}
let block = bitcoin::Block {
header,
txdata: transactions,
};
// block.bip34_block_height()
let hash = block.block_hash();
Ok(Block {
block,
hash: hash.into(),
height: Height::ZERO,
})
}
pub fn coinbase_tag(&self) -> Cow<'_, str> {
String::from_utf8_lossy(
self.txdata
@@ -100,70 +58,28 @@ impl Deref for Block {
#[derive(Debug)]
pub struct ParsedBlock {
block: Block,
position: BlockPosition,
tx_positions: BTreeMap<usize, usize>, // txid -> offset
position: BlkPosition,
tx_positions: Vec<BlkPosition>,
}
impl From<(Block, BlockPosition)> for ParsedBlock {
fn from((block, position): (Block, BlockPosition)) -> Self {
impl From<(Block, BlkPosition, Vec<BlkPosition>)> for ParsedBlock {
fn from((block, position, tx_positions): (Block, BlkPosition, Vec<BlkPosition>)) -> Self {
Self {
block,
position,
tx_positions: BTreeMap::default(),
tx_positions,
}
}
}
impl ParsedBlock {
pub fn position(&self) -> &BlockPosition {
pub fn position(&self) -> &BlkPosition {
&self.position
}
pub fn tx_positions(&self) -> &BTreeMap<usize, usize> {
pub fn tx_positions(&self) -> &Vec<BlkPosition> {
&self.tx_positions
}
// pub fn parse(block_data: &[u8], file_offset: usize) -> Result<Self> {
// let mut cursor = std::io::Cursor::new(block_data);
// let header = Header::consensus_decode(&mut cursor)?;
// // Parse transactions with positions
// let tx_count = bitcoin::VarInt::consensus_decode(&mut cursor)?.0 as usize;
// let mut transactions = Vec::with_capacity(tx_count);
// let mut tx_positions = HashMap::with_capacity(tx_count);
// for _ in 0..tx_count {
// let start = cursor.position() as usize;
// let tx = bitcoin::Transaction::consensus_decode(&mut cursor)?;
// tx_positions.insert(tx.compute_txid(), start);
// transactions.push(tx);
// }
// let block = bitcoin::Block {
// header,
// txdata: transactions,
// };
// // block.bip34_block_height()
// let hash = block.block_hash();
// Ok(Block {
// block,
// hash: hash.into(),
// height: Height::ZERO,
// tx_positions,
// block_start_offset: file_offset,
// })
// }
// pub fn get_absolute_position(&self, txid: &Txid) -> Option<usize> {
// self.tx_positions
// .get(txid)
// .map(|offset| self.block_start_offset + offset)
// }
}
impl Deref for ParsedBlock {
crates/brk_structs/src/structs/blockmeta.rs
-6
View File
@@ -1,6 +0,0 @@
#[derive(Debug, Clone, Copy)]
pub struct BlockPosition {
pub blk_index: u16,
pub offset: usize,
pub len: u32,
}
crates/brk_structs/src/structs/mod.rs
+2 -2
View File
@@ -4,10 +4,10 @@ mod addressbytes;
mod addressbyteshash;
mod anyaddressindex;
mod bitcoin;
mod blkposition;
mod block;
mod blockhash;
mod blockhashprefix;
mod blockmeta;
mod cents;
mod date;
mod dateindex;
@@ -69,10 +69,10 @@ pub use addressbytes::*;
pub use addressbyteshash::*;
pub use anyaddressindex::*;
pub use bitcoin::*;
pub use blkposition::*;
pub use block::*;
pub use blockhash::*;
pub use blockhashprefix::*;
pub use blockmeta::*;
pub use cents::*;
pub use date::*;
pub use dateindex::*;
docs/TODO.md
+1
View File
@@ -132,6 +132,7 @@
- https://fonts.google.com/specimen/Space+Mono
- keep as many files as possible [under 14kb](https://endtimes.dev/why-your-website-should-be-under-14kb-in-size/)
- No classes: https://news.ycombinator.com/item?id=45287155
- __GLOBAL__
- check `TODO`s in codebase
- rename `output` to `txout` or `vout`, `input` to `txin` or `vin`
scripts/update.sh
+1 -1
View File
@@ -4,4 +4,4 @@ cargo clean
rustup update
cargo upgrade --incompatible
cargo update
cargo build --package brk -r
cargo build --package brk