diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 3fbd9f4d5..000000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,14 +0,0 @@ -# Release profile: all targets get native CPU -[profile.release.target.'cfg()'] -rustflags = ["-C", "target-cpu=native"] - -# Release profile: x86_64 gets native CPU + extra features (overrides the above) -[profile.release.target.'cfg(target_arch = "x86_64")'] -rustflags = [ - "-C", "target-cpu=native", - "-C", "target-feature=+bmi1,+bmi2,+avx2" -] - -# Dist profile: x86_64 gets only extra features (no native CPU) -[profile.dist.target.'cfg(target_arch = "x86_64")'] -rustflags = ["-C", "target-feature=+bmi1,+bmi2,+avx2"] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ae24e036..188a40a53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/ +# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist # # Copyright 2022-2024, axodotdev # SPDX-License-Identifier: MIT or Apache-2.0 @@ -58,12 +58,13 @@ jobs: steps: - uses: actions/checkout@v4 with: + persist-credentials: false submodules: recursive - name: Install dist # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh" + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.29.0/cargo-dist-installer.sh | sh" - name: Cache dist uses: actions/upload-artifact@v4 with: @@ -117,6 +118,7 @@ jobs: git config --global core.longpaths true - uses: actions/checkout@v4 with: + persist-credentials: false submodules: recursive - name: Install Rust non-interactively if not already installed if: ${{ matrix.container }} @@ -175,6 +177,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + persist-credentials: false submodules: recursive - name: Install cached dist uses: actions/download-artifact@v4 @@ -224,6 +227,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + persist-credentials: false submodules: recursive - name: Install cached dist uses: actions/download-artifact@v4 @@ -288,4 +292,5 @@ jobs: steps: - uses: actions/checkout@v4 with: + persist-credentials: false submodules: recursive diff --git a/Cargo.lock b/Cargo.lock index 6751a8ca0..ebac3a683 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,9 +364,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcoin" -version = "0.32.6" +version = "0.32.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8929a18b8e33ea6b3c09297b687baaa71fb1b97353243a3f1029fad5c59c5b" +checksum = "0fda569d741b895131a88ee5589a467e73e9c4718e958ac9308e4f7dc44b6945" dependencies = [ "base58ck", "bech32", @@ -3459,9 +3459,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "2.0.2" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2297bf4643b32dc85bf5f622c9a41a268db6c6a6a2a2e3b405958a5b014ad9e1" +checksum = "2ef3d82b018f786967b1a5d34a08ebc3c7a9ab35b5bcbe3e2e057a0a453f26c8" [[package]] name = "rayon" @@ -4231,9 +4231,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1" +checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" dependencies = [ "indexmap 2.10.0", "serde", @@ -4255,9 +4255,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" dependencies = [ "winnow", ] diff --git a/Cargo.toml b/Cargo.toml index 3f03bb547..99f04524e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ inherits = "release" [workspace.dependencies] axum = "0.8.4" -bitcoin = { version = "0.32.6", features = ["serde"] } +bitcoin = { version = "0.32.7", features = ["serde"] } bitcoincore-rpc = "0.19.0" brk_bundler = { version = "0.0.83", path = "crates/brk_bundler" } brk_cli = { version = "0.0.83", path = "crates/brk_cli" } @@ -63,7 +63,7 @@ pre-release-commit-message = "release: v{{version}}" tag-message = "release: v{{version}}" [workspace.metadata.dist] -cargo-dist-version = "0.28.0" +cargo-dist-version = "0.29.0" ci = "github" installers = [] targets = [ @@ -73,10 +73,10 @@ targets = [ "x86_64-unknown-linux-gnu", ] -[workspace.metadata.dist.github-custom-runners] -global = "ubuntu-latest" -aarch64-apple-darwin.runner = "macos-14" -x86_64-unknown-linux-gnu.runner = "ubuntu-latest" -x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" } -aarch64-unknown-linux-gnu.runner = "ubuntu-latest" -aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" } +# [workspace.metadata.dist.github-custom-runners] +# global = "ubuntu-latest" +# aarch64-apple-darwin.runner = "macos-14" +# x86_64-unknown-linux-gnu.runner = "ubuntu-latest" +# x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" } +# aarch64-unknown-linux-gnu.runner = "ubuntu-latest" +# aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" } diff --git a/README.md b/README.md index 47cc318c9..6be20ce3f 100644 --- a/README.md +++ b/README.md @@ -54,20 +54,22 @@ In contrast, existing alternatives tend to be either [very costly](https://studi ## Crates -- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates -- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance -- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer -- [`brk_structs`](https://crates.io/crates/brk_structs): The Core (Structs and Errors) of the Bitcoin Research Kit -- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher -- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser -- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit -- [`brk_mcp`](https://crates.io/crates/brk_mcp): A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK -- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust -- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine -- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer` -- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) -- [`brk_vec`](https://crates.io/crates/brk_vec): A storeable vec +- [`brk`](https://crates.io/crates/brk): A wrapper around all other `brk-*` crates - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/) +- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a BRK instance +- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer built on top of [`brk_indexer`](https://crates.io/crates/brk_indexer) +- [`brk_error`](https://crates.io/crates/brk_error): Errors used throughout BRK +- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher +- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin indexer built on top of [`brk_parser`](https://crates.io/crates/brk_parser) +- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to find and format data from BRK +- [`brk_logger`](https://crates.io/crates/brk_logger): A thin wrapper around [`env_logger`](https://crates.io/crates/env_logger) +- [`brk_mcp`](https://crates.io/crates/brk_mcp): A bridge for LLMs to access BRK +- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin block parser and iterator built on top of [`bitcoin-rust`](https://crates.io/crates/bitcoin) +- [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK +- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) +- [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK +- [`brk_vecs`](https://crates.io/crates/brk_vecs): A KISS index/value store +- [`brk_vecs_macros`](https://crates.io/crates/brk_vecs_macros): Macros for [`brk_vecs`](https://crates.io/crates/brk_vecs) ## Hosting as a service diff --git a/build.rs b/build.rs new file mode 100644 index 000000000..f765747eb --- /dev/null +++ b/build.rs @@ -0,0 +1,14 @@ +fn main() { + let profile = std::env::var("PROFILE").unwrap_or_default(); + + if profile == "release" { + println!("cargo:rustc-flag=-C"); + println!("cargo:rustc-flag=target-cpu=native"); + + #[cfg(target_arch = "x86_64")] + { + println!("cargo:rustc-flag=-C"); + println!("cargo:rustc-flag=target-feature=+bmi1,+bmi2,+avx2"); + } + } +} diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index b37651c85..0e80a6091 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -7,6 +7,7 @@ homepage.workspace = true repository.workspace = true edition.workspace = true version.workspace = true +build = "../../build.rs" [features] full = [ diff --git a/crates/brk_bundler/Cargo.toml b/crates/brk_bundler/Cargo.toml index 160acc309..f2954b370 100644 --- a/crates/brk_bundler/Cargo.toml +++ b/crates/brk_bundler/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] log = { workspace = true } diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 75d841bde..4664eb7c4 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_cli" -description = "A command line interface to run a Bitcoin Research Kit instance" +description = "A command line interface to run a BRK instance" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] bitcoincore-rpc = { workspace = true } @@ -25,7 +26,7 @@ log = { workspace = true } minreq = { workspace = true } serde = { workspace = true } tokio = { workspace = true } -toml = "0.9.4" +toml = "0.9.5" zip = { version = "4.3.0", default-features = false, features = ["deflate"] } [[bin]] diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 355060750..a3536eee4 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_computer" -description = "A Bitcoin dataset computer, built on top of brk_indexer" +description = "A Bitcoin dataset computer built on top of brk_indexer" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] bitcoin = { workspace = true } diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/computer.rs similarity index 76% rename from crates/brk_computer/examples/main.rs rename to crates/brk_computer/examples/computer.rs index 3dc8b3d32..19e347ba4 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/computer.rs @@ -10,8 +10,11 @@ use brk_vecs::Exit; pub fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); - // let bitcoin_dir = brk_structs::default_bitcoin_path(); - let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin"); + let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap()) + .join("Library") + .join("Application Support") + .join("Bitcoin"); + // let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin"); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( "http://localhost:8332", @@ -24,11 +27,9 @@ pub fn main() -> Result<()> { thread::Builder::new() .stack_size(256 * 1024 * 1024) .spawn(move || -> Result<()> { - let parser = Parser::new(bitcoin_dir.join("blocks"), Path::new("").to_path_buf(), rpc); + let outputs_dir = Path::new("../../_outputs"); - let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs"); - let outputs_dir = _outputs_dir.as_path(); - // let outputs_dir = Path::new("../../_outputs"); + let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc); let mut indexer = Indexer::forced_import(outputs_dir)?; diff --git a/crates/brk_computer/src/all.rs b/crates/brk_computer/src/all.rs deleted file mode 100644 index 1a08b7745..000000000 --- a/crates/brk_computer/src/all.rs +++ /dev/null @@ -1,228 +0,0 @@ -use std::{path::Path, sync::Arc}; - -use brk_error::Result; -use brk_fetcher::Fetcher; -use brk_indexer::Indexer; -use brk_structs::Version; -use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format}; -use log::info; - -use crate::{blocks, cointime, constants, fetched, indexes, market, mining, price, transactions}; - -use super::stateful; - -const VERSION: Version = Version::ONE; - -#[derive(Clone)] -pub struct Vecs { - pub indexes: indexes::Vecs, - pub constants: constants::Vecs, - pub blocks: blocks::Vecs, - pub mining: mining::Vecs, - pub market: market::Vecs, - pub price: Option, - pub transactions: transactions::Vecs, - pub stateful: stateful::Vecs, - pub fetched: Option, - pub cointime: cointime::Vecs, -} - -impl Vecs { - #[allow(clippy::too_many_arguments)] - pub fn import( - file: &Arc, - version: Version, - indexer: &Indexer, - fetcher: Option, - computation: Computation, - format: Format, - fetched_file: &Arc, - states_path: &Path, - ) -> Result { - let indexes = indexes::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - indexer, - computation, - format, - )?; - - let fetched = fetcher.map(|fetcher| { - fetched::Vecs::forced_import( - file, - fetched_file, - fetcher, - version + VERSION + Version::ZERO, - ) - .unwrap() - }); - - let price = fetched.is_some().then(|| { - price::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - ) - .unwrap() - }); - - Ok(Self { - blocks: blocks::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - )?, - mining: mining::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - )?, - constants: constants::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - )?, - market: market::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - )?, - stateful: stateful::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - price.as_ref(), - states_path, - )?, - transactions: transactions::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - indexer, - &indexes, - computation, - format, - price.as_ref(), - )?, - cointime: cointime::Vecs::forced_import( - file, - version + VERSION + Version::ZERO, - computation, - format, - &indexes, - price.as_ref(), - )?, - indexes, - fetched, - price, - }) - } - - pub fn compute( - &mut self, - indexer: &Indexer, - starting_indexes: brk_indexer::Indexes, - exit: &Exit, - ) -> Result<()> { - info!("Computing indexes..."); - let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; - - info!("Computing constants..."); - self.constants - .compute(indexer, &self.indexes, &starting_indexes, exit)?; - - info!("Computing blocks..."); - self.blocks - .compute(indexer, &self.indexes, &starting_indexes, exit)?; - - info!("Computing mining..."); - self.mining - .compute(indexer, &self.indexes, &starting_indexes, exit)?; - - if let Some(fetched) = self.fetched.as_mut() { - info!("Computing fetched..."); - fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?; - - self.price.as_mut().unwrap().compute( - indexer, - &self.indexes, - &starting_indexes, - fetched, - exit, - )?; - } - - info!("Computing transactions..."); - self.transactions.compute( - indexer, - &self.indexes, - &starting_indexes, - self.price.as_ref(), - exit, - )?; - - if let Some(price) = self.price.as_ref() { - info!("Computing market..."); - self.market.compute( - indexer, - &self.indexes, - price, - &mut self.transactions, - &starting_indexes, - exit, - )?; - } - - info!("Computing stateful..."); - self.stateful.compute( - indexer, - &self.indexes, - &self.transactions, - self.price.as_ref(), - &self.market, - &mut starting_indexes, - exit, - )?; - - self.cointime.compute( - indexer, - &self.indexes, - &starting_indexes, - self.price.as_ref(), - &self.transactions, - &self.stateful, - exit, - )?; - - Ok(()) - } - - pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { - [ - self.constants.vecs(), - self.indexes.vecs(), - self.blocks.vecs(), - self.mining.vecs(), - self.market.vecs(), - self.transactions.vecs(), - self.stateful.vecs(), - self.cointime.vecs(), - self.fetched.as_ref().map_or(vec![], |v| v.vecs()), - self.price.as_ref().map_or(vec![], |v| v.vecs()), - ] - .into_iter() - .flatten() - .collect::>() - } -} diff --git a/crates/brk_computer/src/blocks.rs b/crates/brk_computer/src/blocks.rs index ad8cc49fd..a4ca984ab 100644 --- a/crates/brk_computer/src/blocks.rs +++ b/crates/brk_computer/src/blocks.rs @@ -6,7 +6,9 @@ use brk_structs::{ CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version, Weight, }; -use brk_vecs::{AnyCollectableVec, Computation, EagerVec, Exit, File, Format, VecIterator}; +use brk_vecs::{ + AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, VecIterator, +}; use crate::grouped::Source; @@ -43,6 +45,7 @@ impl Vecs { indexes: &indexes::Vecs, ) -> Result { let file = Arc::new(File::open(&parent.join("blocks"))?); + file.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { height_to_interval: EagerVec::forced_import( diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index f244b6266..fb5daf7aa 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc}; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version}; -use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator}; +use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator}; use super::{ Indexes, @@ -56,6 +56,7 @@ impl Vecs { price: Option<&price::Vecs>, ) -> Result { let file = Arc::new(File::open(&parent.join("cointime"))?); + file.set_min_len(PAGE_SIZE * 1_000_000)?; let compute_dollars = price.is_some(); diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index e1ac3a471..639cc540e 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -50,8 +50,11 @@ impl Vecs { exit: &Exit, ) -> Result<()> { let height_to_timestamp = &indexer.vecs.height_to_timestamp; + let index = starting_indexes + .height + .min(Height::from(self.height_to_ohlc_in_cents.len())); height_to_timestamp - .iter_at(starting_indexes.height) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let v = v.into_owned(); self.height_to_ohlc_in_cents.forced_push_at( @@ -71,10 +74,13 @@ impl Vecs { })?; self.height_to_ohlc_in_cents.safe_flush(exit)?; + let index = starting_indexes + .dateindex + .min(DateIndex::from(self.dateindex_to_ohlc_in_cents.len())); let mut prev = None; indexes .dateindex_to_date - .iter_at(starting_indexes.dateindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let d = v.into_owned(); if prev.is_none() { diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index 6b851f1c1..c7340e208 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -12,7 +12,7 @@ use brk_structs::{ }; use brk_vecs::{ AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1, - ComputedVecFrom2, EagerVec, Exit, File, Format, StoredIndex, VecIterator, + ComputedVecFrom2, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, VecIterator, }; const VERSION: Version = Version::ZERO; @@ -106,6 +106,7 @@ impl Vecs { format: Format, ) -> Result { let file = Arc::new(File::open(&parent.join("indexes"))?); + file.set_min_len(PAGE_SIZE * 10_000_000)?; let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1( computation, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 7791c15f0..d8c67e3a5 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -1,6 +1,6 @@ #![doc = include_str!("../README.md")] #![doc = "\n## Example\n\n```rust"] -#![doc = include_str!("../examples/main.rs")] +#![doc = include_str!("../examples/computer.rs")] #![doc = "```"] use std::path::Path; @@ -168,6 +168,7 @@ impl Computer { info!("Computing fetched..."); fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?; + info!("Computing prices..."); self.price.as_mut().unwrap().compute( indexer, &self.indexes, diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index dfe85f6c4..c8d4fb0be 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -4,7 +4,8 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version}; use brk_vecs::{ - AnyCollectableVec, Computation, EagerVec, Exit, File, Format, StoredIndex, VecIterator, + AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, + VecIterator, }; use crate::{ @@ -178,6 +179,7 @@ impl Vecs { indexes: &indexes::Vecs, ) -> Result { let file = Arc::new(File::open(&parent.join("market"))?); + file.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { height_to_marketcap: EagerVec::forced_import( diff --git a/crates/brk_computer/src/mining.rs b/crates/brk_computer/src/mining.rs index 0e327d22d..c37bdacf7 100644 --- a/crates/brk_computer/src/mining.rs +++ b/crates/brk_computer/src/mining.rs @@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc}; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version}; -use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator}; +use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator}; use crate::grouped::Source; @@ -33,6 +33,7 @@ impl Vecs { indexes: &indexes::Vecs, ) -> Result { let file = Arc::new(File::open(&parent.join("mining"))?); + file.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { indexes_to_difficulty: ComputedVecsFromHeight::forced_import( diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs index a5e338478..0a9f90dc4 100644 --- a/crates/brk_computer/src/price.rs +++ b/crates/brk_computer/src/price.rs @@ -7,8 +7,8 @@ use brk_structs::{ OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex, }; use brk_vecs::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, Computation, EagerVec, Exit, File, Format, - GenericStoredVec, RawVec, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File, + Format, GenericStoredVec, PAGE_SIZE, RawVec, }; use crate::{fetched, grouped::Source}; @@ -81,6 +81,7 @@ impl Vecs { indexes: &indexes::Vecs, ) -> Result { let file = Arc::new(File::open(&parent.join("price"))?); + file.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { dateindex_to_ohlc: RawVec::forced_import( @@ -400,9 +401,12 @@ impl Vecs { exit, )?; + let index = starting_indexes + .height + .min(Height::from(self.height_to_ohlc.len())); fetched .height_to_ohlc_in_cents - .iter_at(starting_indexes.height) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { self.height_to_ohlc .forced_push_at(i, OHLCDollars::from(v.into_owned()), exit)?; @@ -438,9 +442,12 @@ impl Vecs { exit, )?; + let index = starting_indexes + .dateindex + .min(DateIndex::from(self.dateindex_to_ohlc.len())); fetched .dateindex_to_ohlc_in_cents - .iter_at(starting_indexes.dateindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { self.dateindex_to_ohlc.forced_push_at( i, @@ -582,10 +589,13 @@ impl Vecs { let mut weekindex_first_iter = self.timeindexes_to_open.weekindex.unwrap_first().iter(); let mut weekindex_max_iter = self.timeindexes_to_high.weekindex.unwrap_max().iter(); let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter(); + let index = starting_indexes + .weekindex + .min(WeekIndex::from(self.weekindex_to_ohlc.len())); self.timeindexes_to_close .weekindex .unwrap_last() - .iter_at(starting_indexes.weekindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = weekindex_first_iter.unwrap_get_inner(i); @@ -617,10 +627,13 @@ impl Vecs { .iter(); let mut difficultyepoch_min_iter = self.chainindexes_to_low.difficultyepoch.unwrap_min().iter(); + let index = starting_indexes + .difficultyepoch + .min(DifficultyEpoch::from(self.difficultyepoch_to_ohlc.len())); self.chainindexes_to_close .difficultyepoch .unwrap_last() - .iter_at(starting_indexes.difficultyepoch) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = difficultyepoch_first_iter.unwrap_get_inner(i); @@ -643,10 +656,13 @@ impl Vecs { let mut monthindex_first_iter = self.timeindexes_to_open.monthindex.unwrap_first().iter(); let mut monthindex_max_iter = self.timeindexes_to_high.monthindex.unwrap_max().iter(); let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter(); + let index = starting_indexes + .monthindex + .min(MonthIndex::from(self.monthindex_to_ohlc.len())); self.timeindexes_to_close .monthindex .unwrap_last() - .iter_at(starting_indexes.monthindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = monthindex_first_iter.unwrap_get_inner(i); @@ -670,10 +686,13 @@ impl Vecs { self.timeindexes_to_open.quarterindex.unwrap_first().iter(); let mut quarterindex_max_iter = self.timeindexes_to_high.quarterindex.unwrap_max().iter(); let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter(); + let index = starting_indexes + .quarterindex + .min(QuarterIndex::from(self.quarterindex_to_ohlc.len())); self.timeindexes_to_close .quarterindex .unwrap_last() - .iter_at(starting_indexes.quarterindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = quarterindex_first_iter.unwrap_get_inner(i); @@ -697,10 +716,13 @@ impl Vecs { self.timeindexes_to_open.semesterindex.unwrap_first().iter(); let mut semesterindex_max_iter = self.timeindexes_to_high.semesterindex.unwrap_max().iter(); let mut semesterindex_min_iter = self.timeindexes_to_low.semesterindex.unwrap_min().iter(); + let index = starting_indexes + .semesterindex + .min(SemesterIndex::from(self.semesterindex_to_ohlc.len())); self.timeindexes_to_close .semesterindex .unwrap_last() - .iter_at(starting_indexes.semesterindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = semesterindex_first_iter.unwrap_get_inner(i); @@ -723,10 +745,13 @@ impl Vecs { let mut yearindex_first_iter = self.timeindexes_to_open.yearindex.unwrap_first().iter(); let mut yearindex_max_iter = self.timeindexes_to_high.yearindex.unwrap_max().iter(); let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter(); + let index = starting_indexes + .yearindex + .min(YearIndex::from(self.yearindex_to_ohlc.len())); self.timeindexes_to_close .yearindex .unwrap_last() - .iter_at(starting_indexes.yearindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = yearindex_first_iter.unwrap_get_inner(i); @@ -752,10 +777,13 @@ impl Vecs { let mut decadeindex_first_iter = self.timeindexes_to_open.decadeindex.unwrap_first().iter(); let mut decadeindex_max_iter = self.timeindexes_to_high.decadeindex.unwrap_max().iter(); let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter(); + let index = starting_indexes + .decadeindex + .min(DecadeIndex::from(self.decadeindex_to_ohlc.len())); self.timeindexes_to_close .decadeindex .unwrap_last() - .iter_at(starting_indexes.decadeindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); let open = decadeindex_first_iter.unwrap_get_inner(i); @@ -906,9 +934,12 @@ impl Vecs { let mut height_first_iter = self.chainindexes_to_open_in_sats.height.iter(); let mut height_max_iter = self.chainindexes_to_high_in_sats.height.iter(); let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter(); + let index = starting_indexes + .height + .min(Height::from(self.height_to_ohlc_in_sats.len())); self.chainindexes_to_close_in_sats .height - .iter_at(starting_indexes.height) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.height_to_ohlc_in_sats.forced_push_at( @@ -943,11 +974,14 @@ impl Vecs { .as_ref() .unwrap() .iter(); + let index = starting_indexes + .dateindex + .min(DateIndex::from(self.dateindex_to_ohlc_in_sats.len())); self.timeindexes_to_close_in_sats .dateindex .as_ref() .unwrap() - .iter_at(starting_indexes.dateindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.dateindex_to_ohlc_in_sats.forced_push_at( @@ -979,10 +1013,13 @@ impl Vecs { .weekindex .unwrap_min() .iter(); + let index = starting_indexes + .weekindex + .min(WeekIndex::from(self.weekindex_to_ohlc_in_sats.len())); self.timeindexes_to_close_in_sats .weekindex .unwrap_last() - .iter_at(starting_indexes.weekindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.weekindex_to_ohlc_in_sats.forced_push_at( @@ -1014,10 +1051,13 @@ impl Vecs { .difficultyepoch .unwrap_min() .iter(); + let index = starting_indexes.difficultyepoch.min(DifficultyEpoch::from( + self.difficultyepoch_to_ohlc_in_sats.len(), + )); self.chainindexes_to_close_in_sats .difficultyepoch .unwrap_last() - .iter_at(starting_indexes.difficultyepoch) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.difficultyepoch_to_ohlc_in_sats.forced_push_at( @@ -1049,10 +1089,13 @@ impl Vecs { .monthindex .unwrap_min() .iter(); + let index = starting_indexes + .monthindex + .min(MonthIndex::from(self.monthindex_to_ohlc_in_sats.len())); self.timeindexes_to_close_in_sats .monthindex .unwrap_last() - .iter_at(starting_indexes.monthindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.monthindex_to_ohlc_in_sats.forced_push_at( @@ -1084,10 +1127,13 @@ impl Vecs { .quarterindex .unwrap_min() .iter(); + let index = starting_indexes + .quarterindex + .min(QuarterIndex::from(self.quarterindex_to_ohlc_in_sats.len())); self.timeindexes_to_close_in_sats .quarterindex .unwrap_last() - .iter_at(starting_indexes.quarterindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.quarterindex_to_ohlc_in_sats.forced_push_at( @@ -1119,10 +1165,13 @@ impl Vecs { .semesterindex .unwrap_min() .iter(); + let index = starting_indexes.semesterindex.min(SemesterIndex::from( + self.semesterindex_to_ohlc_in_sats.len(), + )); self.timeindexes_to_close_in_sats .semesterindex .unwrap_last() - .iter_at(starting_indexes.semesterindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.semesterindex_to_ohlc_in_sats.forced_push_at( @@ -1154,10 +1203,13 @@ impl Vecs { .yearindex .unwrap_min() .iter(); + let index = starting_indexes + .yearindex + .min(YearIndex::from(self.yearindex_to_ohlc_in_sats.len())); self.timeindexes_to_close_in_sats .yearindex .unwrap_last() - .iter_at(starting_indexes.yearindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.yearindex_to_ohlc_in_sats.forced_push_at( @@ -1192,10 +1244,13 @@ impl Vecs { .decadeindex .unwrap_min() .iter(); + let index = starting_indexes + .decadeindex + .min(DecadeIndex::from(self.decadeindex_to_ohlc_in_sats.len())); self.timeindexes_to_close_in_sats .decadeindex .unwrap_last() - .iter_at(starting_indexes.decadeindex) + .iter_at(index) .try_for_each(|(i, v)| -> Result<()> { let close = v.into_owned(); self.decadeindex_to_ohlc_in_sats.forced_push_at( diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 9d76341e0..3c463171e 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -11,7 +11,7 @@ use brk_structs::{ }; use brk_vecs::{ AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File, - Format, GenericStoredVec, RawVec, Reader, Stamp, StoredIndex, VecIterator, + Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator, }; use log::info; use rayon::prelude::*; @@ -94,6 +94,8 @@ impl Vecs { states_path: &Path, ) -> Result { let file = Arc::new(File::open(&parent.join("stateful"))?); + file.set_min_len(PAGE_SIZE * 20_000_000)?; + file.set_min_regions(50_000)?; let compute_dollars = price.is_some(); diff --git a/crates/brk_computer/src/states/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs index 161712172..f5bfa602a 100644 --- a/crates/brk_computer/src/states/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -22,10 +22,13 @@ pub struct PriceToAmount { impl PriceToAmount { pub fn forced_import(path: &Path, name: &str) -> Self { - Self::import(path, name).unwrap_or_else(|_| Self { - pathbuf: Self::path_(path, name), - height: None, - state: State::default(), + Self::import(path, name).unwrap_or_else(|_| { + // dbg!(e); + Self { + pathbuf: Self::path_(path, name), + height: None, + state: State::default(), + } }) } @@ -33,7 +36,7 @@ impl PriceToAmount { let path = Self::path_(path, name); fs::create_dir_all(&path)?; - let state = State::deserialize(&fs::read(&path)?)?; + let state = State::deserialize(&fs::read(Self::path_state_(&path))?)?; Ok(Self { height: Height::try_from(Self::path_height_(&path).as_path()).ok(), @@ -63,10 +66,14 @@ impl PriceToAmount { } pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) { - let amount = self.state.get_mut(&price).unwrap(); - *amount -= supply_state.value; - if *amount == Sats::ZERO { - self.state.remove(&price); + if let Some(amount) = self.state.get_mut(&price) { + *amount -= supply_state.value; + if *amount == Sats::ZERO { + self.state.remove(&price); + } + } else { + dbg!(&self.state, price, &self.pathbuf); + unreachable!(); } } @@ -117,11 +124,11 @@ impl State { let mut buffer = Vec::with_capacity(8 + len * 16); - buffer.extend_from_slice(len.as_bytes()); + buffer.extend(len.as_bytes()); self.iter().for_each(|(key, value)| { - buffer.extend_from_slice(key.as_bytes()); - buffer.extend_from_slice(value.as_bytes()); + buffer.extend(key.as_bytes()); + buffer.extend(value.as_bytes()); }); buffer diff --git a/crates/brk_computer/src/transactions.rs b/crates/brk_computer/src/transactions.rs index d6b9aa615..a5f41c933 100644 --- a/crates/brk_computer/src/transactions.rs +++ b/crates/brk_computer/src/transactions.rs @@ -8,8 +8,8 @@ use brk_structs::{ }; use brk_vecs::{ AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec, - ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, StoredIndex, - VecIterator, + ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, PAGE_SIZE, + StoredIndex, VecIterator, }; use crate::grouped::{ @@ -91,6 +91,7 @@ impl Vecs { price: Option<&price::Vecs>, ) -> Result { let file = Arc::new(File::open(&parent.join("transactions"))?); + file.set_min_len(PAGE_SIZE * 10_000_000)?; let compute_dollars = price.is_some(); diff --git a/crates/brk_error/Cargo.toml b/crates/brk_error/Cargo.toml index 9a3987e28..a5d73625c 100644 --- a/crates/brk_error/Cargo.toml +++ b/crates/brk_error/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_error" -description = "Errors used throughout the Bitcoin Research Kit" +description = "Errors used throughout BRK" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] brk_vecs = { workspace = true } diff --git a/crates/brk_fetcher/Cargo.toml b/crates/brk_fetcher/Cargo.toml index 10bc4ad8f..127cec832 100644 --- a/crates/brk_fetcher/Cargo.toml +++ b/crates/brk_fetcher/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] brk_error = { workspace = true } diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 04cd81972..e91dad8e1 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_indexer" -description = "A Bitcoin Core indexer built on top of brk_parser" +description = "A Bitcoin indexer built on top of brk_parser" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] bitcoin = { workspace = true } diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 2f714f326..0626da6e3 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -21,7 +21,7 @@ fn main() -> Result<()> { let blocks_dir = bitcoin_dir.join("blocks"); - let outputs_dir = Path::new("./_outputs"); + let outputs_dir = Path::new("../../_outputs"); fs::create_dir_all(outputs_dir)?; // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index c1314789f..7889ecd4b 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -27,7 +27,7 @@ pub use stores::*; pub use vecs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1_000; -const COLLISIONS_CHECKED_UP_TO: Height = Height::new(907_000); +const COLLISIONS_CHECKED_UP_TO: Height = Height::new(908_700); const VERSION: Version = Version::ONE; #[derive(Clone)] diff --git a/crates/brk_interface/Cargo.toml b/crates/brk_interface/Cargo.toml index 62b717241..37cd9d845 100644 --- a/crates/brk_interface/Cargo.toml +++ b/crates/brk_interface/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_interface" -description = "An interface to BRK's engine" +description = "An interface to find and format data from BRK" license.workspace = true edition.workspace = true version.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] brk_computer = { workspace = true } diff --git a/crates/brk_logger/Cargo.toml b/crates/brk_logger/Cargo.toml index 23054c86d..5638abf5b 100644 --- a/crates/brk_logger/Cargo.toml +++ b/crates/brk_logger/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_logger" -description = "A clean logger used in the Bitcoin Research Kit" +description = "A thin wrapper around env_logger" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] env_logger = "0.11.8" diff --git a/crates/brk_mcp/Cargo.toml b/crates/brk_mcp/Cargo.toml index 3b8c450e7..96c023002 100644 --- a/crates/brk_mcp/Cargo.toml +++ b/crates/brk_mcp/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_mcp" -description = "A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK" +description = "A bridge for LLMs to access BRK" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] axum = { workspace = true } diff --git a/crates/brk_parser/Cargo.toml b/crates/brk_parser/Cargo.toml index 3fbfa52b8..daaa69a0d 100644 --- a/crates/brk_parser/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brk_parser" -description = "A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust" +description = "A very fast Bitcoin block parser and iterator built on top of bitcoin-rust" keywords = ["bitcoin", "block", "iterator"] categories = ["cryptography::cryptocurrencies", "encoding"] version.workspace = true @@ -8,6 +8,7 @@ edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] bitcoin = { workspace = true } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 8a150ca9d..ef56abdf4 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_server" -description = "A crate that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_computer and brk_interface" +description = "A server with an API for anything from BRK" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] axum = { workspace = true } diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index a61cc431d..5c0184fe1 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -8,6 +8,7 @@ edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] brk_error = { workspace = true } diff --git a/crates/brk_structs/Cargo.toml b/crates/brk_structs/Cargo.toml index 76501a417..5500c11f0 100644 --- a/crates/brk_structs/Cargo.toml +++ b/crates/brk_structs/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "brk_structs" -description = "Structs used throughout the Bitcoin Research Kit" +description = "Structs used throughout BRK" version.workspace = true edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] bitcoin = { workspace = true } @@ -15,7 +16,7 @@ brk_vecs = {workspace = true} byteview = { workspace = true } derive_deref = { workspace = true } jiff = { workspace = true } -rapidhash = "2.0.2" +rapidhash = "3.0.0" serde = { workspace = true } serde_bytes = { workspace = true } zerocopy = { workspace = true } diff --git a/crates/brk_vecs/Cargo.toml b/crates/brk_vecs/Cargo.toml index 431f69681..45b0478e7 100644 --- a/crates/brk_vecs/Cargo.toml +++ b/crates/brk_vecs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brk_vecs" -description = "A simple index/value store" +description = "A KISS index/value store" keywords = ["vec", "disk", "data"] categories = ["database"] version.workspace = true @@ -8,6 +8,7 @@ edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [dependencies] brk_vecs_macros = { workspace = true } diff --git a/crates/brk_vecs/src/file/mod.rs b/crates/brk_vecs/src/file/mod.rs index 15f4a9464..5464f69c7 100644 --- a/crates/brk_vecs/src/file/mod.rs +++ b/crates/brk_vecs/src/file/mod.rs @@ -27,6 +27,7 @@ use crate::{Error, Result}; pub const PAGE_SIZE: u64 = 4096; pub const PAGE_SIZE_MINUS_1: u64 = PAGE_SIZE - 1; +const GB: u64 = 1024 * 1024 * 1024; #[derive(Debug)] pub struct File { @@ -156,6 +157,7 @@ impl File { self.write_all_to_region_at_(identifier, data, Some(at), false) } + #[inline] pub fn truncate_write_all_to_region( &self, identifier: Identifier, @@ -165,7 +167,6 @@ impl File { self.write_all_to_region_at_(identifier, data, Some(at), true) } - // NEW fn write_all_to_region_at_( &self, identifier: Identifier, @@ -194,7 +195,7 @@ impl File { }); let write_start = start + at.unwrap_or(len); - if at.is_some_and(|at| at >= reserved) { + if at.is_some_and(|at| at > reserved) { return Err(Error::Str("Invalid at parameter")); } @@ -455,23 +456,50 @@ impl File { regions.flush() } + /// Do not write, right after as there might be a race condition pub fn punch_holes(&self) -> Result<()> { let file = self.file.write(); let mmap = self.mmap.read(); + let regions = self.regions.read(); let layout = self.layout.read(); - Self::punch_holes_(&file, &mmap, &layout) + Self::punch_holes_(&file, &mmap, ®ions, &layout) } - fn punch_holes_(file: &fs::File, mmap: &MmapMut, layout: &Layout) -> Result<()> { + fn punch_holes_( + file: &fs::File, + mmap: &MmapMut, + regions: &Regions, + layout: &Layout, + ) -> Result<()> { + regions + .index_to_region() + .iter() + .flatten() + .try_for_each(|region_lock| -> Result<()> { + let region = region_lock.read(); + let start = region.start(); + let len = region.len(); + let ceil_len = Self::ceil_number_to_page_size_multiple(len); + assert!(len <= ceil_len); + let reserved = region.reserved(); + if ceil_len > reserved { + panic!() + } else if ceil_len < reserved { + let start = start + ceil_len; + let hole = reserved - ceil_len; + if Self::approx_has_punchable_data(mmap, start, hole) { + info!("Punching a hole of {hole} bytes at {start}..."); + Self::punch_hole_(file, start, hole)?; + } + } + Ok(()) + })?; + layout .start_to_hole() .par_iter() .try_for_each(|(&start, &hole)| -> Result<()> { - assert!(start % PAGE_SIZE == 0); - assert!(hole % PAGE_SIZE == 0); - let has_old_data = - mmap[start as usize] != 0 || mmap[(start + hole - PAGE_SIZE) as usize] != 0; - if has_old_data { + if Self::approx_has_punchable_data(mmap, start, hole) { info!("Punching a hole of {hole} bytes at {start}..."); Self::punch_hole_(file, start, hole) } else { @@ -480,6 +508,49 @@ impl File { }) } + fn approx_has_punchable_data(mmap: &MmapMut, start: u64, len: u64) -> bool { + assert!(start % PAGE_SIZE == 0); + assert!(len % PAGE_SIZE == 0); + + let min = start as usize; + let max = (start + len) as usize; + let check = |start, end| { + assert!(start >= min); + assert!(end < max); + mmap[start] != 0 || mmap[end] != 0 + }; + + // Check first page (first and last byte) + let first_page_start = start as usize; + let first_page_end = (start + PAGE_SIZE - 1) as usize; + if check(first_page_start, first_page_end) { + return true; + } + + // Check last page (first and last byte) + let last_page_start = (start + len - PAGE_SIZE) as usize; + let last_page_end = (start + len - 1) as usize; + if check(last_page_start, last_page_end) { + return true; + } + + // For large lengths, check at 1GB intervals + if len > GB { + let num_gb_checks = (len / GB) as usize; + for i in 1..num_gb_checks { + let gb_boundary = start + (i as u64 * GB); + let page_start = gb_boundary as usize; + let page_end = (gb_boundary + PAGE_SIZE - 1) as usize; + + if check(page_start, page_end) { + return true; + } + } + } + + false + } + #[inline] fn punch_hole(&self, start: u64, length: u64) -> Result<()> { let file = self.file.write(); diff --git a/crates/brk_vecs/src/traits/generic.rs b/crates/brk_vecs/src/traits/generic.rs index 909bdd8e1..8f27c152f 100644 --- a/crates/brk_vecs/src/traits/generic.rs +++ b/crates/brk_vecs/src/traits/generic.rs @@ -137,10 +137,10 @@ where } if self.pushed_len() * Self::SIZE_OF_T >= MAX_CACHE_SIZE { - self.safe_flush(exit) - } else { - Ok(()) + self.safe_flush(exit)?; } + + Ok(()) } #[inline] diff --git a/crates/brk_vecs/src/variants/eager.rs b/crates/brk_vecs/src/variants/eager.rs index 015474ae9..26508eff7 100644 --- a/crates/brk_vecs/src/variants/eager.rs +++ b/crates/brk_vecs/src/variants/eager.rs @@ -44,7 +44,7 @@ where } pub fn inner_version(&self) -> Version { - self.version() + self.0.header().vec_version() } fn update_computed_version(&mut self, computed_version: Version) { diff --git a/crates/brk_vecs/src/variants/stored/mod.rs b/crates/brk_vecs/src/variants/stored/mod.rs index d1064188f..53053d02b 100644 --- a/crates/brk_vecs/src/variants/stored/mod.rs +++ b/crates/brk_vecs/src/variants/stored/mod.rs @@ -39,10 +39,9 @@ where } if format.is_compressed() { - todo!(); - // Ok(Self::Compressed(CompressedVec::forced_import( - // file, name, version, - // )?)) + Ok(Self::Compressed(CompressedVec::forced_import( + file, name, version, + )?)) } else { Ok(Self::Raw(RawVec::forced_import(file, name, version)?)) } diff --git a/crates/brk_vecs_macros/Cargo.toml b/crates/brk_vecs_macros/Cargo.toml index e4254bd58..3ecfbfc5a 100644 --- a/crates/brk_vecs_macros/Cargo.toml +++ b/crates/brk_vecs_macros/Cargo.toml @@ -8,6 +8,7 @@ edition.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true +build = "../../build.rs" [lib] proc-macro = true diff --git a/docker/.env.example b/docker/.env.example index 8454693c4..c5687df65 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -14,16 +14,6 @@ BTC_RPC_PORT=8332 # BTC_RPC_USER=your_rpc_username # BTC_RPC_PASSWORD=your_rpc_password -# BRK configuration -# Services to run: all, processor, or server -BRK_SERVICES=all - -# Computation mode: lazy (compute on demand) or eager (precompute and save) -BRK_COMPUTATION=lazy - -# Data format: raw (faster) or compressed (saves disk space) -BRK_FORMAT=raw - # Enable price fetching from exchanges BRK_FETCH=true @@ -39,4 +29,4 @@ BRK_MCP=true # Option 2: Use a bind mount to a local directory # Uncomment and set this to use a specific directory on your host # Also uncomment the corresponding line in docker-compose.yml -# BRK_DATA_DIR=/path/to/brk/data \ No newline at end of file +# BRK_DATA_DIR=/path/to/brk/data diff --git a/docker/Dockerfile b/docker/Dockerfile index ca1d387f4..94dea392c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -55,6 +55,3 @@ WORKDIR /home/brk # Default entrypoint ENTRYPOINT ["brk"] - -# Default command (can be overridden) -CMD ["--services", "all"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 68259fdbd..f0d411bde 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,7 +21,7 @@ This guide explains how to run BRK using Docker and Docker Compose. ```bash docker compose -f docker/docker-compose.yml up -d ``` - + Or from the docker directory: ```bash cd docker && docker compose up -d @@ -62,8 +62,6 @@ cd docker && docker compose up -d | `BTC_RPC_USER` | Bitcoin RPC username | - | | `BTC_RPC_PASSWORD` | Bitcoin RPC password | - | | `BRK_DATA_VOLUME` | Docker volume name for BRK data | `brk-data` | -| `BRK_COMPUTATION` | Computation mode (`lazy`, `eager`) | `lazy` | -| `BRK_FORMAT` | Data format (`raw`, `compressed`) | `raw` | | `BRK_FETCH` | Enable price fetching | `true` | | `BRK_MCP` | Enable MCP for AI/LLM | `true` | @@ -81,8 +79,6 @@ BTC_RPC_USER=your_username BTC_RPC_PASSWORD=your_password # BRK settings -BRK_COMPUTATION=lazy -BRK_FORMAT=raw BRK_FETCH=true BRK_MCP=true ``` @@ -96,7 +92,7 @@ BRK will automatically use the `.cookie` file from your Bitcoin Core directory. Set `BTC_RPC_USER` and `BTC_RPC_PASSWORD` in your `docker/.env` file. #### Network Connectivity -- **Same host**: +- **Same host**: - If Bitcoin Core is running natively (not in Docker): Use `host.docker.internal` on macOS/Windows or `172.17.0.1` on Linux - If Bitcoin Core is also in Docker: Use the service name or container IP - **Remote host**: Use the actual IP address or hostname @@ -210,11 +206,9 @@ docker compose -f docker/docker-compose.yml logs -f #### Slow indexing - Ensure adequate disk space for indexed data - a minimum of 3GB/s is recommended - Monitor memory usage during initial indexing -- Use `BRK_COMPUTATION=lazy` to reduce memory usage #### Out of memory - Increase Docker's memory limit -- Use `BRK_COMPUTATION=lazy` mode - Monitor container resource usage: `docker stats` ### Permission Issues @@ -260,4 +254,4 @@ docker run --rm -v brk_brk-data:/target -v \"$(pwd)\":/backup alpine tar xzf /ba # Start container docker compose -f docker/docker-compose.yml up -d -``` \ No newline at end of file +``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index aa2b01633..b449c46c0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,7 +11,7 @@ services: container_name: brk restart: unless-stopped ports: - - 7070:3110 # Map host port 7070 to container port 3110 + - 7070:3110 # Map host port 7070 to container port 3110 volumes: # Bitcoin Core data directory (read-only) # For access to raw block data @@ -25,20 +25,18 @@ services: # Bitcoin Core configuration - BITCOINDIR=/bitcoin - BLOCKSDIR=/bitcoin/blocks - + # RPC configuration (required for processor) - RPCCONNECT=${BTC_RPC_HOST:-localhost} - RPCPORT=${BTC_RPC_PORT:-8332} # - RPCCOOKIEFILE=/bitcoin/.cookie - + # Username/password authentication - RPCUSER=${BTC_RPC_USER} - RPCPASSWORD=${BTC_RPC_PASSWORD} - + # BRK configuration - BRKDIR=/home/brk/.brk - - COMPUTATION=${BRK_COMPUTATION:-lazy} - - FORMAT=${BRK_FORMAT:-raw} - FETCH=${BRK_FETCH:-true} - MCP=${BRK_MCP:-true} command: @@ -61,4 +59,4 @@ services: volumes: brk-data: - driver: local \ No newline at end of file + driver: local