Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-17 18:29:44 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: compressed vecs work again

Browse Source
This commit is contained in:
nym21
2025-08-05 23:38:43 +02:00
parent e28a0cde55
commit 4740610923
45 changed files with 310 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
-14
View File
@@ -1,14 +0,0 @@
# Release profile: all targets get native CPU
[profile.release.target.'cfg()']
rustflags = ["-C", "target-cpu=native"]
# Release profile: x86_64 gets native CPU + extra features (overrides the above)
[profile.release.target.'cfg(target_arch = "x86_64")']
rustflags = [
"-C", "target-cpu=native",
"-C", "target-feature=+bmi1,+bmi2,+avx2"
]
# Dist profile: x86_64 gets only extra features (no native CPU)
[profile.dist.target.'cfg(target_arch = "x86_64")']
rustflags = ["-C", "target-feature=+bmi1,+bmi2,+avx2"]
.github/workflows/release.yml
+7 -2
View File
@@ -1,4 +1,4 @@
# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
#
# Copyright 2022-2024, axodotdev
# SPDX-License-Identifier: MIT or Apache-2.0
@@ -58,12 +58,13 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install dist
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.29.0/cargo-dist-installer.sh | sh"
- name: Cache dist
uses: actions/upload-artifact@v4
with:
@@ -117,6 +118,7 @@ jobs:
git config --global core.longpaths true
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install Rust non-interactively if not already installed
if: ${{ matrix.container }}
@@ -175,6 +177,7 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
@@ -224,6 +227,7 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
@@ -288,4 +292,5 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
Cargo.lock
Generated
+8 -8
View File
@@ -364,9 +364,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bitcoin"
version = "0.32.6"
version = "0.32.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad8929a18b8e33ea6b3c09297b687baaa71fb1b97353243a3f1029fad5c59c5b"
checksum = "0fda569d741b895131a88ee5589a467e73e9c4718e958ac9308e4f7dc44b6945"
dependencies = [
"base58ck",
"bech32",
@@ -3459,9 +3459,9 @@ dependencies = [
[[package]]
name = "rapidhash"
version = "2.0.2"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2297bf4643b32dc85bf5f622c9a41a268db6c6a6a2a2e3b405958a5b014ad9e1"
checksum = "2ef3d82b018f786967b1a5d34a08ebc3c7a9ab35b5bcbe3e2e057a0a453f26c8"
[[package]]
name = "rayon"
@@ -4231,9 +4231,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.9.4"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1"
checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8"
dependencies = [
"indexmap 2.10.0",
"serde",
@@ -4255,9 +4255,9 @@ dependencies = [
[[package]]
name = "toml_parser"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30"
checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10"
dependencies = [
"winnow",
]
Cargo.toml
+9 -9
View File
@@ -23,7 +23,7 @@ inherits = "release"
[workspace.dependencies]
axum = "0.8.4"
bitcoin = { version = "0.32.6", features = ["serde"] }
bitcoin = { version = "0.32.7", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_bundler = { version = "0.0.83", path = "crates/brk_bundler" }
brk_cli = { version = "0.0.83", path = "crates/brk_cli" }
@@ -63,7 +63,7 @@ pre-release-commit-message = "release: v{{version}}"
tag-message = "release: v{{version}}"
[workspace.metadata.dist]
cargo-dist-version = "0.28.0"
cargo-dist-version = "0.29.0"
ci = "github"
installers = []
targets = [
@@ -73,10 +73,10 @@ targets = [
"x86_64-unknown-linux-gnu",
]
[workspace.metadata.dist.github-custom-runners]
global = "ubuntu-latest"
aarch64-apple-darwin.runner = "macos-14"
x86_64-unknown-linux-gnu.runner = "ubuntu-latest"
x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
aarch64-unknown-linux-gnu.runner = "ubuntu-latest"
aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
# [workspace.metadata.dist.github-custom-runners]
# global = "ubuntu-latest"
# aarch64-apple-darwin.runner = "macos-14"
# x86_64-unknown-linux-gnu.runner = "ubuntu-latest"
# x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
# aarch64-unknown-linux-gnu.runner = "ubuntu-latest"
# aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
README.md
+15 -13
View File
@@ -54,20 +54,22 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
## Crates
- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer
- [`brk_structs`](https://crates.io/crates/brk_structs): The Core (Structs and Errors) of the Bitcoin Research Kit
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser
- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit
- [`brk_mcp`](https://crates.io/crates/brk_mcp): A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A storeable vec
- [`brk`](https://crates.io/crates/brk): A wrapper around all other `brk-*` crates
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a BRK instance
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer built on top of [`brk_indexer`](https://crates.io/crates/brk_indexer)
- [`brk_error`](https://crates.io/crates/brk_error): Errors used throughout BRK
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin indexer built on top of [`brk_parser`](https://crates.io/crates/brk_parser)
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to find and format data from BRK
- [`brk_logger`](https://crates.io/crates/brk_logger): A thin wrapper around [`env_logger`](https://crates.io/crates/env_logger)
- [`brk_mcp`](https://crates.io/crates/brk_mcp): A bridge for LLMs to access BRK
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin block parser and iterator built on top of [`bitcoin-rust`](https://crates.io/crates/bitcoin)
- [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK
- [`brk_vecs`](https://crates.io/crates/brk_vecs): A KISS index/value store
- [`brk_vecs_macros`](https://crates.io/crates/brk_vecs_macros): Macros for [`brk_vecs`](https://crates.io/crates/brk_vecs)
## Hosting as a service
build.rs
+14
View File
@@ -0,0 +1,14 @@
fn main() {
let profile = std::env::var("PROFILE").unwrap_or_default();
if profile == "release" {
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-cpu=native");
#[cfg(target_arch = "x86_64")]
{
println!("cargo:rustc-flag=-C");
println!("cargo:rustc-flag=target-feature=+bmi1,+bmi2,+avx2");
}
}
}
crates/brk/Cargo.toml
+1
View File
@@ -7,6 +7,7 @@ homepage.workspace = true
repository.workspace = true
edition.workspace = true
version.workspace = true
build = "../../build.rs"
[features]
full = [
crates/brk_bundler/Cargo.toml
+1
View File
@@ -6,6 +6,7 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
log = { workspace = true }
crates/brk_cli/Cargo.toml
+3 -2
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_cli"
description = "A command line interface to run a Bitcoin Research Kit instance"
description = "A command line interface to run a BRK instance"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
bitcoincore-rpc = { workspace = true }
@@ -25,7 +26,7 @@ log = { workspace = true }
minreq = { workspace = true }
serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.4"
toml = "0.9.5"
zip = { version = "4.3.0", default-features = false, features = ["deflate"] }
[[bin]]
crates/brk_computer/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_computer"
description = "A Bitcoin dataset computer, built on top of brk_indexer"
description = "A Bitcoin dataset computer built on top of brk_indexer"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
bitcoin = { workspace = true }
crates/brk_computer/examples/main.rs → crates/brk_computer/examples/computer.rs
+7 -6
View File
@@ -10,8 +10,11 @@ use brk_vecs::Exit;
pub fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = brk_structs::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
@@ -24,11 +27,9 @@ pub fn main() -> Result<()> {
thread::Builder::new()
.stack_size(256 * 1024 * 1024)
.spawn(move || -> Result<()> {
let parser = Parser::new(bitcoin_dir.join("blocks"), Path::new("").to_path_buf(), rpc);
let outputs_dir = Path::new("../../_outputs");
let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs");
let outputs_dir = _outputs_dir.as_path();
// let outputs_dir = Path::new("../../_outputs");
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);
let mut indexer = Indexer::forced_import(outputs_dir)?;
crates/brk_computer/src/all.rs
-228
View File
@@ -1,228 +0,0 @@
use std::{path::Path, sync::Arc};
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::Version;
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format};
use log::info;
use crate::{blocks, cointime, constants, fetched, indexes, market, mining, price, transactions};
use super::stateful;
const VERSION: Version = Version::ONE;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub constants: constants::Vecs,
pub blocks: blocks::Vecs,
pub mining: mining::Vecs,
pub market: market::Vecs,
pub price: Option<price::Vecs>,
pub transactions: transactions::Vecs,
pub stateful: stateful::Vecs,
pub fetched: Option<fetched::Vecs>,
pub cointime: cointime::Vecs,
}
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn import(
file: &Arc<File>,
version: Version,
indexer: &Indexer,
fetcher: Option<Fetcher>,
computation: Computation,
format: Format,
fetched_file: &Arc<File>,
states_path: &Path,
) -> Result<Self> {
let indexes = indexes::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
indexer,
computation,
format,
)?;
let fetched = fetcher.map(|fetcher| {
fetched::Vecs::forced_import(
file,
fetched_file,
fetcher,
version + VERSION + Version::ZERO,
)
.unwrap()
});
let price = fetched.is_some().then(|| {
price::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)
.unwrap()
});
Ok(Self {
blocks: blocks::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
mining: mining::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
constants: constants::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
market: market::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
stateful: stateful::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
price.as_ref(),
states_path,
)?,
transactions: transactions::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
indexer,
&indexes,
computation,
format,
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
price.as_ref(),
)?,
indexes,
fetched,
price,
})
}
pub fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> Result<()> {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
self.constants
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing blocks...");
self.blocks
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing mining...");
self.mining
.compute(indexer, &self.indexes, &starting_indexes, exit)?;
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
self.price.as_mut().unwrap().compute(
indexer,
&self.indexes,
&starting_indexes,
fetched,
exit,
)?;
}
info!("Computing transactions...");
self.transactions.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
exit,
)?;
if let Some(price) = self.price.as_ref() {
info!("Computing market...");
self.market.compute(
indexer,
&self.indexes,
price,
&mut self.transactions,
&starting_indexes,
exit,
)?;
}
info!("Computing stateful...");
self.stateful.compute(
indexer,
&self.indexes,
&self.transactions,
self.price.as_ref(),
&self.market,
&mut starting_indexes,
exit,
)?;
self.cointime.compute(
indexer,
&self.indexes,
&starting_indexes,
self.price.as_ref(),
&self.transactions,
&self.stateful,
exit,
)?;
Ok(())
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.constants.vecs(),
self.indexes.vecs(),
self.blocks.vecs(),
self.mining.vecs(),
self.market.vecs(),
self.transactions.vecs(),
self.stateful.vecs(),
self.cointime.vecs(),
self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
self.price.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
crates/brk_computer/src/blocks.rs
+4 -1
View File
@@ -6,7 +6,9 @@ use brk_structs::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version,
Weight,
};
use brk_vecs::{AnyCollectableVec, Computation, EagerVec, Exit, File, Format, VecIterator};
use brk_vecs::{
AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, VecIterator,
};
use crate::grouped::Source;
@@ -43,6 +45,7 @@ impl Vecs {
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("blocks"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_interval: EagerVec::forced_import(
crates/brk_computer/src/cointime.rs
+2 -1
View File
@@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator};
use super::{
Indexes,
@@ -56,6 +56,7 @@ impl Vecs {
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("cointime"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let compute_dollars = price.is_some();
crates/brk_computer/src/fetched.rs
+8 -2
View File
@@ -50,8 +50,11 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
let height_to_timestamp = &indexer.vecs.height_to_timestamp;
let index = starting_indexes
.height
.min(Height::from(self.height_to_ohlc_in_cents.len()));
height_to_timestamp
.iter_at(starting_indexes.height)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let v = v.into_owned();
self.height_to_ohlc_in_cents.forced_push_at(
@@ -71,10 +74,13 @@ impl Vecs {
})?;
self.height_to_ohlc_in_cents.safe_flush(exit)?;
let index = starting_indexes
.dateindex
.min(DateIndex::from(self.dateindex_to_ohlc_in_cents.len()));
let mut prev = None;
indexes
.dateindex_to_date
.iter_at(starting_indexes.dateindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let d = v.into_owned();
if prev.is_none() {
crates/brk_computer/src/indexes.rs
+2 -1
View File
@@ -12,7 +12,7 @@ use brk_structs::{
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Exit, File, Format, StoredIndex, VecIterator,
ComputedVecFrom2, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, VecIterator,
};
const VERSION: Version = Version::ZERO;
@@ -106,6 +106,7 @@ impl Vecs {
format: Format,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("indexes"))?);
file.set_min_len(PAGE_SIZE * 10_000_000)?;
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation,
crates/brk_computer/src/lib.rs
+2 -1
View File
@@ -1,6 +1,6 @@
#![doc = include_str!("../README.md")]
#![doc = "\n## Example\n\n```rust"]
#![doc = include_str!("../examples/main.rs")]
#![doc = include_str!("../examples/computer.rs")]
#![doc = "```"]
use std::path::Path;
@@ -168,6 +168,7 @@ impl Computer {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
info!("Computing prices...");
self.price.as_mut().unwrap().compute(
indexer,
&self.indexes,
crates/brk_computer/src/market.rs
+3 -1
View File
@@ -4,7 +4,8 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version};
use brk_vecs::{
AnyCollectableVec, Computation, EagerVec, Exit, File, Format, StoredIndex, VecIterator,
AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex,
VecIterator,
};
use crate::{
@@ -178,6 +179,7 @@ impl Vecs {
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("market"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_marketcap: EagerVec::forced_import(
crates/brk_computer/src/mining.rs
+2 -1
View File
@@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, VecIterator};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator};
use crate::grouped::Source;
@@ -33,6 +33,7 @@ impl Vecs {
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("mining"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
crates/brk_computer/src/price.rs
+75 -20
View File
@@ -7,8 +7,8 @@ use brk_structs::{
OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, Computation, EagerVec, Exit, File, Format,
GenericStoredVec, RawVec,
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File,
Format, GenericStoredVec, PAGE_SIZE, RawVec,
};
use crate::{fetched, grouped::Source};
@@ -81,6 +81,7 @@ impl Vecs {
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("price"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
dateindex_to_ohlc: RawVec::forced_import(
@@ -400,9 +401,12 @@ impl Vecs {
exit,
)?;
let index = starting_indexes
.height
.min(Height::from(self.height_to_ohlc.len()));
fetched
.height_to_ohlc_in_cents
.iter_at(starting_indexes.height)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
self.height_to_ohlc
.forced_push_at(i, OHLCDollars::from(v.into_owned()), exit)?;
@@ -438,9 +442,12 @@ impl Vecs {
exit,
)?;
let index = starting_indexes
.dateindex
.min(DateIndex::from(self.dateindex_to_ohlc.len()));
fetched
.dateindex_to_ohlc_in_cents
.iter_at(starting_indexes.dateindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
self.dateindex_to_ohlc.forced_push_at(
i,
@@ -582,10 +589,13 @@ impl Vecs {
let mut weekindex_first_iter = self.timeindexes_to_open.weekindex.unwrap_first().iter();
let mut weekindex_max_iter = self.timeindexes_to_high.weekindex.unwrap_max().iter();
let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter();
let index = starting_indexes
.weekindex
.min(WeekIndex::from(self.weekindex_to_ohlc.len()));
self.timeindexes_to_close
.weekindex
.unwrap_last()
.iter_at(starting_indexes.weekindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = weekindex_first_iter.unwrap_get_inner(i);
@@ -617,10 +627,13 @@ impl Vecs {
.iter();
let mut difficultyepoch_min_iter =
self.chainindexes_to_low.difficultyepoch.unwrap_min().iter();
let index = starting_indexes
.difficultyepoch
.min(DifficultyEpoch::from(self.difficultyepoch_to_ohlc.len()));
self.chainindexes_to_close
.difficultyepoch
.unwrap_last()
.iter_at(starting_indexes.difficultyepoch)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = difficultyepoch_first_iter.unwrap_get_inner(i);
@@ -643,10 +656,13 @@ impl Vecs {
let mut monthindex_first_iter = self.timeindexes_to_open.monthindex.unwrap_first().iter();
let mut monthindex_max_iter = self.timeindexes_to_high.monthindex.unwrap_max().iter();
let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter();
let index = starting_indexes
.monthindex
.min(MonthIndex::from(self.monthindex_to_ohlc.len()));
self.timeindexes_to_close
.monthindex
.unwrap_last()
.iter_at(starting_indexes.monthindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = monthindex_first_iter.unwrap_get_inner(i);
@@ -670,10 +686,13 @@ impl Vecs {
self.timeindexes_to_open.quarterindex.unwrap_first().iter();
let mut quarterindex_max_iter = self.timeindexes_to_high.quarterindex.unwrap_max().iter();
let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter();
let index = starting_indexes
.quarterindex
.min(QuarterIndex::from(self.quarterindex_to_ohlc.len()));
self.timeindexes_to_close
.quarterindex
.unwrap_last()
.iter_at(starting_indexes.quarterindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = quarterindex_first_iter.unwrap_get_inner(i);
@@ -697,10 +716,13 @@ impl Vecs {
self.timeindexes_to_open.semesterindex.unwrap_first().iter();
let mut semesterindex_max_iter = self.timeindexes_to_high.semesterindex.unwrap_max().iter();
let mut semesterindex_min_iter = self.timeindexes_to_low.semesterindex.unwrap_min().iter();
let index = starting_indexes
.semesterindex
.min(SemesterIndex::from(self.semesterindex_to_ohlc.len()));
self.timeindexes_to_close
.semesterindex
.unwrap_last()
.iter_at(starting_indexes.semesterindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = semesterindex_first_iter.unwrap_get_inner(i);
@@ -723,10 +745,13 @@ impl Vecs {
let mut yearindex_first_iter = self.timeindexes_to_open.yearindex.unwrap_first().iter();
let mut yearindex_max_iter = self.timeindexes_to_high.yearindex.unwrap_max().iter();
let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter();
let index = starting_indexes
.yearindex
.min(YearIndex::from(self.yearindex_to_ohlc.len()));
self.timeindexes_to_close
.yearindex
.unwrap_last()
.iter_at(starting_indexes.yearindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = yearindex_first_iter.unwrap_get_inner(i);
@@ -752,10 +777,13 @@ impl Vecs {
let mut decadeindex_first_iter = self.timeindexes_to_open.decadeindex.unwrap_first().iter();
let mut decadeindex_max_iter = self.timeindexes_to_high.decadeindex.unwrap_max().iter();
let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter();
let index = starting_indexes
.decadeindex
.min(DecadeIndex::from(self.decadeindex_to_ohlc.len()));
self.timeindexes_to_close
.decadeindex
.unwrap_last()
.iter_at(starting_indexes.decadeindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
let open = decadeindex_first_iter.unwrap_get_inner(i);
@@ -906,9 +934,12 @@ impl Vecs {
let mut height_first_iter = self.chainindexes_to_open_in_sats.height.iter();
let mut height_max_iter = self.chainindexes_to_high_in_sats.height.iter();
let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter();
let index = starting_indexes
.height
.min(Height::from(self.height_to_ohlc_in_sats.len()));
self.chainindexes_to_close_in_sats
.height
.iter_at(starting_indexes.height)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.height_to_ohlc_in_sats.forced_push_at(
@@ -943,11 +974,14 @@ impl Vecs {
.as_ref()
.unwrap()
.iter();
let index = starting_indexes
.dateindex
.min(DateIndex::from(self.dateindex_to_ohlc_in_sats.len()));
self.timeindexes_to_close_in_sats
.dateindex
.as_ref()
.unwrap()
.iter_at(starting_indexes.dateindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.dateindex_to_ohlc_in_sats.forced_push_at(
@@ -979,10 +1013,13 @@ impl Vecs {
.weekindex
.unwrap_min()
.iter();
let index = starting_indexes
.weekindex
.min(WeekIndex::from(self.weekindex_to_ohlc_in_sats.len()));
self.timeindexes_to_close_in_sats
.weekindex
.unwrap_last()
.iter_at(starting_indexes.weekindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.weekindex_to_ohlc_in_sats.forced_push_at(
@@ -1014,10 +1051,13 @@ impl Vecs {
.difficultyepoch
.unwrap_min()
.iter();
let index = starting_indexes.difficultyepoch.min(DifficultyEpoch::from(
self.difficultyepoch_to_ohlc_in_sats.len(),
));
self.chainindexes_to_close_in_sats
.difficultyepoch
.unwrap_last()
.iter_at(starting_indexes.difficultyepoch)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.difficultyepoch_to_ohlc_in_sats.forced_push_at(
@@ -1049,10 +1089,13 @@ impl Vecs {
.monthindex
.unwrap_min()
.iter();
let index = starting_indexes
.monthindex
.min(MonthIndex::from(self.monthindex_to_ohlc_in_sats.len()));
self.timeindexes_to_close_in_sats
.monthindex
.unwrap_last()
.iter_at(starting_indexes.monthindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.monthindex_to_ohlc_in_sats.forced_push_at(
@@ -1084,10 +1127,13 @@ impl Vecs {
.quarterindex
.unwrap_min()
.iter();
let index = starting_indexes
.quarterindex
.min(QuarterIndex::from(self.quarterindex_to_ohlc_in_sats.len()));
self.timeindexes_to_close_in_sats
.quarterindex
.unwrap_last()
.iter_at(starting_indexes.quarterindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.quarterindex_to_ohlc_in_sats.forced_push_at(
@@ -1119,10 +1165,13 @@ impl Vecs {
.semesterindex
.unwrap_min()
.iter();
let index = starting_indexes.semesterindex.min(SemesterIndex::from(
self.semesterindex_to_ohlc_in_sats.len(),
));
self.timeindexes_to_close_in_sats
.semesterindex
.unwrap_last()
.iter_at(starting_indexes.semesterindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.semesterindex_to_ohlc_in_sats.forced_push_at(
@@ -1154,10 +1203,13 @@ impl Vecs {
.yearindex
.unwrap_min()
.iter();
let index = starting_indexes
.yearindex
.min(YearIndex::from(self.yearindex_to_ohlc_in_sats.len()));
self.timeindexes_to_close_in_sats
.yearindex
.unwrap_last()
.iter_at(starting_indexes.yearindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.yearindex_to_ohlc_in_sats.forced_push_at(
@@ -1192,10 +1244,13 @@ impl Vecs {
.decadeindex
.unwrap_min()
.iter();
let index = starting_indexes
.decadeindex
.min(DecadeIndex::from(self.decadeindex_to_ohlc_in_sats.len()));
self.timeindexes_to_close_in_sats
.decadeindex
.unwrap_last()
.iter_at(starting_indexes.decadeindex)
.iter_at(index)
.try_for_each(|(i, v)| -> Result<()> {
let close = v.into_owned();
self.decadeindex_to_ohlc_in_sats.forced_push_at(
crates/brk_computer/src/stateful/mod.rs
+3 -1
View File
@@ -11,7 +11,7 @@ use brk_structs::{
};
use brk_vecs::{
AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File,
Format, GenericStoredVec, RawVec, Reader, Stamp, StoredIndex, VecIterator,
Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator,
};
use log::info;
use rayon::prelude::*;
@@ -94,6 +94,8 @@ impl Vecs {
states_path: &Path,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("stateful"))?);
file.set_min_len(PAGE_SIZE * 20_000_000)?;
file.set_min_regions(50_000)?;
let compute_dollars = price.is_some();
crates/brk_computer/src/states/price_to_amount.rs
+19 -12
View File
@@ -22,10 +22,13 @@ pub struct PriceToAmount {
impl PriceToAmount {
pub fn forced_import(path: &Path, name: &str) -> Self {
Self::import(path, name).unwrap_or_else(|_| Self {
pathbuf: Self::path_(path, name),
height: None,
state: State::default(),
Self::import(path, name).unwrap_or_else(|_| {
// dbg!(e);
Self {
pathbuf: Self::path_(path, name),
height: None,
state: State::default(),
}
})
}
@@ -33,7 +36,7 @@ impl PriceToAmount {
let path = Self::path_(path, name);
fs::create_dir_all(&path)?;
let state = State::deserialize(&fs::read(&path)?)?;
let state = State::deserialize(&fs::read(Self::path_state_(&path))?)?;
Ok(Self {
height: Height::try_from(Self::path_height_(&path).as_path()).ok(),
@@ -63,10 +66,14 @@ impl PriceToAmount {
}
pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) {
let amount = self.state.get_mut(&price).unwrap();
*amount -= supply_state.value;
if *amount == Sats::ZERO {
self.state.remove(&price);
if let Some(amount) = self.state.get_mut(&price) {
*amount -= supply_state.value;
if *amount == Sats::ZERO {
self.state.remove(&price);
}
} else {
dbg!(&self.state, price, &self.pathbuf);
unreachable!();
}
}
@@ -117,11 +124,11 @@ impl State {
let mut buffer = Vec::with_capacity(8 + len * 16);
buffer.extend_from_slice(len.as_bytes());
buffer.extend(len.as_bytes());
self.iter().for_each(|(key, value)| {
buffer.extend_from_slice(key.as_bytes());
buffer.extend_from_slice(value.as_bytes());
buffer.extend(key.as_bytes());
buffer.extend(value.as_bytes());
});
buffer
crates/brk_computer/src/transactions.rs
+3 -2
View File
@@ -8,8 +8,8 @@ use brk_structs::{
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, StoredIndex,
VecIterator,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, PAGE_SIZE,
StoredIndex, VecIterator,
};
use crate::grouped::{
@@ -91,6 +91,7 @@ impl Vecs {
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("transactions"))?);
file.set_min_len(PAGE_SIZE * 10_000_000)?;
let compute_dollars = price.is_some();
crates/brk_error/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_error"
description = "Errors used throughout the Bitcoin Research Kit"
description = "Errors used throughout BRK"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
brk_vecs = { workspace = true }
crates/brk_fetcher/Cargo.toml
+1
View File
@@ -6,6 +6,7 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
brk_error = { workspace = true }
crates/brk_indexer/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_indexer"
description = "A Bitcoin Core indexer built on top of brk_parser"
description = "A Bitcoin indexer built on top of brk_parser"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
bitcoin = { workspace = true }
crates/brk_indexer/examples/indexer.rs
+1 -1
View File
@@ -21,7 +21,7 @@ fn main() -> Result<()> {
let blocks_dir = bitcoin_dir.join("blocks");
let outputs_dir = Path::new("./_outputs");
let outputs_dir = Path::new("../../_outputs");
fs::create_dir_all(outputs_dir)?;
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
crates/brk_indexer/src/lib.rs
+1 -1
View File
@@ -27,7 +27,7 @@ pub use stores::*;
pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(907_000);
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(908_700);
const VERSION: Version = Version::ONE;
#[derive(Clone)]
crates/brk_interface/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_interface"
description = "An interface to BRK's engine"
description = "An interface to find and format data from BRK"
license.workspace = true
edition.workspace = true
version.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
brk_computer = { workspace = true }
crates/brk_logger/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_logger"
description = "A clean logger used in the Bitcoin Research Kit"
description = "A thin wrapper around env_logger"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
env_logger = "0.11.8"
crates/brk_mcp/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_mcp"
description = "A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK"
description = "A bridge for LLMs to access BRK"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
axum = { workspace = true }
crates/brk_parser/Cargo.toml
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "brk_parser"
description = "A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust"
description = "A very fast Bitcoin block parser and iterator built on top of bitcoin-rust"
keywords = ["bitcoin", "block", "iterator"]
categories = ["cryptography::cryptocurrencies", "encoding"]
version.workspace = true
@@ -8,6 +8,7 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
bitcoin = { workspace = true }
crates/brk_server/Cargo.toml
+2 -1
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_server"
description = "A crate that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_computer and brk_interface"
description = "A server with an API for anything from BRK"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
axum = { workspace = true }
crates/brk_store/Cargo.toml
+1
View File
@@ -8,6 +8,7 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
brk_error = { workspace = true }
crates/brk_structs/Cargo.toml
+3 -2
View File
@@ -1,11 +1,12 @@
[package]
name = "brk_structs"
description = "Structs used throughout the Bitcoin Research Kit"
description = "Structs used throughout BRK"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
bitcoin = { workspace = true }
@@ -15,7 +16,7 @@ brk_vecs = {workspace = true}
byteview = { workspace = true }
derive_deref = { workspace = true }
jiff = { workspace = true }
rapidhash = "2.0.2"
rapidhash = "3.0.0"
serde = { workspace = true }
serde_bytes = { workspace = true }
zerocopy = { workspace = true }
crates/brk_vecs/Cargo.toml
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "brk_vecs"
description = "A simple index/value store"
description = "A KISS index/value store"
keywords = ["vec", "disk", "data"]
categories = ["database"]
version.workspace = true
@@ -8,6 +8,7 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[dependencies]
brk_vecs_macros = { workspace = true }
crates/brk_vecs/src/file/mod.rs
+80 -9
View File
@@ -27,6 +27,7 @@ use crate::{Error, Result};
pub const PAGE_SIZE: u64 = 4096;
pub const PAGE_SIZE_MINUS_1: u64 = PAGE_SIZE - 1;
const GB: u64 = 1024 * 1024 * 1024;
#[derive(Debug)]
pub struct File {
@@ -156,6 +157,7 @@ impl File {
self.write_all_to_region_at_(identifier, data, Some(at), false)
}
#[inline]
pub fn truncate_write_all_to_region(
&self,
identifier: Identifier,
@@ -165,7 +167,6 @@ impl File {
self.write_all_to_region_at_(identifier, data, Some(at), true)
}
// NEW
fn write_all_to_region_at_(
&self,
identifier: Identifier,
@@ -194,7 +195,7 @@ impl File {
});
let write_start = start + at.unwrap_or(len);
if at.is_some_and(|at| at >= reserved) {
if at.is_some_and(|at| at > reserved) {
return Err(Error::Str("Invalid at parameter"));
}
@@ -455,23 +456,50 @@ impl File {
regions.flush()
}
/// Do not write, right after as there might be a race condition
pub fn punch_holes(&self) -> Result<()> {
let file = self.file.write();
let mmap = self.mmap.read();
let regions = self.regions.read();
let layout = self.layout.read();
Self::punch_holes_(&file, &mmap, &layout)
Self::punch_holes_(&file, &mmap, &regions, &layout)
}
fn punch_holes_(file: &fs::File, mmap: &MmapMut, layout: &Layout) -> Result<()> {
fn punch_holes_(
file: &fs::File,
mmap: &MmapMut,
regions: &Regions,
layout: &Layout,
) -> Result<()> {
regions
.index_to_region()
.iter()
.flatten()
.try_for_each(|region_lock| -> Result<()> {
let region = region_lock.read();
let start = region.start();
let len = region.len();
let ceil_len = Self::ceil_number_to_page_size_multiple(len);
assert!(len <= ceil_len);
let reserved = region.reserved();
if ceil_len > reserved {
panic!()
} else if ceil_len < reserved {
let start = start + ceil_len;
let hole = reserved - ceil_len;
if Self::approx_has_punchable_data(mmap, start, hole) {
info!("Punching a hole of {hole} bytes at {start}...");
Self::punch_hole_(file, start, hole)?;
}
}
Ok(())
})?;
layout
.start_to_hole()
.par_iter()
.try_for_each(|(&start, &hole)| -> Result<()> {
assert!(start % PAGE_SIZE == 0);
assert!(hole % PAGE_SIZE == 0);
let has_old_data =
mmap[start as usize] != 0 || mmap[(start + hole - PAGE_SIZE) as usize] != 0;
if has_old_data {
if Self::approx_has_punchable_data(mmap, start, hole) {
info!("Punching a hole of {hole} bytes at {start}...");
Self::punch_hole_(file, start, hole)
} else {
@@ -480,6 +508,49 @@ impl File {
})
}
fn approx_has_punchable_data(mmap: &MmapMut, start: u64, len: u64) -> bool {
assert!(start % PAGE_SIZE == 0);
assert!(len % PAGE_SIZE == 0);
let min = start as usize;
let max = (start + len) as usize;
let check = |start, end| {
assert!(start >= min);
assert!(end < max);
mmap[start] != 0 || mmap[end] != 0
};
// Check first page (first and last byte)
let first_page_start = start as usize;
let first_page_end = (start + PAGE_SIZE - 1) as usize;
if check(first_page_start, first_page_end) {
return true;
}
// Check last page (first and last byte)
let last_page_start = (start + len - PAGE_SIZE) as usize;
let last_page_end = (start + len - 1) as usize;
if check(last_page_start, last_page_end) {
return true;
}
// For large lengths, check at 1GB intervals
if len > GB {
let num_gb_checks = (len / GB) as usize;
for i in 1..num_gb_checks {
let gb_boundary = start + (i as u64 * GB);
let page_start = gb_boundary as usize;
let page_end = (gb_boundary + PAGE_SIZE - 1) as usize;
if check(page_start, page_end) {
return true;
}
}
}
false
}
#[inline]
fn punch_hole(&self, start: u64, length: u64) -> Result<()> {
let file = self.file.write();
crates/brk_vecs/src/traits/generic.rs
+3 -3
View File
@@ -137,10 +137,10 @@ where
}
if self.pushed_len() * Self::SIZE_OF_T >= MAX_CACHE_SIZE {
self.safe_flush(exit)
} else {
Ok(())
self.safe_flush(exit)?;
}
Ok(())
}
#[inline]
crates/brk_vecs/src/variants/eager.rs
+1 -1
View File
@@ -44,7 +44,7 @@ where
}
pub fn inner_version(&self) -> Version {
self.version()
self.0.header().vec_version()
}
fn update_computed_version(&mut self, computed_version: Version) {
crates/brk_vecs/src/variants/stored/mod.rs
+3 -4
View File
@@ -39,10 +39,9 @@ where
}
if format.is_compressed() {
todo!();
// Ok(Self::Compressed(CompressedVec::forced_import(
// file, name, version,
// )?))
Ok(Self::Compressed(CompressedVec::forced_import(
file, name, version,
)?))
} else {
Ok(Self::Raw(RawVec::forced_import(file, name, version)?))
}
crates/brk_vecs_macros/Cargo.toml
+1
View File
@@ -8,6 +8,7 @@ edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
build = "../../build.rs"
[lib]
proc-macro = true
docker/.env.example
+1 -11
View File
@@ -14,16 +14,6 @@ BTC_RPC_PORT=8332
# BTC_RPC_USER=your_rpc_username
# BTC_RPC_PASSWORD=your_rpc_password
# BRK configuration
# Services to run: all, processor, or server
BRK_SERVICES=all
# Computation mode: lazy (compute on demand) or eager (precompute and save)
BRK_COMPUTATION=lazy
# Data format: raw (faster) or compressed (saves disk space)
BRK_FORMAT=raw
# Enable price fetching from exchanges
BRK_FETCH=true
@@ -39,4 +29,4 @@ BRK_MCP=true
# Option 2: Use a bind mount to a local directory
# Uncomment and set this to use a specific directory on your host
# Also uncomment the corresponding line in docker-compose.yml
# BRK_DATA_DIR=/path/to/brk/data
# BRK_DATA_DIR=/path/to/brk/data
docker/Dockerfile
-3
View File
@@ -55,6 +55,3 @@ WORKDIR /home/brk
# Default entrypoint
ENTRYPOINT ["brk"]
# Default command (can be overridden)
CMD ["--services", "all"]
docker/README.md
+3 -9
View File
@@ -21,7 +21,7 @@ This guide explains how to run BRK using Docker and Docker Compose.
```bash
docker compose -f docker/docker-compose.yml up -d
```
Or from the docker directory:
```bash
cd docker && docker compose up -d
@@ -62,8 +62,6 @@ cd docker && docker compose up -d
| `BTC_RPC_USER` | Bitcoin RPC username | - |
| `BTC_RPC_PASSWORD` | Bitcoin RPC password | - |
| `BRK_DATA_VOLUME` | Docker volume name for BRK data | `brk-data` |
| `BRK_COMPUTATION` | Computation mode (`lazy`, `eager`) | `lazy` |
| `BRK_FORMAT` | Data format (`raw`, `compressed`) | `raw` |
| `BRK_FETCH` | Enable price fetching | `true` |
| `BRK_MCP` | Enable MCP for AI/LLM | `true` |
@@ -81,8 +79,6 @@ BTC_RPC_USER=your_username
BTC_RPC_PASSWORD=your_password
# BRK settings
BRK_COMPUTATION=lazy
BRK_FORMAT=raw
BRK_FETCH=true
BRK_MCP=true
```
@@ -96,7 +92,7 @@ BRK will automatically use the `.cookie` file from your Bitcoin Core directory.
Set `BTC_RPC_USER` and `BTC_RPC_PASSWORD` in your `docker/.env` file.
#### Network Connectivity
- **Same host**:
- **Same host**:
- If Bitcoin Core is running natively (not in Docker): Use `host.docker.internal` on macOS/Windows or `172.17.0.1` on Linux
- If Bitcoin Core is also in Docker: Use the service name or container IP
- **Remote host**: Use the actual IP address or hostname
@@ -210,11 +206,9 @@ docker compose -f docker/docker-compose.yml logs -f
#### Slow indexing
- Ensure adequate disk space for indexed data - a minimum of 3GB/s is recommended
- Monitor memory usage during initial indexing
- Use `BRK_COMPUTATION=lazy` to reduce memory usage
#### Out of memory
- Increase Docker's memory limit
- Use `BRK_COMPUTATION=lazy` mode
- Monitor container resource usage: `docker stats`
### Permission Issues
@@ -260,4 +254,4 @@ docker run --rm -v brk_brk-data:/target -v \"$(pwd)\":/backup alpine tar xzf /ba
# Start container
docker compose -f docker/docker-compose.yml up -d
```
```
docker/docker-compose.yml
+5 -7
View File
@@ -11,7 +11,7 @@ services:
container_name: brk
restart: unless-stopped
ports:
- 7070:3110 # Map host port 7070 to container port 3110
- 7070:3110 # Map host port 7070 to container port 3110
volumes:
# Bitcoin Core data directory (read-only)
# For access to raw block data
@@ -25,20 +25,18 @@ services:
# Bitcoin Core configuration
- BITCOINDIR=/bitcoin
- BLOCKSDIR=/bitcoin/blocks
# RPC configuration (required for processor)
- RPCCONNECT=${BTC_RPC_HOST:-localhost}
- RPCPORT=${BTC_RPC_PORT:-8332}
# - RPCCOOKIEFILE=/bitcoin/.cookie
# Username/password authentication
- RPCUSER=${BTC_RPC_USER}
- RPCPASSWORD=${BTC_RPC_PASSWORD}
# BRK configuration
- BRKDIR=/home/brk/.brk
- COMPUTATION=${BRK_COMPUTATION:-lazy}
- FORMAT=${BRK_FORMAT:-raw}
- FETCH=${BRK_FETCH:-true}
- MCP=${BRK_MCP:-true}
command:
@@ -61,4 +59,4 @@ services:
volumes:
brk-data:
driver: local
driver: local