This commit is contained in:
nym21
2025-11-20 20:35:54 +01:00
parent 8467e218ae
commit c8c62b504b
21 changed files with 432 additions and 405 deletions
-4
View File
@@ -10,12 +10,10 @@ rust-version.workspace = true
build = "build.rs"
[features]
default = ["cli"]
full = [
"bencher",
"binder",
"bundler",
"cli",
"computer",
"error",
"fetcher",
@@ -36,7 +34,6 @@ full = [
bencher = ["brk_bencher"]
binder = ["brk_binder"]
bundler = ["brk_bundler"]
cli = ["brk_cli"]
computer = ["brk_computer"]
error = ["brk_error"]
fetcher = ["brk_fetcher"]
@@ -58,7 +55,6 @@ types = ["brk_types"]
brk_bencher = { workspace = true, optional = true }
brk_binder = { workspace = true, optional = true }
brk_bundler = { workspace = true, optional = true }
brk_cli = { workspace = true, optional = true }
brk_computer = { workspace = true, optional = true }
brk_error = { workspace = true, optional = true }
brk_fetcher = { workspace = true, optional = true }
-4
View File
@@ -12,10 +12,6 @@ pub use brk_binder as binder;
#[doc(inline)]
pub use brk_bundler as bundler;
#[cfg(feature = "cli")]
#[doc(inline)]
pub use brk_cli as cli;
#[cfg(feature = "computer")]
#[doc(inline)]
pub use brk_computer as computer;
-1
View File
@@ -1 +0,0 @@
use brk_cli::main;
+5 -5
View File
@@ -340,7 +340,7 @@ impl Visualizer {
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("disk_chart.svg");
let output_path = crate_path.join("disk.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
@@ -404,7 +404,7 @@ impl Visualizer {
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("memory_chart.svg");
let output_path = crate_path.join("memory.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
@@ -628,7 +628,7 @@ impl Visualizer {
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("io_read_chart.svg");
let output_path = crate_path.join("io_read.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
@@ -710,7 +710,7 @@ impl Visualizer {
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("io_write_chart.svg");
let output_path = crate_path.join("io_write.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
@@ -792,7 +792,7 @@ impl Visualizer {
crate_name: &str,
runs: &[BenchmarkRun],
) -> Result<()> {
let output_path = crate_path.join("progress_chart.svg");
let output_path = crate_path.join("progress.svg");
let root = SVGBackend::new(&output_path, SIZE).into_drawing_area();
root.fill(&BG_COLOR)?;
+1 -1
View File
@@ -12,7 +12,7 @@ build = "build.rs"
[dependencies]
log = { workspace = true }
notify = "8.2.0"
rolldown = { version = "0.3.0", package = "brk_rolldown" }
rolldown = { version = "0.4.0", package = "brk_rolldown" }
# brk_rolldown = { path = "../../../rolldown/crates/rolldown"}
sugar_path = "1.2.1"
tokio = { workspace = true }
+1 -1
View File
@@ -23,7 +23,7 @@ brk_reader = { workspace = true }
brk_rpc = { workspace = true }
brk_server = { workspace = true }
vecdb = { workspace = true }
clap = { version = "4.5.51", features = ["derive", "string"] }
clap = { version = "4.5.53", features = ["derive", "string"] }
color-eyre = "0.6.5"
log = { workspace = true }
minreq = { workspace = true }
-154
View File
@@ -1,154 +0,0 @@
#![doc = include_str!("../README.md")]
use std::{
fs,
io::Cursor,
path::Path,
thread::{self, sleep},
time::Duration,
};
use brk_binder::generate_js_files;
use brk_bundler::bundle;
use brk_computer::Computer;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_query::AsyncQuery;
use brk_reader::Reader;
use brk_server::{Server, VERSION};
use log::info;
use vecdb::Exit;
mod config;
mod paths;
mod website;
use crate::{config::Config, paths::*};
pub fn main() -> color_eyre::Result<()> {
// Can't increase main thread's stack size, thus we need to use another thread
thread::Builder::new()
.stack_size(512 * 1024 * 1024)
.spawn(run)?
.join()
.unwrap()
}
pub fn run() -> color_eyre::Result<()> {
color_eyre::install()?;
fs::create_dir_all(dot_brk_path())?;
brk_logger::init(Some(&dot_brk_log_path()))?;
let config = Config::import()?;
let client = config.rpc()?;
let exit = Exit::new();
exit.set_ctrlc_handler();
let reader = Reader::new(config.blocksdir(), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&config.brkdir())?;
let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
let query = AsyncQuery::build(&reader, &indexer, &computer);
let website = config.website();
let downloads_path = config.downloads_dir();
let future = async move {
let bundle_path = if website.is_some() {
let websites_dev_path = Path::new("../../websites");
let modules_dev_path = Path::new("../../modules");
let websites_path;
let modules_path;
if fs::exists(websites_dev_path)? && fs::exists(modules_dev_path)? {
websites_path = websites_dev_path.to_path_buf();
modules_path = modules_dev_path.to_path_buf();
} else {
let downloaded_brk_path = downloads_path.join(format!("brk-{VERSION}"));
let downloaded_websites_path = downloaded_brk_path.join("websites");
let downloaded_modules_path = downloaded_brk_path.join("modules");
if !fs::exists(&downloaded_websites_path)? {
info!("Downloading source from Github...");
let url = format!(
"https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
);
let response = minreq::get(url).send()?;
let bytes = response.as_bytes();
let cursor = Cursor::new(bytes);
let mut zip = zip::ZipArchive::new(cursor).unwrap();
zip.extract(downloads_path).unwrap();
}
websites_path = downloaded_websites_path;
modules_path = downloaded_modules_path;
}
generate_js_files(query.inner(), &modules_path)?;
Some(
bundle(
&modules_path,
&websites_path,
website.to_folder_name(),
true,
)
.await?,
)
} else {
None
};
let server = Server::new(&query, bundle_path);
tokio::spawn(async move {
server.serve(true).await.unwrap();
});
Ok(()) as Result<()>
};
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?;
let _handle = runtime.spawn(future);
loop {
client.wait_for_synced_node()?;
let last_height = client.get_last_height()?;
info!("{} blocks found.", u32::from(last_height) + 1);
let starting_indexes = if config.check_collisions() {
indexer.checked_index(&blocks, &client, &exit)?
} else {
indexer.index(&blocks, &client, &exit)?
};
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
info!("Waiting for new blocks...");
while last_height == client.get_last_height()? {
sleep(Duration::from_secs(1))
}
}
}
+154 -1
View File
@@ -1 +1,154 @@
use brk_cli::main;
#![doc = include_str!("../README.md")]
use std::{
fs,
io::Cursor,
path::Path,
thread::{self, sleep},
time::Duration,
};
use brk_binder::generate_js_files;
use brk_bundler::bundle;
use brk_computer::Computer;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_query::AsyncQuery;
use brk_reader::Reader;
use brk_server::{Server, VERSION};
use log::info;
use vecdb::Exit;
mod config;
mod paths;
mod website;
use crate::{config::Config, paths::*};
pub fn main() -> color_eyre::Result<()> {
// Can't increase main thread's stack size, thus we need to use another thread
thread::Builder::new()
.stack_size(512 * 1024 * 1024)
.spawn(run)?
.join()
.unwrap()
}
pub fn run() -> color_eyre::Result<()> {
color_eyre::install()?;
fs::create_dir_all(dot_brk_path())?;
brk_logger::init(Some(&dot_brk_log_path()))?;
let config = Config::import()?;
let client = config.rpc()?;
let exit = Exit::new();
exit.set_ctrlc_handler();
let reader = Reader::new(config.blocksdir(), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&config.brkdir())?;
let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
let query = AsyncQuery::build(&reader, &indexer, &computer);
let website = config.website();
let downloads_path = config.downloads_dir();
let future = async move {
let bundle_path = if website.is_some() {
let websites_dev_path = Path::new("../../websites");
let modules_dev_path = Path::new("../../modules");
let websites_path;
let modules_path;
if fs::exists(websites_dev_path)? && fs::exists(modules_dev_path)? {
websites_path = websites_dev_path.to_path_buf();
modules_path = modules_dev_path.to_path_buf();
} else {
let downloaded_brk_path = downloads_path.join(format!("brk-{VERSION}"));
let downloaded_websites_path = downloaded_brk_path.join("websites");
let downloaded_modules_path = downloaded_brk_path.join("modules");
if !fs::exists(&downloaded_websites_path)? {
info!("Downloading source from Github...");
let url = format!(
"https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
);
let response = minreq::get(url).send()?;
let bytes = response.as_bytes();
let cursor = Cursor::new(bytes);
let mut zip = zip::ZipArchive::new(cursor).unwrap();
zip.extract(downloads_path).unwrap();
}
websites_path = downloaded_websites_path;
modules_path = downloaded_modules_path;
}
generate_js_files(query.inner(), &modules_path)?;
Some(
bundle(
&modules_path,
&websites_path,
website.to_folder_name(),
true,
)
.await?,
)
} else {
None
};
let server = Server::new(&query, bundle_path);
tokio::spawn(async move {
server.serve(true).await.unwrap();
});
Ok(()) as Result<()>
};
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?;
let _handle = runtime.spawn(future);
loop {
client.wait_for_synced_node()?;
let last_height = client.get_last_height()?;
info!("{} blocks found.", u32::from(last_height) + 1);
let starting_indexes = if config.check_collisions() {
indexer.checked_index(&blocks, &client, &exit)?
} else {
indexer.index(&blocks, &client, &exit)?
};
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
info!("Waiting for new blocks...");
while last_height == client.get_last_height()? {
sleep(Duration::from_secs(1))
}
}
}
@@ -5,7 +5,6 @@ use brk_computer::Computer;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::{Indexer, Indexes};
use brk_iterator::Blocks;
use brk_reader::Reader;
use brk_rpc::{Auth, Client};
use log::{debug, info};
@@ -37,9 +36,7 @@ fn run() -> Result<()> {
let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&outputs_dir)?;
let indexer = Indexer::forced_import(&outputs_dir)?;
let fetcher = Fetcher::import(true, None)?;
@@ -0,0 +1,97 @@
use std::{env, path::Path, thread};
use brk_computer::Computer;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_types::TxIndex;
use vecdb::{AnyStoredVec, Exit, GenericStoredVec};
pub fn main() -> Result<()> {
// Can't increase main thread's stack size, thus we need to use another thread
thread::Builder::new()
.stack_size(512 * 1024 * 1024)
.spawn(run)?
.join()
.unwrap()
}
fn run() -> Result<()> {
brk_logger::init(Some(Path::new(".log")))?;
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs");
let indexer = Indexer::forced_import(&outputs_dir)?;
let fetcher = Fetcher::import(true, None)?;
let exit = Exit::new();
exit.set_ctrlc_handler();
let computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
let txindex = TxIndex::new(134217893);
dbg!(
indexer
.vecs
.txindex_to_txid
.read_once(txindex)
.unwrap()
.to_string()
);
let first_txinindex = indexer.vecs.txindex_to_first_txinindex.read_once(txindex)?;
dbg!(first_txinindex);
let first_txoutindex = indexer
.vecs
.txindex_to_first_txoutindex
.read_once(txindex)?;
dbg!(first_txoutindex);
let input_count = *computer.indexes.txindex_to_input_count.read_once(txindex)?;
dbg!(input_count);
let output_count = *computer
.indexes
.txindex_to_output_count
.read_once(txindex)?;
dbg!(output_count);
dbg!(
computer
.indexes
.txinindex_to_txoutindex
.read_once(first_txinindex)
);
dbg!(
computer
.indexes
.txinindex_to_txoutindex
.read_once(first_txinindex + 1)
);
dbg!(computer.chain.txinindex_to_value.read_once(first_txinindex));
dbg!(
computer
.chain
.txinindex_to_value
.read_once(first_txinindex + 1)
);
dbg!(indexer.vecs.txoutindex_to_value.read_once(first_txoutindex));
dbg!(
indexer
.vecs
.txoutindex_to_value
.read_once(first_txoutindex + 1)
);
dbg!(computer.chain.txindex_to_input_value.read_once(txindex));
dbg!(computer.chain.txindex_to_input_value.read_once(txindex));
dbg!(computer.chain.txindex_to_output_value.read_once(txindex));
// dbg!(computer.indexes.txindex_to_txindex.ge(txindex));
dbg!(
computer
.indexes
.txinindex_to_txoutindex
.region()
.meta()
.len()
);
Ok(())
}
+22 -46
View File
@@ -10,8 +10,8 @@ use brk_types::{
TxVersion, Version, WeekIndex, Weight, YearIndex,
};
use vecdb::{
AnyVec, Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, IterableVec,
LazyVecFrom1, LazyVecFrom2, PAGE_SIZE, TypedVecIterator, VecIndex, unlikely,
Database, EagerVec, Exit, GenericStoredVec, IterableCloneableVec, IterableVec, LazyVecFrom1,
LazyVecFrom2, PAGE_SIZE, TypedVecIterator, VecIndex, unlikely,
};
use crate::grouped::{
@@ -1311,20 +1311,6 @@ impl Vecs {
exit,
)?;
// Debug: verify the computed txinindex_to_value
dbg!("txinindex_to_value first 20:");
for i in 0..20.min(self.txinindex_to_value.len()) {
let val = self.txinindex_to_value.read_at_unwrap_once(i);
dbg!((TxInIndex::from(i), val));
}
// Debug: verify the computed txindex_to_input_count
dbg!("txindex_to_input_count first 20:");
for i in 0..20.min(indexes.txindex_to_input_count.len()) {
let val = indexes.txindex_to_input_count.read_at_unwrap_once(i);
dbg!((TxInIndex::from(i), val));
}
self.txindex_to_input_value.compute_sum_from_indexes(
starting_indexes.txindex,
&indexer.vecs.txindex_to_first_txinindex,
@@ -1333,12 +1319,6 @@ impl Vecs {
exit,
)?;
// Debug: verify the computed input values
for i in 0..10.min(self.txindex_to_input_value.len()) {
let val = self.txindex_to_input_value.read_at_unwrap_once(i);
dbg!((TxIndex::from(i), "input_value", val));
}
self.txindex_to_output_value.compute_sum_from_indexes(
starting_indexes.txindex,
&indexer.vecs.txindex_to_first_txoutindex,
@@ -1347,26 +1327,17 @@ impl Vecs {
exit,
)?;
// Debug: verify the computed output values
for i in 0..10.min(self.txindex_to_output_value.len()) {
let val = self.txindex_to_output_value.read_at_unwrap_once(i);
dbg!((TxIndex::from(i), "output_value", val));
}
self.txindex_to_fee.compute_transform2(
starting_indexes.txindex,
&self.txindex_to_input_value,
&self.txindex_to_output_value,
|(i, input, output, ..)| {
(
i,
if unlikely(input.is_max()) {
Sats::ZERO
} else {
dbg!((i, input, output));
input.checked_sub(output).unwrap()
},
)
let fee = if unlikely(input.is_max()) {
Sats::ZERO
} else {
input - output
};
(i, fee)
},
exit,
)?;
@@ -1381,11 +1352,12 @@ impl Vecs {
self.indexes_to_sent
.compute_all(indexes, price, starting_indexes, exit, |v| {
v.compute_sum_from_indexes(
v.compute_filtered_sum_from_indexes(
starting_indexes.height,
&indexer.vecs.height_to_first_txindex,
&indexes.height_to_txindex_count,
&self.txindex_to_input_value,
|sats| !sats.is_max(),
exit,
)?;
Ok(())
@@ -1472,6 +1444,8 @@ impl Vecs {
},
exit,
)?;
drop(height_to_coinbase_iter);
if let Some(mut height_to_coinbase_iter) = self
.indexes_to_coinbase
.dollars
@@ -1493,18 +1467,20 @@ impl Vecs {
)?;
}
drop(height_to_coinbase_iter);
self.indexes_to_subsidy
.compute_all(indexes, price, starting_indexes, exit, |vec| {
let mut indexes_to_fee_sum_iter =
self.indexes_to_fee.sats.height.unwrap_sum().iter();
vec.compute_transform(
vec.compute_transform2(
starting_indexes.height,
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|(height, coinbase, ..)| {
let fees = indexes_to_fee_sum_iter.get_unwrap(height);
(height, coinbase.checked_sub(fees).unwrap())
self.indexes_to_fee.sats.height.unwrap_sum(),
|(height, coinbase, fees, ..)| {
(
height,
coinbase.checked_sub(fees).unwrap_or_else(|| {
dbg!(height, coinbase, fees);
panic!()
}),
)
},
exit,
)?;
@@ -235,21 +235,17 @@ where
Ok(())
}
pub fn compute<I2>(
pub fn compute<A>(
&mut self,
max_from: I,
source: &impl IterableVec<I2, T>,
first_indexes: &impl IterableVec<I, I2>,
source: &impl IterableVec<A, T>,
first_indexes: &impl IterableVec<I, A>,
count_indexes: &impl IterableVec<I, StoredU64>,
exit: &Exit,
) -> Result<()>
where
I2: VecIndex + VecValue + CheckedSub<I2>,
A: VecIndex + VecValue + CheckedSub<A>,
{
dbg!(source.len());
dbg!(first_indexes.len());
dbg!(count_indexes.len());
self.validate_computed_version_or_reset(
source.version() + first_indexes.version() + count_indexes.version(),
)?;
@@ -399,16 +395,16 @@ where
}
#[allow(clippy::wrong_self_convention)]
pub fn from_aligned<I2>(
pub fn from_aligned<A>(
&mut self,
max_from: I,
source: &EagerVecsBuilder<I2, T>,
first_indexes: &impl IterableVec<I, I2>,
source: &EagerVecsBuilder<A, T>,
first_indexes: &impl IterableVec<I, A>,
count_indexes: &impl IterableVec<I, StoredU64>,
exit: &Exit,
) -> Result<()>
where
I2: VecIndex + VecValue + CheckedSub<I2>,
A: VecIndex + VecValue + CheckedSub<A>,
{
if self.pct90.is_some()
|| self.pct75.is_some()
+7 -8
View File
@@ -438,9 +438,9 @@ impl Vecs {
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> Result<Indexes> {
let idxs = self.compute_(indexer, starting_indexes, exit)?;
let indexes = self.compute_(indexer, starting_indexes, exit)?;
self.db.compact()?;
Ok(idxs)
Ok(indexes)
}
fn compute_(
@@ -460,13 +460,12 @@ impl Vecs {
&indexer.vecs.txinindex_to_outpoint,
|(txinindex, outpoint, ..)| {
if unlikely(outpoint.is_coinbase()) {
(txinindex, TxOutIndex::COINBASE)
} else {
let txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &txindex_to_first_txoutindex_reader)
+ outpoint.vout();
(txinindex, txoutindex)
return (txinindex, TxOutIndex::COINBASE);
}
let txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &txindex_to_first_txoutindex_reader)
+ outpoint.vout();
(txinindex, txoutindex)
},
exit,
)?;
+2 -2
View File
@@ -157,8 +157,6 @@ impl Computer {
info!("Computing indexes...");
let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
return Ok(());
if let Some(fetched) = self.fetched.as_mut() {
info!("Computing fetched...");
fetched.compute(indexer, &self.indexes, &starting_indexes, exit)?;
@@ -201,6 +199,8 @@ impl Computer {
self.market.compute(price, &starting_indexes, exit)?;
}
return Ok(());
// constants.join().unwrap()?;
// chain.join().unwrap()?;
// Ok(())
+3 -3
View File
@@ -3,7 +3,7 @@
use std::{collections::BTreeMap, sync::Arc};
use brk_computer::Computer;
use brk_error::Result;
use brk_error::{Error, Result};
use brk_indexer::Indexer;
use brk_reader::Reader;
use brk_traversable::TreeNode;
@@ -191,8 +191,8 @@ impl Query {
Format::JSON => {
let mut values = metrics
.iter()
.map(|vec| vec.collect_range_json_bytes(from, to))
.collect::<Vec<_>>();
.map(|vec| vec.collect_range_json_bytes(from, to).map_err(Error::from))
.collect::<Result<Vec<_>>>()?;
if values.is_empty() {
return Ok(Output::default(format));
+4 -1
View File
@@ -17,9 +17,12 @@ pub struct FeeRate(f64);
impl From<(Sats, StoredU64)> for FeeRate {
#[inline]
fn from((sats, vsize): (Sats, StoredU64)) -> Self {
if sats.is_zero() {
return Self(0.0);
}
let sats = u64::from(sats);
let vsize = u64::from(vsize);
Self(((sats * 1000 + vsize.checked_sub(1).unwrap()) / vsize) as f64 / 1000.0)
Self((sats * 1000).div_ceil(vsize) as f64 / 1000.0)
}
}
+5
View File
@@ -48,6 +48,11 @@ impl OutPoint {
pub fn is_coinbase(self) -> bool {
self == Self::COINBASE
}
#[inline(always)]
pub fn is_not_coinbase(self) -> bool {
self != Self::COINBASE
}
}
impl std::fmt::Display for OutPoint {
+9 -11
View File
@@ -1,6 +1,6 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Div, Mul, SubAssign},
ops::{Add, AddAssign, Div, Mul, Sub, SubAssign},
};
use bitcoin::Amount;
@@ -78,16 +78,14 @@ impl Sats {
impl Add for Sats {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self::from(self.0 + rhs.0)
// Self::from(
// self.0
// .checked_add(rhs.0)
// .or_else(|| {
// dbg!(rhs);
// panic!();
// })
// .unwrap(),
// )
Self(self.0 + rhs.0)
}
}
impl Sub for Sats {
type Output = Self;
fn sub(self, rhs: Self) -> Self::Output {
Self(self.0 - rhs.0)
}
}