Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-12-08 17:05:38 +01:00
parent f4a1384dc4
commit 60a38b4108
19 changed files with 262 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@@ -602,7 +602,6 @@ dependencies = [
"color-eyre",
"log",
"minreq",
"rlimit",
"serde",
"tokio",
"toml",
@@ -632,6 +631,7 @@ dependencies = [
"log",
"pco",
"rayon",
"rlimit",
"rustc-hash",
"serde",
"smallvec",
@@ -1246,7 +1246,7 @@ version = "0.0.111"
dependencies = [
"bitcoin",
"brk_error",
"byteview 0.9.1",
"byteview 0.6.1",
"derive_deref",
"itoa",
"jiff",
@@ -4697,9 +4697,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "simd-adler32"
version = "0.3.7"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "simd-json"
@@ -5104,9 +5104,9 @@ dependencies = [
[[package]]
name = "tower-http"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456"
checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
"async-compression",
"bitflags 2.10.0",

4
Cargo.toml
View File

@@ -57,8 +57,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" }
brk_types = { version = "0.0.111", path = "crates/brk_types" }
brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
# byteview = "=0.6.1"
byteview = "0.9.1"
byteview = "=0.6.1"
# byteview = "0.9.1"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall2 = { version = "2.11.8", package = "brk_fjall" }

1
crates/brk_cli/Cargo.toml
View File

@@ -31,7 +31,6 @@ serde = { workspace = true }
tokio = { workspace = true }
toml = "0.9.8"
zip = { version = "6.0.0", default-features = false, features = ["deflate"] }
rlimit = "0.10.2"
[[bin]]
name = "brk"

7
crates/brk_cli/src/main.rs
View File

@@ -28,13 +28,6 @@ mod website;
use crate::{config::Config, paths::*};
pub fn main() -> color_eyre::Result<()> {
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit(
rlimit::Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
// Can't increase main thread's stack size, thus we need to use another thread
thread::Builder::new()
.stack_size(512 * 1024 * 1024)

1
crates/brk_computer/Cargo.toml
View File

@@ -26,6 +26,7 @@ derive_deref = { workspace = true }
log = { workspace = true }
pco = "0.4.7"
rayon = { workspace = true }
rlimit = "0.10.2"
rustc-hash = { workspace = true }
serde = { workspace = true }
smallvec = "1.15.1"

70
crates/brk_computer/src/indexes.rs
View File

@@ -108,11 +108,15 @@ impl Vecs {
let version = parent_version + VERSION;
macro_rules! eager {
($name:expr) => { EagerVec::forced_import(&db, $name, version)? };
($name:expr) => {
EagerVec::forced_import(&db, $name, version)?
};
}
macro_rules! lazy {
($name:expr, $source:expr) => {
LazyVecFrom1::init($name, version, $source.boxed_clone(), |index, _| Some(index))
LazyVecFrom1::init($name, version, $source.boxed_clone(), |index, _| {
Some(index)
})
};
}
@@ -120,18 +124,54 @@ impl Vecs {
txinindex_to_txoutindex: eager!("txoutindex"),
txoutindex_to_txoutindex: lazy!("txoutindex", indexer.vecs.txoutindex_to_value),
txinindex_to_txinindex: lazy!("txinindex", indexer.vecs.txinindex_to_outpoint),
p2pk33addressindex_to_p2pk33addressindex: lazy!("p2pk33addressindex", indexer.vecs.p2pk33addressindex_to_p2pk33bytes),
p2pk65addressindex_to_p2pk65addressindex: lazy!("p2pk65addressindex", indexer.vecs.p2pk65addressindex_to_p2pk65bytes),
p2pkhaddressindex_to_p2pkhaddressindex: lazy!("p2pkhaddressindex", indexer.vecs.p2pkhaddressindex_to_p2pkhbytes),
p2shaddressindex_to_p2shaddressindex: lazy!("p2shaddressindex", indexer.vecs.p2shaddressindex_to_p2shbytes),
p2traddressindex_to_p2traddressindex: lazy!("p2traddressindex", indexer.vecs.p2traddressindex_to_p2trbytes),
p2wpkhaddressindex_to_p2wpkhaddressindex: lazy!("p2wpkhaddressindex", indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes),
p2wshaddressindex_to_p2wshaddressindex: lazy!("p2wshaddressindex", indexer.vecs.p2wshaddressindex_to_p2wshbytes),
p2aaddressindex_to_p2aaddressindex: lazy!("p2aaddressindex", indexer.vecs.p2aaddressindex_to_p2abytes),
p2msoutputindex_to_p2msoutputindex: lazy!("p2msoutputindex", indexer.vecs.p2msoutputindex_to_txindex),
emptyoutputindex_to_emptyoutputindex: lazy!("emptyoutputindex", indexer.vecs.emptyoutputindex_to_txindex),
unknownoutputindex_to_unknownoutputindex: lazy!("unknownoutputindex", indexer.vecs.unknownoutputindex_to_txindex),
opreturnindex_to_opreturnindex: lazy!("opreturnindex", indexer.vecs.opreturnindex_to_txindex),
p2pk33addressindex_to_p2pk33addressindex: lazy!(
"p2pk33addressindex",
indexer.vecs.p2pk33addressindex_to_p2pk33bytes
),
p2pk65addressindex_to_p2pk65addressindex: lazy!(
"p2pk65addressindex",
indexer.vecs.p2pk65addressindex_to_p2pk65bytes
),
p2pkhaddressindex_to_p2pkhaddressindex: lazy!(
"p2pkhaddressindex",
indexer.vecs.p2pkhaddressindex_to_p2pkhbytes
),
p2shaddressindex_to_p2shaddressindex: lazy!(
"p2shaddressindex",
indexer.vecs.p2shaddressindex_to_p2shbytes
),
p2traddressindex_to_p2traddressindex: lazy!(
"p2traddressindex",
indexer.vecs.p2traddressindex_to_p2trbytes
),
p2wpkhaddressindex_to_p2wpkhaddressindex: lazy!(
"p2wpkhaddressindex",
indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes
),
p2wshaddressindex_to_p2wshaddressindex: lazy!(
"p2wshaddressindex",
indexer.vecs.p2wshaddressindex_to_p2wshbytes
),
p2aaddressindex_to_p2aaddressindex: lazy!(
"p2aaddressindex",
indexer.vecs.p2aaddressindex_to_p2abytes
),
p2msoutputindex_to_p2msoutputindex: lazy!(
"p2msoutputindex",
indexer.vecs.p2msoutputindex_to_txindex
),
emptyoutputindex_to_emptyoutputindex: lazy!(
"emptyoutputindex",
indexer.vecs.emptyoutputindex_to_txindex
),
unknownoutputindex_to_unknownoutputindex: lazy!(
"unknownoutputindex",
indexer.vecs.unknownoutputindex_to_txindex
),
opreturnindex_to_opreturnindex: lazy!(
"opreturnindex",
indexer.vecs.opreturnindex_to_txindex
),
txindex_to_txindex: lazy!("txindex", indexer.vecs.txindex_to_txid),
txindex_to_input_count: eager!("input_count"),
txindex_to_output_count: eager!("output_count"),
@@ -653,7 +693,7 @@ impl Vecs {
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct Indexes {
indexes: brk_indexer::Indexes,
pub dateindex: DateIndex,

63
crates/brk_computer/src/lib.rs
View File

@@ -56,6 +56,14 @@ impl Computer {
indexer: &Indexer,
fetcher: Option<Fetcher>,
) -> Result<Self> {
info!("Increasing number of open files...");
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit(
rlimit::Resource::NOFILE,
no_file_limit.0.max(10_000),
no_file_limit.1,
)?;
info!("Importing computer...");
let import_start = Instant::now();
@@ -181,7 +189,7 @@ impl Computer {
info!("Computed prices in {:?}", i.elapsed());
}
std::thread::scope(|scope| -> Result<()> {
thread::scope(|scope| -> Result<()> {
let blks = scope.spawn(|| -> Result<()> {
info!("Computing BLKs metadata...");
let i = Instant::now();
@@ -227,29 +235,38 @@ impl Computer {
Ok(())
})?;
info!("Computing pools...");
let i = Instant::now();
self.pools.compute(
indexer,
&self.indexes,
&starting_indexes,
&self.chain,
self.price.as_ref(),
exit,
)?;
info!("Computed pools in {:?}", i.elapsed());
let starting_indexes_clone = starting_indexes.clone();
thread::scope(|scope| -> Result<()> {
let pools = scope.spawn(|| -> Result<()> {
info!("Computing pools...");
let i = Instant::now();
self.pools.compute(
indexer,
&self.indexes,
&starting_indexes_clone,
&self.chain,
self.price.as_ref(),
exit,
)?;
info!("Computed pools in {:?}", i.elapsed());
Ok(())
});
info!("Computing stateful...");
let i = Instant::now();
self.stateful.compute(
indexer,
&self.indexes,
&self.chain,
self.price.as_ref(),
&mut starting_indexes,
exit,
)?;
info!("Computed stateful in {:?}", i.elapsed());
info!("Computing stateful...");
let i = Instant::now();
self.stateful.compute(
indexer,
&self.indexes,
&self.chain,
self.price.as_ref(),
&mut starting_indexes,
exit,
)?;
info!("Computed stateful in {:?}", i.elapsed());
pools.join().unwrap()?;
Ok(())
})?;
info!("Computing cointime...");
let i = Instant::now();

8
crates/brk_computer/src/stateful/address_cohorts.rs
View File

@@ -57,7 +57,7 @@ impl Vecs {
[
self.0
.ge_amount
.iter_mut()
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
@@ -71,7 +71,7 @@ impl Vecs {
.collect::<Vec<_>>(),
self.0
.lt_amount
.iter_mut()
.par_iter_mut()
.map(|vecs| {
let filter = vecs.filter().clone();
(
@@ -98,7 +98,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.iter_mut()
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
@@ -116,7 +116,7 @@ impl Vecs {
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.0.iter_mut().try_for_each(|v| {
self.0.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,

22
crates/brk_computer/src/stateful/utxo_cohorts.rs
View File

@@ -511,8 +511,8 @@ impl Vecs {
let by_size_range = &self.0.amount_range;
[(&mut self.0.all, by_date_range.iter().collect::<Vec<_>>())]
.into_iter()
.chain(self.0.min_age.iter_mut().map(|vecs| {
.into_par_iter()
.chain(self.0.min_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
@@ -522,7 +522,7 @@ impl Vecs {
.collect::<Vec<_>>(),
)
}))
.chain(self.0.max_age.iter_mut().map(|vecs| {
.chain(self.0.max_age.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
@@ -532,7 +532,7 @@ impl Vecs {
.collect::<Vec<_>>(),
)
}))
.chain(self.0.term.iter_mut().map(|vecs| {
.chain(self.0.term.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
@@ -542,7 +542,7 @@ impl Vecs {
.collect::<Vec<_>>(),
)
}))
.chain(self.0.ge_amount.iter_mut().map(|vecs| {
.chain(self.0.ge_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
@@ -552,7 +552,7 @@ impl Vecs {
.collect::<Vec<_>>(),
)
}))
.chain(self.0.lt_amount.iter_mut().map(|vecs| {
.chain(self.0.lt_amount.par_iter_mut().map(|vecs| {
let filter = vecs.filter().clone();
(
vecs,
@@ -574,7 +574,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.iter_mut()
self.par_iter_mut()
.try_for_each(|v| v.compute_rest_part1(indexes, price, starting_indexes, exit))
}
@@ -592,7 +592,7 @@ impl Vecs {
dateindex_to_realized_cap: Option<&impl IterableVec<DateIndex, Dollars>>,
exit: &Exit,
) -> Result<()> {
self.iter_mut().try_for_each(|v| {
self.par_iter_mut().try_for_each(|v| {
v.compute_rest_part2(
indexes,
price,
@@ -624,9 +624,9 @@ impl Vecs {
/// Reset aggregate cohorts' price_to_amount when starting from scratch
pub fn reset_aggregate_price_to_amount(&mut self) -> Result<()> {
self.0.iter_aggregate_mut().try_for_each(|v| {
v.price_to_amount.reset()
})
self.0
.iter_aggregate_mut()
.try_for_each(|v| v.price_to_amount.reset())
}
/// Import aggregate cohorts' price_to_amount from disk when resuming from a checkpoint.

10
crates/brk_grouper/src/address.rs
View File

@@ -39,6 +39,16 @@ impl<T> AddressGroups<T> {
.chain(self.lt_amount.iter_mut())
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
self.ge_amount
.par_iter_mut()
.chain(self.amount_range.par_iter_mut())
.chain(self.lt_amount.par_iter_mut())
}
pub fn iter_separate_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.amount_range.iter_mut()
}

23
crates/brk_grouper/src/by_ge_amount.rs
View File

@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use brk_types::Sats;
use rayon::prelude::*;
use super::{AmountFilter, Filter};
@@ -79,4 +80,26 @@ impl<T> ByGreatEqualAmount<T> {
]
.into_iter()
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
[
&mut self._1sat,
&mut self._10sats,
&mut self._100sats,
&mut self._1k_sats,
&mut self._10k_sats,
&mut self._100k_sats,
&mut self._1m_sats,
&mut self._10m_sats,
&mut self._1btc,
&mut self._10btc,
&mut self._100btc,
&mut self._1k_btc,
&mut self._10k_btc,
]
.into_par_iter()
}
}

23
crates/brk_grouper/src/by_lt_amount.rs
View File

@@ -1,5 +1,6 @@
use brk_traversable::Traversable;
use brk_types::Sats;
use rayon::prelude::*;
use super::{AmountFilter, Filter};
@@ -79,4 +80,26 @@ impl<T> ByLowerThanAmount<T> {
]
.into_iter()
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
[
&mut self._10sats,
&mut self._100sats,
&mut self._1k_sats,
&mut self._10k_sats,
&mut self._100k_sats,
&mut self._1m_sats,
&mut self._10m_sats,
&mut self._1btc,
&mut self._10btc,
&mut self._100btc,
&mut self._1k_btc,
&mut self._10k_btc,
&mut self._100k_btc,
]
.into_par_iter()
}
}

49
crates/brk_grouper/src/by_max_age.rs
View File

@@ -1,5 +1,6 @@
use super::{Filter, TimeFilter};
use brk_traversable::Traversable;
use rayon::prelude::*;
#[derive(Default, Clone, Traversable)]
pub struct ByMaxAge<T> {
@@ -52,24 +53,9 @@ impl<T> ByMaxAge<T> {
pub fn iter(&self) -> impl Iterator<Item = &T> {
[
&self._1w,
&self._1m,
&self._2m,
&self._3m,
&self._4m,
&self._5m,
&self._6m,
&self._1y,
&self._2y,
&self._3y,
&self._4y,
&self._5y,
&self._6y,
&self._7y,
&self._8y,
&self._10y,
&self._12y,
&self._15y,
&self._1w, &self._1m, &self._2m, &self._3m, &self._4m, &self._5m, &self._6m, &self._1y,
&self._2y, &self._3y, &self._4y, &self._5y, &self._6y, &self._7y, &self._8y,
&self._10y, &self._12y, &self._15y,
]
.into_iter()
}
@@ -97,4 +83,31 @@ impl<T> ByMaxAge<T> {
]
.into_iter()
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
[
&mut self._1w,
&mut self._1m,
&mut self._2m,
&mut self._3m,
&mut self._4m,
&mut self._5m,
&mut self._6m,
&mut self._1y,
&mut self._2y,
&mut self._3y,
&mut self._4y,
&mut self._5y,
&mut self._6y,
&mut self._7y,
&mut self._8y,
&mut self._10y,
&mut self._12y,
&mut self._15y,
]
.into_par_iter()
}
}

49
crates/brk_grouper/src/by_min_age.rs
View File

@@ -1,4 +1,5 @@
use brk_traversable::Traversable;
use rayon::prelude::*;
use super::{Filter, TimeFilter};
@@ -53,24 +54,9 @@ impl<T> ByMinAge<T> {
pub fn iter(&self) -> impl Iterator<Item = &T> {
[
&self._1d,
&self._1w,
&self._1m,
&self._2m,
&self._3m,
&self._4m,
&self._5m,
&self._6m,
&self._1y,
&self._2y,
&self._3y,
&self._4y,
&self._5y,
&self._6y,
&self._7y,
&self._8y,
&self._10y,
&self._12y,
&self._1d, &self._1w, &self._1m, &self._2m, &self._3m, &self._4m, &self._5m, &self._6m,
&self._1y, &self._2y, &self._3y, &self._4y, &self._5y, &self._6y, &self._7y, &self._8y,
&self._10y, &self._12y,
]
.into_iter()
}
@@ -98,4 +84,31 @@ impl<T> ByMinAge<T> {
]
.into_iter()
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
[
&mut self._1d,
&mut self._1w,
&mut self._1m,
&mut self._2m,
&mut self._3m,
&mut self._4m,
&mut self._5m,
&mut self._6m,
&mut self._1y,
&mut self._2y,
&mut self._3y,
&mut self._4y,
&mut self._5y,
&mut self._6y,
&mut self._7y,
&mut self._8y,
&mut self._10y,
&mut self._12y,
]
.into_par_iter()
}
}

17
crates/brk_grouper/src/utxo.rs
View File

@@ -67,6 +67,23 @@ impl<T> UTXOGroups<T> {
.chain(self.type_.iter_mut())
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
[&mut self.all]
.into_par_iter()
.chain(self.term.par_iter_mut())
.chain(self.max_age.par_iter_mut())
.chain(self.min_age.par_iter_mut())
.chain(self.ge_amount.par_iter_mut())
.chain(self.age_range.par_iter_mut())
.chain(self.epoch.par_iter_mut())
.chain(self.amount_range.par_iter_mut())
.chain(self.lt_amount.par_iter_mut())
.chain(self.type_.par_iter_mut())
}
pub fn iter_separate_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.age_range
.iter_mut()

8
crates/brk_indexer/src/lib.rs
View File

@@ -12,16 +12,16 @@ mod constants;
mod indexes;
mod processor;
mod readers;
// mod stores_v2;
mod stores_v3;
mod stores_v2;
// mod stores_v3;
mod vecs;
use constants::*;
pub use indexes::*;
pub use processor::*;
pub use readers::*;
// pub use stores_v2::*;
pub use stores_v3::*;
pub use stores_v2::*;
// pub use stores_v3::*;
pub use vecs::*;
#[derive(Clone)]

4
crates/brk_indexer/src/stores_v3.rs
View File

@@ -60,7 +60,7 @@ impl Stores {
version,
Mode3::PushOnly,
Kind3::Random,
10,
21,
)
};
@@ -125,7 +125,7 @@ impl Stores {
version,
Mode3::PushOnly,
Kind3::Random,
10,
21,
)?,
})
}

2
crates/brk_server/Cargo.toml
View File

@@ -31,5 +31,5 @@ schemars = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
tower-http = { version = "0.6.7", features = ["compression-full", "trace"] }
tower-http = { version = "0.6.8", features = ["compression-full", "trace"] }
tracing = "0.1.43"

3
crates/brk_store/src/fjall_v3/mod.rs
View File

@@ -29,8 +29,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3);
pub fn open_fjall3_database(path: &Path) -> fjall3::Result<Database> {
Database::builder(path.join("fjall"))
.max_write_buffer_size(256 * 1_024 * 1_024)
.cache_size(4 * 1024 * 1024 * 1024)
.cache_size(1024 * 1024 * 1024)
.open()
}