Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 06:01:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: only raw format + global: fixes

Browse Source
This commit is contained in:
nym21
2025-06-12 12:33:43 +02:00
parent 339c00d815
commit 54ea6cc53b
17 changed files with 109 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4 -4
View File
@@ -1698,9 +1698,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "memchr"
version = "2.7.4"
version = "2.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
[[package]]
name = "memmap2"
@@ -3352,9 +3352,9 @@ dependencies = [
[[package]]
name = "windows-link"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
checksum = "d3bfe459f85da17560875b8bf1423d6f113b7a87a5d942e7da0ac71be7c61f8b"
[[package]]
name = "windows-result"
crates/brk_cli/src/query.rs
+1 -1
View File
@@ -10,7 +10,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let format = config.format();
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
indexer.import_vecs()?;
let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
crates/brk_cli/src/run.rs
+30 -15
View File
@@ -7,7 +7,7 @@ use std::{
use bitcoincore_rpc::{self, Auth, Client, RpcApi};
use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
@@ -29,7 +29,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let format = config.format();
let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
indexer.import_stores()?;
indexer.import_vecs()?;
@@ -109,62 +109,77 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
pub struct RunConfig {
/// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
bitcoindir: Option<String>,
/// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
blocksdir: Option<String>,
/// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
/// Executed by the runner, default: all, saved
/// Activated services, default: all, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
mode: Option<Mode>,
services: Option<Services>,
/// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
computation: Option<Computation>,
/// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: compressed, saved
#[arg(short, long, value_name = "FORMAT")]
/// Format of computed datasets, `compressed` to save disk space, `raw` to prioritize speed, default: `compressed`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
format: Option<Format>,
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short = 'F', long, value_name = "BOOL")]
fetch: Option<bool>,
/// Website served by the server (if active), default: default, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
website: Option<Website>,
/// Bitcoin RPC ip, default: localhost, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "IP")]
rpcconnect: Option<String>,
/// Bitcoin RPC port, default: 8332, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PORT")]
rpcport: Option<u16>,
/// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PATH")]
rpccookiefile: Option<String>,
/// Bitcoin RPC username, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "USERNAME")]
rpcuser: Option<String>,
/// Bitcoin RPC password, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "PASSWORD")]
rpcpassword: Option<String>,
/// Delay between runs, default: 0, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "SECONDS")]
delay: Option<u64>,
/// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "BOOL")]
check_collisions: Option<bool>,
}
@@ -192,8 +207,8 @@ impl RunConfig {
config_saved.brkdir = Some(brkdir);
}
if let Some(mode) = config_args.mode.take() {
config_saved.mode = Some(mode);
if let Some(services) = config_args.services.take() {
config_saved.services = Some(services);
}
if let Some(computation) = config_args.computation.take() {
@@ -255,7 +270,7 @@ impl RunConfig {
// info!("Configuration {{");
// info!(" bitcoindir: {:?}", config.bitcoindir);
// info!(" brkdir: {:?}", config.brkdir);
// info!(" mode: {:?}", config.mode);
// info!(" services: {:?}", config.services);
// info!(" website: {:?}", config.website);
// info!(" rpcconnect: {:?}", config.rpcconnect);
// info!(" rpcport: {:?}", config.rpcport);
@@ -375,13 +390,13 @@ impl RunConfig {
}
pub fn process(&self) -> bool {
self.mode
.is_none_or(|m| m == Mode::All || m == Mode::Processor)
self.services
.is_none_or(|m| m == Services::All || m == Services::Processor)
}
pub fn serve(&self) -> bool {
self.mode
.is_none_or(|m| m == Mode::All || m == Mode::Server)
self.services
.is_none_or(|m| m == Services::All || m == Services::Server)
}
fn path_cookiefile(&self) -> PathBuf {
@@ -449,7 +464,7 @@ impl RunConfig {
PartialOrd,
Ord,
)]
pub enum Mode {
pub enum Services {
#[default]
All,
Processor,
crates/brk_computer/examples/main.rs
+1 -1
View File
@@ -33,7 +33,7 @@ pub fn main() -> color_eyre::Result<()> {
let format = Format::Raw;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
let mut indexer = Indexer::new(outputs_dir, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
crates/brk_computer/src/vecs/grouped/value_from_txindex.rs
+4
View File
@@ -226,7 +226,11 @@ impl ComputedValueVecsFromTxindex {
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
self.sats.vecs(),
vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
self.bitcoin.vecs(),
self.dollars_txindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
]
.into_iter()
crates/brk_computer/src/vecs/mod.rs
+1
View File
@@ -124,6 +124,7 @@ impl Vecs {
fetcher: Option<&mut Fetcher>,
exit: &Exit,
) -> color_eyre::Result<()> {
info!("Computing indexes...");
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
info!("Computing constants...");
crates/brk_core/src/utils/mod.rs
+2
View File
@@ -3,9 +3,11 @@ mod checked_sub;
mod paths;
mod pause;
mod rlimit;
mod serde;
pub use bytes::*;
pub use checked_sub::*;
pub use paths::*;
pub use pause::*;
pub use rlimit::*;
pub use serde::*;
crates/brk_core/src/utils/serde.rs
+12
View File
@@ -0,0 +1,12 @@
use serde::{Deserialize, Deserializer};
pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de> + Default,
{
match T::deserialize(deserializer) {
Ok(v) => Ok(v),
Err(_) => Ok(T::default()),
}
}
crates/brk_indexer/examples/main.rs
+1 -2
View File
@@ -4,7 +4,6 @@ use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vec::Format;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -25,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
let mut indexer = Indexer::new(outputs, false)?;
indexer.import_stores()?;
indexer.import_vecs()?;
crates/brk_indexer/src/lib.rs
+2 -9
View File
@@ -19,7 +19,7 @@ use brk_core::{
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_parser::Parser;
use brk_vec::{AnyVec, Format, VecIterator};
use brk_vec::{AnyVec, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -42,21 +42,15 @@ pub struct Indexer {
vecs: Option<Vecs>,
stores: Option<Stores>,
check_collisions: bool,
format: Format,
}
impl Indexer {
pub fn new(
outputs_dir: &Path,
format: Format,
check_collisions: bool,
) -> color_eyre::Result<Self> {
pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result<Self> {
setrlimit()?;
Ok(Self {
path: outputs_dir.to_owned(),
vecs: None,
stores: None,
format,
check_collisions,
})
}
@@ -65,7 +59,6 @@ impl Indexer {
self.vecs = Some(Vecs::forced_import(
&self.path.join("vecs/indexed"),
VERSION + Version::ZERO,
self.format,
)?);
Ok(())
}
crates/brk_indexer/src/vecs.rs
+34 -38
View File
@@ -66,11 +66,7 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(
path: &Path,
version: Version,
format: Format,
) -> color_eyre::Result<Self> {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
@@ -78,7 +74,7 @@ impl Vecs {
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_blockhash: IndexedVec::forced_import(
path,
@@ -90,145 +86,145 @@ impl Vecs {
path,
"difficulty",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_emptyoutputindex: IndexedVec::forced_import(
path,
"first_emptyoutputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_inputindex: IndexedVec::forced_import(
path,
"first_inputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_opreturnindex: IndexedVec::forced_import(
path,
"first_opreturnindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_outputindex: IndexedVec::forced_import(
path,
"first_outputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2aindex: IndexedVec::forced_import(
path,
"first_p2aindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2msindex: IndexedVec::forced_import(
path,
"first_p2msindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2pk33index: IndexedVec::forced_import(
path,
"first_p2pk33index",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2pk65index: IndexedVec::forced_import(
path,
"first_p2pk65index",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2pkhindex: IndexedVec::forced_import(
path,
"first_p2pkhindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2shindex: IndexedVec::forced_import(
path,
"first_p2shindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2trindex: IndexedVec::forced_import(
path,
"first_p2trindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2wpkhindex: IndexedVec::forced_import(
path,
"first_p2wpkhindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_p2wshindex: IndexedVec::forced_import(
path,
"first_p2wshindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_txindex: IndexedVec::forced_import(
path,
"first_txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_first_unknownoutputindex: IndexedVec::forced_import(
path,
"first_unknownoutputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_timestamp: IndexedVec::forced_import(
path,
"timestamp",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_total_size: IndexedVec::forced_import(
path,
"total_size",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
height_to_weight: IndexedVec::forced_import(
path,
"weight",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
inputindex_to_outputindex: IndexedVec::forced_import(
path,
"outputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
opreturnindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
outputindex_to_outputtype: IndexedVec::forced_import(
path,
"outputtype",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
outputindex_to_outputtypeindex: IndexedVec::forced_import(
path,
"outputtypeindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
outputindex_to_value: IndexedVec::forced_import(
path,
"value",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
p2aindex_to_p2abytes: IndexedVec::forced_import(
path,
@@ -240,7 +236,7 @@ impl Vecs {
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
path,
@@ -288,13 +284,13 @@ impl Vecs {
path,
"base_size",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_first_inputindex: IndexedVec::forced_import(
path,
"first_inputindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_first_outputindex: IndexedVec::forced_import(
path,
@@ -306,19 +302,19 @@ impl Vecs {
path,
"is_explicitly_rbf",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_rawlocktime: IndexedVec::forced_import(
path,
"rawlocktime",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_total_size: IndexedVec::forced_import(
path,
"total_size",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
txindex_to_txid: IndexedVec::forced_import(
path,
@@ -330,13 +326,13 @@ impl Vecs {
path,
"txversion",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
unknownoutputindex_to_txindex: IndexedVec::forced_import(
path,
"txindex",
version + VERSION + Version::ZERO,
format,
Format::Raw,
)?,
})
}
crates/brk_query/examples/main.rs
+1 -1
View File
@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
let format = Format::Compressed;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
let mut indexer = Indexer::new(outputs_dir, true)?;
indexer.import_vecs()?;
let mut computer = Computer::new(outputs_dir, None, format);
crates/brk_server/examples/main.rs
+1 -1
View File
@@ -31,7 +31,7 @@ pub fn main() -> color_eyre::Result<()> {
let format = Format::Compressed;
let mut indexer = Indexer::new(outputs_dir, format, true)?;
let mut indexer = Indexer::new(outputs_dir, true)?;
indexer.import_stores()?;
indexer.import_vecs()?;
crates/brk_vec/src/variants/compressed.rs
+6 -2
View File
@@ -21,7 +21,9 @@ use crate::{
const ONE_KIB: usize = 1024;
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB;
const VERSION: Version = Version::ONE;
#[derive(Debug)]
pub struct CompressedVec<I, T> {
@@ -39,7 +41,9 @@ where
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
/// Same as import but will reset the folder under certain errors, so be careful !
pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
pub fn forced_import(path: &Path, mut version: Version) -> Result<Self> {
version = version + VERSION;
let res = Self::import(path, version);
match res {
Err(Error::WrongEndian)
websites/default/index.html
+5 -16
View File
@@ -1140,22 +1140,11 @@
window.document.documentElement.dataset.display = "standalone";
}
window.addEventListener("load", () => {
navigator.serviceWorker
.register("/service-worker.js")
.then((registration) => {
console.log(
"Service Worker registered with scope:",
registration.scope,
);
})
.catch((error) => {
console.error("Service Worker registration failed:", error);
});
});
// if ("serviceWorker" in navigator) {
// navigator.serviceWorker.register("/service-worker.js");
// }
if ("serviceWorker" in navigator) {
window.addEventListener("load", () => {
navigator.serviceWorker.register("/service-worker.js");
});
}
</script>
<!-- --- -->
websites/default/scripts/vecid-to-indexes.js
+3 -1
View File
@@ -2,7 +2,7 @@
// File auto-generated, any modifications will be overwritten
//
export const VERSION = "v0.0.48";
export const VERSION = "v0.0.52";
/** @typedef {0} DateIndex */
/** @typedef {1} DecadeIndex */
@@ -1251,6 +1251,7 @@ export function createVecIdToIndexes() {
"fee-75p": [5],
"fee-90p": [5],
"fee-average": [0, 1, 2, 5, 7, 19, 22, 23],
"fee-in-btc": [20],
"fee-in-btc-10p": [5],
"fee-in-btc-25p": [5],
"fee-in-btc-75p": [5],
@@ -1260,6 +1261,7 @@ export function createVecIdToIndexes() {
"fee-in-btc-median": [5],
"fee-in-btc-min": [0, 1, 2, 5, 7, 19, 22, 23],
"fee-in-btc-sum": [0, 1, 2, 5, 7, 19, 22, 23],
"fee-in-usd": [20],
"fee-in-usd-10p": [5],
"fee-in-usd-25p": [5],
"fee-in-usd-75p": [5],
websites/default/tsconfig.json
+1
View File
@@ -1,5 +1,6 @@
{
"compilerOptions": {
"allowJs": true,
"checkJs": true,
"strict": true,
"target": "ESNext",