Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-08 05:09:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-03-10 11:42:15 +01:00
parent f9f7172702
commit 9428beeae5
44 changed files with 1348 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/brk_computer/Cargo.toml
View File

@@ -9,6 +9,7 @@ repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }

4
crates/brk_computer/examples/main.rs
View File

@@ -1,6 +1,7 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::{
@@ -14,7 +15,8 @@ pub fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let bitcoin_dir = Path::new("../../../bitcoin");
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),

97
crates/brk_computer/src/lib.rs
View File

@@ -11,7 +11,6 @@ pub use brk_parser::rpc;
mod storage;
use brk_core::Date;
use log::info;
use storage::{Stores, Vecs};
@@ -53,101 +52,7 @@ impl Computer {
) -> color_eyre::Result<()> {
info!("Computing...");
let height_count = indexer.vecs().height_to_size.len();
let txindexes_count = indexer.vecs().txindex_to_txid.len();
let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
// self.vecs.txindex_to_last_txinindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// txinindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// &mut self.vecs.txindex_to_last_txinindex,
// exit,
// )?;
// self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// txoutindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// &mut self.vecs.txindex_to_last_txoutindex,
// exit,
// )?;
self.mut_vecs().height_to_height.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
|_, height| height,
exit,
)?;
self.mut_vecs().height_to_date.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
|timestamp, _| Date::from(*timestamp),
exit,
)?;
// self.vecs.height_to_last_txindex.compute_last_index_from_first(
// starting_indexes.height,
// &mut indexer.vecs().height_to_first_txindex,
// height_count,
// exit,
// )?;
// self.vecs.txindex_to_height.compute_inverse_less_to_more(
// starting_indexes.height,
// &mut indexer.vecs().height_to_first_txindex,
// &mut self.vecs.height_to_last_txindex,
// exit,
// )?;
// self.vecs.txindex_to_is_coinbase.compute_is_first_ordered(
// starting_indexes.txindex,
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
// exit,
// )?;
// self.vecs.txindex_to_fee.compute_transform(
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
// )?;
let date_count = self.vecs().height_to_date.len();
// self.vecs.height_to_dateindex.compute(...)
// self.vecs
// .dateindex_to_first_height
// .compute_inverse_more_to_less(&mut self.vecs.height_to_dateindex, exit)?;
// ---
// Date to X
// ---
// ...
// ---
// Month to X
// ---
// ...
// ---
// Year to X
// ---
// ...
self.mut_vecs().compute(indexer, starting_indexes, exit)?;
Ok(())
}

175
crates/brk_computer/src/storage/vecs.rs
View File

@@ -1,175 +0,0 @@
use std::{fs, path::Path};
use brk_core::{
Addressindex, Cents, Close, Date, Dateindex, Dollars, Feerate, Height, High, Low, Open, Sats, Timestamp, Txindex,
Txinindex, Txoutindex,
};
use brk_vec::{AnyStorableVec, StorableVec, Version};
// mod base;
// use base::*;
#[derive(Clone)]
pub struct Vecs {
pub dateindex_to_first_height: StorableVec<Dateindex, Height>,
// pub dateindex_to_last_height: StorableVec<Dateindex, Height>,
// pub height_to_block_interval: StorableVec<Height, Timestamp>,
pub dateindex_to_close_in_cents: StorableVec<Dateindex, Close<Cents>>,
pub dateindex_to_close_in_dollars: StorableVec<Dateindex, Close<Dollars>>,
pub dateindex_to_high_in_cents: StorableVec<Dateindex, High<Cents>>,
pub dateindex_to_high_in_dollars: StorableVec<Dateindex, High<Dollars>>,
pub dateindex_to_low_in_cents: StorableVec<Dateindex, Low<Cents>>,
pub dateindex_to_low_in_dollars: StorableVec<Dateindex, Low<Dollars>>,
pub dateindex_to_open_in_cents: StorableVec<Dateindex, Open<Cents>>,
pub dateindex_to_open_in_dollars: StorableVec<Dateindex, Open<Dollars>>,
pub height_to_close_in_cents: StorableVec<Height, Close<Cents>>,
pub height_to_close_in_dollars: StorableVec<Height, Close<Dollars>>,
pub height_to_height: StorableVec<Height, Height>,
pub height_to_high_in_cents: StorableVec<Height, High<Cents>>,
pub height_to_high_in_dollars: StorableVec<Height, High<Dollars>>,
pub height_to_low_in_cents: StorableVec<Height, Low<Cents>>,
pub height_to_low_in_dollars: StorableVec<Height, Low<Dollars>>,
pub height_to_open_in_cents: StorableVec<Height, Open<Cents>>,
pub height_to_open_in_dollars: StorableVec<Height, Open<Dollars>>,
pub height_to_date: StorableVec<Height, Date>,
pub height_to_dateindex: StorableVec<Height, Dateindex>,
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Height, u32>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
// pub height_to_maxfeerate: StorableVec<Height, Feerate>,
// pub height_to_medianfeerate: StorableVec<Height, Feerate>,
// pub height_to_minfeerate: StorableVec<Height, Feerate>,
// pub height_to_outputcount: StorableVec<Height, u32>,
// pub height_to_subsidy: StorableVec<Height, u32>,
// pub height_to_totalfees: StorableVec<Height, Amount>,
// pub height_to_txcount: StorableVec<Height, u32>,
pub txindex_to_fee: StorableVec<Txindex, Sats>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
pub txindex_to_inputs_count: StorableVec<Txindex, u32>,
pub txindex_to_inputs_sum: StorableVec<Txindex, Sats>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
pub txindex_to_outputs_count: StorableVec<Txindex, u32>,
pub txindex_to_outputs_sum: StorableVec<Txindex, Sats>,
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_first_height: StorableVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::from(1),
)?,
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
dateindex_to_close_in_cents: StorableVec::import(
&path.join("dateindex_to_close_in_cents"),
Version::from(1),
)?,
dateindex_to_close_in_dollars: StorableVec::import(
&path.join("dateindex_to_close_in_dollars"),
Version::from(1),
)?,
dateindex_to_high_in_cents: StorableVec::import(
&path.join("dateindex_to_high_in_cents"),
Version::from(1),
)?,
dateindex_to_high_in_dollars: StorableVec::import(
&path.join("dateindex_to_high_in_dollars"),
Version::from(1),
)?,
dateindex_to_low_in_cents: StorableVec::import(&path.join("dateindex_to_low_in_cents"), Version::from(1))?,
dateindex_to_low_in_dollars: StorableVec::import(
&path.join("dateindex_to_low_in_dollars"),
Version::from(1),
)?,
dateindex_to_open_in_cents: StorableVec::import(
&path.join("dateindex_to_open_in_cents"),
Version::from(1),
)?,
dateindex_to_open_in_dollars: StorableVec::import(
&path.join("dateindex_to_open_in_dollars"),
Version::from(1),
)?,
height_to_close_in_cents: StorableVec::import(&path.join("height_to_close_in_cents"), Version::from(1))?,
height_to_close_in_dollars: StorableVec::import(
&path.join("height_to_close_in_dollars"),
Version::from(1),
)?,
height_to_height: StorableVec::import(&path.join("height_to_height"), Version::from(1))?,
height_to_high_in_cents: StorableVec::import(&path.join("height_to_high_in_cents"), Version::from(1))?,
height_to_high_in_dollars: StorableVec::import(&path.join("height_to_high_in_dollars"), Version::from(1))?,
height_to_low_in_cents: StorableVec::import(&path.join("height_to_low_in_cents"), Version::from(1))?,
height_to_low_in_dollars: StorableVec::import(&path.join("height_to_low_in_dollars"), Version::from(1))?,
height_to_open_in_cents: StorableVec::import(&path.join("height_to_open_in_cents"), Version::from(1))?,
height_to_open_in_dollars: StorableVec::import(&path.join("height_to_open_in_dollars"), Version::from(1))?,
height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?,
height_to_dateindex: StorableVec::forced_import(&path.join("height_to_dateindex"), Version::from(1))?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::forced_import(
// &path.join("height_to_last_addressindex"),
// Version::from(1),
// )?,
height_to_last_txindex: StorableVec::forced_import(&path.join("height_to_last_txindex"), Version::from(1))?,
// height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?,
// height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?,
txindex_to_fee: StorableVec::forced_import(&path.join("txindex_to_fee"), Version::from(1))?,
txindex_to_height: StorableVec::forced_import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_is_coinbase: StorableVec::forced_import(&path.join("txindex_to_is_coinbase"), Version::from(1))?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
txindex_to_inputs_count: StorableVec::forced_import(
&path.join("txindex_to_inputs_count"),
Version::from(1),
)?,
txindex_to_inputs_sum: StorableVec::forced_import(&path.join("txindex_to_inputs_sum"), Version::from(1))?,
txindex_to_last_txinindex: StorableVec::forced_import(
&path.join("txindex_to_last_txinindex"),
Version::from(1),
)?,
txindex_to_last_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
)?,
txindex_to_outputs_count: StorableVec::forced_import(
&path.join("txindex_to_outputs_count"),
Version::from(1),
)?,
txindex_to_outputs_sum: StorableVec::forced_import(&path.join("txindex_to_outputs_sum"), Version::from(1))?,
})
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![
&self.height_to_date as &dyn AnyStorableVec,
&self.height_to_height,
// &self.dateindex_to_close_in_dollars,
// &self.dateindex_to_high_in_cents,
// &self.dateindex_to_high_in_dollars,
// &self.dateindex_to_low_in_cents,
// &self.dateindex_to_low_in_dollars,
// &self.dateindex_to_open_in_cents,
// &self.dateindex_to_open_in_dollars,
// &self.height_to_close_in_cents,
// &self.height_to_close_in_dollars,
// &self.height_to_high_in_cents,
// &self.height_to_high_in_dollars,
// &self.height_to_low_in_cents,
// &self.height_to_low_in_dollars,
// &self.height_to_open_in_cents,
// &self.height_to_open_in_dollars,
]
}
}

246
crates/brk_computer/src/storage/vecs/indexes.rs Normal file
View File

@@ -0,0 +1,246 @@
use std::{fs, ops::Deref, path::Path};
use brk_core::{Date, Dateindex, Height};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, StorableVec, Value, Version};
#[derive(Clone)]
pub struct Vecs {
pub dateindex_to_date: StorableVec<Dateindex, Date>,
pub dateindex_to_dateindex: StorableVec<Dateindex, Dateindex>,
pub dateindex_to_first_height: StorableVec<Dateindex, Height>,
pub dateindex_to_last_height: StorableVec<Dateindex, Height>,
pub height_to_real_date: StorableVec<Height, Date>,
pub height_to_fixed_date: StorableVec<Height, Date>,
pub height_to_height: StorableVec<Height, Height>,
pub height_to_dateindex: StorableVec<Height, Dateindex>,
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_date: StorableVec::forced_import(
&path.join("dateindex_to_date"),
Version::from(1),
)?,
dateindex_to_dateindex: StorableVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::from(1),
)?,
dateindex_to_first_height: StorableVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::from(1),
)?,
dateindex_to_last_height: StorableVec::forced_import(
&path.join("dateindex_to_last_height"),
Version::from(1),
)?,
height_to_real_date: StorableVec::forced_import(
&path.join("height_to_real_date"),
Version::from(1),
)?,
height_to_fixed_date: StorableVec::forced_import(
&path.join("height_to_fixed_date"),
Version::from(1),
)?,
height_to_dateindex: StorableVec::forced_import(
&path.join("height_to_dateindex"),
Version::from(1),
)?,
height_to_height: StorableVec::forced_import(
&path.join("height_to_height"),
Version::from(1),
)?,
})
}
pub fn compute(
&mut self,
indexer: &mut Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> color_eyre::Result<Indexes> {
self.height_to_height.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
|(h, ..)| (h, h),
exit,
)?;
self.height_to_real_date.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
|(h, t, ..)| (h, Date::from(*t)),
exit,
)?;
self.height_to_fixed_date.compute_transform(
starting_indexes.height,
&mut self.height_to_real_date,
|(h, d, s, ..)| {
let d = h
.decremented()
.and_then(|h| s.read(h).ok())
.flatten()
.map_or(*d, |prev_d| if prev_d > d { *prev_d } else { *d });
(h, d)
},
exit,
)?;
self.height_to_dateindex.compute_transform(
starting_indexes.height,
&mut self.height_to_fixed_date,
|(h, d, ..)| (h, Dateindex::try_from(*d).unwrap()),
exit,
)?;
let starting_dateindex = self
.height_to_dateindex
.get(starting_indexes.height.decremented().unwrap_or_default())?
.map(Value::into_inner)
.unwrap_or_default();
self.dateindex_to_first_height
.compute_inverse_more_to_less(
starting_indexes.height,
&mut self.height_to_dateindex,
exit,
)?;
let date_len = self.dateindex_to_first_height.len();
self.dateindex_to_last_height
.compute_last_index_from_first(
starting_dateindex,
&mut self.dateindex_to_first_height,
date_len,
exit,
)?;
self.dateindex_to_dateindex.compute_transform(
starting_dateindex,
&mut self.dateindex_to_first_height,
|(di, ..)| (di, di),
exit,
)?;
self.dateindex_to_date.compute_transform(
starting_dateindex,
&mut self.dateindex_to_dateindex,
|(di, ..)| (di, Date::from(di)),
exit,
)?;
// let height_count = indexer.vecs().height_to_size.len();
// let txindexes_count = indexer.vecs().txindex_to_txid.len();
// let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
// let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
// let date_count = self.vecs().height_to_date.len();
// self.vecs.txindex_to_last_txinindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// txinindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// &mut self.vecs.txindex_to_last_txinindex,
// exit,
// )?;
// self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// txoutindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// &mut self.vecs.txindex_to_last_txoutindex,
// exit,
// )?;
// self.vecs.height_to_last_txindex.compute_last_index_from_first(
// starting_indexes.height,
// &mut indexer.vecs().height_to_first_txindex,
// height_count,
// exit,
// )?;
// self.vecs.txindex_to_height.compute_inverse_less_to_more(
// starting_indexes.height,
// &mut indexer.vecs().height_to_first_txindex,
// &mut self.vecs.height_to_last_txindex,
// exit,
// )?;
// self.vecs.txindex_to_is_coinbase.compute_is_first_ordered(
// starting_indexes.txindex,
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
// exit,
// )?;
// self.vecs.txindex_to_fee.compute_transform(
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
// )?;
// self.vecs.height_to_dateindex.compute(...)
// ---
// Date to X
// ---
// ...
// ---
// Month to X
// ---
// ...
// ---
// Year to X
// ---
// ...
Ok(Indexes::from((starting_indexes, starting_dateindex)))
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![
&self.dateindex_to_date as &dyn AnyStorableVec,
&self.dateindex_to_dateindex,
&self.dateindex_to_first_height,
&self.dateindex_to_last_height,
&self.height_to_real_date,
&self.height_to_fixed_date,
&self.height_to_height,
&self.height_to_dateindex,
]
}
}
pub struct Indexes {
indexes: brk_indexer::Indexes,
pub dateindex: Dateindex,
}
impl Deref for Indexes {
type Target = brk_indexer::Indexes;
fn deref(&self) -> &Self::Target {
&self.indexes
}
}
impl From<(brk_indexer::Indexes, Dateindex)> for Indexes {
fn from((indexes, dateindex): (brk_indexer::Indexes, Dateindex)) -> Self {
Self { indexes, dateindex }
}
}

302
crates/brk_computer/src/storage/vecs/marketprice.rs Normal file
View File

@@ -0,0 +1,302 @@
use std::{fs, path::Path};
use brk_core::{Cents, Close, Dateindex, Dollars, Height, High, Low, OHLCCents, OHLCDollars, Open};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, StorableVec, Value, Version};
use super::indexes::{self, Indexes};
#[derive(Clone)]
pub struct Vecs {
pub dateindex_to_ohlc_in_cents: StorableVec<Dateindex, OHLCCents>,
pub dateindex_to_ohlc: StorableVec<Dateindex, OHLCDollars>,
pub dateindex_to_close_in_cents: StorableVec<Dateindex, Close<Cents>>,
pub dateindex_to_close: StorableVec<Dateindex, Close<Dollars>>,
pub dateindex_to_high_in_cents: StorableVec<Dateindex, High<Cents>>,
pub dateindex_to_high: StorableVec<Dateindex, High<Dollars>>,
pub dateindex_to_low_in_cents: StorableVec<Dateindex, Low<Cents>>,
pub dateindex_to_low: StorableVec<Dateindex, Low<Dollars>>,
pub dateindex_to_open_in_cents: StorableVec<Dateindex, Open<Cents>>,
pub dateindex_to_open: StorableVec<Dateindex, Open<Dollars>>,
pub height_to_ohlc_in_cents: StorableVec<Height, OHLCCents>,
pub height_to_ohlc: StorableVec<Height, OHLCDollars>,
pub height_to_close_in_cents: StorableVec<Height, Close<Cents>>,
pub height_to_close: StorableVec<Height, Close<Dollars>>,
pub height_to_high_in_cents: StorableVec<Height, High<Cents>>,
pub height_to_high: StorableVec<Height, High<Dollars>>,
pub height_to_low_in_cents: StorableVec<Height, Low<Cents>>,
pub height_to_low: StorableVec<Height, Low<Dollars>>,
pub height_to_open_in_cents: StorableVec<Height, Open<Cents>>,
pub height_to_open: StorableVec<Height, Open<Dollars>>,
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
dateindex_to_ohlc_in_cents: StorableVec::import(
&path.join("dateindex_to_ohlc_in_cents"),
Version::from(1),
)?,
dateindex_to_ohlc: StorableVec::import(
&path.join("dateindex_to_ohlc"),
Version::from(1),
)?,
dateindex_to_close_in_cents: StorableVec::import(
&path.join("dateindex_to_close_in_cents"),
Version::from(1),
)?,
dateindex_to_close: StorableVec::import(
&path.join("dateindex_to_close"),
Version::from(1),
)?,
dateindex_to_high_in_cents: StorableVec::import(
&path.join("dateindex_to_high_in_cents"),
Version::from(1),
)?,
dateindex_to_high: StorableVec::import(
&path.join("dateindex_to_high"),
Version::from(1),
)?,
dateindex_to_low_in_cents: StorableVec::import(
&path.join("dateindex_to_low_in_cents"),
Version::from(1),
)?,
dateindex_to_low: StorableVec::import(
&path.join("dateindex_to_low"),
Version::from(1),
)?,
dateindex_to_open_in_cents: StorableVec::import(
&path.join("dateindex_to_open_in_cents"),
Version::from(1),
)?,
dateindex_to_open: StorableVec::import(
&path.join("dateindex_to_open"),
Version::from(1),
)?,
height_to_ohlc_in_cents: StorableVec::import(
&path.join("height_to_ohlc_in_cents"),
Version::from(1),
)?,
height_to_ohlc: StorableVec::import(&path.join("height_to_ohlc"), Version::from(1))?,
height_to_close_in_cents: StorableVec::import(
&path.join("height_to_close_in_cents"),
Version::from(1),
)?,
height_to_close: StorableVec::import(&path.join("height_to_close"), Version::from(1))?,
height_to_high_in_cents: StorableVec::import(
&path.join("height_to_high_in_cents"),
Version::from(1),
)?,
height_to_high: StorableVec::import(&path.join("height_to_high"), Version::from(1))?,
height_to_low_in_cents: StorableVec::import(
&path.join("height_to_low_in_cents"),
Version::from(1),
)?,
height_to_low: StorableVec::import(&path.join("height_to_low"), Version::from(1))?,
height_to_open_in_cents: StorableVec::import(
&path.join("height_to_open_in_cents"),
Version::from(1),
)?,
height_to_open: StorableVec::import(&path.join("height_to_open"), Version::from(1))?,
})
}
pub fn compute(
&mut self,
indexer: &mut Indexer,
indexes: &mut indexes::Vecs,
starting_indexes: Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
let mut fetcher = Fetcher::import(None)?;
self.height_to_ohlc_in_cents.compute_transform(
starting_indexes.height,
&mut indexer.mut_vecs().height_to_timestamp,
|(h, t, _, height_to_timestamp)| {
let ohlc = fetcher
.get_height(
h,
*t,
h.decremented().map(|prev_h| {
height_to_timestamp
.get(prev_h)
.unwrap()
.map(Value::into_inner)
.unwrap()
}),
)
.unwrap();
(h, ohlc)
},
exit,
)?;
self.height_to_open_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.height_to_high_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.height_to_low_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.height_to_close_in_cents.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.height_to_ohlc.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
self.height_to_open.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.height_to_high.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.height_to_low.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.height_to_close.compute_transform(
starting_indexes.height,
&mut self.height_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.dateindex_to_ohlc_in_cents.compute_transform(
starting_indexes.dateindex,
&mut indexes.dateindex_to_date,
|(di, d, ..)| {
let ohlc = fetcher.get_date(*d).unwrap();
(di, ohlc)
},
exit,
)?;
self.dateindex_to_open_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.dateindex_to_high_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.dateindex_to_low_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.dateindex_to_close_in_cents.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
self.dateindex_to_ohlc.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc_in_cents,
|(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)),
exit,
)?;
self.dateindex_to_open.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.open),
exit,
)?;
self.dateindex_to_high.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.high),
exit,
)?;
self.dateindex_to_low.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.low),
exit,
)?;
self.dateindex_to_close.compute_transform(
starting_indexes.dateindex,
&mut self.dateindex_to_ohlc,
|(di, ohlc, ..)| (di, ohlc.close),
exit,
)?;
Ok(())
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
vec![
&self.dateindex_to_close as &dyn AnyStorableVec,
&self.dateindex_to_close_in_cents,
&self.dateindex_to_high,
&self.dateindex_to_high_in_cents,
&self.dateindex_to_low,
&self.dateindex_to_low_in_cents,
&self.dateindex_to_ohlc,
&self.dateindex_to_ohlc_in_cents,
&self.dateindex_to_open,
&self.dateindex_to_open_in_cents,
&self.height_to_close,
&self.height_to_close_in_cents,
&self.height_to_high,
&self.height_to_high_in_cents,
&self.height_to_low,
&self.height_to_low_in_cents,
&self.height_to_ohlc,
&self.height_to_ohlc_in_cents,
&self.height_to_open,
&self.height_to_open_in_cents,
]
}
}

202
crates/brk_computer/src/storage/vecs/mod.rs Normal file
View File

@@ -0,0 +1,202 @@
use std::{fs, path::Path};
use brk_core::{Height, Sats, Txindex, Txinindex, Txoutindex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyStorableVec, StorableVec, Version};
mod indexes;
mod marketprice;
#[derive(Clone)]
pub struct Vecs {
pub indexes: indexes::Vecs,
pub marketprice: marketprice::Vecs,
// pub height_to_block_interval: StorableVec<Height, Timestamp>,
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Height, u32>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
// pub height_to_maxfeerate: StorableVec<Height, Feerate>,
// pub height_to_medianfeerate: StorableVec<Height, Feerate>,
// pub height_to_minfeerate: StorableVec<Height, Feerate>,
// pub height_to_outputcount: StorableVec<Height, u32>,
// pub height_to_subsidy: StorableVec<Height, u32>,
// pub height_to_totalfees: StorableVec<Height, Amount>,
// pub height_to_txcount: StorableVec<Height, u32>,
pub txindex_to_fee: StorableVec<Txindex, Sats>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
pub txindex_to_inputs_count: StorableVec<Txindex, u32>,
pub txindex_to_inputs_sum: StorableVec<Txindex, Sats>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
pub txindex_to_outputs_count: StorableVec<Txindex, u32>,
pub txindex_to_outputs_sum: StorableVec<Txindex, Sats>,
}
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
indexes: indexes::Vecs::import(path)?,
marketprice: marketprice::Vecs::import(path)?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::forced_import(
// &path.join("height_to_last_addressindex"),
// Version::from(1),
// )?,
height_to_last_txindex: StorableVec::forced_import(
&path.join("height_to_last_txindex"),
Version::from(1),
)?,
// height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?,
// height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?,
txindex_to_fee: StorableVec::forced_import(
&path.join("txindex_to_fee"),
Version::from(1),
)?,
txindex_to_height: StorableVec::forced_import(
&path.join("txindex_to_height"),
Version::from(1),
)?,
txindex_to_is_coinbase: StorableVec::forced_import(
&path.join("txindex_to_is_coinbase"),
Version::from(1),
)?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
txindex_to_inputs_count: StorableVec::forced_import(
&path.join("txindex_to_inputs_count"),
Version::from(1),
)?,
txindex_to_inputs_sum: StorableVec::forced_import(
&path.join("txindex_to_inputs_sum"),
Version::from(1),
)?,
txindex_to_last_txinindex: StorableVec::forced_import(
&path.join("txindex_to_last_txinindex"),
Version::from(1),
)?,
txindex_to_last_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
)?,
txindex_to_outputs_count: StorableVec::forced_import(
&path.join("txindex_to_outputs_count"),
Version::from(1),
)?,
txindex_to_outputs_sum: StorableVec::forced_import(
&path.join("txindex_to_outputs_sum"),
Version::from(1),
)?,
})
}
pub fn compute(
&mut self,
indexer: &mut Indexer,
starting_indexes: brk_indexer::Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
self.marketprice
.compute(indexer, &mut self.indexes, starting_indexes, exit)?;
// self.mut_vecs().height_to_ohlc
// let height_count = indexer.vecs().height_to_size.len();
// let txindexes_count = indexer.vecs().txindex_to_txid.len();
// let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
// let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
// let date_count = self.vecs().height_to_date.len();
// self.vecs.txindex_to_last_txinindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// txinindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txinindex,
// &mut self.vecs.txindex_to_last_txinindex,
// exit,
// )?;
// self.vecs.txindex_to_last_txoutindex.compute_last_index_from_first(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// txoutindexes_count,
// exit,
// )?;
// self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
// starting_indexes.txindex,
// &mut indexer.vecs().txindex_to_first_txoutindex,
// &mut self.vecs.txindex_to_last_txoutindex,
// exit,
// )?;
// self.vecs.height_to_last_txindex.compute_last_index_from_first(
// starting_indexes.height,
// &mut indexer.vecs().height_to_first_txindex,
// height_count,
// exit,
// )?;
// self.vecs.txindex_to_height.compute_inverse_less_to_more(
// starting_indexes.height,
// &mut indexer.vecs().height_to_first_txindex,
// &mut self.vecs.height_to_last_txindex,
// exit,
// )?;
// self.vecs.txindex_to_is_coinbase.compute_is_first_ordered(
// starting_indexes.txindex,
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
// exit,
// )?;
// self.vecs.txindex_to_fee.compute_transform(
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs().height_to_first_txindex,
// )?;
// self.vecs.height_to_dateindex.compute(...)
// ---
// Date to X
// ---
// ...
// ---
// Month to X
// ---
// ...
// ---
// Year to X
// ---
// ...
Ok(())
}
pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
[self.indexes.as_any_vecs(), self.marketprice.as_any_vecs()].concat()
}
}

0
crates/brk_computer/src/storage/vecs_/base.rs
View File

0
crates/brk_computer/src/storage/vecs_/mod.rs
View File

9
crates/brk_core/src/structs/addressindex.rs
View File

@@ -1,7 +1,6 @@
use std::ops::Add;
use byteview::ByteView;
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -15,8 +14,6 @@ use crate::Error;
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
@@ -29,16 +26,12 @@ pub struct Addressindex(u32);
impl Addressindex {
pub const BYTES: usize = size_of::<Self>();
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
Self(self.0 + 1)
}
}

8
crates/brk_core/src/structs/addresstypeindex.rs
View File

@@ -12,8 +12,6 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
@@ -24,16 +22,12 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
pub struct Addresstypeindex(u32);
impl Addresstypeindex {
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
Self(self.0 + 1)
}
pub fn copy_then_increment(&mut self) -> Self {

2
crates/brk_core/src/structs/bitcoin.rs
View File

@@ -9,6 +9,6 @@ impl Bitcoin {
impl From<Sats> for Bitcoin {
fn from(value: Sats) -> Self {
Self((*value as f64) / Self::ONE.0)
Self(f64::from(value) / Self::ONE.0)
}
}

8
crates/brk_core/src/structs/cents.rs
View File

@@ -1,4 +1,3 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -13,7 +12,6 @@ use super::Dollars;
Eq,
PartialOrd,
Ord,
Deref,
FromBytes,
Immutable,
IntoBytes,
@@ -27,3 +25,9 @@ impl From<Dollars> for Cents {
Self((*value * 100.0).floor() as u64)
}
}
impl From<Cents> for f64 {
fn from(value: Cents) -> Self {
value.0 as f64
}
}

14
crates/brk_core/src/structs/date.rs
View File

@@ -60,11 +60,15 @@ impl From<Timestamp> for Date {
impl From<Dateindex> for Date {
fn from(value: Dateindex) -> Self {
Self::from(
Self::INDEX_ZERO_
.checked_add(Span::new().days(i64::from(value)))
.unwrap(),
)
if value == Dateindex::default() {
Date::INDEX_ZERO
} else {
Self::from(
Self::INDEX_ONE_
.checked_add(Span::new().days(i64::from(value) - 1))
.unwrap(),
)
}
}
}

18
crates/brk_core/src/structs/dateindex.rs
View File

@@ -9,10 +9,26 @@ use crate::Error;
use super::Date;
#[derive(
Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Dateindex(u16);
impl Dateindex {
pub const BYTES: usize = size_of::<Self>();
}
impl From<Dateindex> for usize {
fn from(value: Dateindex) -> Self {
value.0 as usize

12
crates/brk_core/src/structs/dollars.rs
View File

@@ -4,7 +4,9 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Cents;
#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
#[derive(
Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
)]
pub struct Dollars(f64);
impl From<f64> for Dollars {
@@ -15,6 +17,12 @@ impl From<f64> for Dollars {
impl From<Cents> for Dollars {
fn from(value: Cents) -> Self {
Self((*value as f64) / 100.0)
Self(f64::from(value) / 100.0)
}
}
impl From<Dollars> for f64 {
fn from(value: Dollars) -> Self {
value.0
}
}

4
crates/brk_core/src/structs/feerate.rs
View File

@@ -1,4 +1,2 @@
use derive_deref::Deref;
#[derive(Debug, Deref, Clone, Copy)]
#[derive(Debug, Clone, Copy)]
pub struct Feerate(f32);

54
crates/brk_core/src/structs/height.rs
View File

@@ -1,19 +1,18 @@
use std::{
fmt::{self, Debug},
ops::{Add, AddAssign, Rem, Sub},
ops::{Add, AddAssign, Rem},
};
use bitcoincore_rpc::{Client, RpcApi};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
#[derive(
Debug,
Clone,
Copy,
Deref,
DerefMut,
PartialEq,
Eq,
PartialOrd,
@@ -49,11 +48,11 @@ impl Height {
}
pub fn decrement(&mut self) {
self.0 -= 1;
*self = self.decremented().unwrap();
}
pub fn decremented(self) -> Self {
Self(self.0.checked_sub(1).unwrap_or_default())
pub fn decremented(self) -> Option<Self> {
self.checked_sub(1_u32)
}
pub fn is_zero(self) -> bool {
@@ -63,7 +62,7 @@ impl Height {
impl PartialEq<u64> for Height {
fn eq(&self, other: &u64) -> bool {
**self == *other as u32
self.0 == *other as u32
}
}
@@ -87,35 +86,25 @@ impl Add<usize> for Height {
type Output = Height;
fn add(self, rhs: usize) -> Self::Output {
Self::from(*self + rhs as u32)
Self::from(self.0 + rhs as u32)
}
}
impl Sub<Height> for Height {
type Output = Height;
fn sub(self, rhs: Height) -> Self::Output {
Self::from(*self - *rhs)
impl CheckedSub<Height> for Height {
fn checked_sub(self, rhs: Height) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Height::from)
}
}
impl Sub<i32> for Height {
type Output = Height;
fn sub(self, rhs: i32) -> Self::Output {
Self::from(*self - rhs as u32)
impl CheckedSub<u32> for Height {
fn checked_sub(self, rhs: u32) -> Option<Self> {
self.0.checked_sub(rhs).map(Height::from)
}
}
impl Sub<u32> for Height {
type Output = Height;
fn sub(self, rhs: u32) -> Self::Output {
Self::from(*self - rhs)
}
}
impl Sub<usize> for Height {
type Output = Height;
fn sub(self, rhs: usize) -> Self::Output {
Self::from(*self - rhs as u32)
impl CheckedSub<usize> for Height {
fn checked_sub(self, rhs: usize) -> Option<Self> {
self.0.checked_sub(rhs as u32).map(Height::from)
}
}
@@ -141,7 +130,7 @@ impl Rem<usize> for Height {
impl fmt::Display for Height {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", **self)
write!(f, "{}", self.0)
}
}
@@ -168,6 +157,11 @@ impl From<Height> for usize {
}
}
impl From<Height> for u32 {
fn from(value: Height) -> Self {
value.0
}
}
impl From<Height> for u64 {
fn from(value: Height) -> Self {
value.0 as u64
@@ -189,7 +183,7 @@ impl From<bitcoin::locktime::absolute::Height> for Height {
impl From<Height> for bitcoin::locktime::absolute::Height {
fn from(value: Height) -> Self {
bitcoin::locktime::absolute::Height::from_consensus(*value).unwrap()
bitcoin::locktime::absolute::Height::from_consensus(value.0).unwrap()
}
}

1
crates/brk_core/src/structs/locktime.rs
View File

@@ -5,6 +5,7 @@ use super::{Height, Timestamp};
#[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)]
#[repr(C)]
#[allow(warnings)]
pub enum LockTime {
Height(Height),
Timestamp(Timestamp),

105
crates/brk_core/src/structs/ohlc.rs
View File

@@ -2,9 +2,86 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Cents;
use super::{Cents, Dollars};
pub type OHLCCents = (Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>);
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
#[repr(C)]
pub struct OHLCCents {
pub open: Open<Cents>,
pub high: High<Cents>,
pub low: Low<Cents>,
pub close: Close<Cents>,
}
impl From<(Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>)> for OHLCCents {
fn from(value: (Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>)) -> Self {
Self {
open: value.0,
high: value.1,
low: value.2,
close: value.3,
}
}
}
impl From<Close<Cents>> for OHLCCents {
fn from(value: Close<Cents>) -> Self {
Self {
open: Open::from(value),
high: High::from(value),
low: Low::from(value),
close: value,
}
}
}
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
#[repr(C)]
pub struct OHLCDollars {
pub open: Open<Dollars>,
pub high: High<Dollars>,
pub low: Low<Dollars>,
pub close: Close<Dollars>,
}
impl From<(Open<Dollars>, High<Dollars>, Low<Dollars>, Close<Dollars>)> for OHLCDollars {
fn from(value: (Open<Dollars>, High<Dollars>, Low<Dollars>, Close<Dollars>)) -> Self {
Self {
open: value.0,
high: value.1,
low: value.2,
close: value.3,
}
}
}
impl From<Close<Dollars>> for OHLCDollars {
fn from(value: Close<Dollars>) -> Self {
Self {
open: Open::from(value),
high: High::from(value),
low: Low::from(value),
close: value,
}
}
}
impl From<OHLCCents> for OHLCDollars {
fn from(value: OHLCCents) -> Self {
Self::from(&value)
}
}
impl From<&OHLCCents> for OHLCDollars {
fn from(value: &OHLCCents) -> Self {
Self {
open: value.open.into(),
high: value.high.into(),
low: value.low.into(),
close: value.close.into(),
}
}
}
#[derive(
Debug,
@@ -39,6 +116,12 @@ where
}
}
impl From<Open<Cents>> for Open<Dollars> {
fn from(value: Open<Cents>) -> Self {
Self(Dollars::from(*value))
}
}
#[derive(
Debug,
Default,
@@ -72,6 +155,12 @@ where
}
}
impl From<High<Cents>> for High<Dollars> {
fn from(value: High<Cents>) -> Self {
Self(Dollars::from(*value))
}
}
#[derive(
Debug,
Default,
@@ -105,6 +194,12 @@ where
}
}
impl From<Low<Cents>> for Low<Dollars> {
fn from(value: Low<Cents>) -> Self {
Self(Dollars::from(*value))
}
}
#[derive(
Debug,
Default,
@@ -128,3 +223,9 @@ impl<T> From<T> for Close<T> {
Self(value)
}
}
impl From<Close<Cents>> for Close<Dollars> {
fn from(value: Close<Cents>) -> Self {
Self(Dollars::from(*value))
}
}

21
crates/brk_core/src/structs/sats.rs
View File

@@ -4,7 +4,6 @@ use std::{
};
use bitcoin::Amount;
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -18,8 +17,6 @@ use super::Height;
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
@@ -40,7 +37,7 @@ impl Sats {
impl Add for Sats {
type Output = Sats;
fn add(self, rhs: Sats) -> Self::Output {
Sats::from(*self + *rhs)
Sats::from(self.0 + rhs.0)
}
}
@@ -53,7 +50,7 @@ impl AddAssign for Sats {
impl Sub for Sats {
type Output = Sats;
fn sub(self, rhs: Sats) -> Self::Output {
Sats::from(*self - *rhs)
Sats::from(self.0 - rhs.0)
}
}
@@ -66,27 +63,27 @@ impl SubAssign for Sats {
impl Mul<Sats> for Sats {
type Output = Sats;
fn mul(self, rhs: Sats) -> Self::Output {
Sats::from(*self * *rhs)
Sats::from(self.0 * rhs.0)
}
}
impl Mul<u64> for Sats {
type Output = Sats;
fn mul(self, rhs: u64) -> Self::Output {
Sats::from(*self * rhs)
Sats::from(self.0 * rhs)
}
}
impl Mul<Height> for Sats {
type Output = Sats;
fn mul(self, rhs: Height) -> Self::Output {
Sats::from(*self * *rhs as u64)
Sats::from(self.0 * u64::from(rhs))
}
}
impl Sum for Sats {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
let sats: u64 = iter.map(|sats| *sats).sum();
let sats: u64 = iter.map(|sats| sats.0).sum();
Sats::from(sats)
}
}
@@ -107,3 +104,9 @@ impl From<Sats> for Amount {
Self::from_sat(value.0)
}
}
impl From<Sats> for f64 {
fn from(value: Sats) -> Self {
value.0 as f64
}
}

4
crates/brk_core/src/structs/txindex.rs
View File

@@ -30,10 +30,6 @@ impl Txindex {
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txindex> for Txindex {

4
crates/brk_core/src/structs/txinindex.rs
View File

@@ -29,10 +29,6 @@ impl Txinindex {
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txinindex> for Txinindex {

4
crates/brk_core/src/structs/txoutindex.rs
View File

@@ -32,10 +32,6 @@ impl Txoutindex {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn is_coinbase(self) -> bool {
self == Self::COINBASE
}

3
crates/brk_core/src/utils/checked_sub.rs Normal file
View File

@@ -0,0 +1,3 @@
pub trait CheckedSub<Rhs = Self>: Sized {
fn checked_sub(self, rhs: Rhs) -> Option<Self>;
}

2
crates/brk_core/src/utils/mod.rs
View File

@@ -1,7 +1,9 @@
mod checked_sub;
mod paths;
mod pause;
mod rlimit;
pub use checked_sub::*;
pub use paths::*;
pub use pause::*;
pub use rlimit::*;

28
crates/brk_core/src/utils/paths.rs
View File

@@ -1,4 +1,7 @@
use std::path::{Path, PathBuf};
use std::{
env,
path::{Path, PathBuf},
};
pub fn path_dot_brk() -> PathBuf {
let home = std::env::var("HOME").unwrap();
@@ -8,3 +11,26 @@ pub fn path_dot_brk() -> PathBuf {
pub fn path_dot_brk_log() -> PathBuf {
path_dot_brk().join("log")
}
pub fn default_brk() -> PathBuf {
path_dot_brk()
}
pub fn default_bitcoin_path() -> PathBuf {
if env::consts::OS == "macos" {
default_mac_bitcoin_path()
} else {
default_linux_bitcoin_path()
}
}
fn default_linux_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap()).join(".bitcoin")
}
fn default_mac_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin")
}

2
crates/brk_fetcher/examples/main.rs
View File

@@ -10,7 +10,7 @@ fn main() -> color_eyre::Result<()> {
dbg!(fetcher.get_date(Date::new(2025, 1, 1))?);
dbg!(fetcher.get_height(
880_000_u32.into(),
881_000_u32.into(),
1740683986.into(),
Some(1740683000.into())
)?);

67
crates/brk_fetcher/src/fetchers/kibo.rs
View File

@@ -1,7 +1,7 @@
use std::{collections::BTreeMap, str::FromStr};
use brk_core::{Date, Height, OHLCCents};
use color_eyre::eyre::ContextCompat;
use brk_core::{CheckedSub, Date, Height, OHLCCents};
use color_eyre::eyre::{ContextCompat, eyre};
use log::info;
use serde_json::Value;
@@ -14,56 +14,43 @@ pub struct Kibo {
}
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api";
const RETRIES: usize = 10;
impl Kibo {
fn get_base_url(try_index: usize) -> &'static str {
if try_index < RETRIES / 2 {
KIBO_OFFICIAL_URL
} else {
KIBO_OFFICIAL_BACKUP_URL
}
}
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
let key = height.checked_sub(height % 10_000).unwrap_or_default();
#[allow(clippy::map_entry)]
if !self.height_to_ohlc_vec.contains_key(&height)
|| ((usize::from(height) + self.height_to_ohlc_vec.get(&height).unwrap().len())
<= usize::from(height))
if !self.height_to_ohlc_vec.contains_key(&key)
|| ((key + self.height_to_ohlc_vec.get(&key).unwrap().len()) <= height)
{
self.height_to_ohlc_vec.insert(
height,
Self::fetch_height_prices(height).inspect_err(|e| {
key,
Self::fetch_height_prices(key).inspect_err(|e| {
dbg!(e);
})?,
);
}
dbg!(&self.height_to_ohlc_vec.keys());
self.height_to_ohlc_vec
.get(&height)
.get(&key)
.unwrap()
.get(usize::from(height))
.get(usize::from(height.checked_sub(key).unwrap()))
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
info!("Fetching Kibo height prices...");
info!("Fetching Kibo height {height} prices...");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let url = format!("{base_url}/height-to-price?chunk={}", height);
|_| {
let url = format!("{KIBO_OFFICIAL_URL}/height-to-price?chunk={}", height);
let body: Value = minreq::get(url).send()?.json()?;
let vec = body
.as_object()
body.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
@@ -75,9 +62,7 @@ impl Kibo {
.context("Expect to be an array")?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()?;
Ok(vec)
.collect::<Result<Vec<_>, _>>()
},
30,
RETRIES,
@@ -107,19 +92,21 @@ impl Kibo {
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date in kibo"))
.ok_or({
dbg!(date);
eyre!("Couldn't find date in kibo")
})
}
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching Kibo date prices...");
info!("Fetching Kibo date {year} prices...");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value = minreq::get(format!("{base_url}/date-to-price?chunk={}", year))
.send()?
.json()?;
|_| {
let body: Value =
minreq::get(format!("{KIBO_OFFICIAL_URL}/date-to-price?chunk={}", year))
.send()?
.json()?;
body.as_object()
.context("Expect to be an object")?
@@ -156,11 +143,11 @@ impl Kibo {
)))
};
Ok((
Ok(OHLCCents::from((
Open::from(get_value("open")?),
High::from(get_value("high")?),
Low::from(get_value("low")?),
Close::from(get_value("close")?),
))
)))
}
}

17
crates/brk_fetcher/src/lib.rs
View File

@@ -106,12 +106,7 @@ impl Fetcher {
let previous_ohlc = previous_ohlc.unwrap();
let mut final_ohlc = (
Open::from(previous_ohlc.3),
High::from(previous_ohlc.3),
Low::from(previous_ohlc.3),
previous_ohlc.3,
);
let mut final_ohlc = OHLCCents::from(previous_ohlc.close);
let start = previous_timestamp.unwrap_or(Timestamp::from(0));
let end = timestamp;
@@ -119,15 +114,15 @@ impl Fetcher {
// Otherwise it's a re-org
if start < end {
tree.range(start..=end).skip(1).for_each(|(_, ohlc)| {
if ohlc.1 > final_ohlc.1 {
final_ohlc.1 = ohlc.1
if ohlc.high > final_ohlc.high {
final_ohlc.high = ohlc.high
}
if ohlc.2 < final_ohlc.2 {
final_ohlc.2 = ohlc.2
if ohlc.low < final_ohlc.low {
final_ohlc.low = ohlc.low
}
final_ohlc.3 = ohlc.3;
final_ohlc.close = ohlc.close;
});
}

4
crates/brk_indexer/examples/main.rs
View File

@@ -1,5 +1,6 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::{Indexer, rpc::RpcApi};
use brk_parser::{
@@ -13,7 +14,8 @@ fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let bitcoin_dir = Path::new("../../../bitcoin");
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),

40
crates/brk_indexer/src/indexes.rs
View File

@@ -1,7 +1,8 @@
use bitcoincore_rpc::Client;
use brk_core::{
Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex,
P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
Addressindex, BlockHash, CheckedSub, Emptyindex, Height, Multisigindex, Opreturnindex,
P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex,
Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use color_eyre::eyre::ContextCompat;
@@ -32,8 +33,10 @@ pub struct Indexes {
impl Indexes {
pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> {
let height = self.height;
vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?;
vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?;
vecs.height_to_first_txindex
.push_if_needed(height, self.txindex)?;
vecs.height_to_first_txinindex
.push_if_needed(height, self.txinindex)?;
vecs.height_to_first_txoutindex
.push_if_needed(height, self.txoutindex)?;
vecs.height_to_first_addressindex
@@ -54,8 +57,10 @@ impl Indexes {
.push_if_needed(height, self.p2pk65index)?;
vecs.height_to_first_p2pkhindex
.push_if_needed(height, self.p2pkhindex)?;
vecs.height_to_first_p2shindex.push_if_needed(height, self.p2shindex)?;
vecs.height_to_first_p2trindex.push_if_needed(height, self.p2trindex)?;
vecs.height_to_first_p2shindex
.push_if_needed(height, self.p2shindex)?;
vecs.height_to_first_p2trindex
.push_if_needed(height, self.p2trindex)?;
vecs.height_to_first_p2wpkhindex
.push_if_needed(height, self.p2wpkhindex)?;
vecs.height_to_first_p2wshindex
@@ -77,9 +82,11 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
// Height at which we wanna start: min last saved + 1 or 0
let starting_height = vecs.starting_height().min(stores.starting_height());
let range = starting_height
.checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
.unwrap_or_default()..*starting_height;
let range = u32::from(
starting_height
.checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
.unwrap_or_default(),
)..u32::from(starting_height);
// But we also need to check the chain and start earlier in case of a reorg
let height = range // ..= because of last saved + 1
@@ -107,8 +114,14 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,
height,
multisigindex: *vecs.height_to_first_multisigindex.get(height)?.context("")?,
opreturnindex: *vecs.height_to_first_opreturnindex.get(height)?.context("")?,
multisigindex: *vecs
.height_to_first_multisigindex
.get(height)?
.context("")?,
opreturnindex: *vecs
.height_to_first_opreturnindex
.get(height)?
.context("")?,
p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?,
p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?,
p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?,
@@ -116,7 +129,10 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?,
p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?,
p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?,
pushonlyindex: *vecs.height_to_first_pushonlyindex.get(height)?.context("")?,
pushonlyindex: *vecs
.height_to_first_pushonlyindex
.get(height)?
.context("")?,
txindex: *vecs.height_to_first_txindex.get(height)?.context("")?,
txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?,
txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?,

2
crates/brk_indexer/src/lib.rs
View File

@@ -83,7 +83,7 @@ impl Indexer {
self.stores
.as_mut()
.unwrap()
.rollback_if_needed(self.vecs.as_ref().unwrap(), &starting_indexes)?;
.rollback_if_needed(self.vecs.as_mut().unwrap(), &starting_indexes)?;
self.vecs
.as_mut()
.unwrap()

93
crates/brk_indexer/src/stores/mod.rs
View File

@@ -1,6 +1,9 @@
use std::{path::Path, thread};
use brk_core::{AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, Txindex};
use brk_core::{
AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix,
Txindex,
};
use brk_vec::{Value, Version};
use crate::Indexes;
@@ -23,12 +26,14 @@ pub struct Stores {
impl Stores {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
thread::scope(|scope| {
let addresshash_to_addressindex =
scope.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::from(1)));
let blockhash_prefix_to_height =
scope.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1)));
let txid_prefix_to_txindex =
scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
let addresshash_to_addressindex = scope.spawn(|| {
Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))
});
let blockhash_prefix_to_height = scope.spawn(|| {
Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))
});
let txid_prefix_to_txindex = scope
.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
Ok(Self {
addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?,
@@ -38,22 +43,29 @@ impl Stores {
})
}
pub fn rollback_if_needed(&mut self, vecs: &Vecs, starting_indexes: &Indexes) -> color_eyre::Result<()> {
pub fn rollback_if_needed(
&mut self,
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> color_eyre::Result<()> {
vecs.height_to_blockhash
.iter_from(starting_indexes.height, |(_, blockhash)| {
.iter_from(starting_indexes.height, |(_, blockhash, ..)| {
let blockhash_prefix = BlockHashPrefix::from(blockhash);
self.blockhash_prefix_to_height.remove(blockhash_prefix);
Ok(())
})?;
vecs.txindex_to_txid
.iter_from(starting_indexes.txindex, |(_txindex, txid)| {
.iter_from(starting_indexes.txindex, |(_txindex, txid, ..)| {
let txid_prefix = TxidPrefix::from(txid);
self.txid_prefix_to_txindex.remove(txid_prefix);
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2pk65index
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes
@@ -67,7 +79,10 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2pk33index.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2pk33index
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes
@@ -81,9 +96,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2pkhindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2pkhindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2pkhindex_to_p2pkhaddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
self.addresshash_to_addressindex.remove(hash);
@@ -91,9 +113,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2shindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2shindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2shindex_to_p2shaddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2shindex_to_p2shaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2SH));
self.addresshash_to_addressindex.remove(hash);
@@ -101,9 +130,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2trindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2trindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2trindex_to_p2traddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2trindex_to_p2traddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2TR));
self.addresshash_to_addressindex.remove(hash);
@@ -111,7 +147,10 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2wpkhindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2wpkhindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes
@@ -125,9 +164,16 @@ impl Stores {
}
}
if let Some(index) = vecs.height_to_first_p2wshindex.get(starting_indexes.height)? {
if let Some(index) = vecs
.height_to_first_p2wshindex
.get(starting_indexes.height)?
{
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2wshindex_to_p2wshaddressbytes.get(index)?.map(Value::into_inner) {
while let Some(typedbytes) = vecs
.p2wshindex_to_p2wshaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
self.addresshash_to_addressindex.remove(hash);
@@ -135,7 +181,7 @@ impl Stores {
}
}
self.commit(starting_indexes.height.decremented())?;
self.commit(starting_indexes.height.decremented().unwrap())?;
Ok(())
}
@@ -158,7 +204,8 @@ impl Stores {
scope.spawn(|| self.addresshash_to_addressindex.commit(height));
let blockhash_prefix_to_height_commit_handle =
scope.spawn(|| self.blockhash_prefix_to_height.commit(height));
let txid_prefix_to_txindex_commit_handle = scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
let txid_prefix_to_txindex_commit_handle =
scope.spawn(|| self.txid_prefix_to_txindex.commit(height));
addresshash_to_addressindex_commit_handle.join().unwrap()?;
blockhash_prefix_to_height_commit_handle.join().unwrap()?;

146
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -1,10 +1,11 @@
use std::{fs, io, path::Path};
use brk_core::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex,
Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex,
P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height,
LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes,
P2PK65index, P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes,
P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats,
Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
};
use brk_vec::{AnyStorableVec, Version};
use rayon::prelude::*;
@@ -75,9 +76,18 @@ impl Vecs {
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
)?,
addressindex_to_height: StorableVec::import(&path.join("addressindex_to_height"), Version::from(1))?,
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?,
height_to_difficulty: StorableVec::import(&path.join("height_to_difficulty"), Version::from(1))?,
addressindex_to_height: StorableVec::import(
&path.join("addressindex_to_height"),
Version::from(1),
)?,
height_to_blockhash: StorableVec::import(
&path.join("height_to_blockhash"),
Version::from(1),
)?,
height_to_difficulty: StorableVec::import(
&path.join("height_to_difficulty"),
Version::from(1),
)?,
height_to_first_addressindex: StorableVec::import(
&path.join("height_to_first_addressindex"),
Version::from(1),
@@ -98,8 +108,14 @@ impl Vecs {
&path.join("height_to_first_pushonlyindex"),
Version::from(1),
)?,
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?,
height_to_first_txinindex: StorableVec::import(&path.join("height_to_first_txinindex"), Version::from(1))?,
height_to_first_txindex: StorableVec::import(
&path.join("height_to_first_txindex"),
Version::from(1),
)?,
height_to_first_txinindex: StorableVec::import(
&path.join("height_to_first_txinindex"),
Version::from(1),
)?,
height_to_first_txoutindex: StorableVec::import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
@@ -120,8 +136,14 @@ impl Vecs {
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
)?,
height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?,
height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?,
height_to_first_p2shindex: StorableVec::import(
&path.join("height_to_first_p2shindex"),
Version::from(1),
)?,
height_to_first_p2trindex: StorableVec::import(
&path.join("height_to_first_p2trindex"),
Version::from(1),
)?,
height_to_first_p2wpkhindex: StorableVec::import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
@@ -131,8 +153,14 @@ impl Vecs {
Version::from(1),
)?,
height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?,
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?,
height_to_timestamp: StorableVec::import(
&path.join("height_to_timestamp"),
Version::from(1),
)?,
height_to_weight: StorableVec::import(
&path.join("height_to_weight"),
Version::from(1),
)?,
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
Version::from(1),
@@ -169,27 +197,48 @@ impl Vecs {
&path.join("txindex_to_first_txoutindex"),
Version::from(1),
)?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_locktime: StorableVec::import(&path.join("txindex_to_locktime"), Version::from(1))?,
txindex_to_height: StorableVec::import(
&path.join("txindex_to_height"),
Version::from(1),
)?,
txindex_to_locktime: StorableVec::import(
&path.join("txindex_to_locktime"),
Version::from(1),
)?,
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?,
txindex_to_base_size: StorableVec::import(&path.join("txindex_to_base_size"), Version::from(1))?,
txindex_to_total_size: StorableVec::import(&path.join("txindex_to_total_size"), Version::from(1))?,
txindex_to_base_size: StorableVec::import(
&path.join("txindex_to_base_size"),
Version::from(1),
)?,
txindex_to_total_size: StorableVec::import(
&path.join("txindex_to_total_size"),
Version::from(1),
)?,
txindex_to_is_explicitly_rbf: StorableVec::import(
&path.join("txindex_to_is_explicitly_rbf"),
Version::from(1),
)?,
txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?,
txinindex_to_txoutindex: StorableVec::import(&path.join("txinindex_to_txoutindex"), Version::from(1))?,
txindex_to_txversion: StorableVec::import(
&path.join("txindex_to_txversion"),
Version::from(1),
)?,
txinindex_to_txoutindex: StorableVec::import(
&path.join("txinindex_to_txoutindex"),
Version::from(1),
)?,
txoutindex_to_addressindex: StorableVec::import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
)?,
txoutindex_to_value: StorableVec::import(&path.join("txoutindex_to_value"), Version::from(1))?,
txoutindex_to_value: StorableVec::import(
&path.join("txoutindex_to_value"),
Version::from(1),
)?,
})
}
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
let saved_height = starting_indexes.height.decremented();
let saved_height = starting_indexes.height.decremented().unwrap_or_default();
// We don't want to override the starting indexes so we cut from n + 1
let height = starting_indexes.height.incremented();
@@ -218,7 +267,8 @@ impl Vecs {
.truncate_if_needed(height, saved_height)?;
self.height_to_first_pushonlyindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txindex.truncate_if_needed(height, saved_height)?;
self.height_to_first_txindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txinindex
.truncate_if_needed(height, saved_height)?;
self.height_to_first_txoutindex
@@ -243,11 +293,16 @@ impl Vecs {
..
} = starting_indexes;
self.height_to_blockhash.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty.truncate_if_needed(height, saved_height)?;
self.height_to_size.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp.truncate_if_needed(height, saved_height)?;
self.height_to_weight.truncate_if_needed(height, saved_height)?;
self.height_to_blockhash
.truncate_if_needed(height, saved_height)?;
self.height_to_difficulty
.truncate_if_needed(height, saved_height)?;
self.height_to_size
.truncate_if_needed(height, saved_height)?;
self.height_to_timestamp
.truncate_if_needed(height, saved_height)?;
self.height_to_weight
.truncate_if_needed(height, saved_height)?;
self.addressindex_to_addresstype
.truncate_if_needed(addressindex, saved_height)?;
@@ -275,12 +330,18 @@ impl Vecs {
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_first_txoutindex
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_height.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_locktime.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txid.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_base_size.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_total_size.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_height
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_locktime
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txid
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_txversion
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_base_size
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_total_size
.truncate_if_needed(txindex, saved_height)?;
self.txindex_to_is_explicitly_rbf
.truncate_if_needed(txindex, saved_height)?;
@@ -289,7 +350,8 @@ impl Vecs {
self.txoutindex_to_addressindex
.truncate_if_needed(txoutindex, saved_height)?;
self.txoutindex_to_value.truncate_if_needed(txoutindex, saved_height)?;
self.txoutindex_to_value
.truncate_if_needed(txoutindex, saved_height)?;
Ok(())
}
@@ -351,13 +413,21 @@ impl Vecs {
Addressbytes::P2PK33(bytes) => self
.p2pk33index_to_p2pk33addressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2PKH(bytes) => self
.p2pkhindex_to_p2pkhaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2SH(bytes) => self
.p2shindex_to_p2shaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2WPKH(bytes) => self
.p2wpkhindex_to_p2wpkhaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2WSH(bytes) => self
.p2wshindex_to_p2wshaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2TR(bytes) => self
.p2trindex_to_p2traddressbytes
.push_if_needed(index.into(), bytes),
}
}

7
crates/brk_parser/examples/main.rs
View File

@@ -1,13 +1,12 @@
use std::path::Path;
use bitcoincore_rpc::{Auth, Client};
use brk_core::Height;
use brk_core::{Height, default_bitcoin_path};
use brk_parser::Parser;
fn main() {
let i = std::time::Instant::now();
let bitcoin_dir = Path::new("../../../bitcoin");
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(
Client::new(
"http://localhost:8332",

8
crates/brk_query/src/lib.rs
View File

@@ -23,8 +23,8 @@ use tree::VecIdToIndexToVec;
pub struct Query<'a> {
pub vecid_to_index_to_vec: VecIdToIndexToVec<'a>,
indexer: &'a Indexer,
computer: &'a Computer,
_indexer: &'a Indexer,
_computer: &'a Computer,
}
impl<'a> Query<'a> {
@@ -45,8 +45,8 @@ impl<'a> Query<'a> {
Self {
vecid_to_index_to_vec: vecs,
indexer,
computer,
_indexer: indexer,
_computer: computer,
}
}

4
crates/brk_server/Cargo.toml
View File

@@ -21,9 +21,9 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.56.0", features = ["codegen", "minifier"] }
oxc = { version = "0.56.5", features = ["codegen", "minifier"] }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { version = "1.43.0", features = ["full"] }
tokio = { version = "1.44.0", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full"] }
zip = "2.2.3"

6
crates/brk_server/examples/main.rs
View File

@@ -1,6 +1,7 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::{
@@ -15,9 +16,10 @@ pub fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
let process = false;
let process = true;
let bitcoin_dir = default_bitcoin_path();
let bitcoin_dir = Path::new("../../../bitcoin");
let rpc = Box::leak(Box::new(rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),

1
crates/brk_vec/Cargo.toml
View File

@@ -9,6 +9,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_exit = { workspace = true }
memmap2 = "0.9.5"
rayon = { workspace = true }

82
crates/brk_vec/src/lib.rs
View File

@@ -16,6 +16,7 @@ use std::{
sync::OnceLock,
};
use brk_core::CheckedSub;
use brk_exit::Exit;
pub use memmap2;
use rayon::prelude::*;
@@ -258,6 +259,17 @@ where
}
#[inline]
pub fn read_(&mut self, index: usize) -> Result<Option<&T>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.pushed.get(index));
}
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(Error::IndexTooLow) => {}
Err(error) => return Err(error),
}
let byte_index = Self::index_to_byte_index(index);
if self.file_position != byte_index {
self.file_position = self.seek(Self::index_to_byte_index(index))?;
@@ -279,21 +291,20 @@ where
self.read_(len - 1)
}
pub fn iter<F>(&self, f: F) -> Result<()>
pub fn iter<F>(&mut self, f: F) -> Result<()>
where
F: FnMut((I, &T)) -> Result<()>,
F: FnMut((I, &T, &mut Self)) -> Result<()>,
{
self.iter_from(I::default(), f)
}
pub fn iter_from<F>(&self, mut index: I, mut f: F) -> Result<()>
pub fn iter_from<F>(&mut self, mut index: I, mut f: F) -> Result<()>
where
F: FnMut((I, &T)) -> Result<()>,
F: FnMut((I, &T, &mut Self)) -> Result<()>,
{
let mut file = self.open_file()?;
let disk_len = I::from(Self::read_disk_len_(&file)?);
let pushed_len = I::from(self.pushed_len());
Self::seek_(
&mut file,
@@ -303,18 +314,12 @@ where
let mut buf = Self::create_buffer();
while index < disk_len {
f((index, Self::read_exact(&mut file, &mut buf)?))?;
f((index, Self::read_exact(&mut file, &mut buf)?, self))?;
index = index + 1;
}
let disk_len = Self::i_to_usize(disk_len)?;
let mut i = I::default();
while i < pushed_len {
f((
i + disk_len,
self.pushed.get(Self::i_to_usize(i)?).as_ref().unwrap(),
))?;
i = i + 1;
if self.pushed_len() != 0 {
unreachable!();
}
Ok(())
@@ -573,24 +578,26 @@ where
std::any::type_name::<I>()
}
pub fn compute_transform<A, F>(
pub fn compute_transform<A, B, F>(
&mut self,
max_from: I,
other: &mut StorableVec<I, A>,
t: F,
max_from: A,
other: &mut StorableVec<A, B>,
mut t: F,
exit: &Exit,
) -> Result<()>
where
A: StoredType,
F: Fn(&A, I) -> T,
A: StoredIndex,
B: StoredType,
F: FnMut((A, &B, &mut Self, &mut StorableVec<A, B>)) -> (I, T),
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version + other.version,
)?;
let index = max_from.min(I::from(self.len()));
other.iter_from(index, |(i, a)| {
self.push_and_flush_if_needed(i, t(a, i), exit)
let index = max_from.min(A::from(self.len()));
other.iter_from(index, |(a, b, other)| {
let (i, v) = t((a, b, self, other));
self.push_and_flush_if_needed(i, v, exit)
})?;
Ok(self.safe_flush(exit)?)
@@ -611,7 +618,14 @@ where
)?;
let index = max_from.min(self.read_last()?.cloned().unwrap_or_default());
other.iter_from(index, |(v, i)| self.push_and_flush_if_needed(*i, v, exit))?;
other.iter_from(index, |(v, i, ..)| {
let i = *i;
if self.read(i).unwrap().is_none_or(|old_v| *old_v > v) {
self.push_and_flush_if_needed(i, v, exit)
} else {
Ok(())
}
})?;
Ok(self.safe_flush(exit)?)
}
@@ -632,7 +646,7 @@ where
)?;
let index = max_from.min(T::from(self.len()));
first_indexes.iter_from(index, |(value, first_index)| {
first_indexes.iter_from(index, |(value, first_index, ..)| {
let first_index = Self::i_to_usize(*first_index)?;
let last_index = Self::i_to_usize(*last_indexes.read(value)?.unwrap())?;
(first_index..last_index)
@@ -650,7 +664,7 @@ where
exit: &Exit,
) -> Result<()>
where
T: Copy + From<usize> + Sub<T, Output = T> + StoredIndex,
T: Copy + From<usize> + CheckedSub<T> + StoredIndex,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version + first_indexes.version,
@@ -659,15 +673,19 @@ where
let index = max_from.min(I::from(self.len()));
let one = T::from(1);
let mut prev_index: Option<I> = None;
first_indexes.iter_from(index, |(i, v)| {
first_indexes.iter_from(index, |(i, v, ..)| {
if let Some(prev_index) = prev_index {
self.push_and_flush_if_needed(prev_index, *v - one, exit)?;
self.push_and_flush_if_needed(prev_index, v.checked_sub(one).unwrap(), exit)?;
}
prev_index.replace(i);
Ok(())
})?;
if let Some(prev_index) = prev_index {
self.push_and_flush_if_needed(prev_index, T::from(final_len) - one, exit)?;
self.push_and_flush_if_needed(
prev_index,
T::from(final_len).checked_sub(one).unwrap(),
exit,
)?;
}
Ok(self.safe_flush(exit)?)
@@ -690,7 +708,7 @@ where
)?;
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(i, first_index)| {
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.read(i)?.unwrap();
let count = *last_index + 1_usize - *first_index;
self.push_and_flush_if_needed(i, count.into(), exit)
@@ -716,7 +734,7 @@ where
)?;
let index = max_from.min(I::from(self.len()));
self_to_other.iter_from(index, |(i, other)| {
self_to_other.iter_from(index, |(i, other, ..)| {
self.push_and_flush_if_needed(
i,
T::from(other_to_self.read(*other)?.unwrap() == &i),
@@ -745,7 +763,7 @@ where
)?;
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(index, first_index)| {
first_indexes.iter_from(index, |(index, first_index, ..)| {
let last_index = last_indexes.read(index)?.unwrap();
let count = *last_index + 1_usize - *first_index;
self.push_and_flush_if_needed(index, count.into(), exit)