Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-08 13:19:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-04-03 14:31:39 +02:00
parent 4c2da31bb3
commit e8c34dd59b
51 changed files with 2561 additions and 5501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/brk_cli/src/query.rs
View File

@@ -1,5 +1,4 @@
use brk_computer::Computer;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_query::{Index, Output, Params as QueryParams, Query, Tabled, Value};
use tabled::settings::Style;

6
crates/brk_computer/src/storage/stores.rs
View File

@@ -12,8 +12,10 @@ pub struct Stores {
impl Stores {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?;
let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?;
let address_to_utxos_received =
Store::import(&path.join("address_to_utxos_received"), Version::ONE)?;
let address_to_utxos_spent =
Store::import(&path.join("address_to_utxos_spent"), Version::ONE)?;
Ok(Self {
address_to_utxos_received,

14
crates/brk_computer/src/storage/vecs/base.rs
View File

@@ -137,7 +137,7 @@ where
F: FnMut((A, B, &mut Self, &mut brk_vec::StorableVec<A, B>)) -> (I, T),
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + other.version(),
Version::ZERO + self.version() + other.version(),
)?;
let index = max_from.min(A::from(self.len()));
@@ -160,7 +160,7 @@ where
T: StoredIndex,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + other.version(),
Version::ZERO + self.version() + other.version(),
)?;
let index = max_from.min(self.vec.get_last()?.cloned().unwrap_or_default());
@@ -188,7 +188,7 @@ where
T: StoredIndex,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
Version::ZERO + self.version() + first_indexes.version() + last_indexes.version(),
)?;
let index = max_from.min(T::from(self.len()));
@@ -213,7 +213,7 @@ where
T: Copy + From<usize> + CheckedSub<T> + StoredIndex,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + first_indexes.version(),
Version::ZERO + self.version() + first_indexes.version(),
)?;
let index = max_from.min(I::from(self.len()));
@@ -250,7 +250,7 @@ where
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
Version::ZERO + self.version() + first_indexes.version() + last_indexes.version(),
)?;
let index = max_from.min(I::from(self.len()));
@@ -278,7 +278,7 @@ where
A: StoredIndex + StoredType,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + self_to_other.version() + other_to_self.version(),
Version::ZERO + self.version() + self_to_other.version() + other_to_self.version(),
)?;
let index = max_from.min(I::from(self.len()));
@@ -302,7 +302,7 @@ where
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
self.validate_computed_version_or_reset_file(
Version::from(0) + self.version() + first_indexes.version() + last_indexes.version(),
Version::ZERO + self.version() + first_indexes.version() + last_indexes.version(),
)?;
let index = max_from.min(I::from(self.len()));

6
crates/brk_computer/src/storage/vecs/blocks.rs
View File

@@ -25,7 +25,7 @@ impl Vecs {
Ok(Self {
height_to_block_interval: StorableVec::forced_import(
&path.join("height_to_block_interval"),
Version::from(1),
Version::ONE,
compressed,
)?,
indexes_to_block_interval_stats: StorableVecsStatsFromHeight::forced_import(
@@ -38,12 +38,12 @@ impl Vecs {
)?,
dateindex_to_block_count: StorableVec::forced_import(
&path.join("dateindex_to_block_count"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_total_block_count: StorableVec::forced_import(
&path.join("dateindex_to_total_block_count"),
Version::from(1),
Version::ONE,
compressed,
)?,
})

96
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -71,242 +71,242 @@ impl Vecs {
Ok(Self {
dateindex_to_date: StorableVec::forced_import(
&path.join("dateindex_to_date"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_dateindex: StorableVec::forced_import(
&path.join("dateindex_to_dateindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_first_height: StorableVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_last_height: StorableVec::forced_import(
&path.join("dateindex_to_last_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_real_date: StorableVec::forced_import(
&path.join("height_to_real_date"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_fixed_date: StorableVec::forced_import(
&path.join("height_to_fixed_date"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_dateindex: StorableVec::forced_import(
&path.join("height_to_dateindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_height: StorableVec::forced_import(
&path.join("height_to_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_last_txindex: StorableVec::forced_import(
&path.join("height_to_last_txindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_last_txinindex: StorableVec::forced_import(
&path.join("txindex_to_last_txinindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_last_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
difficultyepoch_to_first_height: StorableVec::forced_import(
&path.join("difficultyepoch_to_first_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
difficultyepoch_to_last_height: StorableVec::forced_import(
&path.join("difficultyepoch_to_last_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
halvingepoch_to_first_height: StorableVec::forced_import(
&path.join("halvingepoch_to_first_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
halvingepoch_to_last_height: StorableVec::forced_import(
&path.join("halvingepoch_to_last_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
weekindex_to_first_dateindex: StorableVec::forced_import(
&path.join("weekindex_to_first_dateindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
weekindex_to_last_dateindex: StorableVec::forced_import(
&path.join("weekindex_to_last_dateindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_first_dateindex: StorableVec::forced_import(
&path.join("monthindex_to_first_dateindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_last_dateindex: StorableVec::forced_import(
&path.join("monthindex_to_last_dateindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
yearindex_to_first_monthindex: StorableVec::forced_import(
&path.join("yearindex_to_first_monthindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
yearindex_to_last_monthindex: StorableVec::forced_import(
&path.join("yearindex_to_last_monthindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
decadeindex_to_first_yearindex: StorableVec::forced_import(
&path.join("decadeindex_to_first_yearindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
decadeindex_to_last_yearindex: StorableVec::forced_import(
&path.join("decadeindex_to_last_yearindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_weekindex: StorableVec::forced_import(
&path.join("dateindex_to_weekindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_monthindex: StorableVec::forced_import(
&path.join("dateindex_to_monthindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_yearindex: StorableVec::forced_import(
&path.join("monthindex_to_yearindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
yearindex_to_decadeindex: StorableVec::forced_import(
&path.join("yearindex_to_decadeindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_difficultyepoch: StorableVec::forced_import(
&path.join("height_to_difficultyepoch"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_halvingepoch: StorableVec::forced_import(
&path.join("height_to_halvingepoch"),
Version::from(1),
Version::ONE,
compressed,
)?,
weekindex_to_weekindex: StorableVec::forced_import(
&path.join("weekindex_to_weekindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_monthindex: StorableVec::forced_import(
&path.join("monthindex_to_monthindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
yearindex_to_yearindex: StorableVec::forced_import(
&path.join("yearindex_to_yearindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
decadeindex_to_decadeindex: StorableVec::forced_import(
&path.join("decadeindex_to_decadeindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
difficultyepoch_to_difficultyepoch: StorableVec::forced_import(
&path.join("difficultyepoch_to_difficultyepoch"),
Version::from(1),
Version::ONE,
compressed,
)?,
halvingepoch_to_halvingepoch: StorableVec::forced_import(
&path.join("halvingepoch_to_halvingepoch"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_timestamp: StorableVec::forced_import(
&path.join("dateindex_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
decadeindex_to_timestamp: StorableVec::forced_import(
&path.join("decadeindex_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
difficultyepoch_to_timestamp: StorableVec::forced_import(
&path.join("difficultyepoch_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
halvingepoch_to_timestamp: StorableVec::forced_import(
&path.join("halvingepoch_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_timestamp: StorableVec::forced_import(
&path.join("monthindex_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
weekindex_to_timestamp: StorableVec::forced_import(
&path.join("weekindex_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
yearindex_to_timestamp: StorableVec::forced_import(
&path.join("yearindex_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_fixed_timestamp: StorableVec::forced_import(
&path.join("height_to_fixed_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_quarterindex: StorableVec::forced_import(
&path.join("monthindex_to_quarterindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
quarterindex_to_first_monthindex: StorableVec::forced_import(
&path.join("quarterindex_to_first_monthindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
quarterindex_to_last_monthindex: StorableVec::forced_import(
&path.join("quarterindex_to_last_monthindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
quarterindex_to_quarterindex: StorableVec::forced_import(
&path.join("quarterindex_to_quarterindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
quarterindex_to_timestamp: StorableVec::forced_import(
&path.join("quarterindex_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
})

58
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -66,112 +66,112 @@ impl Vecs {
Ok(Self {
dateindex_to_ohlc_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_ohlc_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_ohlc: StorableVec::forced_import(
&path.join("dateindex_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_close_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_close_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_close: StorableVec::forced_import(
&path.join("dateindex_to_close"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_high_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_high_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_high: StorableVec::forced_import(
&path.join("dateindex_to_high"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_low_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_low_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_low: StorableVec::forced_import(
&path.join("dateindex_to_low"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_open_in_cents: StorableVec::forced_import(
&path.join("dateindex_to_open_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_open: StorableVec::forced_import(
&path.join("dateindex_to_open"),
Version::from(1),
Version::ONE,
compressed,
)?,
dateindex_to_sats_per_dollar: StorableVec::forced_import(
&path.join("dateindex_to_sats_per_dollar"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_ohlc_in_cents: StorableVec::forced_import(
&path.join("height_to_ohlc_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_ohlc: StorableVec::forced_import(
&path.join("height_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_close_in_cents: StorableVec::forced_import(
&path.join("height_to_close_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_close: StorableVec::forced_import(
&path.join("height_to_close"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_high_in_cents: StorableVec::forced_import(
&path.join("height_to_high_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_high: StorableVec::forced_import(
&path.join("height_to_high"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_low_in_cents: StorableVec::forced_import(
&path.join("height_to_low_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_low: StorableVec::forced_import(
&path.join("height_to_low"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_open_in_cents: StorableVec::forced_import(
&path.join("height_to_open_in_cents"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_open: StorableVec::forced_import(
&path.join("height_to_open"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_sats_per_dollar: StorableVec::forced_import(
&path.join("height_to_sats_per_dollar"),
Version::from(1),
Version::ONE,
compressed,
)?,
timeindexes_to_open: StorableVecsStatsFromDate::forced_import(
@@ -226,33 +226,33 @@ impl Vecs {
)?,
weekindex_to_ohlc: StorableVec::forced_import(
&path.join("weekindex_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
difficultyepoch_to_ohlc: StorableVec::forced_import(
&path.join("difficultyepoch_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
monthindex_to_ohlc: StorableVec::forced_import(
&path.join("monthindex_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
quarterindex_to_ohlc: StorableVec::forced_import(
&path.join("quarterindex_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
yearindex_to_ohlc: StorableVec::forced_import(
&path.join("yearindex_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
// halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::from(1), compressed)?,
// halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ONE, compressed)?,
decadeindex_to_ohlc: StorableVec::forced_import(
&path.join("decadeindex_to_ohlc"),
Version::from(1),
Version::ONE,
compressed,
)?,
})

23
crates/brk_computer/src/storage/vecs/stats/generic.rs
View File

@@ -44,43 +44,42 @@ where
let s = Self {
first: options.first.then(|| {
StorableVec::forced_import(&prefix("first"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&prefix("first"), Version::ONE, compressed).unwrap()
}),
last: options.last.then(|| {
StorableVec::forced_import(
&path.with_file_name(format!("{key}_to_{name}")),
Version::from(1),
Version::ONE,
compressed,
)
.unwrap()
}),
min: options.min.then(|| {
StorableVec::forced_import(&suffix("min"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("min"), Version::ONE, compressed).unwrap()
}),
max: options.max.then(|| {
StorableVec::forced_import(&suffix("max"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("max"), Version::ONE, compressed).unwrap()
}),
median: options.median.then(|| {
StorableVec::forced_import(&suffix("median"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("median"), Version::ONE, compressed).unwrap()
}),
average: options.average.then(|| {
StorableVec::forced_import(&suffix("average"), Version::from(1), compressed)
.unwrap()
StorableVec::forced_import(&suffix("average"), Version::ONE, compressed).unwrap()
}),
sum: options.sum.then(|| {
StorableVec::forced_import(&suffix("sum"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("sum"), Version::ONE, compressed).unwrap()
}),
_90p: options._90p.then(|| {
StorableVec::forced_import(&suffix("90p"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("90p"), Version::ONE, compressed).unwrap()
}),
_75p: options._75p.then(|| {
StorableVec::forced_import(&suffix("75p"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("75p"), Version::ONE, compressed).unwrap()
}),
_25p: options._25p.then(|| {
StorableVec::forced_import(&suffix("25p"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("25p"), Version::ONE, compressed).unwrap()
}),
_10p: options._10p.then(|| {
StorableVec::forced_import(&suffix("10p"), Version::from(1), compressed).unwrap()
StorableVec::forced_import(&suffix("10p"), Version::ONE, compressed).unwrap()
}),
};

34
crates/brk_computer/src/storage/vecs/transactions.rs
View File

@@ -33,52 +33,52 @@ impl Vecs {
fs::create_dir_all(path)?;
Ok(Self {
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::ONE)?,
// height_to_input_count: StorableVec::forced_import(
// &path.join("height_to_input_count"),
// Version::from(1),
// Version::ONE,
// )?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::ONE)?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::ONE)?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::ONE)?,
// height_to_output_count: StorableVec::forced_import(
// &path.join("height_to_output_count"),
// Version::from(1),
// Version::ONE,
// )?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ONE)?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::ONE)?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::ONE)?,
// txindex_to_fee: StorableVec::forced_import(
// &path.join("txindex_to_fee"),
// Version::from(1),
// Version::ONE,
// )?,
txindex_to_is_coinbase: StorableVec::forced_import(
&path.join("txindex_to_is_coinbase"),
Version::from(1),
Version::ONE,
compressed,
)?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::ONE)?,
txindex_to_inputs_count: StorableVec::forced_import(
&path.join("txindex_to_inputs_count"),
Version::from(1),
Version::ONE,
compressed,
)?,
// txindex_to_inputs_sum: StorableVec::forced_import(
// &path.join("txindex_to_inputs_sum"),
// Version::from(1),
// Version::ONE,
// )?,
txindex_to_outputs_count: StorableVec::forced_import(
&path.join("txindex_to_outputs_count"),
Version::from(1),
Version::ONE,
compressed,
)?,
// txindex_to_outputs_sum: StorableVec::forced_import(
// &path.join("txindex_to_outputs_sum"),
// Version::from(1),
// Version::ONE,
// )?,
// txinindex_to_value: StorableVec::forced_import(
// &path.join("txinindex_to_value"),
// Version::from(1),
// Version::ONE,
// compressed,
// )?,
})

3
crates/brk_indexer/src/stores/base.rs
View File

@@ -27,6 +27,7 @@ pub struct Store<Key, Value> {
}
const CHECK_COLLISISONS: bool = true;
const FJALL_VERSION: Version = Version::TWO;
impl<K, V> Store<K, V>
where
@@ -35,6 +36,8 @@ where
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
{
pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
let version = FJALL_VERSION + version;
let meta = StoreMeta::checked_open(path, version)?;
let keyspace = match Self::open_keyspace(path) {

14
crates/brk_indexer/src/stores/mod.rs
View File

@@ -26,14 +26,12 @@ pub struct Stores {
impl Stores {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
thread::scope(|scope| {
let addresshash_to_addressindex = scope.spawn(|| {
Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))
});
let blockhash_prefix_to_height = scope.spawn(|| {
Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))
});
let txid_prefix_to_txindex = scope
.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1)));
let addresshash_to_addressindex = scope
.spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::ONE));
let blockhash_prefix_to_height = scope
.spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::ONE));
let txid_prefix_to_txindex =
scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::ONE));
Ok(Self {
addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?,

86
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -71,217 +71,217 @@ impl Vecs {
Ok(Self {
addressindex_to_addresstype: StorableVec::forced_import(
&path.join("addressindex_to_addresstype"),
Version::from(1),
Version::ONE,
compressed,
)?,
addressindex_to_addresstypeindex: StorableVec::forced_import(
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
addressindex_to_height: StorableVec::forced_import(
&path.join("addressindex_to_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_blockhash: StorableVec::forced_import(
&path.join("height_to_blockhash"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
height_to_difficulty: StorableVec::forced_import(
&path.join("height_to_difficulty"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_addressindex: StorableVec::forced_import(
&path.join("height_to_first_addressindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_emptyindex: StorableVec::forced_import(
&path.join("height_to_first_emptyindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_multisigindex: StorableVec::forced_import(
&path.join("height_to_first_multisigindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_opreturnindex: StorableVec::forced_import(
&path.join("height_to_first_opreturnindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_pushonlyindex: StorableVec::forced_import(
&path.join("height_to_first_pushonlyindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_txindex: StorableVec::forced_import(
&path.join("height_to_first_txindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_txinindex: StorableVec::forced_import(
&path.join("height_to_first_txinindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_txoutindex: StorableVec::forced_import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_unknownindex: StorableVec::forced_import(
&path.join("height_to_first_unkownindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2pk33index: StorableVec::forced_import(
&path.join("height_to_first_p2pk33index"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2pk65index: StorableVec::forced_import(
&path.join("height_to_first_p2pk65index"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2pkhindex: StorableVec::forced_import(
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2shindex: StorableVec::forced_import(
&path.join("height_to_first_p2shindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2trindex: StorableVec::forced_import(
&path.join("height_to_first_p2trindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2wpkhindex: StorableVec::forced_import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_first_p2wshindex: StorableVec::forced_import(
&path.join("height_to_first_p2wshindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_size: StorableVec::forced_import(
&path.join("height_to_size"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_timestamp: StorableVec::forced_import(
&path.join("height_to_timestamp"),
Version::from(1),
Version::ONE,
compressed,
)?,
height_to_weight: StorableVec::forced_import(
&path.join("height_to_weight"),
Version::from(1),
Version::ONE,
compressed,
)?,
p2pk33index_to_p2pk33addressbytes: StorableVec::forced_import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
p2pk65index_to_p2pk65addressbytes: StorableVec::forced_import(
&path.join("p2pk65index_to_p2pk65addressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
p2pkhindex_to_p2pkhaddressbytes: StorableVec::forced_import(
&path.join("p2pkhindex_to_p2pkhaddressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
p2shindex_to_p2shaddressbytes: StorableVec::forced_import(
&path.join("p2shindex_to_p2shaddressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
p2trindex_to_p2traddressbytes: StorableVec::forced_import(
&path.join("p2trindex_to_p2traddressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::forced_import(
&path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
p2wshindex_to_p2wshaddressbytes: StorableVec::forced_import(
&path.join("p2wshindex_to_p2wshaddressbytes"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
txindex_to_first_txinindex: StorableVec::forced_import(
&path.join("txindex_to_first_txinindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_first_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_first_txoutindex"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
txindex_to_height: StorableVec::forced_import(
&path.join("txindex_to_height"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_locktime: StorableVec::forced_import(
&path.join("txindex_to_locktime"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_txid: StorableVec::forced_import(
&path.join("txindex_to_txid"),
Version::from(1),
Version::ONE,
Compressed::NO,
)?,
txindex_to_base_size: StorableVec::forced_import(
&path.join("txindex_to_base_size"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_total_size: StorableVec::forced_import(
&path.join("txindex_to_total_size"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_is_explicitly_rbf: StorableVec::forced_import(
&path.join("txindex_to_is_explicitly_rbf"),
Version::from(1),
Version::ONE,
compressed,
)?,
txindex_to_txversion: StorableVec::forced_import(
&path.join("txindex_to_txversion"),
Version::from(1),
Version::ONE,
compressed,
)?,
txinindex_to_txoutindex: StorableVec::forced_import(
&path.join("txinindex_to_txoutindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
txoutindex_to_addressindex: StorableVec::forced_import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
Version::ONE,
compressed,
)?,
txoutindex_to_value: StorableVec::forced_import(
&path.join("txoutindex_to_value"),
Version::from(1),
Version::ONE,
compressed,
)?,
})

4
crates/brk_server/Cargo.toml
View File

@@ -24,5 +24,7 @@ minreq = { workspace = true }
oxc = { version = "0.62.0", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { version = "1.44.1", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full"] }
tower-http = { version = "0.6.2", features = ["compression-full", "trace"] }
zip = "2.5.0"
tracing = "0.1.41"
tracing-subscriber = "0.3.19"

18
crates/brk_server/src/api/query/mod.rs
View File

@@ -1,17 +1,12 @@
use std::time::Instant;
use axum::{
Json,
extract::{Query as AxumQuery, State},
http::{HeaderMap, StatusCode, Uri},
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
};
use brk_query::{Format, Index, Output, Params};
use crate::{
log_result,
traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
};
use crate::traits::{HeaderMapExtended, ModifiedState, ResponseExtended};
use super::AppState;
@@ -21,21 +16,14 @@ pub use dts::*;
pub async fn handler(
headers: HeaderMap,
uri: Uri,
query: AxumQuery<Params>,
State(app_state): State<AppState>,
) -> Response {
let instant = Instant::now();
match req_to_response_res(headers, query, app_state) {
Ok(response) => {
log_result(response.status(), &uri, instant);
response
}
Ok(response) => response,
Err(error) => {
let mut response =
(StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
log_result(response.status(), &uri, instant);
response.headers_mut().insert_cors();
response
}

22
crates/brk_server/src/files/file.rs
View File

@@ -1,15 +1,15 @@
use std::{fs, path::Path, time::Instant};
use std::{fs, path::Path};
use axum::{
body::Body,
extract::{self, State},
http::{HeaderMap, StatusCode, Uri},
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
};
use log::{error, info};
use crate::{
AppState, log_result,
AppState,
traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
};
@@ -18,24 +18,18 @@ use super::minify::minify_js;
pub async fn file_handler(
headers: HeaderMap,
State(app_state): State<AppState>,
uri: Uri,
path: extract::Path<String>,
) -> Response {
any_handler(headers, app_state, uri, Some(path))
any_handler(headers, app_state, Some(path))
}
pub async fn index_handler(
headers: HeaderMap,
State(app_state): State<AppState>,
uri: Uri,
) -> Response {
any_handler(headers, app_state, uri, None)
pub async fn index_handler(headers: HeaderMap, State(app_state): State<AppState>) -> Response {
any_handler(headers, app_state, None)
}
fn any_handler(
headers: HeaderMap,
app_state: AppState,
uri: Uri,
path: Option<extract::Path<String>>,
) -> Response {
let website_path = app_state
@@ -44,8 +38,6 @@ fn any_handler(
.expect("Should never reach here is websites_path is None")
.join(app_state.website.to_folder_name());
let instant = Instant::now();
let response = if let Some(path) = path.as_ref() {
let path = path.0.replace("..", "").replace("\\", "");
@@ -72,8 +64,6 @@ fn any_handler(
path_to_response(&headers, &website_path.join("index.html"))
};
log_result(response.status(), &uri, instant);
response
}

65
crates/brk_server/src/lib.rs
View File

@@ -7,13 +7,15 @@ use std::{
fs,
io::Cursor,
path::{Path, PathBuf},
time::Instant,
time::{Duration, Instant},
};
use api::{ApiRoutes, DTS};
use axum::{
Json, Router,
http::{StatusCode, Uri},
body::Body,
http::{Request, Response, StatusCode, Uri},
middleware::Next,
routing::get,
serve,
};
@@ -26,13 +28,14 @@ use files::FilesRoutes;
use log::{error, info};
pub use tokio;
use tokio::net::TcpListener;
use tower_http::compression::CompressionLayer;
use tower_http::{compression::CompressionLayer, trace::TraceLayer};
mod api;
mod files;
mod traits;
pub use files::Website;
use tracing::Span;
#[derive(Clone)]
pub struct AppState {
@@ -63,14 +66,16 @@ impl Server {
} else {
let downloads_path = dot_brk_path().join(DOWNLOADS);
let downloaded_websites_path = downloads_path.join("brk-main").join(WEBSITES);
let version = format!("v{}", env!("CARGO_PKG_VERSION"));
let downloaded_websites_path = downloads_path.join(&version).join(WEBSITES);
if !fs::exists(&downloaded_websites_path)? {
info!("Downloading websites from Github...");
let url = format!(
"https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{}.zip",
env!("CARGO_PKG_VERSION")
"https://github.com/bitcoinresearchkit/brk/archive/refs/tags/{}.zip",
version
);
let response = minreq::get(url).send()?;
@@ -108,12 +113,47 @@ impl Server {
.gzip(true)
.zstd(true);
let response_uri_layer = axum::middleware::from_fn(
async |request: Request<Body>, next: Next| -> Response<Body> {
let uri = request.uri().clone();
let mut response = next.run(request).await;
response.extensions_mut().insert(uri);
response
},
);
let trace_layer = TraceLayer::new_for_http()
.on_request(())
.on_response(
|response: &Response<Body>, latency: Duration, _span: &Span| {
let latency = latency.bright_black();
let status = response.status();
let uri = response.extensions().get::<Uri>().unwrap();
match status {
StatusCode::INTERNAL_SERVER_ERROR => {
error!("{} {} {:?}", status.as_u16().red(), uri, latency)
}
StatusCode::NOT_MODIFIED => {
info!("{} {} {:?}", status.as_u16().bright_black(), uri, latency)
}
StatusCode::OK => {
info!("{} {} {:?}", status.as_u16().green(), uri, latency)
}
_ => error!("{} {} {:?}", status.as_u16().red(), uri, latency),
}
},
)
.on_body_chunk(())
.on_eos(());
let router = Router::new()
.add_api_routes()
.add_website_routes(state.website)
.route("/version", get(Json(env!("CARGO_PKG_VERSION"))))
.with_state(state)
.layer(compression_layer);
.layer(compression_layer)
.layer(response_uri_layer)
.layer(trace_layer);
let mut port = 3110;
@@ -135,14 +175,3 @@ impl Server {
Ok(())
}
}
pub fn log_result(code: StatusCode, uri: &Uri, instant: Instant) {
let time = format!("{}µs", instant.elapsed().as_micros());
let time = time.bright_black();
match code {
StatusCode::INTERNAL_SERVER_ERROR => error!("{} {} {}", code.as_u16().red(), uri, time),
StatusCode::NOT_MODIFIED => info!("{} {} {}", code.as_u16().bright_black(), uri, time),
StatusCode::OK => info!("{} {} {}", code.as_u16().green(), uri, time),
_ => error!("{} {} {}", code.as_u16().red(), uri, time),
}
}

15
crates/brk_server/src/traits/header_map.rs
View File

@@ -17,9 +17,9 @@ pub enum ModifiedState {
}
pub trait HeaderMapExtended {
fn get_scheme(&self) -> &str;
fn _get_scheme(&self) -> &str;
fn get_host(&self) -> &str;
fn check_if_host_is_any_local(&self) -> bool;
fn check_if_host_is_local(&self) -> bool;
fn check_if_host_is_0000(&self) -> bool;
fn check_if_host_is_localhost(&self) -> bool;
@@ -30,8 +30,7 @@ pub trait HeaderMapExtended {
-> color_eyre::Result<(ModifiedState, DateTime)>;
fn insert_cache_control_immutable(&mut self);
#[allow(unused)]
fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
fn insert_last_modified(&mut self, date: DateTime);
fn insert_content_disposition_attachment(&mut self);
@@ -53,8 +52,8 @@ pub trait HeaderMapExtended {
}
impl HeaderMapExtended for HeaderMap {
fn get_scheme(&self) -> &str {
if self.check_if_host_is_any_local() {
fn _get_scheme(&self) -> &str {
if self.check_if_host_is_local() {
"http"
} else {
"https"
@@ -65,7 +64,7 @@ impl HeaderMapExtended for HeaderMap {
self[HOST].to_str().unwrap()
}
fn check_if_host_is_any_local(&self) -> bool {
fn check_if_host_is_local(&self) -> bool {
self.check_if_host_is_localhost() || self.check_if_host_is_0000()
}
@@ -95,7 +94,7 @@ impl HeaderMapExtended for HeaderMap {
self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap());
}
fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64) {
fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64) {
self.insert(
header::CACHE_CONTROL,
format!(

6
crates/brk_vec/examples/main.rs
View File

@@ -7,7 +7,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StorableVec<usize, u32> =
StorableVec::forced_import(Path::new("./vec"), Version::from(1), Compressed::YES)?;
StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?;
(0..21_u32).for_each(|v| {
vec.push(v);
@@ -21,7 +21,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StorableVec<usize, u32> =
StorableVec::forced_import(Path::new("./vec"), Version::from(1), Compressed::YES)?;
StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?;
dbg!(vec.get(0)?);
dbg!(vec.get(0)?);
@@ -43,7 +43,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StorableVec<usize, u32> =
StorableVec::forced_import(Path::new("./vec"), Version::from(1), Compressed::YES)?;
StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?;
vec.enable_large_cache();

4
crates/brk_vec/src/structs/version.rs
View File

@@ -15,6 +15,10 @@ use crate::{Error, Result};
pub struct Version(u32);
impl Version {
pub const ZERO: Self = Self(0);
pub const ONE: Self = Self(1);
pub const TWO: Self = Self(2);
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.as_bytes())
}