Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-10 14:19:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-04-02 16:20:17 +02:00
parent 0bb869fb33
commit a07b641adb
25 changed files with 583 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/brk_cli/src/query.rs
View File

@@ -1,4 +1,5 @@
use brk_computer::Computer;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_query::{Index, Output, Params as QueryParams, Query, Tabled, Value};
use tabled::settings::Style;
@@ -13,7 +14,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
let mut indexer = Indexer::new(config.indexeddir(), compressed, config.check_collisions())?;
indexer.import_vecs()?;
let mut computer = Computer::new(config.computeddir(), None, compressed);
let mut computer = Computer::new(config.computeddir(), config.fetcher(), compressed);
computer.import_vecs()?;
let query = Query::build(&indexer, &computer);

11
crates/brk_cli/src/run.rs
View File

@@ -32,11 +32,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
indexer.import_stores()?;
indexer.import_vecs()?;
let fetcher = config
.fetch()
.then(|| Fetcher::import(Some(config.harsdir().as_path())).unwrap());
let mut computer = Computer::new(config.computeddir(), fetcher, compressed);
let mut computer = Computer::new(config.computeddir(), config.fetcher(), compressed);
computer.import_stores()?;
computer.import_vecs()?;
@@ -401,6 +397,11 @@ impl RunConfig {
self.fetch.is_none_or(|b| b)
}
pub fn fetcher(&self) -> Option<Fetcher> {
self.fetch()
.then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap())
}
pub fn compressed(&self) -> bool {
self.compressed.is_none_or(|b| b)
}

89
crates/brk_computer/src/storage/vecs/indexes.rs
View File

@@ -1,8 +1,8 @@
use std::{fs, ops::Deref, path::Path};
use brk_core::{
Date, Dateindex, Decadeindex, Difficultyepoch, Halvingepoch, Height, Monthindex, Timestamp,
Txindex, Txinindex, Txoutindex, Weekindex, Yearindex,
Date, Dateindex, Decadeindex, Difficultyepoch, Halvingepoch, Height, Monthindex, Quarterindex,
Timestamp, Txindex, Txinindex, Txoutindex, Weekindex, Yearindex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
@@ -35,8 +35,8 @@ pub struct Vecs {
pub halvingepoch_to_timestamp: StorableVec<Halvingepoch, Timestamp>,
pub height_to_dateindex: StorableVec<Height, Dateindex>,
pub height_to_difficultyepoch: StorableVec<Height, Difficultyepoch>,
pub height_to_fixed_timestamp: StorableVec<Height, Timestamp>,
pub height_to_fixed_date: StorableVec<Height, Date>,
pub height_to_fixed_timestamp: StorableVec<Height, Timestamp>,
pub height_to_halvingepoch: StorableVec<Height, Halvingepoch>,
pub height_to_height: StorableVec<Height, Height>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
@@ -44,8 +44,13 @@ pub struct Vecs {
pub monthindex_to_first_dateindex: StorableVec<Monthindex, Dateindex>,
pub monthindex_to_last_dateindex: StorableVec<Monthindex, Dateindex>,
pub monthindex_to_monthindex: StorableVec<Monthindex, Monthindex>,
pub monthindex_to_quarterindex: StorableVec<Monthindex, Quarterindex>,
pub monthindex_to_timestamp: StorableVec<Monthindex, Timestamp>,
pub monthindex_to_yearindex: StorableVec<Monthindex, Yearindex>,
pub quarterindex_to_first_monthindex: StorableVec<Quarterindex, Monthindex>,
pub quarterindex_to_last_monthindex: StorableVec<Quarterindex, Monthindex>,
pub quarterindex_to_quarterindex: StorableVec<Quarterindex, Quarterindex>,
pub quarterindex_to_timestamp: StorableVec<Quarterindex, Timestamp>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
pub weekindex_to_first_dateindex: StorableVec<Weekindex, Dateindex>,
@@ -279,6 +284,31 @@ impl Vecs {
Version::from(1),
compressed,
)?,
monthindex_to_quarterindex: StorableVec::forced_import(
&path.join("monthindex_to_quarterindex"),
Version::from(1),
compressed,
)?,
quarterindex_to_first_monthindex: StorableVec::forced_import(
&path.join("quarterindex_to_first_monthindex"),
Version::from(1),
compressed,
)?,
quarterindex_to_last_monthindex: StorableVec::forced_import(
&path.join("quarterindex_to_last_monthindex"),
Version::from(1),
compressed,
)?,
quarterindex_to_quarterindex: StorableVec::forced_import(
&path.join("quarterindex_to_quarterindex"),
Version::from(1),
compressed,
)?,
quarterindex_to_timestamp: StorableVec::forced_import(
&path.join("quarterindex_to_timestamp"),
Version::from(1),
compressed,
)?,
})
}
@@ -509,6 +539,52 @@ impl Vecs {
// ---
let starting_quarterindex = self
.monthindex_to_quarterindex
.get(starting_monthindex)?
.copied()
.unwrap_or_default();
self.monthindex_to_quarterindex.compute_transform(
starting_monthindex,
self.monthindex_to_monthindex.mut_vec(),
|(mi, ..)| (mi, Quarterindex::from(mi)),
exit,
)?;
self.quarterindex_to_first_monthindex
.compute_inverse_more_to_less(
starting_monthindex,
self.monthindex_to_quarterindex.mut_vec(),
exit,
)?;
// let quarter_count = self.quarterindex_to_first_monthindex.len();
self.quarterindex_to_last_monthindex
.compute_last_index_from_first(
starting_quarterindex,
self.quarterindex_to_first_monthindex.mut_vec(),
month_count,
exit,
)?;
self.quarterindex_to_quarterindex.compute_transform(
starting_quarterindex,
self.quarterindex_to_first_monthindex.mut_vec(),
|(yi, ..)| (yi, yi),
exit,
)?;
self.quarterindex_to_timestamp.compute_transform(
starting_quarterindex,
self.quarterindex_to_first_monthindex.mut_vec(),
|(i, m, ..)| (i, *self.monthindex_to_timestamp.get(m).unwrap().unwrap()),
exit,
)?;
// ---
let starting_yearindex = self
.monthindex_to_yearindex
.get(starting_monthindex)?
@@ -700,6 +776,7 @@ impl Vecs {
dateindex: starting_dateindex,
weekindex: starting_weekindex,
monthindex: starting_monthindex,
quarterindex: starting_quarterindex,
yearindex: starting_yearindex,
decadeindex: starting_decadeindex,
difficultyepoch: starting_difficultyepoch,
@@ -752,6 +829,11 @@ impl Vecs {
self.weekindex_to_timestamp.any_vec(),
self.yearindex_to_timestamp.any_vec(),
self.height_to_fixed_timestamp.any_vec(),
self.monthindex_to_quarterindex.any_vec(),
self.quarterindex_to_first_monthindex.any_vec(),
self.quarterindex_to_last_monthindex.any_vec(),
self.quarterindex_to_quarterindex.any_vec(),
self.quarterindex_to_timestamp.any_vec(),
]
}
}
@@ -761,6 +843,7 @@ pub struct Indexes {
pub dateindex: Dateindex,
pub weekindex: Weekindex,
pub monthindex: Monthindex,
pub quarterindex: Quarterindex,
pub yearindex: Yearindex,
pub decadeindex: Decadeindex,
pub difficultyepoch: Difficultyepoch,

55
crates/brk_computer/src/storage/vecs/marketprice.rs
View File

@@ -2,7 +2,7 @@ use std::{fs, path::Path};
use brk_core::{
Cents, Close, Dateindex, Decadeindex, Difficultyepoch, Dollars, Height, High, Low, Monthindex,
OHLCCents, OHLCDollars, Open, Sats, Weekindex, Yearindex,
OHLCCents, OHLCDollars, Open, Quarterindex, Sats, Weekindex, Yearindex,
};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
@@ -53,6 +53,7 @@ pub struct Vecs {
pub weekindex_to_ohlc: StorableVec<Weekindex, OHLCDollars>,
pub difficultyepoch_to_ohlc: StorableVec<Difficultyepoch, OHLCDollars>,
pub monthindex_to_ohlc: StorableVec<Monthindex, OHLCDollars>,
pub quarterindex_to_ohlc: StorableVec<Quarterindex, OHLCDollars>,
pub yearindex_to_ohlc: StorableVec<Yearindex, OHLCDollars>,
// pub halvingepoch_to_ohlc: StorableVec<Halvingepoch, OHLCDollars>,
pub decadeindex_to_ohlc: StorableVec<Decadeindex, OHLCDollars>,
@@ -238,6 +239,11 @@ impl Vecs {
Version::from(1),
compressed,
)?,
quarterindex_to_ohlc: StorableVec::forced_import(
&path.join("quarterindex_to_ohlc"),
Version::from(1),
compressed,
)?,
yearindex_to_ohlc: StorableVec::forced_import(
&path.join("yearindex_to_ohlc"),
Version::from(1),
@@ -623,6 +629,52 @@ impl Vecs {
exit,
)?;
self.quarterindex_to_ohlc.compute_transform(
starting_indexes.quarterindex,
self.timeindexes_to_close
.quarterindex
.last
.as_mut()
.unwrap()
.mut_vec(),
|(i, close, ..)| {
(
i,
OHLCDollars {
open: *self
.timeindexes_to_open
.quarterindex
.first
.as_mut()
.unwrap()
.get(i)
.unwrap()
.unwrap(),
high: *self
.timeindexes_to_high
.quarterindex
.max
.as_mut()
.unwrap()
.get(i)
.unwrap()
.unwrap(),
low: *self
.timeindexes_to_low
.quarterindex
.min
.as_mut()
.unwrap()
.get(i)
.unwrap()
.unwrap(),
close,
},
)
},
exit,
)?;
self.yearindex_to_ohlc.compute_transform(
starting_indexes.yearindex,
self.timeindexes_to_close
@@ -750,6 +802,7 @@ impl Vecs {
self.weekindex_to_ohlc.any_vec(),
self.difficultyepoch_to_ohlc.any_vec(),
self.monthindex_to_ohlc.any_vec(),
self.quarterindex_to_ohlc.any_vec(),
self.yearindex_to_ohlc.any_vec(),
// self.halvingepoch_to_ohlc.any_vec(),
self.decadeindex_to_ohlc.any_vec(),

13
crates/brk_computer/src/storage/vecs/stats/from_date.rs
View File

@@ -1,6 +1,6 @@
use std::path::Path;
use brk_core::{Dateindex, Decadeindex, Monthindex, Weekindex, Yearindex};
use brk_core::{Dateindex, Decadeindex, Monthindex, Quarterindex, Weekindex, Yearindex};
use brk_exit::Exit;
use brk_vec::{AnyStorableVec, Compressed};
@@ -15,6 +15,7 @@ where
{
pub weekindex: StorableVecBuilder<Weekindex, T>,
pub monthindex: StorableVecBuilder<Monthindex, T>,
pub quarterindex: StorableVecBuilder<Quarterindex, T>,
pub yearindex: StorableVecBuilder<Yearindex, T>,
pub decadeindex: StorableVecBuilder<Decadeindex, T>,
}
@@ -34,6 +35,7 @@ where
Ok(Self {
weekindex: StorableVecBuilder::forced_import(path, compressed, options)?,
monthindex: StorableVecBuilder::forced_import(path, compressed, options)?,
quarterindex: StorableVecBuilder::forced_import(path, compressed, options)?,
yearindex: StorableVecBuilder::forced_import(path, compressed, options)?,
decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?,
})
@@ -62,6 +64,14 @@ where
exit,
)?;
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
&mut self.monthindex,
indexes.quarterindex_to_first_monthindex.mut_vec(),
indexes.quarterindex_to_last_monthindex.mut_vec(),
exit,
)?;
self.yearindex.from_aligned(
starting_indexes.yearindex,
&mut self.monthindex,
@@ -85,6 +95,7 @@ where
[
self.weekindex.as_any_vecs(),
self.monthindex.as_any_vecs(),
self.quarterindex.as_any_vecs(),
self.yearindex.as_any_vecs(),
self.decadeindex.as_any_vecs(),
]

15
crates/brk_computer/src/storage/vecs/stats/from_height.rs
View File

@@ -1,6 +1,8 @@
use std::path::Path;
use brk_core::{Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Weekindex, Yearindex};
use brk_core::{
Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Quarterindex, Weekindex, Yearindex,
};
use brk_exit::Exit;
use brk_vec::{AnyStorableVec, Compressed};
@@ -17,6 +19,7 @@ where
pub weekindex: StorableVecBuilder<Weekindex, T>,
pub difficultyepoch: StorableVecBuilder<Difficultyepoch, T>,
pub monthindex: StorableVecBuilder<Monthindex, T>,
pub quarterindex: StorableVecBuilder<Quarterindex, T>,
pub yearindex: StorableVecBuilder<Yearindex, T>,
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
pub decadeindex: StorableVecBuilder<Decadeindex, T>,
@@ -41,6 +44,7 @@ where
weekindex: StorableVecBuilder::forced_import(path, compressed, options)?,
difficultyepoch: StorableVecBuilder::forced_import(path, compressed, options)?,
monthindex: StorableVecBuilder::forced_import(path, compressed, options)?,
quarterindex: StorableVecBuilder::forced_import(path, compressed, options)?,
yearindex: StorableVecBuilder::forced_import(path, compressed, options)?,
// halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?,
decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?,
@@ -78,6 +82,14 @@ where
exit,
)?;
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
&mut self.monthindex,
indexes.quarterindex_to_first_monthindex.mut_vec(),
indexes.quarterindex_to_last_monthindex.mut_vec(),
exit,
)?;
self.yearindex.from_aligned(
starting_indexes.yearindex,
&mut self.monthindex,
@@ -111,6 +123,7 @@ where
self.weekindex.as_any_vecs(),
self.difficultyepoch.as_any_vecs(),
self.monthindex.as_any_vecs(),
self.quarterindex.as_any_vecs(),
self.yearindex.as_any_vecs(),
// self.halvingepoch.as_any_vecs(),
self.decadeindex.as_any_vecs(),

2
crates/brk_core/src/structs/mod.rs
View File

@@ -18,6 +18,7 @@ mod height;
mod locktime;
mod monthindex;
mod ohlc;
mod quarterindex;
mod sats;
mod timestamp;
mod txid;
@@ -52,6 +53,7 @@ pub use height::*;
pub use locktime::*;
pub use monthindex::*;
pub use ohlc::*;
pub use quarterindex::*;
pub use sats::*;
pub use timestamp::*;
pub use txid::*;

64
crates/brk_core/src/structs/quarterindex.rs Normal file
View File

@@ -0,0 +1,64 @@
use std::{fmt::Debug, ops::Add};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use super::Monthindex;
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
Serialize,
Deserialize,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Quarterindex(u16);
impl From<u16> for Quarterindex {
fn from(value: u16) -> Self {
Self(value)
}
}
impl From<usize> for Quarterindex {
fn from(value: usize) -> Self {
Self(value as u16)
}
}
impl From<Quarterindex> for usize {
fn from(value: Quarterindex) -> Self {
value.0 as usize
}
}
impl Add<usize> for Quarterindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(self.0 + rhs as u16)
}
}
impl From<Monthindex> for Quarterindex {
fn from(value: Monthindex) -> Self {
Self((usize::from(value) / 3) as u16)
}
}
impl CheckedSub for Quarterindex {
fn checked_sub(self, rhs: Self) -> Option<Self> {
self.0.checked_sub(rhs.0).map(Self)
}
}

68
crates/brk_query/src/index.rs
View File

@@ -19,6 +19,7 @@ pub enum Index {
Txoutindex,
Weekindex,
Monthindex,
Quarterindex,
Yearindex,
Decadeindex,
Difficultyepoch,
@@ -26,11 +27,18 @@ pub enum Index {
}
impl Index {
pub fn all() -> [Self; 19] {
pub fn all() -> [Self; 20] {
[
Self::Addressindex,
Self::Dateindex,
Self::Height,
Self::Dateindex,
Self::Weekindex,
Self::Difficultyepoch,
Self::Monthindex,
Self::Quarterindex,
Self::Yearindex,
Self::Decadeindex,
Self::Halvingepoch,
Self::Addressindex,
Self::P2PK33index,
Self::P2PK65index,
Self::P2PKHindex,
@@ -41,37 +49,32 @@ impl Index {
Self::Txindex,
Self::Txinindex,
Self::Txoutindex,
Self::Weekindex,
Self::Monthindex,
Self::Yearindex,
Self::Decadeindex,
Self::Difficultyepoch,
Self::Halvingepoch,
]
}
pub fn possible_values(&self) -> &[&str] {
// Always have the "correct" id at the end
match self {
Self::Dateindex => &["d", "date", "di", "dateindex"],
Self::Height => &["h", "height"],
Self::Txindex => &["txi", "txindex"],
Self::Txinindex => &["txini", "txinindex"],
Self::Txoutindex => &["txouti", "txoutindex"],
Self::Addressindex => &["addri", "addressindex"],
Self::P2PK33index => &["p2pk33i", "p2pk33index"],
Self::P2PK65index => &["p2pk65i", "p2pk65index"],
Self::P2PKHindex => &["p2pkhi", "p2pkhindex"],
Self::P2SHindex => &["p2shi", "p2shindex"],
Self::P2TRindex => &["p2tri", "p2trindex"],
Self::P2WPKHindex => &["p2wpkhi", "p2wpkhindex"],
Self::P2WSHindex => &["p2wshi", "p2wshindex"],
Self::Weekindex => &["w", "wi", "week", "weekindex"],
Self::Monthindex => &["m", "mi", "month", "monthindex"],
Self::Yearindex => &["y", "yi", "year", "yearindex"],
Self::Decadeindex => &["decade", "decadeindex"],
Self::Dateindex => &["d", "date", "dateindex"],
Self::Weekindex => &["w", "week", "weekindex"],
Self::Difficultyepoch => &["difficulty", "difficultyepoch"],
Self::Halvingepoch => &["halving", "halvingepoch"],
Self::Monthindex => &["m", "month", "monthindex"],
Self::Quarterindex => &["q", "quarter", "quarterindex"],
Self::Yearindex => &["y", "year", "yearindex"],
Self::Decadeindex => &["decade", "decadeindex"],
Self::Halvingepoch => &["h", "halving", "halvingepoch"],
Self::Txindex => &["tx", "txindex"],
Self::Txinindex => &["txin", "txinindex"],
Self::Txoutindex => &["txout", "txoutindex"],
Self::Addressindex => &["a", "address", "addressindex"],
Self::P2PK33index => &["p2pk33", "p2pk33index"],
Self::P2PK65index => &["p2pk65", "p2pk65index"],
Self::P2PKHindex => &["p2pkh", "p2pkhindex"],
Self::P2SHindex => &["p2sh", "p2shindex"],
Self::P2TRindex => &["p2tr", "p2trindex"],
Self::P2WPKHindex => &["p2wpkh", "p2wpkhindex"],
Self::P2WSHindex => &["p2wsh", "p2wshindex"],
}
}
@@ -81,6 +84,18 @@ impl Index {
.flat_map(|i| i.possible_values().iter().map(|s| s.to_string()))
.collect::<Vec<_>>()
}
pub fn serialize_short(&self) -> String {
self.possible_values()
.iter()
.find(|str| str.len() > 1)
.unwrap()
.to_string()
}
pub fn serialize_long(&self) -> String {
self.possible_values().last().unwrap().to_string()
}
}
impl TryFrom<&str> for Index {
@@ -106,6 +121,7 @@ impl TryFrom<&str> for Index {
v if (Self::Decadeindex).possible_values().contains(&v) => Self::Decadeindex,
v if (Self::Difficultyepoch).possible_values().contains(&v) => Self::Difficultyepoch,
v if (Self::Halvingepoch).possible_values().contains(&v) => Self::Halvingepoch,
v if (Self::Quarterindex).possible_values().contains(&v) => Self::Quarterindex,
_ => return Err(eyre!("Bad index")),
})
}

18
crates/brk_query/src/lib.rs
View File

@@ -13,39 +13,39 @@ mod index;
mod output;
mod params;
mod table;
mod tree;
mod vec_trees;
pub use format::Format;
pub use index::Index;
pub use output::{Output, Value};
pub use params::Params;
pub use table::Tabled;
use tree::VecIdToIndexToVec;
use vec_trees::VecTrees;
pub struct Query<'a> {
pub vecid_to_index_to_vec: VecIdToIndexToVec<'a>,
pub vec_trees: VecTrees<'a>,
_indexer: &'a Indexer,
_computer: &'a Computer,
}
impl<'a> Query<'a> {
pub fn build(indexer: &'a Indexer, computer: &'a Computer) -> Self {
let mut vecs = VecIdToIndexToVec::default();
let mut vec_trees = VecTrees::default();
indexer
.vecs()
.as_any_vecs()
.into_iter()
.for_each(|vec| vecs.insert(vec));
.for_each(|vec| vec_trees.insert(vec));
computer
.vecs()
.as_any_vecs()
.into_iter()
.for_each(|vec| vecs.insert(vec));
.for_each(|vec| vec_trees.insert(vec));
Self {
vecid_to_index_to_vec: vecs,
vec_trees,
_indexer: indexer,
_computer: computer,
}
@@ -65,11 +65,11 @@ impl<'a> Query<'a> {
.collect::<Vec<_>>()
})
.map(|mut id| {
let mut res = self.vecid_to_index_to_vec.get(&id);
let mut res = self.vec_trees.id_to_index_to_vec.get(&id);
if res.is_none() {
if let Ok(index) = Index::try_from(id.as_str()) {
id = index.possible_values().last().unwrap().to_string();
res = self.vecid_to_index_to_vec.get(&id)
res = self.vec_trees.id_to_index_to_vec.get(&id)
}
}
(id, res)

44
crates/brk_query/src/tree.rs
View File

@@ -1,44 +0,0 @@
use std::collections::BTreeMap;
use brk_vec::AnyStorableVec;
use derive_deref::{Deref, DerefMut};
use super::index::Index;
#[derive(Default, Deref, DerefMut)]
pub struct VecIdToIndexToVec<'a>(BTreeMap<String, IndexToVec<'a>>);
impl<'a> VecIdToIndexToVec<'a> {
// Not the most performant or type safe but only built once so that's okay
pub fn insert(&mut self, vec: &'a dyn AnyStorableVec) {
let file_name = vec.file_name();
let split = file_name.split("_to_").collect::<Vec<_>>();
if split.len() != 2 {
panic!();
}
let str = vec
.index_type_to_string()
.split("::")
.last()
.unwrap()
.to_lowercase();
let index = Index::try_from(str.as_str())
.inspect_err(|_| {
dbg!(&str);
})
.unwrap();
if split[0] != index.to_string().to_lowercase() {
dbg!(&file_name, split[0], index.to_string());
panic!();
}
let key = split[1].to_string().replace("_", "-");
let prev = self.entry(key.clone()).or_default().insert(index, vec);
if prev.is_some() {
dbg!(&key, str, file_name);
panic!()
}
}
}
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyStorableVec>);

93
crates/brk_query/src/vec_trees.rs Normal file
View File

@@ -0,0 +1,93 @@
use std::collections::BTreeMap;
use brk_vec::AnyStorableVec;
use derive_deref::{Deref, DerefMut};
use super::index::Index;
#[derive(Default)]
pub struct VecTrees<'a> {
pub id_to_index_to_vec: BTreeMap<String, IndexToVec<'a>>,
pub index_to_id_to_vec: BTreeMap<Index, IdToVec<'a>>,
}
impl<'a> VecTrees<'a> {
// Not the most performant or type safe but only built once so that's okay
pub fn insert(&mut self, vec: &'a dyn AnyStorableVec) {
let file_name = vec.file_name();
let split = file_name.split("_to_").collect::<Vec<_>>();
if split.len() != 2 {
panic!();
}
let str = vec
.index_type_to_string()
.split("::")
.last()
.unwrap()
.to_lowercase();
let index = Index::try_from(str.as_str())
.inspect_err(|_| {
dbg!(&str);
})
.unwrap();
if split[0] != index.to_string().to_lowercase() {
dbg!(&file_name, split[0], index.to_string());
panic!();
}
let key = split[1].to_string().replace("_", "-");
let prev = self
.id_to_index_to_vec
.entry(key.clone())
.or_default()
.insert(index.clone(), vec);
if prev.is_some() {
dbg!(&key, str, file_name);
panic!()
}
let prev = self
.index_to_id_to_vec
.entry(index)
.or_default()
.insert(key.clone(), vec);
if prev.is_some() {
dbg!(&key, str, file_name);
panic!()
}
}
pub fn serialize_id_to_index_to_vec(&self) -> BTreeMap<String, Vec<String>> {
self.id_to_index_to_vec
.iter()
.map(|(id, index_to_vec)| {
(
id.to_string(),
index_to_vec
.keys()
.map(|i| i.serialize_long())
.collect::<Vec<_>>(),
)
})
.collect()
}
pub fn serialize_index_to_id_to_vec(&self) -> BTreeMap<String, Vec<String>> {
self.index_to_id_to_vec
.iter()
.map(|(index, id_to_vec)| {
(
index.serialize_long(),
id_to_vec
.keys()
.map(|id| id.to_string())
.collect::<Vec<_>>(),
)
})
.collect()
}
}
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyStorableVec>);
#[derive(Default, Deref, DerefMut)]
pub struct IdToVec<'a>(BTreeMap<String, &'a dyn AnyStorableVec>);

49
crates/brk_server/README.md
View File

@@ -39,3 +39,52 @@ A crate that serves Bitcoin data and swappable front-ends, built on top of `brk_
The file handler, will serve the website specified by the user if any, which can be *no website*, *kibo.money* or *custom* (which is a blank folder for people to experiment). If a website is specified and the server is ran outside of the brk project and thus can't find the requested website, it will download the whole project with the correct version from Github and store it in `.brk` to be able to serve to website. This is due to the crate size limit on [crates.io](https://crates.io) and the various shenanigans that need to be done to have a website in a crate.
The API uses `brk_query` and so inherites all of its features including formats.
## Endpoints
### `GET /api/vecs/indexes`
A list of all possible vec indexes and their accepted format
### `GET /api/vecs/ids`
A list of all possible vec ids
### `GET /api/vecs/id-to-indexes`
A list of all possible vec ids and their supported vec indexes
### `GET /api/vecs/index-to-ids`
A list of all possible vec indexes and their supported vec ids
### `GET /api/query`
This endpoint retrieves data based on the specified vector index and values.
**Parameters:**
| Parameter | Type | Required | Description |
| --- | --- | --- | --- |
| `index` | `VecIndex` | Yes | The vector index to query. |
| `values` | `VecId[]` | Yes | A comma or space-separated list of vector IDs to retrieve. |
| `from` | `unsigned int` | No | The starting index for pagination (default is 0). |
| `to` | `unsigned int` | No | The ending index for pagination (default is the total number of results). |
| `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
**Examples:**
```
GET /api/query?index=date&values=ohlc
GET /api/query?index=week&values=ohlc,block-interval-average&from=0&to=20&format=md
```
### `GET /version`
The version of the server and thus BRK.
### `GET /*`
Catch all.
When no pattern is found, the server will look for a match inside the folder of the chosen website, if any.

46
crates/brk_server/src/api/mod.rs
View File

@@ -1,4 +1,11 @@
use axum::{Router, routing::get};
use std::collections::BTreeMap;
use axum::{
Router,
extract::State,
response::{IntoResponse, Response},
routing::get,
};
use super::AppState;
@@ -14,5 +21,42 @@ pub trait ApiRoutes {
impl ApiRoutes for Router<AppState> {
fn add_api_routes(self) -> Self {
self.route("/api/query", get(query::handler))
.route("/api/vecs/ids", get(vecids_handler))
.route("/api/vecs/indexes", get(vecindexes_handler))
.route("/api/vecs/id-to-indexes", get(vecid_to_vecindexes_handler))
.route("/api/vecs/index-to-ids", get(vecindex_to_vecids_handler))
}
}
pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
axum::Json(
app_state
.query
.vec_trees
.id_to_index_to_vec
.keys()
.collect::<Vec<_>>(),
)
.into_response()
}
pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
axum::Json(
app_state
.query
.vec_trees
.index_to_id_to_vec
.keys()
.map(|i| (i.to_string().to_lowercase(), i.possible_values()))
.collect::<BTreeMap<_, _>>(),
)
.into_response()
}
pub async fn vecid_to_vecindexes_handler(State(app_state): State<AppState>) -> Response {
axum::Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
}
pub async fn vecindex_to_vecids_handler(State(app_state): State<AppState>) -> Response {
axum::Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
}

3
crates/brk_server/src/api/query/dts.rs
View File

@@ -64,7 +64,8 @@ impl DTS for Query<'static> {
contents += "\n\n return {\n";
self.vecid_to_index_to_vec
self.vec_trees
.id_to_index_to_vec
.iter()
.for_each(|(id, index_to_vec)| {
let indexes = index_to_vec

2
crates/brk_server/src/lib.rs
View File

@@ -4,9 +4,11 @@
#![doc = "```"]
use std::{
collections::BTreeMap,
fs,
io::Cursor,
path::{Path, PathBuf},
sync::Arc,
time::Instant,
};