Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-09 14:41:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

server: smart generic vec routes build

Browse Source
This commit is contained in:
nym21
2025-02-12 14:11:04 +01:00
parent eaf76e27f5
commit 269c64e4ed
21 changed files with 278 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+2
View File
@@ -2627,6 +2627,8 @@ name = "storable_vec"
version = "0.1.2"
dependencies = [
"memmap2",
"serde",
"serde_json",
"zerocopy 0.8.17",
]
Cargo.toml
+2 -1
View File
@@ -25,6 +25,7 @@ logger = { path = "logger" }
rayon = "1.10.0"
rlimit = { version = "0.10.2" }
serde = { version = "1.0.217", features = ["derive"] }
serde_json = { version = "1.0.138", features = ["float_roundtrip"] }
server = { path = "server", package = "berver" }
storable_vec = { path = "storable_vec" }
storable_vec = { path = "storable_vec", features = ["json"] }
zerocopy = { version = "0.8.17", features = ["derive"] }
computer/src/lib.rs
+4 -2
View File
@@ -83,9 +83,11 @@ impl Computer<SINGLE_THREAD> {
let date_count = self.vecs.height_to_date.len();
// self.vecs.height_to_dateindex.compute(...)
self.vecs
.date_to_first_height
.compute_inverse_more_to_less(&mut self.vecs.height_to_date)?;
.dateindex_to_first_height
.compute_inverse_more_to_less(&mut self.vecs.height_to_dateindex)?;
// ---
// Date to X
computer/src/storage/storable_vecs.rs
+14 -6
View File
@@ -3,17 +3,21 @@ use std::{fs, path::Path};
use indexer::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use storable_vec::{StorableVec, Version};
use crate::structs::{Date, Feerate};
use crate::{
structs::{Date, Feerate},
Dateindex,
};
// mod base;
// use base::*;
pub struct StorableVecs<const MODE: u8> {
pub date_to_first_height: StorableVec<Date, Height, MODE>,
// pub date_to_last_height: StorableVec<Date, Height, MODE>,
pub dateindex_to_first_height: StorableVec<Dateindex, Height, MODE>,
// pub dateindex_to_last_height: StorableVec<Dateindex, Height, MODE>,
// pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
pub height_to_date: StorableVec<Height, Date, MODE>,
pub height_to_dateindex: StorableVec<Height, Dateindex, MODE>,
// pub height_to_fee: StorableVec<Txindex, Amount, MODE>,
// pub height_to_inputcount: StorableVec<Height, u32, MODE>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex, MODE>,
@@ -43,9 +47,13 @@ impl<const MODE: u8> StorableVecs<MODE> {
fs::create_dir_all(path)?;
Ok(Self {
date_to_first_height: StorableVec::forced_import(&path.join("date_to_first_height"), Version::from(1))?,
dateindex_to_first_height: StorableVec::forced_import(
&path.join("dateindex_to_first_height"),
Version::from(1),
)?,
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?,
height_to_dateindex: StorableVec::forced_import(&path.join("height_to_dateindex"), Version::from(1))?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::forced_import(
@@ -87,10 +95,10 @@ impl<const MODE: u8> StorableVecs<MODE> {
}
// pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] {
// [&self.date_to_first_height]
// [&self.dateindex_to_first_height]
// }
// pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] {
// [&mut self.date_to_first_height]
// [&mut self.dateindex_to_first_height]
// }
}
computer/src/structs/date.rs
+45 -61
View File
@@ -6,25 +6,25 @@ use jiff::{civil::Date as Date_, tz::TimeZone, Span};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, PartialEq, Eq, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Date {
year: u16,
month: u8,
day: u8,
}
pub struct Date(u32);
impl Date {
const INDEX_ZERO: Self = Self {
year: 2009,
month: 1,
day: 3,
};
const INDEX_ZERO: Self = Self(20090103);
const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3);
const INDEX_ONE: Self = Self {
year: 2009,
month: 1,
day: 9,
};
const INDEX_ONE: Self = Self(20090109);
const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9);
pub fn year(&self) -> u16 {
(self.0 / 1_00_00) as u16
}
pub fn month(&self) -> u8 {
((self.0 % 1_00_00) / 1_00) as u8
}
pub fn day(&self) -> u8 {
(self.0 % 1_00) as u8
}
}
impl Default for Date {
@@ -33,31 +33,15 @@ impl Default for Date {
}
}
impl PartialOrd for Date {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for Date {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
Date_::from(*self).cmp(&Date_::from(*other))
}
}
impl From<Date_> for Date {
fn from(value: Date_) -> Self {
Self {
year: value.year() as u16,
month: value.month() as u8,
day: value.day() as u8,
}
Self(value.year() as u32 * 1_00_00 + value.month() as u32 * 1_00 + value.day() as u32)
}
}
impl From<Date> for Date_ {
fn from(value: Date) -> Self {
Self::new(value.year as i16, value.month as i8, value.day as i8).unwrap()
Self::new(value.year() as i16, value.month() as i8, value.day() as i8).unwrap()
}
}
@@ -67,33 +51,33 @@ impl From<Timestamp> for Date {
}
}
impl TryFrom<Date> for usize {
type Error = color_eyre::Report;
fn try_from(value: Date) -> Result<Self, Self::Error> {
let value_ = Date_::from(value);
if value_ < Date::INDEX_ZERO_ {
Err(eyre!("Date is too early"))
} else if value == Date::INDEX_ZERO {
Ok(0)
} else if value_ < Date::INDEX_ONE_ {
Err(eyre!("Date is between first and second"))
} else if value == Date::INDEX_ONE {
Ok(1)
} else {
Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1)
}
}
}
// impl TryFrom<Date> for usize {
// type Error = color_eyre::Report;
// fn try_from(value: Date) -> Result<Self, Self::Error> {
// let value_ = Date_::from(value);
// if value_ < Date::INDEX_ZERO_ {
// Err(eyre!("Date is too early"))
// } else if value == Date::INDEX_ZERO {
// Ok(0)
// } else if value_ < Date::INDEX_ONE_ {
// Err(eyre!("Date is between first and second"))
// } else if value == Date::INDEX_ONE {
// Ok(1)
// } else {
// Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1)
// }
// }
// }
impl From<usize> for Date {
fn from(value: usize) -> Self {
Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap())
}
}
// impl From<usize> for Date {
// fn from(value: usize) -> Self {
// Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap())
// }
// }
impl Add<usize> for Date {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap())
}
}
// impl Add<usize> for Date {
// type Output = Self;
// fn add(self, rhs: usize) -> Self::Output {
// Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap())
// }
// }
computer/src/structs/dateindex.rs
+25
View File
@@ -0,0 +1,25 @@
use std::ops::Add;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Dateindex(u16);
impl From<Dateindex> for usize {
fn from(value: Dateindex) -> Self {
value.0 as usize
}
}
impl From<usize> for Dateindex {
fn from(value: usize) -> Self {
Self(value as u16)
}
}
impl Add<usize> for Dateindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u16)
}
}
computer/src/structs/mod.rs
+2
View File
@@ -1,9 +1,11 @@
mod addressindextxoutindex;
mod date;
mod dateindex;
mod feerate;
mod unit;
pub use addressindextxoutindex::*;
pub use date::*;
pub use dateindex::*;
pub use feerate::*;
pub use unit::*;
indexer/src/storage/storable_vecs/base.rs
+6 -6
View File
@@ -5,7 +5,7 @@ use std::{
path::{Path, PathBuf},
};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use storable_vec::{StoredIndex, StoredType, Version};
use super::Height;
@@ -17,8 +17,8 @@ pub struct StorableVec<I, T, const MODE: u8> {
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
pub fn import(path: &Path, version: Version) -> storable_vec::Result<Self> {
Ok(Self {
@@ -65,15 +65,15 @@ impl<I, T, const MODE: u8> DerefMut for StorableVec<I, T, MODE> {
}
}
pub trait AnyStorableVec {
pub trait AnyStorableVec: Send + Sync {
fn height(&self) -> storable_vec::Result<Height>;
fn flush(&mut self, height: Height) -> io::Result<()>;
}
impl<I, T, const MODE: u8> AnyStorableVec for StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
fn height(&self) -> storable_vec::Result<Height> {
self.height()
indexer/src/storage/storable_vecs/mod.rs
+47 -47
View File
@@ -2,7 +2,7 @@ use std::{fs, io, path::Path};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::{Version, CACHED_GETS};
use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Emptyindex, Height, LockTime,
@@ -296,67 +296,67 @@ impl<const MODE: u8> StorableVecs<MODE> {
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_slice()
self.as_mut_any_vec_slice()
.into_par_iter()
.try_for_each(|vec| vec.flush(height))
}
pub fn min_height(&self) -> color_eyre::Result<Option<Height>> {
pub fn min_height(&mut self) -> color_eyre::Result<Option<Height>> {
Ok(self
.as_slice()
.as_mut_any_vec_slice()
.into_iter()
.map(|vec| vec.height().unwrap_or_default())
.min())
}
pub fn as_slice(&self) -> [&dyn AnyStorableVec; 40] {
pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 40] {
[
&self.addressindex_to_addresstype as &dyn AnyStorableVec,
&self.addressindex_to_addresstypeindex,
&self.addressindex_to_height,
&self.height_to_blockhash,
&self.height_to_difficulty,
&self.height_to_first_addressindex,
&self.height_to_first_emptyindex,
&self.height_to_first_multisigindex,
&self.height_to_first_opreturnindex,
&self.height_to_first_pushonlyindex,
&self.height_to_first_txindex,
&self.height_to_first_txinindex,
&self.height_to_first_txoutindex,
&self.height_to_first_unknownindex,
&self.height_to_first_p2pk33index,
&self.height_to_first_p2pk65index,
&self.height_to_first_p2pkhindex,
&self.height_to_first_p2shindex,
&self.height_to_first_p2trindex,
&self.height_to_first_p2wpkhindex,
&self.height_to_first_p2wshindex,
&self.height_to_size,
&self.height_to_timestamp,
&self.height_to_weight,
&self.p2pk33index_to_p2pk33addressbytes,
&self.p2pk65index_to_p2pk65addressbytes,
&self.p2pkhindex_to_p2pkhaddressbytes,
&self.p2shindex_to_p2shaddressbytes,
&self.p2trindex_to_p2traddressbytes,
&self.p2wpkhindex_to_p2wpkhaddressbytes,
&self.p2wshindex_to_p2wshaddressbytes,
&self.txindex_to_first_txinindex,
&self.txindex_to_first_txoutindex,
&self.txindex_to_height,
&self.txindex_to_locktime,
&self.txindex_to_txid,
&self.txindex_to_txversion,
&self.txinindex_to_txoutindex,
&self.txoutindex_to_addressindex,
&self.txoutindex_to_amount,
&*self.addressindex_to_addresstype as &dyn AnyJsonStorableVec,
&*self.addressindex_to_addresstypeindex,
&*self.addressindex_to_height,
&*self.height_to_blockhash,
&*self.height_to_difficulty,
&*self.height_to_first_addressindex,
&*self.height_to_first_emptyindex,
&*self.height_to_first_multisigindex,
&*self.height_to_first_opreturnindex,
&*self.height_to_first_pushonlyindex,
&*self.height_to_first_txindex,
&*self.height_to_first_txinindex,
&*self.height_to_first_txoutindex,
&*self.height_to_first_unknownindex,
&*self.height_to_first_p2pk33index,
&*self.height_to_first_p2pk65index,
&*self.height_to_first_p2pkhindex,
&*self.height_to_first_p2shindex,
&*self.height_to_first_p2trindex,
&*self.height_to_first_p2wpkhindex,
&*self.height_to_first_p2wshindex,
&*self.height_to_size,
&*self.height_to_timestamp,
&*self.height_to_weight,
&*self.p2pk33index_to_p2pk33addressbytes,
&*self.p2pk65index_to_p2pk65addressbytes,
&*self.p2pkhindex_to_p2pkhaddressbytes,
&*self.p2shindex_to_p2shaddressbytes,
&*self.p2trindex_to_p2traddressbytes,
&*self.p2wpkhindex_to_p2wpkhaddressbytes,
&*self.p2wshindex_to_p2wshaddressbytes,
&*self.txindex_to_first_txinindex,
&*self.txindex_to_first_txoutindex,
&*self.txindex_to_height,
&*self.txindex_to_locktime,
&*self.txindex_to_txid,
&*self.txindex_to_txversion,
&*self.txinindex_to_txoutindex,
&*self.txoutindex_to_addressindex,
&*self.txoutindex_to_amount,
]
}
pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyStorableVec + Send + Sync); 40] {
pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 40] {
[
&mut self.addressindex_to_addresstype as &mut (dyn AnyStorableVec + Send + Sync),
&mut self.addressindex_to_addresstype as &mut dyn AnyStorableVec,
&mut self.addressindex_to_addresstypeindex,
&mut self.addressindex_to_height,
&mut self.height_to_blockhash,
server/Cargo.toml
+1 -12
View File
@@ -4,17 +4,6 @@ version = "0.6.0"
edition = "2021"
[dependencies]
# bitcoin_hashes = { version = "0.15.0" }
# biter = { path = "./src/crates/biter" }
# chrono = { version = "0.4.39", features = ["serde"] }
# clap = { version = "4.5.26", features = ["derive"] }
# ctrlc = { version = "3.4.5", features = ["termination"] }
# inferno = "0.12.1"
# itertools = "0.13.0"
# ordered-float = "4.6.0"
# rayon = "1.10.0"
# rlimit = "0.10.2"
# serde_json = "1.0.135"
axum = "0.8.1"
color-eyre = { workspace = true }
computer = { workspace = true }
@@ -26,7 +15,7 @@ oxc = { version = "0.49.0", features = ["codegen", "minifier"] }
regex = "1.11.1"
reqwest = { version = "0.12.12", features = ["blocking", "json"] }
serde = { workspace = true }
serde_json = { version = "1.0.138", features = ["float_roundtrip"] }
serde_json = { workspace = true }
storable_vec = { workspace = true }
tokio = { version = "1.43.0", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full"] }
server/src/api/mod.rs
+1 -1
View File
@@ -100,7 +100,7 @@ fn req_to_response_res(
let values = ids
.iter()
.flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap()))
.map(|vec| -> storable_vec::Result<Vec<Value>> { vec.collect_range(from, to) })
.map(|vec| -> storable_vec::Result<Vec<Value>> { vec.collect_range_values(from, to) })
.collect::<storable_vec::Result<Vec<_>>>()?;
if ids.is_empty() {
server/src/api/structs/index.rs
+2 -78
View File
@@ -1,15 +1,9 @@
use std::fmt::{self, Debug};
use computer::Dateindex;
use indexer::{
Addressindex, Height, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Txindex,
Txinindex, Txoutindex,
};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum Index {
Addressindex,
DateIndex,
Dateindex,
Height,
P2PK33index,
P2PK65index,
@@ -27,7 +21,7 @@ impl TryFrom<&str> for Index {
type Error = ();
fn try_from(value: &str) -> Result<Self, Self::Error> {
Ok(match value {
"d" | "date" => Self::Date,
"d" | "date" | "dateindex" => Self::Dateindex,
"h" | "height" => Self::Height,
"txi" | "txindex" => Self::Txindex,
"txini" | "txinindex" => Self::Txinindex,
@@ -50,73 +44,3 @@ impl fmt::Display for Index {
Debug::fmt(self, f)
}
}
pub trait IndexTypeToIndexEnum {
fn to_enum() -> Index;
}
impl IndexTypeToIndexEnum for Addressindex {
fn to_enum() -> Index {
Index::Addressindex
}
}
impl IndexTypeToIndexEnum for Dateindex {
fn to_enum() -> Index {
Index::DateIndex
}
}
impl IndexTypeToIndexEnum for Height {
fn to_enum() -> Index {
Index::Height
}
}
impl IndexTypeToIndexEnum for Txindex {
fn to_enum() -> Index {
Index::Txindex
}
}
impl IndexTypeToIndexEnum for Txinindex {
fn to_enum() -> Index {
Index::Txinindex
}
}
impl IndexTypeToIndexEnum for Txoutindex {
fn to_enum() -> Index {
Index::Txoutindex
}
}
impl IndexTypeToIndexEnum for P2PK33index {
fn to_enum() -> Index {
Index::P2PK33index
}
}
impl IndexTypeToIndexEnum for P2PK65index {
fn to_enum() -> Index {
Index::P2PK65index
}
}
impl IndexTypeToIndexEnum for P2PKHindex {
fn to_enum() -> Index {
Index::P2PKHindex
}
}
impl IndexTypeToIndexEnum for P2SHindex {
fn to_enum() -> Index {
Index::P2SHindex
}
}
impl IndexTypeToIndexEnum for P2TRindex {
fn to_enum() -> Index {
Index::P2TRindex
}
}
impl IndexTypeToIndexEnum for P2WPKHindex {
fn to_enum() -> Index {
Index::P2WPKHindex
}
}
impl IndexTypeToIndexEnum for P2WSHindex {
fn to_enum() -> Index {
Index::P2WSHindex
}
}
server/src/api/structs/kind.rs
-52
View File
@@ -1,52 +0,0 @@
use std::collections::BTreeSet;
use color_eyre::eyre::{eyre, ContextCompat};
use serde::Deserialize;
use crate::structs::{AnyMap, Date, Height, MapKey};
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
pub enum Kind {
Date,
Height,
// Timestamp,
// Last,
}
impl TryFrom<&String> for Kind {
type Error = color_eyre::Report;
fn try_from(str: &String) -> Result<Self, Self::Error> {
Ok(
match str
.to_lowercase()
.chars()
.next()
.context("Expect kind to have first letter")?
{
'd' => Self::Date,
'h' => Self::Height,
// 't' => Self::Timestamp,
// 'l' => Self::Last,
_ => return Err(eyre!("Bad kind")),
},
)
}
}
impl From<&(dyn AnyMap + Send + Sync)> for BTreeSet<Kind> {
fn from(map: &(dyn AnyMap + Send + Sync)) -> Self {
let mut s = Self::new();
if map.key_name() == Date::map_name() {
s.insert(Kind::Date);
}
if map.key_name() == Height::map_name() {
s.insert(Kind::Height);
s.insert(Kind::Timestamp);
}
if map.last_value().is_some() {
s.insert(Kind::Last);
}
s
}
}
server/src/lib.rs
+18 -89
View File
@@ -1,19 +1,15 @@
use std::{collections::BTreeMap, time::Instant};
use api::{
structs::{Index, IndexTypeToIndexEnum},
ApiRoutes,
};
use axum::{body::Body, response::IntoResponse, routing::get, serve, Router};
use api::{structs::Index, ApiRoutes};
use axum::{routing::get, serve, Router};
use color_eyre::owo_colors::OwoColorize;
use computer::Computer;
use derive_deref::{Deref, DerefMut};
use indexer::Indexer;
use logger::{error, info};
use oxc::syntax::identifier::LF;
use reqwest::StatusCode;
use serde::Serialize;
use serde_json::Value;
use storable_vec::{StorableVecIndex, StorableVecType, STATELESS};
use storable_vec::{AnyJsonStorableVec, STATELESS};
use tokio::net::TcpListener;
use tower_http::compression::CompressionLayer;
use website::WebsiteRoutes;
@@ -33,17 +29,18 @@ pub struct AppState {
pub struct VecIdToIndexToVec(BTreeMap<String, IndexToVec>);
impl VecIdToIndexToVec {
pub fn insert<I, T>(&mut self, vec: &'static storable_vec::StorableVec<I, T, STATELESS>)
where
I: StorableVecIndex + IndexTypeToIndexEnum + Send + Sync,
T: StorableVecType + Send + Sync + Serialize,
{
pub fn insert(&mut self, vec: &'static dyn AnyJsonStorableVec) {
let file_name = vec.file_name();
let split = file_name.split("_to_").collect::<Vec<_>>();
if split.len() != 2 {
panic!();
}
let index = vec.key_to_enum();
let str = vec.index_type_to_string().split("::").last().unwrap().to_lowercase();
let index = Index::try_from(str.as_str())
.inspect_err(|_| {
dbg!(str);
})
.unwrap();
if split[0] != index.to_string().to_lowercase() {
dbg!(split[0], index.to_string());
panic!();
@@ -58,80 +55,7 @@ impl VecIdToIndexToVec {
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec {
pub index_to_vec: BTreeMap<Index, &'static (dyn AnyStatelessStorableVec + Send + Sync)>,
}
pub trait AnyStatelessStorableVec {
fn key_to_enum(&self) -> Index;
fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> storable_vec::Result<Vec<Value>>;
// fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> storable_vec::Result<Vec<T>>;
}
impl<I, T> AnyStatelessStorableVec for storable_vec::StorableVec<I, T, STATELESS>
where
I: StorableVecIndex + IndexTypeToIndexEnum + Send + Sync,
T: StorableVecType + Send + Sync + Serialize,
{
fn key_to_enum(&self) -> Index {
I::to_enum()
}
fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> storable_vec::Result<Vec<Value>> {
Ok(self
.collect_range(from, to)?
.into_iter()
.map(|v| serde_json::to_value(v).unwrap())
.collect::<Vec<_>>())
}
}
trait StatelessVecs {
fn parse(&'static self, vecs: &mut VecIdToIndexToVec);
}
impl StatelessVecs for Indexer<STATELESS> {
fn parse(&'static self, vecs: &mut VecIdToIndexToVec) {
vecs.insert(&self.vecs.addressindex_to_addresstype);
vecs.insert(&self.vecs.addressindex_to_addresstypeindex);
vecs.insert(&self.vecs.addressindex_to_height);
vecs.insert(&self.vecs.height_to_blockhash);
vecs.insert(&self.vecs.height_to_difficulty);
vecs.insert(&self.vecs.height_to_first_addressindex);
vecs.insert(&self.vecs.height_to_first_emptyindex);
vecs.insert(&self.vecs.height_to_first_multisigindex);
vecs.insert(&self.vecs.height_to_first_opreturnindex);
vecs.insert(&self.vecs.height_to_first_pushonlyindex);
vecs.insert(&self.vecs.height_to_first_txindex);
vecs.insert(&self.vecs.height_to_first_txinindex);
vecs.insert(&self.vecs.height_to_first_txoutindex);
vecs.insert(&self.vecs.height_to_first_unknownindex);
vecs.insert(&self.vecs.height_to_first_p2pk33index);
vecs.insert(&self.vecs.height_to_first_p2pk65index);
vecs.insert(&self.vecs.height_to_first_p2pkhindex);
vecs.insert(&self.vecs.height_to_first_p2shindex);
vecs.insert(&self.vecs.height_to_first_p2trindex);
vecs.insert(&self.vecs.height_to_first_p2wpkhindex);
vecs.insert(&self.vecs.height_to_first_p2wshindex);
vecs.insert(&self.vecs.height_to_size);
vecs.insert(&self.vecs.height_to_timestamp);
vecs.insert(&self.vecs.height_to_weight);
vecs.insert(&self.vecs.p2pk33index_to_p2pk33addressbytes);
vecs.insert(&self.vecs.p2pk65index_to_p2pk65addressbytes);
vecs.insert(&self.vecs.p2pkhindex_to_p2pkhaddressbytes);
vecs.insert(&self.vecs.p2shindex_to_p2shaddressbytes);
vecs.insert(&self.vecs.p2trindex_to_p2traddressbytes);
vecs.insert(&self.vecs.p2wpkhindex_to_p2wpkhaddressbytes);
vecs.insert(&self.vecs.p2wshindex_to_p2wshaddressbytes);
vecs.insert(&self.vecs.txindex_to_first_txinindex);
vecs.insert(&self.vecs.txindex_to_first_txoutindex);
vecs.insert(&self.vecs.txindex_to_height);
vecs.insert(&self.vecs.txindex_to_locktime);
vecs.insert(&self.vecs.txindex_to_txid);
vecs.insert(&self.vecs.txindex_to_txversion);
vecs.insert(&self.vecs.txinindex_to_txoutindex);
vecs.insert(&self.vecs.txoutindex_to_addressindex);
vecs.insert(&self.vecs.txoutindex_to_amount);
}
pub index_to_vec: BTreeMap<Index, &'static dyn AnyJsonStorableVec>,
}
pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) -> color_eyre::Result<()> {
@@ -141,7 +65,12 @@ pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) ->
let indexer = Box::leak(Box::new(indexer));
let computer = Box::leak(Box::new(computer));
let vecs = Box::leak(Box::new(VecIdToIndexToVec::default()));
indexer.parse(vecs);
indexer
.vecs
.as_any_json_vec_slice()
.into_iter()
.for_each(|vec| vecs.insert(vec));
let state = AppState {
vecs,
storable_vec/Cargo.toml
+5
View File
@@ -7,6 +7,11 @@ keywords = ["vec", "disk", "data"]
categories = ["database"]
edition = "2021"
[features]
json = ["dep:serde", "dep:serde_json"]
[dependencies]
memmap2 = "0.9.5"
zerocopy = { workspace = true }
serde = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
storable_vec/src/lib.rs
+39 -33
View File
@@ -56,7 +56,6 @@ pub struct StorableVec<I, T, const MODE: u8> {
file: File,
/// **Number of values NOT number of bytes**
file_len: usize,
/// Only for SINGLE_THREAD
file_position: u64,
buf: Buffer,
/// Only for CACHED_GETS
@@ -78,8 +77,8 @@ const MAX_CACHE_SIZE: usize = usize::MAX;
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
pub const SIZE_OF_T: usize = size_of::<T>();
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
@@ -102,20 +101,23 @@ where
pub fn import(path: &Path, version: Version) -> Result<Self> {
fs::create_dir_all(path)?;
let path_version = Self::path_version_(path);
if MODE != STATELESS {
let path_version = Self::path_version_(path);
if let Ok(prev_version) = Version::try_from(path_version.as_path()) {
if prev_version != version {
if prev_version.swap_bytes() == version {
return Err(Error::WrongEndian);
if let Ok(prev_version) = Version::try_from(path_version.as_path()) {
if prev_version != version {
if prev_version.swap_bytes() == version {
return Err(Error::WrongEndian);
}
return Err(Error::DifferentVersion {
found: prev_version,
expected: version,
});
}
return Err(Error::DifferentVersion {
found: prev_version,
expected: version,
});
}
version.write(&path_version)?;
}
version.write(&path_version)?;
let file = Self::open_file_(&Self::path_vec_(path))?;
@@ -336,12 +338,16 @@ where
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
pub fn index_type_to_string(&self) -> &str {
std::any::type_name::<I>()
}
}
impl<I, T> StorableVec<I, T, CACHED_GETS>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
@@ -420,8 +426,8 @@ where
const FLUSH_EVERY: usize = 10_000;
impl<I, T> StorableVec<I, T, SINGLE_THREAD>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
pub fn get(&mut self, index: I) -> Result<&T> {
self.get_(Self::i_to_usize(index)?)
@@ -499,7 +505,7 @@ where
pub fn compute_transform<A, F>(&mut self, other: &mut StorableVec<I, A, SINGLE_THREAD>, t: F) -> Result<()>
where
A: StorableVecType,
A: StoredType,
F: Fn(&A) -> T,
{
other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))?;
@@ -508,8 +514,8 @@ where
pub fn compute_inverse_more_to_less(&mut self, other: &mut StorableVec<T, I, SINGLE_THREAD>) -> Result<()>
where
I: StorableVecType,
T: StorableVecIndex,
I: StoredType,
T: StoredIndex,
{
let index = self.last()?.cloned().unwrap_or_default();
other.iter_from(index, |(v, i)| self.push_if_needed(*i, v))?;
@@ -522,8 +528,8 @@ where
last_indexes: &mut StorableVec<T, I, SINGLE_THREAD>,
) -> Result<()>
where
I: StorableVecType,
T: StorableVecIndex,
I: StoredType,
T: StoredIndex,
{
first_indexes.iter_from(T::from(self.len()), |(value, first_index)| {
let first_index = Self::i_to_usize(*first_index)?;
@@ -539,7 +545,7 @@ where
final_len: usize,
) -> Result<()>
where
T: Copy + From<usize> + Sub<T, Output = T> + StorableVecIndex,
T: Copy + From<usize> + Sub<T, Output = T> + StoredIndex,
{
let one = T::from(1);
let mut prev_index: Option<I> = None;
@@ -563,8 +569,8 @@ where
) -> Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
let last_index = last_indexes.get(i)?;
@@ -580,9 +586,9 @@ where
other_to_self: &mut StorableVec<A, I, SINGLE_THREAD>,
) -> Result<()>
where
I: StorableVecType,
I: StoredType,
T: From<bool>,
A: StorableVecIndex + StorableVecType,
A: StoredIndex + StoredType,
{
self_to_other.iter_from(I::from(self.len()), |(i, other)| {
self.push_if_needed(i, T::from(other_to_self.get(*other)? == &i))
@@ -597,8 +603,8 @@ where
) -> Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + 'static,
F: Fn(&T2) -> T,
{
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
@@ -612,8 +618,8 @@ where
impl<I, T> StorableVec<I, T, STATELESS>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
#[inline]
pub fn get(&self, index: I) -> Result<Option<T>> {
@@ -664,8 +670,8 @@ where
impl<I, T> Clone for StorableVec<I, T, STATELESS>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
fn clone(&self) -> Self {
let path = &self.pathbuf;
storable_vec/src/traits/any.rs
+41 -6
View File
@@ -1,10 +1,12 @@
use std::io;
use std::{io, mem};
use crate::StorableVec;
use crate::{Result, StorableVec, STATELESS};
use super::{StorableVecIndex, StorableVecType};
use super::{StoredIndex, StoredType};
pub trait AnyStorableVec {
pub trait AnyStorableVec: Send + Sync {
fn file_name(&self) -> String;
fn index_type_to_string(&self) -> &str;
fn len(&self) -> usize;
fn is_empty(&self) -> bool;
fn flush(&mut self) -> io::Result<()>;
@@ -12,9 +14,17 @@ pub trait AnyStorableVec {
impl<I, T, const MODE: u8> AnyStorableVec for StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
I: StoredIndex,
T: StoredType,
{
fn file_name(&self) -> String {
self.file_name()
}
fn index_type_to_string(&self) -> &str {
self.index_type_to_string()
}
fn len(&self) -> usize {
self.len()
}
@@ -27,3 +37,28 @@ where
self.flush()
}
}
#[cfg(feature = "json")]
pub trait AnyJsonStorableVec: AnyStorableVec {
fn collect_range_values(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<serde_json::Value>>;
}
impl<I, T, const MODE: u8> AnyJsonStorableVec for StorableVec<I, T, MODE>
where
I: StoredIndex,
T: StoredType + serde::Serialize,
{
fn collect_range_values(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<serde_json::Value>> {
if MODE == STATELESS {
Ok(
unsafe { mem::transmute::<&StorableVec<I, T, MODE>, &StorableVec<I, T, STATELESS>>(self) }
.collect_range(from, to)?
.into_iter()
.map(|v| serde_json::to_value(v).unwrap())
.collect::<Vec<_>>(),
)
} else {
unreachable!()
}
}
}
storable_vec/src/traits/mod.rs
+4 -4
View File
@@ -1,8 +1,8 @@
mod any;
// mod bytes;
mod index;
mod type_;
mod stored_index;
mod stored_type;
pub use any::*;
pub use index::*;
pub use type_::*;
pub use stored_index::*;
pub use stored_type::*;
storable_vec/src/traits/index.rs → storable_vec/src/traits/stored_index.rs
+7 -3
View File
@@ -1,6 +1,6 @@
use std::{fmt::Debug, ops::Add};
pub trait StorableVecIndex
pub trait StoredIndex
where
Self: Debug
+ Default
@@ -12,10 +12,12 @@ where
+ Ord
+ TryInto<usize>
+ From<usize>
+ Add<usize, Output = Self>,
+ Add<usize, Output = Self>
+ Send
+ Sync,
{
}
impl<I> StorableVecIndex for I where
impl<I> StoredIndex for I where
I: Debug
+ Default
+ Copy
@@ -27,5 +29,7 @@ impl<I> StorableVecIndex for I where
+ TryInto<usize>
+ From<usize>
+ Add<usize, Output = Self>
+ Send
+ Sync
{
}
storable_vec/src/traits/stored_type.rs
+13
View File
@@ -0,0 +1,13 @@
use std::fmt::Debug;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
pub trait StoredType
where
Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync,
{
}
impl<T> StoredType for T where
T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync
{
}
storable_vec/src/traits/type_.rs
-10
View File
@@ -1,10 +0,0 @@
use std::fmt::Debug;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
pub trait StorableVecType
where
Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout,
{
}
impl<T> StorableVecType for T where T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout {}