diff --git a/Cargo.lock b/Cargo.lock index 3270679e2..cf1bf751b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2627,6 +2627,8 @@ name = "storable_vec" version = "0.1.2" dependencies = [ "memmap2", + "serde", + "serde_json", "zerocopy 0.8.17", ] diff --git a/Cargo.toml b/Cargo.toml index a4a4a6eb3..d7aee1aa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ logger = { path = "logger" } rayon = "1.10.0" rlimit = { version = "0.10.2" } serde = { version = "1.0.217", features = ["derive"] } +serde_json = { version = "1.0.138", features = ["float_roundtrip"] } server = { path = "server", package = "berver" } -storable_vec = { path = "storable_vec" } +storable_vec = { path = "storable_vec", features = ["json"] } zerocopy = { version = "0.8.17", features = ["derive"] } diff --git a/computer/src/lib.rs b/computer/src/lib.rs index 4cf2cb9ad..775ed732b 100644 --- a/computer/src/lib.rs +++ b/computer/src/lib.rs @@ -83,9 +83,11 @@ impl Computer { let date_count = self.vecs.height_to_date.len(); + // self.vecs.height_to_dateindex.compute(...) + self.vecs - .date_to_first_height - .compute_inverse_more_to_less(&mut self.vecs.height_to_date)?; + .dateindex_to_first_height + .compute_inverse_more_to_less(&mut self.vecs.height_to_dateindex)?; // --- // Date to X diff --git a/computer/src/storage/storable_vecs.rs b/computer/src/storage/storable_vecs.rs index c7835320d..39fe0207b 100644 --- a/computer/src/storage/storable_vecs.rs +++ b/computer/src/storage/storable_vecs.rs @@ -3,17 +3,21 @@ use std::{fs, path::Path}; use indexer::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex}; use storable_vec::{StorableVec, Version}; -use crate::structs::{Date, Feerate}; +use crate::{ + structs::{Date, Feerate}, + Dateindex, +}; // mod base; // use base::*; pub struct StorableVecs { - pub date_to_first_height: StorableVec, - // pub date_to_last_height: StorableVec, + pub dateindex_to_first_height: StorableVec, + // pub dateindex_to_last_height: StorableVec, // pub height_to_block_interval: StorableVec, pub height_to_date: StorableVec, + pub height_to_dateindex: StorableVec, // pub height_to_fee: StorableVec, // pub height_to_inputcount: StorableVec, // pub height_to_last_addressindex: StorableVec, @@ -43,9 +47,13 @@ impl StorableVecs { fs::create_dir_all(path)?; Ok(Self { - date_to_first_height: StorableVec::forced_import(&path.join("date_to_first_height"), Version::from(1))?, + dateindex_to_first_height: StorableVec::forced_import( + &path.join("dateindex_to_first_height"), + Version::from(1), + )?, // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?, height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?, + height_to_dateindex: StorableVec::forced_import(&path.join("height_to_dateindex"), Version::from(1))?, // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?, // height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?, // height_to_last_addressindex: StorableVec::forced_import( @@ -87,10 +95,10 @@ impl StorableVecs { } // pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] { - // [&self.date_to_first_height] + // [&self.dateindex_to_first_height] // } // pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] { - // [&mut self.date_to_first_height] + // [&mut self.dateindex_to_first_height] // } } diff --git a/computer/src/structs/date.rs b/computer/src/structs/date.rs index 583389450..997559a0b 100644 --- a/computer/src/structs/date.rs +++ b/computer/src/structs/date.rs @@ -6,25 +6,25 @@ use jiff::{civil::Date as Date_, tz::TimeZone, Span}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive(Debug, Clone, Copy, PartialEq, Eq, FromBytes, Immutable, IntoBytes, KnownLayout)] -pub struct Date { - year: u16, - month: u8, - day: u8, -} +pub struct Date(u32); impl Date { - const INDEX_ZERO: Self = Self { - year: 2009, - month: 1, - day: 3, - }; + const INDEX_ZERO: Self = Self(20090103); const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3); - const INDEX_ONE: Self = Self { - year: 2009, - month: 1, - day: 9, - }; + const INDEX_ONE: Self = Self(20090109); const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9); + + pub fn year(&self) -> u16 { + (self.0 / 1_00_00) as u16 + } + + pub fn month(&self) -> u8 { + ((self.0 % 1_00_00) / 1_00) as u8 + } + + pub fn day(&self) -> u8 { + (self.0 % 1_00) as u8 + } } impl Default for Date { @@ -33,31 +33,15 @@ impl Default for Date { } } -impl PartialOrd for Date { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Date { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - Date_::from(*self).cmp(&Date_::from(*other)) - } -} - impl From for Date { fn from(value: Date_) -> Self { - Self { - year: value.year() as u16, - month: value.month() as u8, - day: value.day() as u8, - } + Self(value.year() as u32 * 1_00_00 + value.month() as u32 * 1_00 + value.day() as u32) } } impl From for Date_ { fn from(value: Date) -> Self { - Self::new(value.year as i16, value.month as i8, value.day as i8).unwrap() + Self::new(value.year() as i16, value.month() as i8, value.day() as i8).unwrap() } } @@ -67,33 +51,33 @@ impl From for Date { } } -impl TryFrom for usize { - type Error = color_eyre::Report; - fn try_from(value: Date) -> Result { - let value_ = Date_::from(value); - if value_ < Date::INDEX_ZERO_ { - Err(eyre!("Date is too early")) - } else if value == Date::INDEX_ZERO { - Ok(0) - } else if value_ < Date::INDEX_ONE_ { - Err(eyre!("Date is between first and second")) - } else if value == Date::INDEX_ONE { - Ok(1) - } else { - Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1) - } - } -} +// impl TryFrom for usize { +// type Error = color_eyre::Report; +// fn try_from(value: Date) -> Result { +// let value_ = Date_::from(value); +// if value_ < Date::INDEX_ZERO_ { +// Err(eyre!("Date is too early")) +// } else if value == Date::INDEX_ZERO { +// Ok(0) +// } else if value_ < Date::INDEX_ONE_ { +// Err(eyre!("Date is between first and second")) +// } else if value == Date::INDEX_ONE { +// Ok(1) +// } else { +// Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1) +// } +// } +// } -impl From for Date { - fn from(value: usize) -> Self { - Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap()) - } -} +// impl From for Date { +// fn from(value: usize) -> Self { +// Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap()) +// } +// } -impl Add for Date { - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap()) - } -} +// impl Add for Date { +// type Output = Self; +// fn add(self, rhs: usize) -> Self::Output { +// Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap()) +// } +// } diff --git a/computer/src/structs/dateindex.rs b/computer/src/structs/dateindex.rs index e69de29bb..64338bed9 100644 --- a/computer/src/structs/dateindex.rs +++ b/computer/src/structs/dateindex.rs @@ -0,0 +1,25 @@ +use std::ops::Add; + +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] +pub struct Dateindex(u16); + +impl From for usize { + fn from(value: Dateindex) -> Self { + value.0 as usize + } +} + +impl From for Dateindex { + fn from(value: usize) -> Self { + Self(value as u16) + } +} + +impl Add for Dateindex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs as u16) + } +} diff --git a/computer/src/structs/mod.rs b/computer/src/structs/mod.rs index b73b18519..60235dc16 100644 --- a/computer/src/structs/mod.rs +++ b/computer/src/structs/mod.rs @@ -1,9 +1,11 @@ mod addressindextxoutindex; mod date; +mod dateindex; mod feerate; mod unit; pub use addressindextxoutindex::*; pub use date::*; +pub use dateindex::*; pub use feerate::*; pub use unit::*; diff --git a/indexer/src/storage/storable_vecs/base.rs b/indexer/src/storage/storable_vecs/base.rs index e165431ce..c05d21c5a 100644 --- a/indexer/src/storage/storable_vecs/base.rs +++ b/indexer/src/storage/storable_vecs/base.rs @@ -5,7 +5,7 @@ use std::{ path::{Path, PathBuf}, }; -use storable_vec::{StorableVecIndex, StorableVecType, Version}; +use storable_vec::{StoredIndex, StoredType, Version}; use super::Height; @@ -17,8 +17,8 @@ pub struct StorableVec { impl StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { pub fn import(path: &Path, version: Version) -> storable_vec::Result { Ok(Self { @@ -65,15 +65,15 @@ impl DerefMut for StorableVec { } } -pub trait AnyStorableVec { +pub trait AnyStorableVec: Send + Sync { fn height(&self) -> storable_vec::Result; fn flush(&mut self, height: Height) -> io::Result<()>; } impl AnyStorableVec for StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { fn height(&self) -> storable_vec::Result { self.height() diff --git a/indexer/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs index be26308d9..9abab5799 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/indexer/src/storage/storable_vecs/mod.rs @@ -2,7 +2,7 @@ use std::{fs, io, path::Path}; use exit::Exit; use rayon::prelude::*; -use storable_vec::{Version, CACHED_GETS}; +use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS}; use crate::structs::{ Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Emptyindex, Height, LockTime, @@ -296,67 +296,67 @@ impl StorableVecs { } pub fn flush(&mut self, height: Height) -> io::Result<()> { - self.as_mut_slice() + self.as_mut_any_vec_slice() .into_par_iter() .try_for_each(|vec| vec.flush(height)) } - pub fn min_height(&self) -> color_eyre::Result> { + pub fn min_height(&mut self) -> color_eyre::Result> { Ok(self - .as_slice() + .as_mut_any_vec_slice() .into_iter() .map(|vec| vec.height().unwrap_or_default()) .min()) } - pub fn as_slice(&self) -> [&dyn AnyStorableVec; 40] { + pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 40] { [ - &self.addressindex_to_addresstype as &dyn AnyStorableVec, - &self.addressindex_to_addresstypeindex, - &self.addressindex_to_height, - &self.height_to_blockhash, - &self.height_to_difficulty, - &self.height_to_first_addressindex, - &self.height_to_first_emptyindex, - &self.height_to_first_multisigindex, - &self.height_to_first_opreturnindex, - &self.height_to_first_pushonlyindex, - &self.height_to_first_txindex, - &self.height_to_first_txinindex, - &self.height_to_first_txoutindex, - &self.height_to_first_unknownindex, - &self.height_to_first_p2pk33index, - &self.height_to_first_p2pk65index, - &self.height_to_first_p2pkhindex, - &self.height_to_first_p2shindex, - &self.height_to_first_p2trindex, - &self.height_to_first_p2wpkhindex, - &self.height_to_first_p2wshindex, - &self.height_to_size, - &self.height_to_timestamp, - &self.height_to_weight, - &self.p2pk33index_to_p2pk33addressbytes, - &self.p2pk65index_to_p2pk65addressbytes, - &self.p2pkhindex_to_p2pkhaddressbytes, - &self.p2shindex_to_p2shaddressbytes, - &self.p2trindex_to_p2traddressbytes, - &self.p2wpkhindex_to_p2wpkhaddressbytes, - &self.p2wshindex_to_p2wshaddressbytes, - &self.txindex_to_first_txinindex, - &self.txindex_to_first_txoutindex, - &self.txindex_to_height, - &self.txindex_to_locktime, - &self.txindex_to_txid, - &self.txindex_to_txversion, - &self.txinindex_to_txoutindex, - &self.txoutindex_to_addressindex, - &self.txoutindex_to_amount, + &*self.addressindex_to_addresstype as &dyn AnyJsonStorableVec, + &*self.addressindex_to_addresstypeindex, + &*self.addressindex_to_height, + &*self.height_to_blockhash, + &*self.height_to_difficulty, + &*self.height_to_first_addressindex, + &*self.height_to_first_emptyindex, + &*self.height_to_first_multisigindex, + &*self.height_to_first_opreturnindex, + &*self.height_to_first_pushonlyindex, + &*self.height_to_first_txindex, + &*self.height_to_first_txinindex, + &*self.height_to_first_txoutindex, + &*self.height_to_first_unknownindex, + &*self.height_to_first_p2pk33index, + &*self.height_to_first_p2pk65index, + &*self.height_to_first_p2pkhindex, + &*self.height_to_first_p2shindex, + &*self.height_to_first_p2trindex, + &*self.height_to_first_p2wpkhindex, + &*self.height_to_first_p2wshindex, + &*self.height_to_size, + &*self.height_to_timestamp, + &*self.height_to_weight, + &*self.p2pk33index_to_p2pk33addressbytes, + &*self.p2pk65index_to_p2pk65addressbytes, + &*self.p2pkhindex_to_p2pkhaddressbytes, + &*self.p2shindex_to_p2shaddressbytes, + &*self.p2trindex_to_p2traddressbytes, + &*self.p2wpkhindex_to_p2wpkhaddressbytes, + &*self.p2wshindex_to_p2wshaddressbytes, + &*self.txindex_to_first_txinindex, + &*self.txindex_to_first_txoutindex, + &*self.txindex_to_height, + &*self.txindex_to_locktime, + &*self.txindex_to_txid, + &*self.txindex_to_txversion, + &*self.txinindex_to_txoutindex, + &*self.txoutindex_to_addressindex, + &*self.txoutindex_to_amount, ] } - pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyStorableVec + Send + Sync); 40] { + pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 40] { [ - &mut self.addressindex_to_addresstype as &mut (dyn AnyStorableVec + Send + Sync), + &mut self.addressindex_to_addresstype as &mut dyn AnyStorableVec, &mut self.addressindex_to_addresstypeindex, &mut self.addressindex_to_height, &mut self.height_to_blockhash, diff --git a/server/Cargo.toml b/server/Cargo.toml index e402111b2..4e858a313 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -4,17 +4,6 @@ version = "0.6.0" edition = "2021" [dependencies] -# bitcoin_hashes = { version = "0.15.0" } -# biter = { path = "./src/crates/biter" } -# chrono = { version = "0.4.39", features = ["serde"] } -# clap = { version = "4.5.26", features = ["derive"] } -# ctrlc = { version = "3.4.5", features = ["termination"] } -# inferno = "0.12.1" -# itertools = "0.13.0" -# ordered-float = "4.6.0" -# rayon = "1.10.0" -# rlimit = "0.10.2" -# serde_json = "1.0.135" axum = "0.8.1" color-eyre = { workspace = true } computer = { workspace = true } @@ -26,7 +15,7 @@ oxc = { version = "0.49.0", features = ["codegen", "minifier"] } regex = "1.11.1" reqwest = { version = "0.12.12", features = ["blocking", "json"] } serde = { workspace = true } -serde_json = { version = "1.0.138", features = ["float_roundtrip"] } +serde_json = { workspace = true } storable_vec = { workspace = true } tokio = { version = "1.43.0", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full"] } diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 44a709288..0d88c051f 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -100,7 +100,7 @@ fn req_to_response_res( let values = ids .iter() .flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap())) - .map(|vec| -> storable_vec::Result> { vec.collect_range(from, to) }) + .map(|vec| -> storable_vec::Result> { vec.collect_range_values(from, to) }) .collect::>>()?; if ids.is_empty() { diff --git a/server/src/api/structs/index.rs b/server/src/api/structs/index.rs index a8025a659..69d0c7ac0 100644 --- a/server/src/api/structs/index.rs +++ b/server/src/api/structs/index.rs @@ -1,15 +1,9 @@ use std::fmt::{self, Debug}; -use computer::Dateindex; -use indexer::{ - Addressindex, Height, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Txindex, - Txinindex, Txoutindex, -}; - #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum Index { Addressindex, - DateIndex, + Dateindex, Height, P2PK33index, P2PK65index, @@ -27,7 +21,7 @@ impl TryFrom<&str> for Index { type Error = (); fn try_from(value: &str) -> Result { Ok(match value { - "d" | "date" => Self::Date, + "d" | "date" | "dateindex" => Self::Dateindex, "h" | "height" => Self::Height, "txi" | "txindex" => Self::Txindex, "txini" | "txinindex" => Self::Txinindex, @@ -50,73 +44,3 @@ impl fmt::Display for Index { Debug::fmt(self, f) } } - -pub trait IndexTypeToIndexEnum { - fn to_enum() -> Index; -} - -impl IndexTypeToIndexEnum for Addressindex { - fn to_enum() -> Index { - Index::Addressindex - } -} -impl IndexTypeToIndexEnum for Dateindex { - fn to_enum() -> Index { - Index::DateIndex - } -} -impl IndexTypeToIndexEnum for Height { - fn to_enum() -> Index { - Index::Height - } -} -impl IndexTypeToIndexEnum for Txindex { - fn to_enum() -> Index { - Index::Txindex - } -} -impl IndexTypeToIndexEnum for Txinindex { - fn to_enum() -> Index { - Index::Txinindex - } -} -impl IndexTypeToIndexEnum for Txoutindex { - fn to_enum() -> Index { - Index::Txoutindex - } -} -impl IndexTypeToIndexEnum for P2PK33index { - fn to_enum() -> Index { - Index::P2PK33index - } -} -impl IndexTypeToIndexEnum for P2PK65index { - fn to_enum() -> Index { - Index::P2PK65index - } -} -impl IndexTypeToIndexEnum for P2PKHindex { - fn to_enum() -> Index { - Index::P2PKHindex - } -} -impl IndexTypeToIndexEnum for P2SHindex { - fn to_enum() -> Index { - Index::P2SHindex - } -} -impl IndexTypeToIndexEnum for P2TRindex { - fn to_enum() -> Index { - Index::P2TRindex - } -} -impl IndexTypeToIndexEnum for P2WPKHindex { - fn to_enum() -> Index { - Index::P2WPKHindex - } -} -impl IndexTypeToIndexEnum for P2WSHindex { - fn to_enum() -> Index { - Index::P2WSHindex - } -} diff --git a/server/src/api/structs/kind.rs b/server/src/api/structs/kind.rs deleted file mode 100644 index 31e6a852a..000000000 --- a/server/src/api/structs/kind.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::collections::BTreeSet; - -use color_eyre::eyre::{eyre, ContextCompat}; -use serde::Deserialize; - -use crate::structs::{AnyMap, Date, Height, MapKey}; - -#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub enum Kind { - Date, - Height, - // Timestamp, - // Last, -} - -impl TryFrom<&String> for Kind { - type Error = color_eyre::Report; - - fn try_from(str: &String) -> Result { - Ok( - match str - .to_lowercase() - .chars() - .next() - .context("Expect kind to have first letter")? - { - 'd' => Self::Date, - 'h' => Self::Height, - // 't' => Self::Timestamp, - // 'l' => Self::Last, - _ => return Err(eyre!("Bad kind")), - }, - ) - } -} - -impl From<&(dyn AnyMap + Send + Sync)> for BTreeSet { - fn from(map: &(dyn AnyMap + Send + Sync)) -> Self { - let mut s = Self::new(); - if map.key_name() == Date::map_name() { - s.insert(Kind::Date); - } - if map.key_name() == Height::map_name() { - s.insert(Kind::Height); - s.insert(Kind::Timestamp); - } - if map.last_value().is_some() { - s.insert(Kind::Last); - } - s - } -} diff --git a/server/src/lib.rs b/server/src/lib.rs index bf1271520..631e14e2a 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,19 +1,15 @@ use std::{collections::BTreeMap, time::Instant}; -use api::{ - structs::{Index, IndexTypeToIndexEnum}, - ApiRoutes, -}; -use axum::{body::Body, response::IntoResponse, routing::get, serve, Router}; +use api::{structs::Index, ApiRoutes}; +use axum::{routing::get, serve, Router}; use color_eyre::owo_colors::OwoColorize; use computer::Computer; use derive_deref::{Deref, DerefMut}; use indexer::Indexer; use logger::{error, info}; +use oxc::syntax::identifier::LF; use reqwest::StatusCode; -use serde::Serialize; -use serde_json::Value; -use storable_vec::{StorableVecIndex, StorableVecType, STATELESS}; +use storable_vec::{AnyJsonStorableVec, STATELESS}; use tokio::net::TcpListener; use tower_http::compression::CompressionLayer; use website::WebsiteRoutes; @@ -33,17 +29,18 @@ pub struct AppState { pub struct VecIdToIndexToVec(BTreeMap); impl VecIdToIndexToVec { - pub fn insert(&mut self, vec: &'static storable_vec::StorableVec) - where - I: StorableVecIndex + IndexTypeToIndexEnum + Send + Sync, - T: StorableVecType + Send + Sync + Serialize, - { + pub fn insert(&mut self, vec: &'static dyn AnyJsonStorableVec) { let file_name = vec.file_name(); let split = file_name.split("_to_").collect::>(); if split.len() != 2 { panic!(); } - let index = vec.key_to_enum(); + let str = vec.index_type_to_string().split("::").last().unwrap().to_lowercase(); + let index = Index::try_from(str.as_str()) + .inspect_err(|_| { + dbg!(str); + }) + .unwrap(); if split[0] != index.to_string().to_lowercase() { dbg!(split[0], index.to_string()); panic!(); @@ -58,80 +55,7 @@ impl VecIdToIndexToVec { #[derive(Default, Deref, DerefMut)] pub struct IndexToVec { - pub index_to_vec: BTreeMap, -} - -pub trait AnyStatelessStorableVec { - fn key_to_enum(&self) -> Index; - fn collect_range(&self, from: Option, to: Option) -> storable_vec::Result>; - // fn collect_range(&self, from: Option, to: Option) -> storable_vec::Result>; -} - -impl AnyStatelessStorableVec for storable_vec::StorableVec -where - I: StorableVecIndex + IndexTypeToIndexEnum + Send + Sync, - T: StorableVecType + Send + Sync + Serialize, -{ - fn key_to_enum(&self) -> Index { - I::to_enum() - } - - fn collect_range(&self, from: Option, to: Option) -> storable_vec::Result> { - Ok(self - .collect_range(from, to)? - .into_iter() - .map(|v| serde_json::to_value(v).unwrap()) - .collect::>()) - } -} - -trait StatelessVecs { - fn parse(&'static self, vecs: &mut VecIdToIndexToVec); -} - -impl StatelessVecs for Indexer { - fn parse(&'static self, vecs: &mut VecIdToIndexToVec) { - vecs.insert(&self.vecs.addressindex_to_addresstype); - vecs.insert(&self.vecs.addressindex_to_addresstypeindex); - vecs.insert(&self.vecs.addressindex_to_height); - vecs.insert(&self.vecs.height_to_blockhash); - vecs.insert(&self.vecs.height_to_difficulty); - vecs.insert(&self.vecs.height_to_first_addressindex); - vecs.insert(&self.vecs.height_to_first_emptyindex); - vecs.insert(&self.vecs.height_to_first_multisigindex); - vecs.insert(&self.vecs.height_to_first_opreturnindex); - vecs.insert(&self.vecs.height_to_first_pushonlyindex); - vecs.insert(&self.vecs.height_to_first_txindex); - vecs.insert(&self.vecs.height_to_first_txinindex); - vecs.insert(&self.vecs.height_to_first_txoutindex); - vecs.insert(&self.vecs.height_to_first_unknownindex); - vecs.insert(&self.vecs.height_to_first_p2pk33index); - vecs.insert(&self.vecs.height_to_first_p2pk65index); - vecs.insert(&self.vecs.height_to_first_p2pkhindex); - vecs.insert(&self.vecs.height_to_first_p2shindex); - vecs.insert(&self.vecs.height_to_first_p2trindex); - vecs.insert(&self.vecs.height_to_first_p2wpkhindex); - vecs.insert(&self.vecs.height_to_first_p2wshindex); - vecs.insert(&self.vecs.height_to_size); - vecs.insert(&self.vecs.height_to_timestamp); - vecs.insert(&self.vecs.height_to_weight); - vecs.insert(&self.vecs.p2pk33index_to_p2pk33addressbytes); - vecs.insert(&self.vecs.p2pk65index_to_p2pk65addressbytes); - vecs.insert(&self.vecs.p2pkhindex_to_p2pkhaddressbytes); - vecs.insert(&self.vecs.p2shindex_to_p2shaddressbytes); - vecs.insert(&self.vecs.p2trindex_to_p2traddressbytes); - vecs.insert(&self.vecs.p2wpkhindex_to_p2wpkhaddressbytes); - vecs.insert(&self.vecs.p2wshindex_to_p2wshaddressbytes); - vecs.insert(&self.vecs.txindex_to_first_txinindex); - vecs.insert(&self.vecs.txindex_to_first_txoutindex); - vecs.insert(&self.vecs.txindex_to_height); - vecs.insert(&self.vecs.txindex_to_locktime); - vecs.insert(&self.vecs.txindex_to_txid); - vecs.insert(&self.vecs.txindex_to_txversion); - vecs.insert(&self.vecs.txinindex_to_txoutindex); - vecs.insert(&self.vecs.txoutindex_to_addressindex); - vecs.insert(&self.vecs.txoutindex_to_amount); - } + pub index_to_vec: BTreeMap, } pub async fn main(indexer: Indexer, computer: Computer) -> color_eyre::Result<()> { @@ -141,7 +65,12 @@ pub async fn main(indexer: Indexer, computer: Computer) -> let indexer = Box::leak(Box::new(indexer)); let computer = Box::leak(Box::new(computer)); let vecs = Box::leak(Box::new(VecIdToIndexToVec::default())); - indexer.parse(vecs); + + indexer + .vecs + .as_any_json_vec_slice() + .into_iter() + .for_each(|vec| vecs.insert(vec)); let state = AppState { vecs, diff --git a/storable_vec/Cargo.toml b/storable_vec/Cargo.toml index 5604fb80b..dc4fd3c8f 100644 --- a/storable_vec/Cargo.toml +++ b/storable_vec/Cargo.toml @@ -7,6 +7,11 @@ keywords = ["vec", "disk", "data"] categories = ["database"] edition = "2021" +[features] +json = ["dep:serde", "dep:serde_json"] + [dependencies] memmap2 = "0.9.5" zerocopy = { workspace = true } +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } diff --git a/storable_vec/src/lib.rs b/storable_vec/src/lib.rs index 98105b8f1..020873551 100644 --- a/storable_vec/src/lib.rs +++ b/storable_vec/src/lib.rs @@ -56,7 +56,6 @@ pub struct StorableVec { file: File, /// **Number of values NOT number of bytes** file_len: usize, - /// Only for SINGLE_THREAD file_position: u64, buf: Buffer, /// Only for CACHED_GETS @@ -78,8 +77,8 @@ const MAX_CACHE_SIZE: usize = usize::MAX; impl StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { pub const SIZE_OF_T: usize = size_of::(); pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; @@ -102,20 +101,23 @@ where pub fn import(path: &Path, version: Version) -> Result { fs::create_dir_all(path)?; - let path_version = Self::path_version_(path); + if MODE != STATELESS { + let path_version = Self::path_version_(path); - if let Ok(prev_version) = Version::try_from(path_version.as_path()) { - if prev_version != version { - if prev_version.swap_bytes() == version { - return Err(Error::WrongEndian); + if let Ok(prev_version) = Version::try_from(path_version.as_path()) { + if prev_version != version { + if prev_version.swap_bytes() == version { + return Err(Error::WrongEndian); + } + return Err(Error::DifferentVersion { + found: prev_version, + expected: version, + }); } - return Err(Error::DifferentVersion { - found: prev_version, - expected: version, - }); } + + version.write(&path_version)?; } - version.write(&path_version)?; let file = Self::open_file_(&Self::path_vec_(path))?; @@ -336,12 +338,16 @@ where fn path_version_(path: &Path) -> PathBuf { path.join("version") } + + pub fn index_type_to_string(&self) -> &str { + std::any::type_name::() + } } impl StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { #[inline] pub fn get(&self, index: I) -> Result>> { @@ -420,8 +426,8 @@ where const FLUSH_EVERY: usize = 10_000; impl StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { pub fn get(&mut self, index: I) -> Result<&T> { self.get_(Self::i_to_usize(index)?) @@ -499,7 +505,7 @@ where pub fn compute_transform(&mut self, other: &mut StorableVec, t: F) -> Result<()> where - A: StorableVecType, + A: StoredType, F: Fn(&A) -> T, { other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))?; @@ -508,8 +514,8 @@ where pub fn compute_inverse_more_to_less(&mut self, other: &mut StorableVec) -> Result<()> where - I: StorableVecType, - T: StorableVecIndex, + I: StoredType, + T: StoredIndex, { let index = self.last()?.cloned().unwrap_or_default(); other.iter_from(index, |(v, i)| self.push_if_needed(*i, v))?; @@ -522,8 +528,8 @@ where last_indexes: &mut StorableVec, ) -> Result<()> where - I: StorableVecType, - T: StorableVecIndex, + I: StoredType, + T: StoredIndex, { first_indexes.iter_from(T::from(self.len()), |(value, first_index)| { let first_index = Self::i_to_usize(*first_index)?; @@ -539,7 +545,7 @@ where final_len: usize, ) -> Result<()> where - T: Copy + From + Sub + StorableVecIndex, + T: Copy + From + Sub + StoredIndex, { let one = T::from(1); let mut prev_index: Option = None; @@ -563,8 +569,8 @@ where ) -> Result<()> where T: From, - T2: StorableVecType + Copy + Add + Sub + TryInto, - >::Error: error::Error + Send + Sync + 'static, + T2: StoredType + Copy + Add + Sub + TryInto, + >::Error: error::Error + 'static, { first_indexes.iter_from(I::from(self.len()), |(i, first_index)| { let last_index = last_indexes.get(i)?; @@ -580,9 +586,9 @@ where other_to_self: &mut StorableVec, ) -> Result<()> where - I: StorableVecType, + I: StoredType, T: From, - A: StorableVecIndex + StorableVecType, + A: StoredIndex + StoredType, { self_to_other.iter_from(I::from(self.len()), |(i, other)| { self.push_if_needed(i, T::from(other_to_self.get(*other)? == &i)) @@ -597,8 +603,8 @@ where ) -> Result<()> where T: From, - T2: StorableVecType + Copy + Add + Sub + TryInto, - >::Error: error::Error + Send + Sync + 'static, + T2: StoredType + Copy + Add + Sub + TryInto, + >::Error: error::Error + 'static, F: Fn(&T2) -> T, { first_indexes.iter_from(I::from(self.len()), |(i, first_index)| { @@ -612,8 +618,8 @@ where impl StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { #[inline] pub fn get(&self, index: I) -> Result> { @@ -664,8 +670,8 @@ where impl Clone for StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { fn clone(&self) -> Self { let path = &self.pathbuf; diff --git a/storable_vec/src/traits/any.rs b/storable_vec/src/traits/any.rs index 8dcf72a82..5a9e21697 100644 --- a/storable_vec/src/traits/any.rs +++ b/storable_vec/src/traits/any.rs @@ -1,10 +1,12 @@ -use std::io; +use std::{io, mem}; -use crate::StorableVec; +use crate::{Result, StorableVec, STATELESS}; -use super::{StorableVecIndex, StorableVecType}; +use super::{StoredIndex, StoredType}; -pub trait AnyStorableVec { +pub trait AnyStorableVec: Send + Sync { + fn file_name(&self) -> String; + fn index_type_to_string(&self) -> &str; fn len(&self) -> usize; fn is_empty(&self) -> bool; fn flush(&mut self) -> io::Result<()>; @@ -12,9 +14,17 @@ pub trait AnyStorableVec { impl AnyStorableVec for StorableVec where - I: StorableVecIndex, - T: StorableVecType, + I: StoredIndex, + T: StoredType, { + fn file_name(&self) -> String { + self.file_name() + } + + fn index_type_to_string(&self) -> &str { + self.index_type_to_string() + } + fn len(&self) -> usize { self.len() } @@ -27,3 +37,28 @@ where self.flush() } } + +#[cfg(feature = "json")] +pub trait AnyJsonStorableVec: AnyStorableVec { + fn collect_range_values(&self, from: Option, to: Option) -> Result>; +} + +impl AnyJsonStorableVec for StorableVec +where + I: StoredIndex, + T: StoredType + serde::Serialize, +{ + fn collect_range_values(&self, from: Option, to: Option) -> Result> { + if MODE == STATELESS { + Ok( + unsafe { mem::transmute::<&StorableVec, &StorableVec>(self) } + .collect_range(from, to)? + .into_iter() + .map(|v| serde_json::to_value(v).unwrap()) + .collect::>(), + ) + } else { + unreachable!() + } + } +} diff --git a/storable_vec/src/traits/mod.rs b/storable_vec/src/traits/mod.rs index 2a8c9e8ae..c12f4f424 100644 --- a/storable_vec/src/traits/mod.rs +++ b/storable_vec/src/traits/mod.rs @@ -1,8 +1,8 @@ mod any; // mod bytes; -mod index; -mod type_; +mod stored_index; +mod stored_type; pub use any::*; -pub use index::*; -pub use type_::*; +pub use stored_index::*; +pub use stored_type::*; diff --git a/storable_vec/src/traits/index.rs b/storable_vec/src/traits/stored_index.rs similarity index 74% rename from storable_vec/src/traits/index.rs rename to storable_vec/src/traits/stored_index.rs index c99b64320..c0200f1ed 100644 --- a/storable_vec/src/traits/index.rs +++ b/storable_vec/src/traits/stored_index.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, ops::Add}; -pub trait StorableVecIndex +pub trait StoredIndex where Self: Debug + Default @@ -12,10 +12,12 @@ where + Ord + TryInto + From - + Add, + + Add + + Send + + Sync, { } -impl StorableVecIndex for I where +impl StoredIndex for I where I: Debug + Default + Copy @@ -27,5 +29,7 @@ impl StorableVecIndex for I where + TryInto + From + Add + + Send + + Sync { } diff --git a/storable_vec/src/traits/stored_type.rs b/storable_vec/src/traits/stored_type.rs new file mode 100644 index 000000000..f0a7b9e42 --- /dev/null +++ b/storable_vec/src/traits/stored_type.rs @@ -0,0 +1,13 @@ +use std::fmt::Debug; + +use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; + +pub trait StoredType +where + Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync, +{ +} +impl StoredType for T where + T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync +{ +} diff --git a/storable_vec/src/traits/type_.rs b/storable_vec/src/traits/type_.rs deleted file mode 100644 index 06e94efc0..000000000 --- a/storable_vec/src/traits/type_.rs +++ /dev/null @@ -1,10 +0,0 @@ -use std::fmt::Debug; - -use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; - -pub trait StorableVecType -where - Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout, -{ -} -impl StorableVecType for T where T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout {}