diff --git a/Cargo.lock b/Cargo.lock index 536fa3394..e6fd1bc01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,6 +325,14 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "brk" version = "0.0.0" +dependencies = [ + "brk_computer", + "brk_fetcher", + "brk_indexer", + "brk_logger", + "brk_parser", + "brk_server", +] [[package]] name = "brk_cli" @@ -337,6 +345,7 @@ dependencies = [ "brk_fetcher", "brk_indexer", "brk_parser", + "byteview", "color-eyre", "derive_deref", "fjall", @@ -369,6 +378,7 @@ dependencies = [ "bitcoin", "brk_logger", "brk_parser", + "byteview", "color-eyre", "derive_deref", "fjall", @@ -391,7 +401,6 @@ dependencies = [ "color-eyre", "env_logger", "jiff", - "log", ] [[package]] @@ -400,9 +409,9 @@ version = "0.0.0" dependencies = [ "bitcoin", "bitcoincore-rpc", + "byteview", "crossbeam", "derive_deref", - "fjall", "rayon", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index bb50967a5..48a9425e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,9 +10,10 @@ bitcoin = { version = "0.32.5", features = ["serde"] } brk_computer = { version = "0", path = "crates/brk_computer" } brk_fetcher = { version = "0", path = "crates/brk_fetcher" } brk_indexer = { version = "0", path = "crates/brk_indexer" } -brk_parser = { version = "0", path = "crates/brk_parser", features = ["bytes"] } +brk_parser = { version = "0", path = "crates/brk_parser" } brk_logger = { version = "0", path = "crates/brk_logger" } brk_server = { version = "0", path = "crates/brk_server" } +byteview = "0.5.4" color-eyre = "0.6.3" derive_deref = "1.1.1" fjall = "2.6.5" diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index fbcc31153..9e83030cd 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -4,4 +4,22 @@ license.workspace = true edition.workspace = true version.workspace = true +[features] +full = ["computer", "fetcher", "indexer", "logger", "parser", "server"] +computer = ["brk_computer"] +fetcher = ["brk_fetcher"] +indexer = ["brk_indexer"] +logger = ["brk_logger"] +parser = ["brk_parser"] +server = ["brk_server"] + [dependencies] +brk_computer = { workspace = true, optional = true } +brk_fetcher = { workspace = true, optional = true } +brk_indexer = { workspace = true, optional = true } +brk_logger = { workspace = true, optional = true } +brk_parser = { workspace = true, optional = true } +brk_server = { workspace = true, optional = true } + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs new file mode 100644 index 000000000..458340dca --- /dev/null +++ b/crates/brk/src/lib.rs @@ -0,0 +1,35 @@ +#[cfg(feature = "computer")] +pub mod computer { + #[doc(inline)] + pub use brk_computer::*; +} + +#[cfg(feature = "fetcher")] +pub mod fetcher { + #[doc(inline)] + pub use brk_fetcher::*; +} + +#[cfg(feature = "indexer")] +pub mod indexer { + #[doc(inline)] + pub use brk_indexer::*; +} + +#[cfg(feature = "logger")] +pub mod logger { + #[doc(inline)] + pub use brk_logger::*; +} + +#[cfg(feature = "parser")] +pub mod parser { + #[doc(inline)] + pub use brk_parser::*; +} + +#[cfg(feature = "server")] +pub mod server { + #[doc(inline)] + pub use brk_server::*; +} diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 733032898..add4956ff 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -9,6 +9,7 @@ license = { workspace = true } brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_parser = { workspace = true } +byteview = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } fjall = { workspace = true } diff --git a/crates/brk_computer/src/structs/addressindextxoutindex.rs b/crates/brk_computer/src/structs/addressindextxoutindex.rs index e08cbf8b4..11a743b50 100644 --- a/crates/brk_computer/src/structs/addressindextxoutindex.rs +++ b/crates/brk_computer/src/structs/addressindextxoutindex.rs @@ -1,5 +1,5 @@ use brk_indexer::{Addressindex, Txoutindex}; -use fjall::Slice; +use byteview::ByteView; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)] @@ -9,13 +9,13 @@ pub struct AddressindexTxoutindex { txoutindex: Txoutindex, } -impl TryFrom for AddressindexTxoutindex { +impl TryFrom for AddressindexTxoutindex { type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { + fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -impl From for Slice { +impl From for ByteView { fn from(value: AddressindexTxoutindex) -> Self { Self::new(value.as_bytes()) } diff --git a/crates/brk_computer/src/structs/unit.rs b/crates/brk_computer/src/structs/unit.rs index 05b8bb0b8..e9754fb93 100644 --- a/crates/brk_computer/src/structs/unit.rs +++ b/crates/brk_computer/src/structs/unit.rs @@ -1,12 +1,12 @@ -use fjall::Slice; +use byteview::ByteView; pub struct Unit(); -impl From for Unit { - fn from(_: Slice) -> Self { +impl From for Unit { + fn from(_: ByteView) -> Self { Self() } } -impl From for Slice { +impl From for ByteView { fn from(_: Unit) -> Self { Self::new(&[]) } diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index e6526f818..3bb4f35f4 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -9,6 +9,7 @@ license = { workspace = true } bitcoin = { workspace = true } brk_parser = { workspace = true } brk_logger = { workspace = true } +byteview = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } fjall = { workspace = true } diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 4a3258135..ac7fab13a 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -47,7 +47,7 @@ impl Indexer { } impl Indexer { - pub fn index(&mut self, bitcoin_dir: &Path, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> { + pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> { info!("Started indexing..."); let check_collisions = true; @@ -80,8 +80,6 @@ impl Indexer { let mut idxs = starting_indexes; - let parser = Parser::new(bitcoin_dir, rpc); - parser.parse(Some(idxs.height), None) .iter() .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { diff --git a/crates/brk_indexer/src/main.rs b/crates/brk_indexer/src/main.rs index 54b2b56f5..eb3d9d5a5 100644 --- a/crates/brk_indexer/src/main.rs +++ b/crates/brk_indexer/src/main.rs @@ -1,7 +1,10 @@ use std::{path::Path, thread::sleep, time::Duration}; use brk_indexer::{Indexer, rpc::RpcApi}; -use brk_parser::rpc::{self}; +use brk_parser::{ + Parser, + rpc::{self}, +}; use hodor::Exit; use log::info; use storable_vec::CACHED_GETS; @@ -18,6 +21,8 @@ fn main() -> color_eyre::Result<()> { )?)); let exit = Exit::new(); + let parser = Parser::new(data_dir, rpc); + loop { let block_count = rpc.get_blockchain_info().unwrap().blocks as usize; @@ -27,7 +32,7 @@ fn main() -> color_eyre::Result<()> { let mut indexer: Indexer = Indexer::import(Path::new("../../_outputs/indexes"))?; - indexer.index(data_dir, rpc, &exit)?; + indexer.index(&parser, rpc, &exit)?; dbg!(i.elapsed()); diff --git a/crates/brk_indexer/src/storage/fjalls/base.rs b/crates/brk_indexer/src/storage/fjalls/base.rs index 207a88853..6f9e1cafd 100644 --- a/crates/brk_indexer/src/storage/fjalls/base.rs +++ b/crates/brk_indexer/src/storage/fjalls/base.rs @@ -5,9 +5,9 @@ use std::{ }; use brk_parser::Height; +use byteview::ByteView; use fjall::{ - PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace, - TransactionalPartitionHandle, + PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace, TransactionalPartitionHandle, }; use storable_vec::{Value, Version}; use zerocopy::{Immutable, IntoBytes}; @@ -27,9 +27,9 @@ const CHECK_COLLISISONS: bool = true; impl Store where - K: Into + Ord + Immutable + IntoBytes, - V: Into + TryFrom, - >::Error: error::Error + Send + Sync + 'static, + K: Into + Ord + Immutable + IntoBytes, + V: Into + TryFrom, + >::Error: error::Error + Send + Sync + 'static, { pub fn import(path: &Path, version: Version) -> color_eyre::Result { let meta = StoreMeta::checked_open(path, version)?; @@ -62,7 +62,7 @@ where if let Some(v) = self.puts.get(key) { Ok(Some(Value::Ref(v))) } else if let Some(slice) = self.rtx.get(&self.part, key.as_bytes())? { - Ok(Some(Value::Owned(V::try_from(slice)?))) + Ok(Some(Value::Owned(V::try_from(slice.into())?))) } else { Ok(None) } @@ -97,7 +97,7 @@ where mem::take(&mut self.dels) .into_iter() - .for_each(|key| wtx.remove(&self.part, key)); + .for_each(|key| wtx.remove(&self.part, key.into())); mem::take(&mut self.puts).into_iter().for_each(|(key, value)| { if CHECK_COLLISISONS { @@ -106,7 +106,7 @@ where unreachable!(); } } - wtx.insert(&self.part, key, value) + wtx.insert(&self.part, key.into(), value.into()) }); wtx.commit()?; diff --git a/crates/brk_indexer/src/structs/addressindex.rs b/crates/brk_indexer/src/structs/addressindex.rs index 02310ede8..c30d2ac4e 100644 --- a/crates/brk_indexer/src/structs/addressindex.rs +++ b/crates/brk_indexer/src/structs/addressindex.rs @@ -1,7 +1,7 @@ use std::ops::Add; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; -use fjall::Slice; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -68,13 +68,13 @@ impl From for usize { } } -impl TryFrom for Addressindex { +impl TryFrom for Addressindex { type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { + fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -impl From for Slice { +impl From for ByteView { fn from(value: Addressindex) -> Self { Self::new(value.as_bytes()) } diff --git a/crates/brk_indexer/src/structs/compressed.rs b/crates/brk_indexer/src/structs/compressed.rs index fc482eba9..51a00f54d 100644 --- a/crates/brk_indexer/src/structs/compressed.rs +++ b/crates/brk_indexer/src/structs/compressed.rs @@ -1,7 +1,7 @@ use std::hash::Hasher; +use byteview::ByteView; use derive_deref::Deref; -use fjall::Slice; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::{Addressbytes, Addresstype, BlockHash, Txid}; @@ -22,18 +22,18 @@ impl From<[u8; 8]> for AddressHash { Self(value) } } -impl TryFrom for AddressHash { +impl TryFrom for AddressHash { type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { + fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -impl From<&AddressHash> for Slice { +impl From<&AddressHash> for ByteView { fn from(value: &AddressHash) -> Self { Self::new(value.as_bytes()) } } -impl From for Slice { +impl From for ByteView { fn from(value: AddressHash) -> Self { Self::from(&value) } @@ -46,18 +46,18 @@ impl From<&BlockHash> for BlockHashPrefix { Self(copy_first_8bytes(&value[..]).unwrap()) } } -impl TryFrom for BlockHashPrefix { +impl TryFrom for BlockHashPrefix { type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { + fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -impl From<&BlockHashPrefix> for Slice { +impl From<&BlockHashPrefix> for ByteView { fn from(value: &BlockHashPrefix) -> Self { Self::new(value.as_bytes()) } } -impl From for Slice { +impl From for ByteView { fn from(value: BlockHashPrefix) -> Self { Self::from(&value) } @@ -70,18 +70,18 @@ impl From<&Txid> for TxidPrefix { Self(copy_first_8bytes(&value[..]).unwrap()) } } -impl TryFrom for TxidPrefix { +impl TryFrom for TxidPrefix { type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { + fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -impl From<&TxidPrefix> for Slice { +impl From<&TxidPrefix> for ByteView { fn from(value: &TxidPrefix) -> Self { Self::new(value.as_bytes()) } } -impl From for Slice { +impl From for ByteView { fn from(value: TxidPrefix) -> Self { Self::from(&value) } diff --git a/crates/brk_indexer/src/structs/txindex.rs b/crates/brk_indexer/src/structs/txindex.rs index 82b466963..8ca6fba34 100644 --- a/crates/brk_indexer/src/structs/txindex.rs +++ b/crates/brk_indexer/src/structs/txindex.rs @@ -1,7 +1,7 @@ use std::ops::{Add, AddAssign, Sub}; +use byteview::ByteView; use derive_deref::{Deref, DerefMut}; -use fjall::Slice; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -89,13 +89,13 @@ impl From for usize { } } -impl TryFrom for Txindex { +impl TryFrom for Txindex { type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { + fn try_from(value: ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -impl From for Slice { +impl From for ByteView { fn from(value: Txindex) -> Self { Self::new(value.as_bytes()) } diff --git a/crates/brk_logger/Cargo.toml b/crates/brk_logger/Cargo.toml index bbc78223a..fe45f370b 100644 --- a/crates/brk_logger/Cargo.toml +++ b/crates/brk_logger/Cargo.toml @@ -9,4 +9,3 @@ license = { workspace = true } color-eyre = { workspace = true } env_logger = "0.11.6" jiff = { workspace = true } -log = { workspace = true } diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index 123503149..37540bb7e 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -8,7 +8,6 @@ use std::{ use color_eyre::owo_colors::OwoColorize; use env_logger::{Builder, Env}; use jiff::{Timestamp, tz}; -pub use log::{debug, error, info, trace, warn}; #[inline(always)] pub fn init(path: Option<&Path>) { diff --git a/crates/brk_logger/src/main.rs b/crates/brk_logger/src/main.rs index 5414a6614..d2a6728ea 100644 --- a/crates/brk_logger/src/main.rs +++ b/crates/brk_logger/src/main.rs @@ -1,6 +1,3 @@ -use log::info; - fn main() { brk_logger::init(None); - info!("test"); } diff --git a/crates/brk_parser/Cargo.toml b/crates/brk_parser/Cargo.toml index fadde0dc1..e4f685927 100644 --- a/crates/brk_parser/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -8,16 +8,13 @@ version = { workspace = true } edition = { workspace = true } license = { workspace = true } -[features] -bytes = ["dep:fjall", "dep:zerocopy"] - [dependencies] bitcoin = { workspace = true } bitcoincore-rpc = "0.19.0" +byteview = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } derive_deref = { workspace = true } -fjall = { workspace = true, optional = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -zerocopy = { workspace = true, optional = true } +zerocopy = { workspace = true } diff --git a/crates/brk_parser/src/error.rs b/crates/brk_parser/src/error.rs index 0f670f987..0a7562c65 100644 --- a/crates/brk_parser/src/error.rs +++ b/crates/brk_parser/src/error.rs @@ -17,7 +17,6 @@ impl From for Error { } } -#[cfg(feature = "bytes")] impl From> for Error { fn from(_: zerocopy::error::SizeError) -> Self { Self::ZeroCopyError diff --git a/crates/brk_parser/src/height.rs b/crates/brk_parser/src/height.rs index 12071665e..c81c0bc3b 100644 --- a/crates/brk_parser/src/height.rs +++ b/crates/brk_parser/src/height.rs @@ -5,20 +5,34 @@ use std::{ use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; -#[cfg(feature = "bytes")] use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::rpc::{self, RpcApi}; -#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] -#[cfg_attr(feature = "bytes", derive(FromBytes, Immutable, IntoBytes, KnownLayout,))] +#[derive( + Debug, + Clone, + Copy, + Deref, + DerefMut, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + Serialize, + Deserialize, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Height(u32); impl Height { pub const ZERO: Self = Height(0); pub const MAX: Self = Height(u32::MAX); - #[cfg(feature = "bytes")] pub fn write(&self, path: &std::path::Path) -> Result<(), std::io::Error> { std::fs::write(path, self.as_bytes()) } @@ -170,7 +184,6 @@ impl From for bitcoin::locktime::absolute::Height { } } -#[cfg(feature = "bytes")] impl TryFrom<&std::path::Path> for Height { type Error = crate::Error; fn try_from(value: &std::path::Path) -> Result { @@ -178,15 +191,14 @@ impl TryFrom<&std::path::Path> for Height { } } -#[cfg(feature = "bytes")] -impl TryFrom for Height { +impl TryFrom for Height { type Error = crate::Error; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: byteview::ByteView) -> Result { Ok(Self::read_from_bytes(&value)?) } } -#[cfg(feature = "bytes")] -impl From for fjall::Slice { + +impl From for byteview::ByteView { fn from(value: Height) -> Self { Self::new(value.as_bytes()) } diff --git a/crates/storable_vec/Cargo.toml b/crates/storable_vec/Cargo.toml index 9ce44172e..1b22c346b 100644 --- a/crates/storable_vec/Cargo.toml +++ b/crates/storable_vec/Cargo.toml @@ -8,7 +8,7 @@ edition = { workspace = true } license = { workspace = true } [features] -json = ["dep:serde", "dep:serde_json"] +json = ["serde", "serde_json"] [dependencies] memmap2 = "0.9.5"