diff --git a/.gitignore b/.gitignore index 7df94636e..86d71e5f0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ target *\ copy* # Ignored -ignore +_ignore # Editors .vscode @@ -29,3 +29,6 @@ docker/kibo # Types paths.d.ts + +# Outputs +_outputs diff --git a/CHANGELOG.md b/CHANGELOG.md index cc13603fc..501136c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # v0.6.0 | WIP @@ -33,7 +33,7 @@ Added git tags for each version though Markdown won't display formatted on Githu # [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04 -![Image of the kibō Web App version 0.5.0](https://github.com/kibo-money/kibo/blob/main/assets/v0.5.0.jpg) +![Image of the kibō Web App version 0.5.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.5.0.jpg) ## Datasets @@ -100,7 +100,7 @@ Added git tags for each version though Markdown won't display formatted on Githu # [v0.4.0](https://github.com/kibo-money/kibo/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19 -![Image of the kibō Web App version 0.4.0](https://github.com/kibo-money/kibo/blob/main/assets/v0.4.0.jpg) +![Image of the kibō Web App version 0.4.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.4.0.jpg) ## Brand @@ -137,7 +137,7 @@ Added git tags for each version though Markdown won't display formatted on Githu # [v0.3.0](https://github.com/kibo-money/kibo/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26 -![Image of the Satonomics Web App version 0.3.0](https://github.com/kibo-money/kibo/blob/main/assets/v0.3.0.jpg) +![Image of the Satonomics Web App version 0.3.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.3.0.jpg) ## Parser @@ -216,7 +216,7 @@ Added git tags for each version though Markdown won't display formatted on Githu # [v0.2.0](https://github.com/kibo-money/kibo/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08 -![Image of the Satonomics Web App version 0.2.0](https://github.com/kibo-money/kibo/blob/main/assets/v0.2.0.jpg) +![Image of the Satonomics Web App version 0.2.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.2.0.jpg) ## App @@ -252,7 +252,7 @@ Added git tags for each version though Markdown won't display formatted on Githu # [v0.1.1](https://github.com/kibo-money/kibo/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24 -![Image of the Satonomics Web App version 0.1.1](https://github.com/kibo-money/kibo/blob/main/assets/v0.1.1.jpg) +![Image of the Satonomics Web App version 0.1.1](https://github.com/kibo-money/kibo/blob/main/_assets/v0.1.1.jpg) ## Parser @@ -302,8 +302,8 @@ Added git tags for each version though Markdown won't display formatted on Githu # [v0.1.0](https://github.com/kibo-money/kibo/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19 -![Image of the Satonomics Web App version 0.1.0](https://github.com/kibo-money/kibo/blob/main/assets/v0.1.0.jpg) +![Image of the Satonomics Web App version 0.1.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.1.0.jpg) # v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20 -![Image of the Satonomics Web App version 0.0.X](https://github.com/kibo-money/kibo/blob/main/assets/v0.0.X.jpg) +![Image of the Satonomics Web App version 0.0.X](https://github.com/kibo-money/kibo/blob/main/_assets/v0.0.X.jpg) diff --git a/Cargo.lock b/Cargo.lock index 55cedc130..af827c497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,8 +215,11 @@ dependencies = [ "color-eyre", "derive_deref", "exit", + "fjall", "jiff", "rayon", + "storable_vec", + "unsafe_slice_serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index daf8e4a05..947468aa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ bomputer = { path = "computer" } color-eyre = "0.6.3" derive_deref = "1.1.1" exit = { path = "exit" } +fjall = "2.5.0" jiff = "0.1.28" rayon = "1.10.0" storable_vec = { path = "storable_vec" } diff --git a/README.md b/README.md index b7cfd1067..900f36fa9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ - - - kibō + + + kibō diff --git a/assets/dove-orange.svg b/_assets/dove-orange.svg similarity index 100% rename from assets/dove-orange.svg rename to _assets/dove-orange.svg diff --git a/assets/dove-white.svg b/_assets/dove-white.svg similarity index 100% rename from assets/dove-white.svg rename to _assets/dove-white.svg diff --git a/assets/logo-dove-dark.svg b/_assets/logo-dove-dark.svg similarity index 100% rename from assets/logo-dove-dark.svg rename to _assets/logo-dove-dark.svg diff --git a/assets/logo-dove-light.svg b/_assets/logo-dove-light.svg similarity index 100% rename from assets/logo-dove-light.svg rename to _assets/logo-dove-light.svg diff --git a/assets/logo-dove-orange.svg b/_assets/logo-dove-orange.svg similarity index 100% rename from assets/logo-dove-orange.svg rename to _assets/logo-dove-orange.svg diff --git a/assets/logo-full-dark.svg b/_assets/logo-full-dark.svg similarity index 100% rename from assets/logo-full-dark.svg rename to _assets/logo-full-dark.svg diff --git a/assets/logo-full-light.svg b/_assets/logo-full-light.svg similarity index 100% rename from assets/logo-full-light.svg rename to _assets/logo-full-light.svg diff --git a/assets/logo-icon.svg b/_assets/logo-icon.svg similarity index 100% rename from assets/logo-icon.svg rename to _assets/logo-icon.svg diff --git a/assets/logo-long-text-dark.svg b/_assets/logo-long-text-dark.svg similarity index 100% rename from assets/logo-long-text-dark.svg rename to _assets/logo-long-text-dark.svg diff --git a/assets/logo-long-text-light.svg b/_assets/logo-long-text-light.svg similarity index 100% rename from assets/logo-long-text-light.svg rename to _assets/logo-long-text-light.svg diff --git a/assets/logo-short-text-dark.svg b/_assets/logo-short-text-dark.svg similarity index 100% rename from assets/logo-short-text-dark.svg rename to _assets/logo-short-text-dark.svg diff --git a/assets/logo-short-text-light.svg b/_assets/logo-short-text-light.svg similarity index 100% rename from assets/logo-short-text-light.svg rename to _assets/logo-short-text-light.svg diff --git a/assets/logo-stamp-orange.svg b/_assets/logo-stamp-orange.svg similarity index 100% rename from assets/logo-stamp-orange.svg rename to _assets/logo-stamp-orange.svg diff --git a/assets/logo-stamp.svg b/_assets/logo-stamp.svg similarity index 100% rename from assets/logo-stamp.svg rename to _assets/logo-stamp.svg diff --git a/assets/v0.0.X.jpg b/_assets/v0.0.X.jpg similarity index 100% rename from assets/v0.0.X.jpg rename to _assets/v0.0.X.jpg diff --git a/assets/v0.1.0.jpg b/_assets/v0.1.0.jpg similarity index 100% rename from assets/v0.1.0.jpg rename to _assets/v0.1.0.jpg diff --git a/assets/v0.1.1.jpg b/_assets/v0.1.1.jpg similarity index 100% rename from assets/v0.1.1.jpg rename to _assets/v0.1.1.jpg diff --git a/assets/v0.2.0.jpg b/_assets/v0.2.0.jpg similarity index 100% rename from assets/v0.2.0.jpg rename to _assets/v0.2.0.jpg diff --git a/assets/v0.3.0.jpg b/_assets/v0.3.0.jpg similarity index 100% rename from assets/v0.3.0.jpg rename to _assets/v0.3.0.jpg diff --git a/assets/v0.4.0.jpg b/_assets/v0.4.0.jpg similarity index 100% rename from assets/v0.4.0.jpg rename to _assets/v0.4.0.jpg diff --git a/assets/v0.5.0.jpg b/_assets/v0.5.0.jpg similarity index 100% rename from assets/v0.5.0.jpg rename to _assets/v0.5.0.jpg diff --git a/docker/Dockerfile b/_docker/Dockerfile similarity index 100% rename from docker/Dockerfile rename to _docker/Dockerfile diff --git a/docker/build.sh b/_docker/build.sh similarity index 100% rename from docker/build.sh rename to _docker/build.sh diff --git a/docker/cmd.sh b/_docker/cmd.sh similarity index 100% rename from docker/cmd.sh rename to _docker/cmd.sh diff --git a/docker/run.sh b/_docker/run.sh similarity index 100% rename from docker/run.sh rename to _docker/run.sh diff --git a/computer/Cargo.toml b/computer/Cargo.toml index a8f285300..4ea89fad8 100644 --- a/computer/Cargo.toml +++ b/computer/Cargo.toml @@ -4,10 +4,13 @@ version = "0.1.0" edition = "2021" [dependencies] -biter = { workspace = true } bindex = { workspace = true } +biter = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } exit = { workspace = true } +fjall = { workspace = true } jiff = { workspace = true } rayon = { workspace = true } +storable_vec = { workspace = true } +unsafe_slice_serde = { workspace = true } diff --git a/computer/src/lib.rs b/computer/src/lib.rs index ad59cc0b5..c59dbed44 100644 --- a/computer/src/lib.rs +++ b/computer/src/lib.rs @@ -1,36 +1,71 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; -use bindex::Indexer; +use bindex::{Height, Indexer}; use biter::rpc; use exit::Exit; +mod storage; mod structs; +use storage::{Fjalls, StorableVecs}; use structs::*; -pub struct Bomputer; - -impl Bomputer { - pub fn compute() {} +pub struct Computer { + outputs_dir: PathBuf, + vecs: StorableVecs, + trees: Fjalls, } -pub fn main() -> color_eyre::Result<()> { - color_eyre::install()?; +impl Computer { + pub fn import(outputs_dir: &Path) -> color_eyre::Result { + let outputs_dir = outputs_dir.to_owned(); + let computed_dir = outputs_dir.join("computed"); + let vecs = StorableVecs::import(&computed_dir.join("vecs"))?; + let trees = Fjalls::import(&computed_dir.join("fjall"))?; + Ok(Self { + outputs_dir, + vecs, + trees, + }) + } - let data_dir = Path::new("../../bitcoin"); - let rpc = rpc::Client::new( - "http://localhost:8332", - rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), - )?; - let exit = Exit::new(); + pub fn compute(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> { + let mut indexer = Indexer::import(&self.outputs_dir.join("indexes"))?; - let i = std::time::Instant::now(); + if false { + indexer.index(bitcoin_dir, rpc, exit)?; + } - let mut indexer = Indexer::import(Path::new("indexes"))?; + // TODO: Remove all outdated - indexer.index(data_dir, rpc, &exit)?; + // Compute txindex to X - dbg!(i.elapsed()); + // Compute height to X + indexer + .vecs() + .height_to_timestamp + .read_from_(self.vecs.height_to_date.len(), |(_height, timestamp)| { + self.vecs + .height_to_date + .push_if_needed(Height::from(_height), Date::from(timestamp)) + })?; + self.vecs + .height_to_date + .read_from_(self.vecs.date_to_first_height.len(), |(_height, date)| { + self.vecs + .date_to_first_height + .push_if_needed(*date, Height::from(_height)) + })?; - Ok(()) + // Compute date to X + // ... + + // Compute month to X + // ... + + // Compute year to X + // ... + + Ok(()) + } } diff --git a/computer/src/main.rs b/computer/src/main.rs index 164fa7fa1..92623c1ed 100644 --- a/computer/src/main.rs +++ b/computer/src/main.rs @@ -1,13 +1,28 @@ -use structs::Date; +use std::path::Path; + +use biter::rpc; +use bomputer::Computer; +use exit::Exit; mod structs; pub fn main() -> color_eyre::Result<()> { - let date1 = Date::from(jiff::civil::Date::constant(2009, 1, 9)); - let date2 = Date::from(jiff::civil::Date::constant(2009, 1, 31)); - let date3 = Date::from(jiff::civil::Date::constant(2019, 1, 9)); - dbg!(usize::try_from(date1))?; - dbg!(usize::try_from(date2))?; - dbg!(usize::try_from(date3))?; + color_eyre::install()?; + + let data_dir = Path::new("../../bitcoin"); + let rpc = rpc::Client::new( + "http://localhost:8332", + rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), + )?; + let exit = Exit::new(); + + let i = std::time::Instant::now(); + + let mut computer = Computer::import(Path::new("../_outputs"))?; + + computer.compute(data_dir, rpc, &exit)?; + + dbg!(i.elapsed()); + Ok(()) } diff --git a/computer/src/storage/fjalls.rs b/computer/src/storage/fjalls.rs new file mode 100644 index 000000000..d4a5c6d34 --- /dev/null +++ b/computer/src/storage/fjalls.rs @@ -0,0 +1,22 @@ +use std::path::Path; + +use bindex::{Store, Version}; + +use crate::structs::{AddressindexTxoutindex, Unit}; + +pub struct Fjalls { + pub address_txoutindex_in: Store, + pub address_txoutindex_out: Store, +} + +impl Fjalls { + pub fn import(path: &Path) -> color_eyre::Result { + let address_txoutindex_in = Store::import(&path.join("address_txoutindex_in"), Version::from(1))?; + let address_txoutindex_out = Store::import(&path.join("address_txoutindex_out"), Version::from(1))?; + + Ok(Self { + address_txoutindex_in, + address_txoutindex_out, + }) + } +} diff --git a/computer/src/storage/mod.rs b/computer/src/storage/mod.rs new file mode 100644 index 000000000..15d757c6a --- /dev/null +++ b/computer/src/storage/mod.rs @@ -0,0 +1,5 @@ +mod fjalls; +mod storable_vecs; + +pub use fjalls::*; +pub use storable_vecs::*; diff --git a/computer/src/storage/storable_vecs/base.rs b/computer/src/storage/storable_vecs/base.rs new file mode 100644 index 000000000..04e86f108 --- /dev/null +++ b/computer/src/storage/storable_vecs/base.rs @@ -0,0 +1,74 @@ +use std::{ + fmt::Debug, + io, + ops::{Deref, DerefMut}, + path::Path, +}; + +use bindex::{Indexer, Version}; + +use crate::Computer; + +pub struct StorableVec { + vec: bindex::StorableVec, + f: Box storable_vec::Result>>, +} + +impl StorableVec +where + I: TryInto, + T: Sized + Debug + Clone, +{ + pub fn import(path: &Path, version: Version, f: F) -> io::Result + where + F: Fn(&Indexer, &Computer) -> storable_vec::Result> + 'static, + { + let vec = bindex::StorableVec::import(path, version)?; + + Ok(Self { vec, f: Box::new(f) }) + } + + pub fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> { + (self.f)(indexer, computer)? + .into_iter() + .try_for_each(|(i, v)| self.push_if_needed(i, v)) + } + + // pub fn fill(&mut self) { + // self + // .vecs() + // .height_to_timestamp + // .read_iter(move |(_height, timestamp)| { + // let height = Height::from(_height); + // let date = Date::from(timestamp); + // self.vecs.date_to_first_height.push_if_needed(date, height)?; + // Ok(()) + // })?; + // } +} + +impl Deref for StorableVec { + type Target = bindex::StorableVec; + fn deref(&self) -> &Self::Target { + &self.vec + } +} +impl DerefMut for StorableVec { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.vec + } +} + +pub trait AnyComputedStorableVec { + fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()>; +} + +impl AnyComputedStorableVec for StorableVec +where + I: TryInto, + T: Sized + Debug + Clone, +{ + fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> { + self.compute(indexer, computer) + } +} diff --git a/computer/src/storage/storable_vecs/mod.rs b/computer/src/storage/storable_vecs/mod.rs new file mode 100644 index 000000000..27cf5a856 --- /dev/null +++ b/computer/src/storage/storable_vecs/mod.rs @@ -0,0 +1,77 @@ +use std::{fs, path::Path}; + +use bindex::{Addressindex, Amount, Height, StorableVec, Timestamp, Txindex, Txinindex, Txoutindex, Version}; + +use crate::structs::{Date, Feerate}; + +// mod base; + +// use base::*; + +pub struct StorableVecs { + pub date_to_first_height: StorableVec, + // pub height_to_block_interval: StorableVec, + pub height_to_date: StorableVec, + // pub height_to_fee: StorableVec, + // pub height_to_inputcount: StorableVec, + // pub height_to_last_addressindex: StorableVec, + // pub height_to_last_txindex: StorableVec, + // pub height_to_last_txoutindex: StorableVec, + // pub height_to_maxfeerate: StorableVec, + // pub height_to_medianfeerate: StorableVec, + // pub height_to_minfeerate: StorableVec, + // pub height_to_outputcount: StorableVec, + // pub height_to_subsidy: StorableVec, + // pub height_to_totalfees: StorableVec, + // pub height_to_txcount: StorableVec, + pub txindex_to_fee: StorableVec, + // pub txindex_to_feerate: StorableVec, + pub txindex_to_inputcount: StorableVec, + pub txindex_to_last_txinindex: StorableVec, + pub txindex_to_last_txoutindex: StorableVec, + pub txindex_to_outputcount: StorableVec, +} + +impl StorableVecs { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + date_to_first_height: StorableVec::import(&path.join("date_to_first_height"), Version::from(1))?, + // height_to_block_interval: StorableVec::import(&path.join("height_to_block_interval"), Version::from(1))?, + height_to_date: StorableVec::import(&path.join("height_to_date"), Version::from(1))?, + // height_to_fee: StorableVec::import(&path.join("height_to_fee"), Version::from(1))?, + // height_to_inputcount: StorableVec::import(&path.join("height_to_inputcount"), Version::from(1))?, + // height_to_last_addressindex: StorableVec::import( + // &path.join("height_to_last_addressindex"), + // Version::from(1), + // )?, + // height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?, + // height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?, + // height_to_maxfeerate: StorableVec::import(&path.join("height_to_maxfeerate"), Version::from(1))?, + // height_to_medianfeerate: StorableVec::import(&path.join("height_to_medianfeerate"), Version::from(1))?, + // height_to_minfeerate: StorableVec::import(&path.join("height_to_minfeerate"), Version::from(1))?, + // height_to_outputcount: StorableVec::import(&path.join("height_to_outputcount"), Version::from(1))?, + // height_to_subsidy: StorableVec::import(&path.join("height_to_subsidy"), Version::from(1))?, + // height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?, + // height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?, + txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?, + // txindex_to_feerate: StorableVec::import(&path.join("txindex_to_feerate"), Version::from(1))?, + txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?, + txindex_to_last_txinindex: StorableVec::import(&path.join("txindex_to_last_txinindex"), Version::from(1))?, + txindex_to_last_txoutindex: StorableVec::import( + &path.join("txindex_to_last_txoutindex"), + Version::from(1), + )?, + txindex_to_outputcount: StorableVec::import(&path.join("txindex_to_outputcount"), Version::from(1))?, + }) + } + + // pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] { + // [&self.date_to_first_height] + // } + + // pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] { + // [&mut self.date_to_first_height] + // } +} diff --git a/computer/src/structs/addressindextxoutindex.rs b/computer/src/structs/addressindextxoutindex.rs new file mode 100644 index 000000000..bc5ef32a2 --- /dev/null +++ b/computer/src/structs/addressindextxoutindex.rs @@ -0,0 +1,21 @@ +use bindex::{Addressindex, Txoutindex}; +use fjall::Slice; +use unsafe_slice_serde::UnsafeSliceSerde; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct AddressindexTxoutindex { + addressindex: Addressindex, + txoutindex: Txoutindex, +} + +impl TryFrom for AddressindexTxoutindex { + type Error = unsafe_slice_serde::Error; + fn try_from(value: Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From for Slice { + fn from(value: AddressindexTxoutindex) -> Self { + Self::new(value.unsafe_as_slice()) + } +} diff --git a/computer/src/structs/date.rs b/computer/src/structs/date.rs index c603135c1..dda76cdea 100644 --- a/computer/src/structs/date.rs +++ b/computer/src/structs/date.rs @@ -3,7 +3,7 @@ use color_eyre::eyre::eyre; use derive_deref::Deref; use jiff::{civil::Date as _Date, tz::TimeZone}; -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)] pub struct Date(_Date); impl Date { diff --git a/computer/src/structs/mod.rs b/computer/src/structs/mod.rs index 9d48ff76d..b73b18519 100644 --- a/computer/src/structs/mod.rs +++ b/computer/src/structs/mod.rs @@ -1,5 +1,9 @@ +mod addressindextxoutindex; mod date; mod feerate; +mod unit; +pub use addressindextxoutindex::*; pub use date::*; pub use feerate::*; +pub use unit::*; diff --git a/computer/src/structs/unit.rs b/computer/src/structs/unit.rs new file mode 100644 index 000000000..05b8bb0b8 --- /dev/null +++ b/computer/src/structs/unit.rs @@ -0,0 +1,13 @@ +use fjall::Slice; + +pub struct Unit(); +impl From for Unit { + fn from(_: Slice) -> Self { + Self() + } +} +impl From for Slice { + fn from(_: Unit) -> Self { + Self::new(&[]) + } +} diff --git a/indexer/.gitignore b/indexer/.gitignore deleted file mode 100644 index 752861df6..000000000 --- a/indexer/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/database -/indexes diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 6773c2618..4a07f4015 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -10,7 +10,7 @@ biter = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } exit = { workspace = true } -fjall = "2.5.0" +fjall = { workspace = true } jiff = { workspace = true } rapidhash = "1.3.0" rayon = { workspace = true } diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index b286b2866..6ff495281 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -17,7 +17,8 @@ use rayon::prelude::*; mod storage; mod structs; -pub use biter; +pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta}; +pub use structs::Version; use storage::{Fjalls, StorableVecs}; pub use structs::{ @@ -30,27 +31,34 @@ const SNAPSHOT_BLOCK_RANGE: usize = 1000; pub struct Indexer { vecs: StorableVecs, - parts: Fjalls, + trees: Fjalls, } impl Indexer { pub fn import(indexes_dir: &Path) -> color_eyre::Result { - Ok(Self { - vecs: StorableVecs::import(&indexes_dir.join("vecs"))?, - parts: Fjalls::import(&indexes_dir.join("fjall"))?, - }) + let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?; + let trees = Fjalls::import(&indexes_dir.join("fjall"))?; + Ok(Self { vecs, trees }) + } + + pub fn vecs(&self) -> &StorableVecs { + &self.vecs + } + + pub fn trees(&self) -> &Fjalls { + &self.trees } pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> { let check_collisions = true; let vecs = &mut self.vecs; - let parts = &mut self.parts; + let trees = &mut self.trees; let mut height = vecs .min_height() .unwrap_or_default() - .min(parts.min_height()) + .min(trees.min_height()) .and_then(|h| h.checked_sub(UNSAFE_BLOCKS)) .map(Height::from) .unwrap_or_default(); @@ -72,14 +80,14 @@ impl Indexer { let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?; let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?; - let export = |parts: &mut Fjalls, vecs: &mut StorableVecs, height: Height| -> color_eyre::Result<()> { + let export = |trees: &mut Fjalls, vecs: &mut StorableVecs, height: Height| -> color_eyre::Result<()> { println!("Exporting..."); exit.block(); thread::scope(|scope| -> color_eyre::Result<()> { let vecs_handle = scope.spawn(|| vecs.flush(height)); - parts.commit(height)?; + trees.commit(height)?; vecs_handle.join().unwrap()?; Ok(()) })?; @@ -95,18 +103,17 @@ impl Indexer { println!("Processing block {_height}..."); height = Height::from(_height); - let timestamp = Timestamp::try_from(block.header.time)?; if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? { if &blockhash != saved_blockhash.as_ref() { todo!("Rollback not implemented"); - // parts.rollback_from(&mut rtx, height, &exit)?; + // trees.rollback_from(&mut rtx, height, &exit)?; } } let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?; - if parts + if trees .blockhash_prefix_to_height .get(&blockhash_prefix)? .is_some_and(|prev_height| *prev_height != height) @@ -115,12 +122,13 @@ impl Indexer { return Err(eyre!("Collision, expect prefix to need be set yet")); } - parts + trees .blockhash_prefix_to_height .insert_if_needed(blockhash_prefix, height, height); vecs.height_to_blockhash.push_if_needed(height, blockhash)?; - vecs.height_to_timestamp.push_if_needed(height, timestamp)?; + vecs.height_to_difficulty.push_if_needed(height, block.header.difficulty_float())?; + vecs.height_to_timestamp.push_if_needed(height, Timestamp::try_from(block.header.time)?)?; vecs.height_to_size.push_if_needed(height, block.total_size())?; vecs.height_to_weight.push_if_needed(height, block.weight())?; vecs.height_to_first_txindex.push_if_needed(height, txindex_global)?; @@ -193,9 +201,9 @@ impl Indexer { let txid_prefix = TxidPrefix::try_from(&txid)?; let prev_txindex_opt = - if check_collisions && parts.txid_prefix_to_txindex.needs(height) { + if check_collisions && trees.txid_prefix_to_txindex.needs(height) { // Should only find collisions for two txids (duplicates), see below - parts.txid_prefix_to_txindex.get(&txid_prefix)?.map(|v| *v) + trees.txid_prefix_to_txindex.get(&txid_prefix)?.map(|v| *v) } else { None }; @@ -234,7 +242,7 @@ impl Indexer { return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin)))); } - let prev_txindex = if let Some(txindex) = parts + let prev_txindex = if let Some(txindex) = trees .txid_prefix_to_txindex .get(&TxidPrefix::try_from(&outpoint.txid)?)? .map(|v| *v) @@ -312,7 +320,7 @@ impl Indexer { }); let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| { - parts + trees .addresshash_to_addressindex .get(&AddressHash::from((addressbytes, addresstype))) .unwrap() @@ -344,9 +352,9 @@ impl Indexer { let prev_addressbytes = prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?; - if (vecs.addressindex_to_addresstype.hasnt(addressindex) + if (vecs.addressindex_to_addresstype.hasnt(addressindex)? && addresstype != prev_addresstype) - || (parts.addresshash_to_addressindex.needs(height) + || (trees.addresshash_to_addressindex.needs(height) && prev_addressbytes != addressbytes) { let txid = tx.compute_txid(); @@ -496,7 +504,7 @@ impl Indexer { already_added_addresshash .insert(addresshash, addressindex); - parts.addresshash_to_addressindex.insert_if_needed( + trees.addresshash_to_addressindex.insert_if_needed( addresshash, addressindex, height, @@ -583,7 +591,7 @@ impl Indexer { match prev_txindex_opt { None => { - parts + trees .txid_prefix_to_txindex .insert_if_needed(txid_prefix, txindex, height); } @@ -649,13 +657,13 @@ impl Indexer { let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active(); if should_snapshot { - export(parts, vecs, height)?; + export(trees, vecs, height)?; } Ok(()) })?; - export(parts, vecs, height)?; + export(trees, vecs, height)?; sleep(Duration::from_millis(100)); diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 7d6e08eb4..d51f8bac7 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -16,7 +16,7 @@ fn main() -> color_eyre::Result<()> { let i = std::time::Instant::now(); - let mut indexer = Indexer::import(Path::new("indexes"))?; + let mut indexer = Indexer::import(Path::new("../_outputs/indexes"))?; indexer.index(data_dir, rpc, &exit)?; diff --git a/indexer/src/storage/fjalls/base.rs b/indexer/src/storage/fjalls/base.rs index 42a575e4a..8f2767da7 100644 --- a/indexer/src/storage/fjalls/base.rs +++ b/indexer/src/storage/fjalls/base.rs @@ -9,24 +9,24 @@ use unsafe_slice_serde::UnsafeSliceSerde; use crate::structs::{Height, Version}; -use super::Meta; +use super::StoreMeta; -pub struct Partition { - meta: Meta, +pub struct Store { + meta: StoreMeta, keyspace: TransactionalKeyspace, part: TransactionalPartitionHandle, rtx: ReadTransaction, puts: BTreeMap, } -impl Partition +impl Store where K: Into + Ord, V: Into + TryFrom, >::Error: error::Error + Send + Sync + 'static, { pub fn import(path: &Path, version: Version) -> color_eyre::Result { - let meta = Meta::checked_open(path, version)?; + let meta = StoreMeta::checked_open(path, version)?; let keyspace = if let Ok(keyspace) = Self::open_keyspace(path) { keyspace } else { @@ -95,6 +95,9 @@ where pub fn len(&self) -> usize { self.meta.len() + self.puts.len() } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } pub fn has(&self, height: Height) -> bool { self.meta.has(height) diff --git a/indexer/src/storage/fjalls/meta.rs b/indexer/src/storage/fjalls/meta.rs index c26a19033..1372f1d34 100644 --- a/indexer/src/storage/fjalls/meta.rs +++ b/indexer/src/storage/fjalls/meta.rs @@ -7,14 +7,14 @@ use unsafe_slice_serde::UnsafeSliceSerde; use super::{Height, Version}; -pub struct Meta { +pub struct StoreMeta { pathbuf: PathBuf, version: Version, height: Option, len: usize, } -impl Meta { +impl StoreMeta { pub fn checked_open(path: &Path, version: Version) -> color_eyre::Result { fs::create_dir_all(path)?; @@ -40,6 +40,9 @@ impl Meta { pub fn len(&self) -> usize { self.len } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> { self.len = len; diff --git a/indexer/src/storage/fjalls/mod.rs b/indexer/src/storage/fjalls/mod.rs index 2a952a7e4..be75266ea 100644 --- a/indexer/src/storage/fjalls/mod.rs +++ b/indexer/src/storage/fjalls/mod.rs @@ -5,24 +5,25 @@ use crate::{structs::Version, AddressHash, Addressindex, BlockHashPrefix, Height mod base; mod meta; -use base::*; -use meta::*; +pub use base::*; +pub use meta::*; pub struct Fjalls { - pub addresshash_to_addressindex: Partition, - pub blockhash_prefix_to_height: Partition, - pub txid_prefix_to_txindex: Partition, + pub addresshash_to_addressindex: Store, + pub blockhash_prefix_to_height: Store, + pub txid_prefix_to_txindex: Store, } impl Fjalls { pub fn import(path: &Path) -> color_eyre::Result { + let addresshash_to_addressindex = Store::import(&path.join("addresshash_to_addressindex"), Version::from(1))?; + let blockhash_prefix_to_height = Store::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?; + let txid_prefix_to_txindex = Store::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?; + Ok(Self { - addresshash_to_addressindex: Partition::import( - &path.join("addresshash_to_addressindex"), - Version::from(1), - )?, - blockhash_prefix_to_height: Partition::import(&path.join("blockhash_prefix_to_height"), Version::from(1))?, - txid_prefix_to_txindex: Partition::import(&path.join("txid_prefix_to_txindex"), Version::from(1))?, + addresshash_to_addressindex, + blockhash_prefix_to_height, + txid_prefix_to_txindex, }) } diff --git a/indexer/src/storage/storable_vecs/base.rs b/indexer/src/storage/storable_vecs/base.rs index 9149d3b34..41707602c 100644 --- a/indexer/src/storage/storable_vecs/base.rs +++ b/indexer/src/storage/storable_vecs/base.rs @@ -7,6 +7,7 @@ use std::{ use super::{Height, Version}; +#[derive(Debug)] pub struct StorableVec { height: Option, pathbuf: PathBuf, @@ -16,7 +17,7 @@ pub struct StorableVec { impl StorableVec where - I: Into, + I: TryInto, T: Sized + Debug + Clone, { pub fn import(path: &Path, version: Version) -> io::Result { @@ -99,12 +100,12 @@ impl DerefMut for StorableVec { } } -pub trait AnyBindexVec { +pub trait AnyStorableVec { fn height(&self) -> color_eyre::Result; fn flush(&mut self, height: Height) -> io::Result<()>; } -impl AnyBindexVec for StorableVec +impl AnyStorableVec for StorableVec where I: Into, T: Sized + Debug + Clone, diff --git a/indexer/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs index a9ea76211..7d251df42 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/indexer/src/storage/storable_vecs/mod.rs @@ -12,13 +12,14 @@ use crate::structs::{ mod base; -use base::*; +pub use base::*; pub struct StorableVecs { pub addressindex_to_addresstype: StorableVec, pub addressindex_to_addresstypeindex: StorableVec, pub addressindex_to_height: StorableVec, pub height_to_blockhash: StorableVec, + pub height_to_difficulty: StorableVec, pub height_to_first_addressindex: StorableVec, pub height_to_first_emptyindex: StorableVec, pub height_to_first_multisigindex: StorableVec, @@ -54,23 +55,6 @@ pub struct StorableVecs { pub txinindex_to_txoutindex: StorableVec, pub txoutindex_to_addressindex: StorableVec, pub txoutindex_to_amount: StorableVec, - // Can be computed later: - // pub height_to_date: StorableVec, - // pub height_to_totalfees: StorableVec, - // pub height_to_inputcount: StorableVec, - // pub height_to_last_addressindex: StorableVec, - // pub height_to_last_txindex: StorableVec, - // pub height_to_last_txoutindex: StorableVec, - // pub height_to_outputcount: StorableVec, - // pub height_to_txcount: StorableVec, - // pub height_to_subsidy: StorableVec, - // pub height_to_minfeerate: StorableVec, - // pub height_to_maxfeerate: StorableVec, - // pub height_to_medianfeerate: StorableVec, - // pub txindex_to_feerate: StorableVec, - // pub txindex_to_inputcount: StorableVec, - // pub txindex_to_outputcount: StorableVec, - // pub txindex_to_last_txoutindex: StorableVec, } // const UNSAFE_BLOCKS: usize = 100; @@ -90,6 +74,7 @@ impl StorableVecs { )?, addressindex_to_height: StorableVec::import(&path.join("addressindex_to_height"), Version::from(1))?, height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?, + height_to_difficulty: StorableVec::import(&path.join("height_to_difficulty"), Version::from(1))?, height_to_first_addressindex: StorableVec::import( &path.join("height_to_first_addressindex"), Version::from(1), @@ -383,12 +368,13 @@ impl StorableVecs { .min()) } - pub fn as_slice(&self) -> [&dyn AnyBindexVec; 39] { + pub fn as_slice(&self) -> [&dyn AnyStorableVec; 40] { [ - &self.addressindex_to_addresstype as &dyn AnyBindexVec, + &self.addressindex_to_addresstype as &dyn AnyStorableVec, &self.addressindex_to_addresstypeindex, &self.addressindex_to_height, &self.height_to_blockhash, + &self.height_to_difficulty, &self.height_to_first_addressindex, &self.height_to_first_emptyindex, &self.height_to_first_multisigindex, @@ -427,12 +413,13 @@ impl StorableVecs { ] } - pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyBindexVec + Send + Sync); 39] { + pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyStorableVec + Send + Sync); 40] { [ - &mut self.addressindex_to_addresstype as &mut (dyn AnyBindexVec + Send + Sync), + &mut self.addressindex_to_addresstype as &mut (dyn AnyStorableVec + Send + Sync), &mut self.addressindex_to_addresstypeindex, &mut self.addressindex_to_height, &mut self.height_to_blockhash, + &mut self.height_to_difficulty, &mut self.height_to_first_addressindex, &mut self.height_to_first_emptyindex, &mut self.height_to_first_multisigindex, diff --git a/indexer/src/structs/addressindex.rs b/indexer/src/structs/addressindex.rs index 2780cf4d5..02578722b 100644 --- a/indexer/src/structs/addressindex.rs +++ b/indexer/src/structs/addressindex.rs @@ -1,10 +1,9 @@ use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use fjall::Slice; use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Addressindex(u32); -// direct_repr!(Addressindex); impl Addressindex { pub const BYTES: usize = size_of::(); @@ -50,13 +49,13 @@ impl From for usize { } } -impl TryFrom for Addressindex { +impl TryFrom for Addressindex { type Error = unsafe_slice_serde::Error; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Addressindex) -> Self { Self::new(value.unsafe_as_slice()) } diff --git a/indexer/src/structs/addresstypeindex.rs b/indexer/src/structs/addresstypeindex.rs index 7fe615080..971be4455 100644 --- a/indexer/src/structs/addresstypeindex.rs +++ b/indexer/src/structs/addresstypeindex.rs @@ -1,9 +1,7 @@ use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Addresstypeindex(u32); -// direct_repr!(Addresstypeindex); impl Addresstypeindex { pub fn decremented(self) -> Self { diff --git a/indexer/src/structs/amount.rs b/indexer/src/structs/amount.rs index f6ac571bc..6c66fc725 100644 --- a/indexer/src/structs/amount.rs +++ b/indexer/src/structs/amount.rs @@ -5,13 +5,11 @@ use std::{ use biter::bitcoin; use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Height; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Amount(bitcoin::Amount); -// direct_repr!(Amount); impl Amount { pub const ZERO: Self = Self(bitcoin::Amount::ZERO); diff --git a/indexer/src/structs/compressed.rs b/indexer/src/structs/compressed.rs index 4aab25aa1..59acbaa97 100644 --- a/indexer/src/structs/compressed.rs +++ b/indexer/src/structs/compressed.rs @@ -2,14 +2,13 @@ use std::hash::Hasher; use biter::bitcoin::{BlockHash, Txid}; use derive_deref::Deref; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use fjall::Slice; use unsafe_slice_serde::UnsafeSliceSerde; use super::{Addressbytes, Addresstype, SliceExtended}; #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct AddressHash([u8; 8]); -// direct_repr!(AddressHash); impl From<(&Addressbytes, Addresstype)> for AddressHash { fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self { let mut hasher = rapidhash::RapidHasher::default(); @@ -24,18 +23,18 @@ impl From<[u8; 8]> for AddressHash { Self(value) } } -impl TryFrom for AddressHash { +impl TryFrom for AddressHash { type Error = color_eyre::Report; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From<&AddressHash> for fjall::Slice { +impl From<&AddressHash> for Slice { fn from(value: &AddressHash) -> Self { Self::new(value.unsafe_as_slice()) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: AddressHash) -> Self { Self::from(&value) } @@ -43,25 +42,24 @@ impl From for fjall::Slice { #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockHashPrefix([u8; 8]); -// direct_repr!(BlockHashPrefix); impl TryFrom<&BlockHash> for BlockHashPrefix { type Error = color_eyre::Report; fn try_from(value: &BlockHash) -> Result { Ok(Self((&value[..]).read_8x_u8()?)) } } -impl TryFrom for BlockHashPrefix { +impl TryFrom for BlockHashPrefix { type Error = color_eyre::Report; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From<&BlockHashPrefix> for fjall::Slice { +impl From<&BlockHashPrefix> for Slice { fn from(value: &BlockHashPrefix) -> Self { Self::new(value.unsafe_as_slice()) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: BlockHashPrefix) -> Self { Self::from(&value) } @@ -69,25 +67,24 @@ impl From for fjall::Slice { #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct TxidPrefix([u8; 8]); -// direct_repr!(TxidPrefix); impl TryFrom<&Txid> for TxidPrefix { type Error = color_eyre::Report; fn try_from(value: &Txid) -> Result { Ok(Self((&value[..]).read_8x_u8()?)) } } -impl TryFrom for TxidPrefix { +impl TryFrom for TxidPrefix { type Error = color_eyre::Report; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From<&TxidPrefix> for fjall::Slice { +impl From<&TxidPrefix> for Slice { fn from(value: &TxidPrefix) -> Self { Self::new(value.unsafe_as_slice()) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: TxidPrefix) -> Self { Self::from(&value) } diff --git a/indexer/src/structs/height.rs b/indexer/src/structs/height.rs index 1784fc206..ee829a992 100644 --- a/indexer/src/structs/height.rs +++ b/indexer/src/structs/height.rs @@ -6,12 +6,11 @@ use std::{ use biter::rpc::{self, RpcApi}; use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use fjall::Slice; use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)] pub struct Height(u32); -// direct_repr!(Height); impl Height { pub fn write(&self, path: &Path) -> Result<(), io::Error> { @@ -120,13 +119,13 @@ impl TryFrom<&rpc::Client> for Height { } } -impl TryFrom for Height { +impl TryFrom for Height { type Error = unsafe_slice_serde::Error; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Height) -> Self { Self::new(value.unsafe_as_slice()) } diff --git a/indexer/src/structs/timestamp.rs b/indexer/src/structs/timestamp.rs index 3ed62e1ae..4b9fc285e 100644 --- a/indexer/src/structs/timestamp.rs +++ b/indexer/src/structs/timestamp.rs @@ -1,6 +1,6 @@ use derive_deref::Deref; -#[derive(Debug, Deref, Clone)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Timestamp(jiff::Timestamp); impl TryFrom for Timestamp { diff --git a/indexer/src/structs/txindex.rs b/indexer/src/structs/txindex.rs index 974565a02..2ec5b4ac6 100644 --- a/indexer/src/structs/txindex.rs +++ b/indexer/src/structs/txindex.rs @@ -1,12 +1,11 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; +use fjall::Slice; use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txindex(u32); -// direct_repr!(Txindex); impl Txindex { pub fn incremented(self) -> Self { @@ -59,13 +58,13 @@ impl From for usize { } } -impl TryFrom for Txindex { +impl TryFrom for Txindex { type Error = unsafe_slice_serde::Error; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Txindex) -> Self { Self::new(value.unsafe_as_slice()) } diff --git a/indexer/src/structs/txinindex.rs b/indexer/src/structs/txinindex.rs index 03807b5c7..2c8fcfb07 100644 --- a/indexer/src/structs/txinindex.rs +++ b/indexer/src/structs/txinindex.rs @@ -1,13 +1,11 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Vout; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txinindex(u64); -// direct_repr!(Txinindex); impl Txinindex { pub fn incremented(self) -> Self { diff --git a/indexer/src/structs/txoutindex.rs b/indexer/src/structs/txoutindex.rs index 5ab2e7ccf..bf7517f35 100644 --- a/indexer/src/structs/txoutindex.rs +++ b/indexer/src/structs/txoutindex.rs @@ -1,13 +1,11 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; -// use snkrj::{direct_repr, Storable, UnsizedStorable}; use super::Vout; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txoutindex(u64); -// direct_repr!(Txoutindex); impl Txoutindex { pub const COINBASE: Self = Self(u64::MAX); diff --git a/indexer/src/structs/version.rs b/indexer/src/structs/version.rs index 3f01cb414..4d8257e2c 100644 --- a/indexer/src/structs/version.rs +++ b/indexer/src/structs/version.rs @@ -1,5 +1,6 @@ use std::{fs, io, path::Path}; +use fjall::Slice; use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -24,13 +25,13 @@ impl TryFrom<&Path> for Version { } } -impl TryFrom for Version { +impl TryFrom for Version { type Error = color_eyre::Report; - fn try_from(value: fjall::Slice) -> Result { + fn try_from(value: Slice) -> Result { Ok(*Self::unsafe_try_from_slice(&value)?) } } -impl From for fjall::Slice { +impl From for Slice { fn from(value: Version) -> Self { Self::new(value.unsafe_as_slice()) } diff --git a/iterator/src/blk_index_to_blk_recap.rs b/iterator/src/blk_index_to_blk_recap.rs index f080b6d54..5a105bfca 100644 --- a/iterator/src/blk_index_to_blk_recap.rs +++ b/iterator/src/blk_index_to_blk_recap.rs @@ -59,6 +59,15 @@ impl BlkIndexToBlkRecap { self.tree.remove(&blk_index); }); + while self.tree.last_entry().map(|last| *last.key()).is_some_and(|key| { + if key >= self.tree.len() { + self.tree.pop_last(); + true + } else { + false + } + }) {} + self.last_safe_height = self.tree.values().map(|recap| recap.height()).max(); } @@ -68,13 +77,12 @@ impl BlkIndexToBlkRecap { if last_value.height() < start { return Some((*last_key, *last_value)); - } else if let Some((blk_index, _)) = self - .tree - .iter() - .find(|(_, blk_recap)| blk_recap.is_younger_than(start)) + } else if let Some((blk_index, _)) = + self.tree.iter().find(|(_, blk_recap)| blk_recap.is_younger_than(start)) { if *blk_index != 0 { - let blk_index = *blk_index - 1; + // Temporary fix, need to rethink the whole thing + let blk_index = (*blk_index).checked_sub(3).unwrap_or_default(); return Some((blk_index, *self.tree.get(&blk_index).unwrap())); } } @@ -103,13 +111,10 @@ impl BlkIndexToBlkRecap { unreachable!(); } - self.tree - .insert(blk_index, BlkRecap::first(blk_metadata_and_block)); + self.tree.insert(blk_index, BlkRecap::first(blk_metadata_and_block)); } - if self - .last_safe_height - .map_or(true, |safe_height| height >= safe_height) + if self.last_safe_height.map_or(true, |safe_height| height >= safe_height) && (height % TARGET_BLOCKS_PER_MONTH) == 0 { self.export(); diff --git a/iterator/src/lib.rs b/iterator/src/lib.rs index 03afdfebb..ebd20c97d 100644 --- a/iterator/src/lib.rs +++ b/iterator/src/lib.rs @@ -98,7 +98,7 @@ pub fn new( thread::spawn(move || { blk_index_to_blk_path .iter() - .filter(|(blk_index, _)| blk_index >= &&starting_blk_index) + .filter(|(blk_index, _)| **blk_index >= starting_blk_index) .try_for_each(move |(blk_index, blk_path)| { let blk_metadata = BlkMetadata::new(*blk_index, blk_path); @@ -147,38 +147,6 @@ pub fn new( }) }); - // thread::spawn(move || { - // recv_block_reader.iter().par_bridge().try_for_each( - // move |(blk_metadata, mut block_state)| { - // let raw_block = match block_state { - // BlockState::Raw(vec) => vec, - // _ => unreachable!(), - // }; - - // let mut cursor = Cursor::new(raw_block); - - // block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap()); - - // if send_block - // .send(BlkMetadataAndBlock::new( - // blk_metadata, - // match block_state { - // BlockState::Decoded(block) => block, - // _ => unreachable!(), - // }, - // )) - // .is_err() - // { - // return ControlFlow::Break(()); - // } - - // ControlFlow::Continue(()) - // }, - // ); - // }); - - // Can't use the previous code because .send() blocks all the threads if full - // And other .par_iter() are also stuck because of that thread::spawn(move || { let mut bulk = vec![]; @@ -192,7 +160,9 @@ pub fn new( let mut cursor = Cursor::new(raw_block); - *block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap()); + let block = Block::consensus_decode(&mut cursor).unwrap(); + + *block_state = BlockState::Decoded(block); }); bulk.drain(..).try_for_each(|(blk_metadata, block_state)| { @@ -219,6 +189,7 @@ pub fn new( return ControlFlow::Continue(()); } + // Sending in bulk to not lock threads in standby drain_and_send(&mut bulk) }); diff --git a/iterator/src/main.rs b/iterator/src/main.rs index a8bea0066..908ba6127 100644 --- a/iterator/src/main.rs +++ b/iterator/src/main.rs @@ -5,13 +5,13 @@ use bitcoincore_rpc::{Auth, Client}; fn main() { let i = std::time::Instant::now(); - let data_dir = Path::new("../../../bitcoin"); + let data_dir = Path::new("../../bitcoin"); let url = "http://localhost:8332"; let cookie = Path::new(data_dir).join(".cookie"); let auth = Auth::CookieFile(cookie); let rpc = Client::new(url, auth).unwrap(); - let start = Some(810078); + let start = Some(749900); let end = None; biter::new(data_dir, start, end, rpc) diff --git a/run.sh b/run.sh deleted file mode 100755 index 24531425a..000000000 --- a/run.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -# https://stackoverflow.com/questions/31389483/find-and-delete-file-or-folder-older-than-x-days - -if command -v ulimit &> /dev/null; then - echo "Increasing limit of opened files..." - # ulimit -n 1000000 # Can't be $(ulimit -Hn), bitcoind needs some too ! - - # Needed because the datasets tree is too big lol - echo "Increasing stack size..." - ulimit -s $(ulimit -Hs) -fi - -# For Mac OS users -if [ "$(uname)" == "Darwin" ]; then - if mdutil -s / | grep "enabled"; then - echo "Disabling spotlight indexing..." - sudo mdutil -a -i off &> /dev/null - fi - - echo "Thinning local TimeMachine snapshots..." - tmutil thinlocalsnapshots / &> /dev/null -fi - -cargo run -r -- "$@" diff --git a/indexer/rustfmt.toml b/rustfmt.toml similarity index 100% rename from indexer/rustfmt.toml rename to rustfmt.toml diff --git a/storable_vec/src/lib.rs b/storable_vec/src/lib.rs index 9924c827a..df2297f25 100644 --- a/storable_vec/src/lib.rs +++ b/storable_vec/src/lib.rs @@ -50,7 +50,7 @@ const MAX_CACHE_SIZE: usize = 100 * ONE_MB; impl StorableVec where - I: Into, + I: TryInto, T: Sized + Debug + Clone, { pub const SIZE_OF_T: usize = size_of::(); @@ -60,11 +60,11 @@ where pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; pub fn import(path: &Path) -> Result { - let file = Self::open_file(path)?; + let file = Self::open_file_(path)?; let mut this = Self { pathbuf: path.to_owned(), - disk_len: Self::byte_index_to_index(file.metadata()?.len() as usize), + disk_len: Self::byte_index_to_index(Self::file_len(&file)?), file, cache: vec![], pushed: vec![], @@ -81,6 +81,10 @@ where Ok(this) } + fn file_len(file: &File) -> io::Result { + Ok(file.metadata()?.len() as usize) + } + fn reset_cache(&mut self) { // let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize; // self.cache.clear(); @@ -100,7 +104,10 @@ where } } - fn open_file(path: &Path) -> Result { + fn open_file(&self) -> Result { + Self::open_file_(&self.pathbuf).map_err(Error::IO) + } + fn open_file_(path: &Path) -> Result { OpenOptions::new() .read(true) .create(true) @@ -140,7 +147,7 @@ where #[inline] pub fn get(&self, index: I) -> Result>> { - self.get_(index.into()) + self.get_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?) } pub fn get_(&self, index: usize) -> Result>> { match self.index_to_pushed_index(index) { @@ -163,9 +170,7 @@ where let page_index = index / Self::PER_PAGE; let last_index = self.disk_len - 1; let max_page_index = last_index / Self::PER_PAGE; - let min_page_index = (max_page_index + 1) - .checked_sub(self.cache.len()) - .unwrap_or_default(); + let min_page_index = (max_page_index + 1).checked_sub(self.cache.len()).unwrap_or_default(); // let min_open_page = self.min.load(AtomicOrdering::SeqCst); @@ -196,14 +201,11 @@ where T::unsafe_try_from_slice(slice).map_err(Error::UnsafeSliceSerde)?, ))) } else { - let mut file = Self::open_file(&self.pathbuf).map_err(Error::IO)?; - - file.seek(SeekFrom::Start(byte_index as u64)) - .map_err(Error::IO)?; - + let mut file = self.open_file()?; let mut buf = vec![0; Self::SIZE_OF_T]; - file.read_exact(&mut buf).map_err(Error::IO)?; + file.seek(SeekFrom::Start(byte_index as u64)).map_err(Error::IO)?; + file.read_exact(&mut buf).map_err(Error::IO)?; let value = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?; Ok(Some(Value::Owned(value.to_owned()))) @@ -213,10 +215,39 @@ where where T: Default + Clone, { - Ok(self - .get(index)? - .map(|v| (*v).clone()) - .unwrap_or(Default::default())) + Ok(self.get(index)?.map(|v| (*v).clone()).unwrap_or(Default::default())) + } + + pub fn read_iter(&self, f: F) -> Result<()> + where + F: FnMut((usize, &T)) -> Result<()>, + { + self.read_from_(0, f) + } + + pub fn read_from_(&self, index: usize, mut f: F) -> Result<()> + where + F: FnMut((usize, &T)) -> Result<()>, + { + let mut file = self.open_file()?; + let mut buf = vec![0; Self::SIZE_OF_T]; + + let byte_index = Self::index_to_byte_index(index); + + file.seek(SeekFrom::Start(byte_index as u64)).map_err(Error::IO)?; + + let mut i = 0; + + loop { + if file.read_exact(&mut buf).map_err(Error::IO).is_err() { + break; + } + let v = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?; + f((i, v))?; + i += 1; + } + + Ok(()) } #[allow(unused)] @@ -238,7 +269,7 @@ where } pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { - self.push_if_needed_(index.into(), value) + self.push_if_needed_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?, value) } pub fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> { let len = self.len(); @@ -307,15 +338,15 @@ where self.len() == 0 } - pub fn has(&self, index: I) -> bool { - self.has_(index.into()) + pub fn has(&self, index: I) -> Result { + Ok(self.has_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)) } pub fn has_(&self, index: usize) -> bool { index < self.len() } - pub fn hasnt(&self, index: I) -> bool { - self.hasnt_(index.into()) + pub fn hasnt(&self, index: I) -> Result { + Ok(self.hasnt_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)) } pub fn hasnt_(&self, index: usize) -> bool { !self.has_(index) @@ -414,6 +445,7 @@ pub enum Error { IndexTooHigh, ExpectFileToHaveIndex, ExpectVecToHaveIndex, + FailedKeyTryIntoUsize, } impl fmt::Display for Error { // This trait requires `fmt` with this exact signature. @@ -425,6 +457,7 @@ impl fmt::Display for Error { Error::IndexTooHigh => write!(f, "Index too high"), Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"), Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"), + Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"), } } } diff --git a/unsafe_slice_serde/Cargo.toml b/unsafe_slice_serde/Cargo.toml index 60a15b1ff..bc73a8033 100644 --- a/unsafe_slice_serde/Cargo.toml +++ b/unsafe_slice_serde/Cargo.toml @@ -1,3 +1,4 @@ [package] name = "unsafe_slice_serde" version = "0.1.0" +edition = "2021"