diff --git a/parser/Cargo.lock b/parser/Cargo.lock index 5a8466a1e..33a98fde0 100644 --- a/parser/Cargo.lock +++ b/parser/Cargo.lock @@ -170,15 +170,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bincode" version = "2.0.0-rc.3" @@ -301,9 +292,6 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -dependencies = [ - "serde", -] [[package]] name = "bumpalo" @@ -551,15 +539,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "doxygen-rs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" -dependencies = [ - "phf", -] - [[package]] name = "either" version = "1.12.0" @@ -764,44 +743,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "heed" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e" -dependencies = [ - "bitflags 2.6.0", - "byteorder", - "heed-traits", - "heed-types", - "libc", - "lmdb-master-sys", - "once_cell", - "page_size", - "serde", - "synchronoise", - "url", -] - -[[package]] -name = "heed-traits" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" - -[[package]] -name = "heed-types" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" -dependencies = [ - "bincode 1.3.3", - "byteorder", - "heed-traits", - "serde", - "serde_json", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -1103,17 +1044,6 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" -[[package]] -name = "lmdb-master-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24" -dependencies = [ - "cc", - "doxygen-rs", - "libc", -] - [[package]] name = "lock_api" version = "0.4.12" @@ -1304,16 +1234,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -1357,14 +1277,13 @@ name = "parser" version = "0.3.0" dependencies = [ "allocative", - "bincode 2.0.0-rc.3", + "bincode", "bitcoin_hashes", "biter", "chrono", "clap", "color-eyre", "derive_deref", - "heed", "inferno", "itertools", "memory-stats", @@ -1384,48 +1303,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" -dependencies = [ - "phf_shared", - "rand", -] - -[[package]] -name = "phf_macros" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn 2.0.72", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project" version = "1.1.5" @@ -1843,12 +1720,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "slab" version = "0.4.9" @@ -1926,15 +1797,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -[[package]] -name = "synchronoise" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" -dependencies = [ - "crossbeam-queue", -] - [[package]] name = "system-configuration" version = "0.5.1" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e6bf6d940..2df89367b 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -28,4 +28,3 @@ serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.122" toml = "0.8.19" zstd = "0.13.2" -heed = "0.20.3" diff --git a/parser/src/databases/_database_heed.rs b/parser/src/databases/_database_heed.rs deleted file mode 100644 index 7ad0eaef2..000000000 --- a/parser/src/databases/_database_heed.rs +++ /dev/null @@ -1,206 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Debug, - fs, -}; - -use allocative::Allocative; -use derive_deref::{Deref, DerefMut}; - -// https://docs.rs/sanakirja/latest/sanakirja/index.html -// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ -// -// Seems indeed much faster than ReDB and LMDB (heed) -// But a lot has changed code wise between them so a retest wouldn't hurt -// -// Possible compression: https://pijul.org/posts/sanakirja-zstd/ -use sanakirja::{ - btree::{self, page, Db_}, - direct_repr, Commit, Env, Error, MutTxn, RootDb, Storable, UnsizedStorable, -}; - -use crate::io::OUTPUTS_FOLDER_PATH; - -#[derive(Allocative)] -#[allocative(bound = "Key: Allocative, Value: Allocative")] -/// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way. -pub struct DatabaseHeed -where - Key: Ord + Clone + Debug + ?Sized + Storable, - Value: Storable + PartialEq, -{ - pub cached_puts: BTreeMap, - pub cached_dels: BTreeSet, - #[allocative(skip)] - db: Db_>, - #[allocative(skip)] - txn: MutTxn, -} - -const ROOT_DB: usize = 0; -const PAGE_SIZE: u64 = 4096 * 256; // 1mo - Must be a multiplier of 4096 - -impl DatabaseHeed -where - Key: Ord + Clone + Debug + ?Sized + Storable, - Value: Storable + PartialEq, -{ - pub fn open(folder: &str, file: &str) -> color_eyre::Result { - let mut txn = Self::init_txn(folder, file)?; - - let db = txn - .root_db(ROOT_DB) - .unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() }); - - Ok(Self { - cached_puts: BTreeMap::default(), - cached_dels: BTreeSet::default(), - db, - txn, - }) - } - - pub fn iter(&self, callback: &mut F) - where - F: FnMut((&Key, &Value)), - { - btree::iter(&self.txn, &self.db, None) - .unwrap() - .for_each(|entry| callback(entry.unwrap())); - } - - pub fn get(&self, key: &Key) -> Option<&Value> { - if let Some(cached_put) = self.get_from_puts(key) { - return Some(cached_put); - } - - self.db_get(key) - } - - pub fn db_get(&self, key: &Key) -> Option<&Value> { - let option = btree::get(&self.txn, &self.db, key, None).unwrap(); - - if let Some((key_found, v)) = option { - if key == key_found { - return Some(v); - } - } - - None - } - - #[inline(always)] - pub fn get_from_puts(&self, key: &Key) -> Option<&Value> { - self.cached_puts.get(key) - } - - #[inline(always)] - pub fn get_mut_from_puts(&mut self, key: &Key) -> Option<&mut Value> { - self.cached_puts.get_mut(key) - } - - #[inline(always)] - pub fn remove(&mut self, key: &Key) -> Option { - self.remove_from_puts(key).or_else(|| { - self.db_remove(key); - - None - }) - } - - #[inline(always)] - pub fn db_remove(&mut self, key: &Key) { - self.cached_dels.insert(key.clone()); - } - - pub fn update(&mut self, key: Key, value: Value) -> Option { - self.cached_dels.insert(key.clone()); - - self.cached_puts.insert(key, value) - } - - #[inline(always)] - pub fn remove_from_puts(&mut self, key: &Key) -> Option { - self.cached_puts.remove(key) - } - - #[inline(always)] - pub fn insert(&mut self, key: Key, value: Value) -> Option { - self.cached_dels.remove(&key); - - self.unsafe_insert(key, value) - } - - #[inline(always)] - pub fn unsafe_insert(&mut self, key: Key, value: Value) -> Option { - self.cached_puts.insert(key, value) - } - - fn init_txn(folder: &str, file: &str) -> color_eyre::Result> { - let path = databases_folder_path(folder); - - fs::create_dir_all(&path)?; - - let env = unsafe { Env::new_nolock(format!("{path}/{file}"), PAGE_SIZE, 1).unwrap() }; - - let txn = Env::mut_txn_begin(env)?; - - Ok(txn) - } - - pub fn export(mut self) -> color_eyre::Result<(), Error> { - if self.cached_dels.is_empty() && self.cached_puts.is_empty() { - return Ok(()); - } - - self.cached_dels - .into_iter() - .try_for_each(|key| -> Result<(), Error> { - btree::del(&mut self.txn, &mut self.db, &key, None)?; - - Ok(()) - })?; - - self.cached_puts - .into_iter() - .try_for_each(|(key, value)| -> Result<(), Error> { - btree::put(&mut self.txn, &mut self.db, &key, &value)?; - - Ok(()) - })?; - - self.txn.set_root(ROOT_DB, self.db.db.into()); - - self.txn.commit() - } -} - -#[derive( - Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, -)] -pub struct U8x19([u8; 19]); -direct_repr!(U8x19); -impl From<&[u8]> for U8x19 { - fn from(slice: &[u8]) -> Self { - let mut arr = Self::default(); - arr.copy_from_slice(slice); - arr - } -} - -#[derive( - Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deref, DerefMut, Default, Copy, Allocative, -)] -pub struct U8x31([u8; 31]); -direct_repr!(U8x31); -impl From<&[u8]> for U8x31 { - fn from(slice: &[u8]) -> Self { - let mut arr = Self::default(); - arr.copy_from_slice(slice); - arr - } -} - -pub fn databases_folder_path(folder: &str) -> String { - format!("{OUTPUTS_FOLDER_PATH}/databases/{folder}") -} diff --git a/server/Cargo.lock b/server/Cargo.lock index fb81ed3bc..98a4980c6 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -276,15 +276,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bincode" version = "2.0.0-rc.3" @@ -407,9 +398,6 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -dependencies = [ - "serde", -] [[package]] name = "brotli" @@ -687,15 +675,6 @@ dependencies = [ "syn 2.0.60", ] -[[package]] -name = "doxygen-rs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" -dependencies = [ - "phf", -] - [[package]] name = "either" version = "1.12.0" @@ -910,44 +889,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "heed" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e" -dependencies = [ - "bitflags 2.6.0", - "byteorder", - "heed-traits", - "heed-types", - "libc", - "lmdb-master-sys", - "once_cell", - "page_size", - "serde", - "synchronoise", - "url", -] - -[[package]] -name = "heed-traits" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" - -[[package]] -name = "heed-types" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" -dependencies = [ - "bincode 1.3.3", - "byteorder", - "heed-traits", - "serde", - "serde_json", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -1256,17 +1197,6 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" -[[package]] -name = "lmdb-master-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24" -dependencies = [ - "cc", - "doxygen-rs", - "libc", -] - [[package]] name = "lock_api" version = "0.4.11" @@ -1463,16 +1393,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -1526,14 +1446,13 @@ name = "parser" version = "0.3.0" dependencies = [ "allocative", - "bincode 2.0.0-rc.3", + "bincode", "bitcoin_hashes", "biter", "chrono", "clap", "color-eyre", "derive_deref", - "heed", "inferno", "itertools", "memory-stats", @@ -1553,48 +1472,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" -dependencies = [ - "phf_shared", - "rand", -] - -[[package]] -name = "phf_macros" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project" version = "1.1.5" @@ -2053,7 +1930,7 @@ name = "server" version = "0.3.0" dependencies = [ "axum", - "bincode 2.0.0-rc.3", + "bincode", "color-eyre", "derive_deref", "itertools", @@ -2084,12 +1961,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "slab" version = "0.4.9" @@ -2173,15 +2044,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -[[package]] -name = "synchronoise" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" -dependencies = [ - "crossbeam-queue", -] - [[package]] name = "system-configuration" version = "0.5.1"