diff --git a/Cargo.lock b/Cargo.lock index cbd1033fa..0740b9304 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,7 +609,7 @@ dependencies = [ "brk_logger", "brk_parser", "brk_store", - "brk_vec", + "brk_vecs", "color-eyre", "fjall", "libc", diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index bd2ffc2bf..c77e9ee7c 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -15,7 +15,7 @@ brk_exit = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_store = { workspace = true } -brk_vec = { workspace = true } +brk_vecs = { workspace = true } color-eyre = { workspace = true } fjall = { workspace = true } libc = { workspace = true } diff --git a/crates/brk_indexer/examples/main.rs b/crates/brk_indexer/examples/indexer.rs similarity index 100% rename from crates/brk_indexer/examples/main.rs rename to crates/brk_indexer/examples/indexer.rs diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index ced35df24..8b44290ac 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -6,7 +6,7 @@ use brk_core::{ Result, TxIndex, TypeIndex, UnknownOutputIndex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; -use brk_vec::{AnyIndexedVec, AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType}; +use brk_vecs::{AnyIterableVec, AnyStampedVec, AnyVec, StampedVec, StoredIndex, StoredType}; use color_eyre::eyre::ContextCompat; use crate::{Stores, Vecs}; @@ -208,18 +208,18 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { } pub fn starting_index( - height_to_index: &IndexedVec, - index_to_else: &IndexedVec, + height_to_index: &StampedVec, + index_to_else: &StampedVec, starting_height: Height, ) -> Option where I: StoredType + StoredIndex + From, T: StoredType, { - let h = height_to_index.height(); + let h = Height::from(u64::from(height_to_index.stamp())); if h.is_zero() { None - } else if height_to_index.height() + 1_u32 == starting_height { + } else if h + 1_u32 == starting_height { Some(I::from(index_to_else.len())) } else { height_to_index.iter().get_inner(starting_height) diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index a724c58ba..88b8bd8d9 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -1,21 +1,20 @@ #![doc = include_str!("../README.md")] #![doc = "\n## Example\n\n```rust"] -#![doc = include_str!("../examples/main.rs")] +#![doc = include_str!("../examples/indexer.rs")] #![doc = "```"] -use std::{collections::BTreeMap, path::Path, str::FromStr, thread}; +use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread}; +use bitcoin::{Transaction, TxIn, TxOut}; use brk_core::{ AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex, OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit, }; - -use bitcoin::{Transaction, TxIn, TxOut}; use brk_exit::Exit; use brk_parser::Parser; use brk_store::AnyStore; -use brk_vec::{AnyVec, Mmap, VecIterator}; +use brk_vecs::{AnyVec, File, Reader, VecIterator}; use color_eyre::eyre::{ContextCompat, eyre}; use log::{error, info}; use rayon::prelude::*; @@ -33,6 +32,7 @@ const VERSION: Version = Version::ONE; #[derive(Clone)] pub struct Indexer { + pub file: Arc, pub vecs: Vecs, pub stores: Stores, } @@ -41,9 +41,12 @@ impl Indexer { pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result { setrlimit()?; + let file = Arc::new(File::open(&outputs_dir.join("vecs"))?); + Ok(Self { - vecs: Vecs::forced_import(&outputs_dir.join("vecs/indexed"), VERSION + Version::ZERO)?, + vecs: Vecs::forced_import(&file, VERSION + Version::ZERO)?, stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?, + file, }) } @@ -101,67 +104,64 @@ impl Indexer { Ok(true) }; - let mut txindex_to_first_outputindex_mmap_opt = None; - let mut p2pk65addressindex_to_p2pk65bytes_mmap_opt = None; - let mut p2pk33addressindex_to_p2pk33bytes_mmap_opt = None; - let mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt = None; - let mut p2shaddressindex_to_p2shbytes_mmap_opt = None; - let mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt = None; - let mut p2wshaddressindex_to_p2wshbytes_mmap_opt = None; - let mut p2traddressindex_to_p2trbytes_mmap_opt = None; - let mut p2aaddressindex_to_p2abytes_mmap_opt = None; + let mut txindex_to_first_outputindex_reader_opt = None; + let mut p2pk65addressindex_to_p2pk65bytes_reader_opt = None; + let mut p2pk33addressindex_to_p2pk33bytes_reader_opt = None; + let mut p2pkhaddressindex_to_p2pkhbytes_reader_opt = None; + let mut p2shaddressindex_to_p2shbytes_reader_opt = None; + let mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt = None; + let mut p2wshaddressindex_to_p2wshbytes_reader_opt = None; + let mut p2traddressindex_to_p2trbytes_reader_opt = None; + let mut p2aaddressindex_to_p2abytes_reader_opt = None; let reset_mmaps_options = |vecs: &mut Vecs, - txindex_to_first_outputindex_mmap_opt: &mut Option, - p2pk65addressindex_to_p2pk65bytes_mmap_opt: &mut Option, - p2pk33addressindex_to_p2pk33bytes_mmap_opt: &mut Option, - p2pkhaddressindex_to_p2pkhbytes_mmap_opt: &mut Option, - p2shaddressindex_to_p2shbytes_mmap_opt: &mut Option, - p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt: &mut Option, - p2wshaddressindex_to_p2wshbytes_mmap_opt: &mut Option, - p2traddressindex_to_p2trbytes_mmap_opt: &mut Option, - p2aaddressindex_to_p2abytes_mmap_opt: &mut Option| { - txindex_to_first_outputindex_mmap_opt - .replace(vecs.txindex_to_first_outputindex.create_mmap().unwrap()); - p2pk65addressindex_to_p2pk65bytes_mmap_opt.replace( + txindex_to_first_outputindex_reader_opt: &mut Option>, + p2pk65addressindex_to_p2pk65bytes_reader_opt: &mut Option>, + p2pk33addressindex_to_p2pk33bytes_reader_opt: &mut Option>, + p2pkhaddressindex_to_p2pkhbytes_reader_opt: &mut Option>, + p2shaddressindex_to_p2shbytes_reader_opt: &mut Option>, + p2wpkhaddressindex_to_p2wpkhbytes_reader_opt: &mut Option>, + p2wshaddressindex_to_p2wshbytes_reader_opt: &mut Option>, + p2traddressindex_to_p2trbytes_reader_opt: &mut Option>, + p2aaddressindex_to_p2abytes_reader_opt: &mut Option>| { + txindex_to_first_outputindex_reader_opt + .replace(vecs.txindex_to_first_outputindex.create_static_reader()); + p2pk65addressindex_to_p2pk65bytes_reader_opt.replace( vecs.p2pk65addressindex_to_p2pk65bytes - .create_mmap() - .unwrap(), + .create_static_reader(), ); - p2pk33addressindex_to_p2pk33bytes_mmap_opt.replace( + p2pk33addressindex_to_p2pk33bytes_reader_opt.replace( vecs.p2pk33addressindex_to_p2pk33bytes - .create_mmap() - .unwrap(), + .create_static_reader(), ); - p2pkhaddressindex_to_p2pkhbytes_mmap_opt - .replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_mmap().unwrap()); - p2shaddressindex_to_p2shbytes_mmap_opt - .replace(vecs.p2shaddressindex_to_p2shbytes.create_mmap().unwrap()); - p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.replace( + p2pkhaddressindex_to_p2pkhbytes_reader_opt + .replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_static_reader()); + p2shaddressindex_to_p2shbytes_reader_opt + .replace(vecs.p2shaddressindex_to_p2shbytes.create_static_reader()); + p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.replace( vecs.p2wpkhaddressindex_to_p2wpkhbytes - .create_mmap() - .unwrap(), + .create_static_reader(), ); - p2wshaddressindex_to_p2wshbytes_mmap_opt - .replace(vecs.p2wshaddressindex_to_p2wshbytes.create_mmap().unwrap()); - p2traddressindex_to_p2trbytes_mmap_opt - .replace(vecs.p2traddressindex_to_p2trbytes.create_mmap().unwrap()); - p2aaddressindex_to_p2abytes_mmap_opt - .replace(vecs.p2aaddressindex_to_p2abytes.create_mmap().unwrap()); + p2wshaddressindex_to_p2wshbytes_reader_opt + .replace(vecs.p2wshaddressindex_to_p2wshbytes.create_static_reader()); + p2traddressindex_to_p2trbytes_reader_opt + .replace(vecs.p2traddressindex_to_p2trbytes.create_static_reader()); + p2aaddressindex_to_p2abytes_reader_opt + .replace(vecs.p2aaddressindex_to_p2abytes.create_static_reader()); }; reset_mmaps_options( vecs, - &mut txindex_to_first_outputindex_mmap_opt, - &mut p2pk65addressindex_to_p2pk65bytes_mmap_opt, - &mut p2pk33addressindex_to_p2pk33bytes_mmap_opt, - &mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt, - &mut p2shaddressindex_to_p2shbytes_mmap_opt, - &mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt, - &mut p2wshaddressindex_to_p2wshbytes_mmap_opt, - &mut p2traddressindex_to_p2trbytes_mmap_opt, - &mut p2aaddressindex_to_p2abytes_mmap_opt, + &mut txindex_to_first_outputindex_reader_opt, + &mut p2pk65addressindex_to_p2pk65bytes_reader_opt, + &mut p2pk33addressindex_to_p2pk33bytes_reader_opt, + &mut p2pkhaddressindex_to_p2pkhbytes_reader_opt, + &mut p2shaddressindex_to_p2shbytes_reader_opt, + &mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt, + &mut p2wshaddressindex_to_p2wshbytes_reader_opt, + &mut p2traddressindex_to_p2trbytes_reader_opt, + &mut p2aaddressindex_to_p2abytes_reader_opt, ); parser.parse(start, end).iter().try_for_each( @@ -170,15 +170,15 @@ impl Indexer { idxs.height = height; - let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_mmap_opt.as_ref().unwrap(); - let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_mmap_opt.as_ref().unwrap(); - let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_mmap_opt.as_ref().unwrap(); - let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_mmap_opt.as_ref().unwrap(); - let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_mmap_opt.as_ref().unwrap(); - let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.as_ref().unwrap(); - let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_mmap_opt.as_ref().unwrap(); - let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_mmap_opt.as_ref().unwrap(); - let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_mmap_opt.as_ref().unwrap(); + let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_reader_opt.as_ref().unwrap(); + let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_reader_opt.as_ref().unwrap(); + let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_reader_opt.as_ref().unwrap(); + let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_reader_opt.as_ref().unwrap(); + let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_reader_opt.as_ref().unwrap(); + let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.as_ref().unwrap(); + let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_reader_opt.as_ref().unwrap(); + let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_reader_opt.as_ref().unwrap(); + let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_reader_opt.as_ref().unwrap(); // Used to check rapidhash collisions let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO); @@ -721,6 +721,8 @@ impl Indexer { }, )?; + drop(txindex_to_txid_iter); + txindex_to_tx_and_txid .into_iter() .try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> { @@ -737,20 +739,30 @@ impl Indexer { idxs.inputindex += InputIndex::from(inputs_len); idxs.outputindex += OutputIndex::from(outputs_len); + txindex_to_first_outputindex_reader_opt.take(); + p2pk65addressindex_to_p2pk65bytes_reader_opt.take(); + p2pk33addressindex_to_p2pk33bytes_reader_opt.take(); + p2pkhaddressindex_to_p2pkhbytes_reader_opt.take(); + p2shaddressindex_to_p2shbytes_reader_opt.take(); + p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take(); + p2wshaddressindex_to_p2wshbytes_reader_opt.take(); + p2traddressindex_to_p2trbytes_reader_opt.take(); + p2aaddressindex_to_p2abytes_reader_opt.take(); + let exported = export_if_needed(stores, vecs, height, false, exit)?; if exported { reset_mmaps_options( vecs, - &mut txindex_to_first_outputindex_mmap_opt, - &mut p2pk65addressindex_to_p2pk65bytes_mmap_opt, - &mut p2pk33addressindex_to_p2pk33bytes_mmap_opt, - &mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt, - &mut p2shaddressindex_to_p2shbytes_mmap_opt, - &mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt, - &mut p2wshaddressindex_to_p2wshbytes_mmap_opt, - &mut p2traddressindex_to_p2trbytes_mmap_opt, - &mut p2aaddressindex_to_p2abytes_mmap_opt, + &mut txindex_to_first_outputindex_reader_opt, + &mut p2pk65addressindex_to_p2pk65bytes_reader_opt, + &mut p2pk33addressindex_to_p2pk33bytes_reader_opt, + &mut p2pkhaddressindex_to_p2pkhbytes_reader_opt, + &mut p2shaddressindex_to_p2shbytes_reader_opt, + &mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt, + &mut p2wshaddressindex_to_p2wshbytes_reader_opt, + &mut p2traddressindex_to_p2trbytes_reader_opt, + &mut p2aaddressindex_to_p2abytes_reader_opt, ); } diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index ee7e69f48..ed0410bd5 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -5,7 +5,7 @@ use brk_core::{ OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version, }; use brk_store::{AnyStore, Store}; -use brk_vec::{AnyIterableVec, VecIterator}; +use brk_vecs::{AnyIterableVec, VecIterator}; use fjall::{PersistMode, TransactionalKeyspace}; use rayon::prelude::*; diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index d2b8fb125..44bab394c 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::sync::Arc; use brk_core::{ AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex, @@ -8,7 +8,7 @@ use brk_core::{ RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, Txid, TypeIndex, UnknownOutputIndex, Version, Weight, }; -use brk_vec::{AnyCollectableVec, AnyIndexedVec, Format, IndexedVec}; +use brk_vecs::{AnyCollectableVec, AnyStampedVec, File, Format, StampedVec}; use rayon::prelude::*; use crate::Indexes; @@ -17,317 +17,317 @@ const VERSION: Version = Version::ZERO; #[derive(Clone)] pub struct Vecs { - pub emptyoutputindex_to_txindex: IndexedVec, - pub height_to_blockhash: IndexedVec, - pub height_to_difficulty: IndexedVec, - pub height_to_first_emptyoutputindex: IndexedVec, - pub height_to_first_inputindex: IndexedVec, - pub height_to_first_opreturnindex: IndexedVec, - pub height_to_first_outputindex: IndexedVec, - pub height_to_first_p2aaddressindex: IndexedVec, - pub height_to_first_p2msoutputindex: IndexedVec, - pub height_to_first_p2pk33addressindex: IndexedVec, - pub height_to_first_p2pk65addressindex: IndexedVec, - pub height_to_first_p2pkhaddressindex: IndexedVec, - pub height_to_first_p2shaddressindex: IndexedVec, - pub height_to_first_p2traddressindex: IndexedVec, - pub height_to_first_p2wpkhaddressindex: IndexedVec, - pub height_to_first_p2wshaddressindex: IndexedVec, - pub height_to_first_txindex: IndexedVec, - pub height_to_first_unknownoutputindex: IndexedVec, + pub emptyoutputindex_to_txindex: StampedVec, + pub height_to_blockhash: StampedVec, + pub height_to_difficulty: StampedVec, + pub height_to_first_emptyoutputindex: StampedVec, + pub height_to_first_inputindex: StampedVec, + pub height_to_first_opreturnindex: StampedVec, + pub height_to_first_outputindex: StampedVec, + pub height_to_first_p2aaddressindex: StampedVec, + pub height_to_first_p2msoutputindex: StampedVec, + pub height_to_first_p2pk33addressindex: StampedVec, + pub height_to_first_p2pk65addressindex: StampedVec, + pub height_to_first_p2pkhaddressindex: StampedVec, + pub height_to_first_p2shaddressindex: StampedVec, + pub height_to_first_p2traddressindex: StampedVec, + pub height_to_first_p2wpkhaddressindex: StampedVec, + pub height_to_first_p2wshaddressindex: StampedVec, + pub height_to_first_txindex: StampedVec, + pub height_to_first_unknownoutputindex: StampedVec, /// Doesn't guarantee continuity due to possible reorgs - pub height_to_timestamp: IndexedVec, - pub height_to_total_size: IndexedVec, - pub height_to_weight: IndexedVec, + pub height_to_timestamp: StampedVec, + pub height_to_total_size: StampedVec, + pub height_to_weight: StampedVec, /// If outputindex == Outputindex::MAX then it's coinbase - pub inputindex_to_outputindex: IndexedVec, - pub opreturnindex_to_txindex: IndexedVec, - pub outputindex_to_outputtype: IndexedVec, - pub outputindex_to_typeindex: IndexedVec, - pub outputindex_to_value: IndexedVec, - pub p2aaddressindex_to_p2abytes: IndexedVec, - pub p2msoutputindex_to_txindex: IndexedVec, - pub p2pk33addressindex_to_p2pk33bytes: IndexedVec, - pub p2pk65addressindex_to_p2pk65bytes: IndexedVec, - pub p2pkhaddressindex_to_p2pkhbytes: IndexedVec, - pub p2shaddressindex_to_p2shbytes: IndexedVec, - pub p2traddressindex_to_p2trbytes: IndexedVec, - pub p2wpkhaddressindex_to_p2wpkhbytes: IndexedVec, - pub p2wshaddressindex_to_p2wshbytes: IndexedVec, - pub txindex_to_base_size: IndexedVec, - pub txindex_to_first_inputindex: IndexedVec, - pub txindex_to_first_outputindex: IndexedVec, - pub txindex_to_is_explicitly_rbf: IndexedVec, - pub txindex_to_rawlocktime: IndexedVec, - pub txindex_to_total_size: IndexedVec, - pub txindex_to_txid: IndexedVec, - pub txindex_to_txversion: IndexedVec, - pub unknownoutputindex_to_txindex: IndexedVec, + pub inputindex_to_outputindex: StampedVec, + pub opreturnindex_to_txindex: StampedVec, + pub outputindex_to_outputtype: StampedVec, + pub outputindex_to_typeindex: StampedVec, + pub outputindex_to_value: StampedVec, + pub p2aaddressindex_to_p2abytes: StampedVec, + pub p2msoutputindex_to_txindex: StampedVec, + pub p2pk33addressindex_to_p2pk33bytes: StampedVec, + pub p2pk65addressindex_to_p2pk65bytes: StampedVec, + pub p2pkhaddressindex_to_p2pkhbytes: StampedVec, + pub p2shaddressindex_to_p2shbytes: StampedVec, + pub p2traddressindex_to_p2trbytes: StampedVec, + pub p2wpkhaddressindex_to_p2wpkhbytes: StampedVec, + pub p2wshaddressindex_to_p2wshbytes: StampedVec, + pub txindex_to_base_size: StampedVec, + pub txindex_to_first_inputindex: StampedVec, + pub txindex_to_first_outputindex: StampedVec, + pub txindex_to_is_explicitly_rbf: StampedVec, + pub txindex_to_rawlocktime: StampedVec, + pub txindex_to_total_size: StampedVec, + pub txindex_to_txid: StampedVec, + pub txindex_to_txversion: StampedVec, + pub unknownoutputindex_to_txindex: StampedVec, } impl Vecs { - pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result { + pub fn forced_import(file: &Arc, version: Version) -> color_eyre::Result { Ok(Self { - emptyoutputindex_to_txindex: IndexedVec::forced_import( - path, + emptyoutputindex_to_txindex: StampedVec::forced_import( + file, "txindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_blockhash: IndexedVec::forced_import( - path, + height_to_blockhash: StampedVec::forced_import( + file, "blockhash", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_difficulty: IndexedVec::forced_import( - path, + height_to_difficulty: StampedVec::forced_import( + file, "difficulty", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_emptyoutputindex: IndexedVec::forced_import( - path, + height_to_first_emptyoutputindex: StampedVec::forced_import( + file, "first_emptyoutputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_inputindex: IndexedVec::forced_import( - path, + height_to_first_inputindex: StampedVec::forced_import( + file, "first_inputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_opreturnindex: IndexedVec::forced_import( - path, + height_to_first_opreturnindex: StampedVec::forced_import( + file, "first_opreturnindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_outputindex: IndexedVec::forced_import( - path, + height_to_first_outputindex: StampedVec::forced_import( + file, "first_outputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2aaddressindex: IndexedVec::forced_import( - path, + height_to_first_p2aaddressindex: StampedVec::forced_import( + file, "first_p2aaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2msoutputindex: IndexedVec::forced_import( - path, + height_to_first_p2msoutputindex: StampedVec::forced_import( + file, "first_p2msoutputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2pk33addressindex: IndexedVec::forced_import( - path, + height_to_first_p2pk33addressindex: StampedVec::forced_import( + file, "first_p2pk33addressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2pk65addressindex: IndexedVec::forced_import( - path, + height_to_first_p2pk65addressindex: StampedVec::forced_import( + file, "first_p2pk65addressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2pkhaddressindex: IndexedVec::forced_import( - path, + height_to_first_p2pkhaddressindex: StampedVec::forced_import( + file, "first_p2pkhaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2shaddressindex: IndexedVec::forced_import( - path, + height_to_first_p2shaddressindex: StampedVec::forced_import( + file, "first_p2shaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2traddressindex: IndexedVec::forced_import( - path, + height_to_first_p2traddressindex: StampedVec::forced_import( + file, "first_p2traddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2wpkhaddressindex: IndexedVec::forced_import( - path, + height_to_first_p2wpkhaddressindex: StampedVec::forced_import( + file, "first_p2wpkhaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_p2wshaddressindex: IndexedVec::forced_import( - path, + height_to_first_p2wshaddressindex: StampedVec::forced_import( + file, "first_p2wshaddressindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_txindex: IndexedVec::forced_import( - path, + height_to_first_txindex: StampedVec::forced_import( + file, "first_txindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_first_unknownoutputindex: IndexedVec::forced_import( - path, + height_to_first_unknownoutputindex: StampedVec::forced_import( + file, "first_unknownoutputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_timestamp: IndexedVec::forced_import( - path, + height_to_timestamp: StampedVec::forced_import( + file, "timestamp", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_total_size: IndexedVec::forced_import( - path, + height_to_total_size: StampedVec::forced_import( + file, "total_size", version + VERSION + Version::ZERO, Format::Raw, )?, - height_to_weight: IndexedVec::forced_import( - path, + height_to_weight: StampedVec::forced_import( + file, "weight", version + VERSION + Version::ZERO, Format::Raw, )?, - inputindex_to_outputindex: IndexedVec::forced_import( - path, + inputindex_to_outputindex: StampedVec::forced_import( + file, "outputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - opreturnindex_to_txindex: IndexedVec::forced_import( - path, + opreturnindex_to_txindex: StampedVec::forced_import( + file, "txindex", version + VERSION + Version::ZERO, Format::Raw, )?, - outputindex_to_outputtype: IndexedVec::forced_import( - path, + outputindex_to_outputtype: StampedVec::forced_import( + file, "outputtype", version + VERSION + Version::ZERO, Format::Raw, )?, - outputindex_to_typeindex: IndexedVec::forced_import( - path, + outputindex_to_typeindex: StampedVec::forced_import( + file, "typeindex", version + VERSION + Version::ZERO, Format::Raw, )?, - outputindex_to_value: IndexedVec::forced_import( - path, + outputindex_to_value: StampedVec::forced_import( + file, "value", version + VERSION + Version::ZERO, Format::Raw, )?, - p2aaddressindex_to_p2abytes: IndexedVec::forced_import( - path, + p2aaddressindex_to_p2abytes: StampedVec::forced_import( + file, "p2abytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2msoutputindex_to_txindex: IndexedVec::forced_import( - path, + p2msoutputindex_to_txindex: StampedVec::forced_import( + file, "txindex", version + VERSION + Version::ZERO, Format::Raw, )?, - p2pk33addressindex_to_p2pk33bytes: IndexedVec::forced_import( - path, + p2pk33addressindex_to_p2pk33bytes: StampedVec::forced_import( + file, "p2pk33bytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2pk65addressindex_to_p2pk65bytes: IndexedVec::forced_import( - path, + p2pk65addressindex_to_p2pk65bytes: StampedVec::forced_import( + file, "p2pk65bytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2pkhaddressindex_to_p2pkhbytes: IndexedVec::forced_import( - path, + p2pkhaddressindex_to_p2pkhbytes: StampedVec::forced_import( + file, "p2pkhbytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2shaddressindex_to_p2shbytes: IndexedVec::forced_import( - path, + p2shaddressindex_to_p2shbytes: StampedVec::forced_import( + file, "p2shbytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2traddressindex_to_p2trbytes: IndexedVec::forced_import( - path, + p2traddressindex_to_p2trbytes: StampedVec::forced_import( + file, "p2trbytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2wpkhaddressindex_to_p2wpkhbytes: IndexedVec::forced_import( - path, + p2wpkhaddressindex_to_p2wpkhbytes: StampedVec::forced_import( + file, "p2wpkhbytes", version + VERSION + Version::ZERO, Format::Raw, )?, - p2wshaddressindex_to_p2wshbytes: IndexedVec::forced_import( - path, + p2wshaddressindex_to_p2wshbytes: StampedVec::forced_import( + file, "p2wshbytes", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_base_size: IndexedVec::forced_import( - path, + txindex_to_base_size: StampedVec::forced_import( + file, "base_size", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_first_inputindex: IndexedVec::forced_import( - path, + txindex_to_first_inputindex: StampedVec::forced_import( + file, "first_inputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_first_outputindex: IndexedVec::forced_import( - path, + txindex_to_first_outputindex: StampedVec::forced_import( + file, "first_outputindex", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_is_explicitly_rbf: IndexedVec::forced_import( - path, + txindex_to_is_explicitly_rbf: StampedVec::forced_import( + file, "is_explicitly_rbf", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_rawlocktime: IndexedVec::forced_import( - path, + txindex_to_rawlocktime: StampedVec::forced_import( + file, "rawlocktime", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_total_size: IndexedVec::forced_import( - path, + txindex_to_total_size: StampedVec::forced_import( + file, "total_size", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_txid: IndexedVec::forced_import( - path, + txindex_to_txid: StampedVec::forced_import( + file, "txid", version + VERSION + Version::ZERO, Format::Raw, )?, - txindex_to_txversion: IndexedVec::forced_import( - path, + txindex_to_txversion: StampedVec::forced_import( + file, "txversion", version + VERSION + Version::ZERO, Format::Raw, )?, - unknownoutputindex_to_txindex: IndexedVec::forced_import( - path, + unknownoutputindex_to_txindex: StampedVec::forced_import( + file, "txindex", version + VERSION + Version::ZERO, Format::Raw, @@ -357,94 +357,92 @@ impl Vecs { unknownoutputindex, } = starting_indexes; + let stamp = u64::from(saved_height).into(); + self.emptyoutputindex_to_txindex - .truncate_if_needed(emptyoutputindex, saved_height)?; - self.height_to_blockhash - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(emptyoutputindex, stamp)?; + self.height_to_blockhash.truncate_if_needed(height, stamp)?; self.height_to_difficulty - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_emptyoutputindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_inputindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_opreturnindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_outputindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2aaddressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2msoutputindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2pk33addressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2pk65addressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2pkhaddressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2shaddressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2traddressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2wpkhaddressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_p2wshaddressindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_txindex - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; self.height_to_first_unknownoutputindex - .truncate_if_needed(height, saved_height)?; - self.height_to_timestamp - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; + self.height_to_timestamp.truncate_if_needed(height, stamp)?; self.height_to_total_size - .truncate_if_needed(height, saved_height)?; - self.height_to_weight - .truncate_if_needed(height, saved_height)?; + .truncate_if_needed(height, stamp)?; + self.height_to_weight.truncate_if_needed(height, stamp)?; self.inputindex_to_outputindex - .truncate_if_needed(inputindex, saved_height)?; + .truncate_if_needed(inputindex, stamp)?; self.opreturnindex_to_txindex - .truncate_if_needed(opreturnindex, saved_height)?; + .truncate_if_needed(opreturnindex, stamp)?; self.outputindex_to_outputtype - .truncate_if_needed(outputindex, saved_height)?; + .truncate_if_needed(outputindex, stamp)?; self.outputindex_to_typeindex - .truncate_if_needed(outputindex, saved_height)?; + .truncate_if_needed(outputindex, stamp)?; self.outputindex_to_value - .truncate_if_needed(outputindex, saved_height)?; + .truncate_if_needed(outputindex, stamp)?; self.p2aaddressindex_to_p2abytes - .truncate_if_needed(p2aaddressindex, saved_height)?; + .truncate_if_needed(p2aaddressindex, stamp)?; self.p2msoutputindex_to_txindex - .truncate_if_needed(p2msoutputindex, saved_height)?; + .truncate_if_needed(p2msoutputindex, stamp)?; self.p2pk33addressindex_to_p2pk33bytes - .truncate_if_needed(p2pk33addressindex, saved_height)?; + .truncate_if_needed(p2pk33addressindex, stamp)?; self.p2pk65addressindex_to_p2pk65bytes - .truncate_if_needed(p2pk65addressindex, saved_height)?; + .truncate_if_needed(p2pk65addressindex, stamp)?; self.p2pkhaddressindex_to_p2pkhbytes - .truncate_if_needed(p2pkhaddressindex, saved_height)?; + .truncate_if_needed(p2pkhaddressindex, stamp)?; self.p2shaddressindex_to_p2shbytes - .truncate_if_needed(p2shaddressindex, saved_height)?; + .truncate_if_needed(p2shaddressindex, stamp)?; self.p2traddressindex_to_p2trbytes - .truncate_if_needed(p2traddressindex, saved_height)?; + .truncate_if_needed(p2traddressindex, stamp)?; self.p2wpkhaddressindex_to_p2wpkhbytes - .truncate_if_needed(p2wpkhaddressindex, saved_height)?; + .truncate_if_needed(p2wpkhaddressindex, stamp)?; self.p2wshaddressindex_to_p2wshbytes - .truncate_if_needed(p2wshaddressindex, saved_height)?; + .truncate_if_needed(p2wshaddressindex, stamp)?; self.txindex_to_base_size - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; self.txindex_to_first_inputindex - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; self.txindex_to_first_outputindex - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; self.txindex_to_is_explicitly_rbf - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; self.txindex_to_rawlocktime - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; self.txindex_to_total_size - .truncate_if_needed(txindex, saved_height)?; - self.txindex_to_txid - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; + self.txindex_to_txid.truncate_if_needed(txindex, stamp)?; self.txindex_to_txversion - .truncate_if_needed(txindex, saved_height)?; + .truncate_if_needed(txindex, stamp)?; self.unknownoutputindex_to_txindex - .truncate_if_needed(unknownoutputindex, saved_height)?; + .truncate_if_needed(unknownoutputindex, stamp)?; Ok(()) } @@ -481,14 +479,14 @@ impl Vecs { pub fn flush(&mut self, height: Height) -> Result<()> { self.mut_vecs() .into_par_iter() - .try_for_each(|vec| vec.flush(height)) + .try_for_each(|vec| vec.flush(u64::from(height).into())) } pub fn starting_height(&mut self) -> Height { self.mut_vecs() .into_iter() .map(|vec| { - let h = vec.height(); + let h = Height::from(u64::from(vec.stamp())); if h > Height::ZERO { h.incremented() } else { h } }) .min() @@ -544,7 +542,7 @@ impl Vecs { ] } - fn mut_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> { + fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStampedVec> { vec![ &mut self.emptyoutputindex_to_txindex, &mut self.height_to_blockhash, diff --git a/crates/brk_vecs/src/lib.rs b/crates/brk_vecs/src/lib.rs index 3fd098fcf..a9ba1a2cd 100644 --- a/crates/brk_vecs/src/lib.rs +++ b/crates/brk_vecs/src/lib.rs @@ -2,9 +2,8 @@ mod file; mod traits; mod variants; -use file::*; use variants::*; -pub use file::{File, PAGE_SIZE}; +pub use file::{File, PAGE_SIZE, Reader}; pub use traits::*; -pub use variants::{RawVec, Stamp, StampedVec}; +pub use variants::{AnyStampedVec, Format, RawVec, Stamp, StampedVec}; diff --git a/crates/brk_vecs/src/traits/any.rs b/crates/brk_vecs/src/traits/any.rs index 84c516788..572c06ead 100644 --- a/crates/brk_vecs/src/traits/any.rs +++ b/crates/brk_vecs/src/traits/any.rs @@ -1,5 +1,7 @@ use brk_core::{Height, Version}; +use crate::{File, Reader}; + use super::{BoxedVecIterator, StoredIndex, StoredType}; pub fn i64_to_usize(i: i64, len: usize) -> usize { @@ -37,6 +39,30 @@ pub trait AnyVec: Send + Sync { ) } + fn file(&self) -> &File; + + fn region_index(&self) -> usize; + + /// Be careful with deadlocks + /// + /// You'll want to drop the reader before mutable ops + fn create_reader(&self) -> Reader<'_> { + self.create_static_reader() + } + + /// Be careful with deadlocks + /// + /// You'll want to drop the reader before mutable ops + fn create_static_reader(&self) -> Reader<'static> { + unsafe { + std::mem::transmute( + self.file() + .create_region_reader(self.region_index().into()) + .unwrap(), + ) + } + } + #[inline] fn i64_to_usize(&self, i: i64) -> usize { let len = self.len(); diff --git a/crates/brk_vecs/src/traits/generic.rs b/crates/brk_vecs/src/traits/generic.rs index 52c26dcce..e7b4eab84 100644 --- a/crates/brk_vecs/src/traits/generic.rs +++ b/crates/brk_vecs/src/traits/generic.rs @@ -6,7 +6,7 @@ use std::{ use brk_core::{Error, Result}; -use crate::{AnyVec, File, HEADER_OFFSET, Header, file::Reader}; +use crate::{AnyVec, HEADER_OFFSET, Header, file::Reader}; use super::{StoredIndex, StoredType}; @@ -193,30 +193,6 @@ where index < self.len_() } - fn file(&self) -> &File; - - fn region_index(&self) -> usize; - - /// Be careful with deadlocks - /// - /// You'll want to drop the reader before mutable ops - fn create_reader(&self) -> Reader<'_> { - self.create_static_reader() - } - - /// Be careful with deadlocks - /// - /// You'll want to drop the reader before mutable ops - fn create_static_reader(&self) -> Reader<'static> { - unsafe { - std::mem::transmute( - self.file() - .create_region_reader(self.region_index().into()) - .unwrap(), - ) - } - } - fn flush(&mut self) -> Result<()>; fn truncate_if_needed(&mut self, index: I) -> Result<()>; diff --git a/crates/brk_vecs/src/variants/compressed/mod.rs b/crates/brk_vecs/src/variants/compressed/mod.rs index 794dcb29e..2804d686f 100644 --- a/crates/brk_vecs/src/variants/compressed/mod.rs +++ b/crates/brk_vecs/src/variants/compressed/mod.rs @@ -189,14 +189,6 @@ where .cloned()) } - fn file(&self) -> &File { - self.inner.file() - } - - fn region_index(&self) -> usize { - self.inner.region_index() - } - fn header(&self) -> &Header { self.inner.header() } @@ -250,7 +242,7 @@ where // let mut file = file_opt.unwrap_or(self.open_file()?); - let mut pages_meta = self.pages_meta.read(); + // let mut pages_meta = self.pages_meta.read(); // let mut starting_page_index = pages_meta.len(); // let mut values = vec![]; @@ -309,7 +301,7 @@ where // .flat_map(|(v, _)| v) // .collect::>(); - pages_meta.write()?; + // pages_meta.write()?; // if let Some(truncate_at) = truncate_at { // self.file_set_len(&mut file, truncate_at)?; @@ -406,6 +398,14 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } + + fn file(&self) -> &File { + self.inner.file() + } + + fn region_index(&self) -> usize { + self.inner.region_index() + } } impl Clone for CompressedVec { diff --git a/crates/brk_vecs/src/variants/raw/mod.rs b/crates/brk_vecs/src/variants/raw/mod.rs index bebdd995a..5c438a9f0 100644 --- a/crates/brk_vecs/src/variants/raw/mod.rs +++ b/crates/brk_vecs/src/variants/raw/mod.rs @@ -289,14 +289,6 @@ where fn reset(&mut self) -> Result<()> { self.reset_() } - - fn file(&self) -> &File { - &self.file - } - - fn region_index(&self) -> usize { - self.region_index - } } impl AnyVec for RawVec @@ -328,6 +320,14 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } + + fn file(&self) -> &File { + &self.file + } + + fn region_index(&self) -> usize { + self.region_index + } } impl Clone for RawVec { diff --git a/crates/brk_vecs/src/variants/stamped/mod.rs b/crates/brk_vecs/src/variants/stamped/mod.rs index 6da32ae9d..ff9ab898b 100644 --- a/crates/brk_vecs/src/variants/stamped/mod.rs +++ b/crates/brk_vecs/src/variants/stamped/mod.rs @@ -162,6 +162,14 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } + + fn file(&self) -> &File { + self.0.file() + } + + fn region_index(&self) -> usize { + self.0.region_index() + } } pub trait AnyStampedVec: AnyVec { diff --git a/crates/brk_vecs/src/variants/stamped/stamp.rs b/crates/brk_vecs/src/variants/stamped/stamp.rs index 2a9a3fc88..c598f423e 100644 --- a/crates/brk_vecs/src/variants/stamped/stamp.rs +++ b/crates/brk_vecs/src/variants/stamped/stamp.rs @@ -8,3 +8,15 @@ impl Stamp { Self(stamp) } } + +impl From for Stamp { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl From for u64 { + fn from(value: Stamp) -> Self { + value.0 + } +} diff --git a/crates/brk_vecs/src/variants/stored/mod.rs b/crates/brk_vecs/src/variants/stored/mod.rs index d6428e651..48d65a4e8 100644 --- a/crates/brk_vecs/src/variants/stored/mod.rs +++ b/crates/brk_vecs/src/variants/stored/mod.rs @@ -70,22 +70,6 @@ where } } - #[inline] - fn file(&self) -> &File { - match self { - StoredVec::Raw(v) => v.file(), - StoredVec::Compressed(v) => v.file(), - } - } - - #[inline] - fn region_index(&self) -> usize { - match self { - StoredVec::Raw(v) => v.region_index(), - StoredVec::Compressed(v) => v.region_index(), - } - } - #[inline] fn mut_header(&mut self) -> &mut Header { match self { @@ -203,6 +187,22 @@ where fn value_type_to_size_of(&self) -> usize { size_of::() } + + #[inline] + fn file(&self) -> &File { + match self { + StoredVec::Raw(v) => v.file(), + StoredVec::Compressed(v) => v.file(), + } + } + + #[inline] + fn region_index(&self) -> usize { + match self { + StoredVec::Raw(v) => v.region_index(), + StoredVec::Compressed(v) => v.region_index(), + } + } } #[derive(Debug)]