diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 6a1da2142..2f714f326 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -13,21 +13,27 @@ use brk_vecs::Exit; fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log"))); - // let bitcoin_dir = brk_structs::default_bitcoin_path(); - let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); + let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap()) + .join("Library") + .join("Application Support") + .join("Bitcoin"); + // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); + + let blocks_dir = bitcoin_dir.join("blocks"); + let outputs_dir = Path::new("./_outputs"); fs::create_dir_all(outputs_dir)?; - // let outputs_dir = brk_structs::default_brk_path().join("outputs"); // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new( "http://localhost:8332", bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")), )?)); + let exit = Exit::new(); exit.set_ctrlc_handler(); - let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc); + let parser = Parser::new(blocks_dir, outputs_dir.to_path_buf(), rpc); fs::create_dir_all(outputs_dir)?; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index e457e6a81..99c6ad0bc 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -124,7 +124,7 @@ impl Indexer { let mut p2traddressindex_to_p2trbytes_reader_opt = None; let mut p2aaddressindex_to_p2abytes_reader_opt = None; - let reset_mmaps_options = + let reset_readers = |vecs: &mut Vecs, txindex_to_first_outputindex_reader_opt: &mut Option>, p2pk65addressindex_to_p2pk65bytes_reader_opt: &mut Option>, @@ -161,7 +161,7 @@ impl Indexer { .replace(vecs.p2aaddressindex_to_p2abytes.create_static_reader()); }; - reset_mmaps_options( + reset_readers( vecs, &mut txindex_to_first_outputindex_reader_opt, &mut p2pk65addressindex_to_p2pk65bytes_reader_opt, @@ -762,8 +762,10 @@ impl Indexer { p2wshaddressindex_to_p2wshbytes_reader_opt.take(); p2traddressindex_to_p2trbytes_reader_opt.take(); p2aaddressindex_to_p2abytes_reader_opt.take(); + export(stores, vecs, height, exit)?; - reset_mmaps_options( + + reset_readers( vecs, &mut txindex_to_first_outputindex_reader_opt, &mut p2pk65addressindex_to_p2pk65bytes_reader_opt, diff --git a/crates/brk_server/src/api/interface.rs b/crates/brk_server/src/api/interface.rs index 81955d91a..81b3ffac5 100644 --- a/crates/brk_server/src/api/interface.rs +++ b/crates/brk_server/src/api/interface.rs @@ -89,21 +89,7 @@ fn req_to_response_res( Response::new(Body::from(v)) } else { match interface.format(vecs, ¶ms.rest)? { - Output::CSV(s) => { - if let GuardResult::Guard(g) = guard_res { - g.insert(s.clone().into()) - .map_err(|_| Error::QuickCacheError)?; - } - s.into_response() - } - Output::TSV(s) => { - if let GuardResult::Guard(g) = guard_res { - g.insert(s.clone().into()) - .map_err(|_| Error::QuickCacheError)?; - } - s.into_response() - } - Output::MD(s) => { + Output::CSV(s) | Output::TSV(s) | Output::MD(s) => { if let GuardResult::Guard(g) = guard_res { g.insert(s.clone().into()) .map_err(|_| Error::QuickCacheError)?; @@ -143,7 +129,7 @@ fn req_to_response_res( Format::JSON => headers.insert_content_type_application_json(), } } - _ => headers.insert_content_type_application_json(), + None => headers.insert_content_type_application_json(), }; Ok(response) diff --git a/crates/brk_store/src/trait.rs b/crates/brk_store/src/any.rs similarity index 100% rename from crates/brk_store/src/trait.rs rename to crates/brk_store/src/any.rs diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 31ba85fe7..6527e412a 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -19,12 +19,12 @@ use fjall::{ TransactionalPartitionHandle, }; +mod any; mod meta; -mod r#trait; +pub use any::*; use log::info; use meta::*; -pub use r#trait::*; pub struct Store { meta: StoreMeta, diff --git a/crates/brk_vecs/examples/compressed.rs b/crates/brk_vecs/examples/compressed.rs index 8128b2d5f..d5f17fa45 100644 --- a/crates/brk_vecs/examples/compressed.rs +++ b/crates/brk_vecs/examples/compressed.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path, sync::Arc}; +use std::{borrow::Cow, collections::BTreeSet, fs, path::Path, sync::Arc}; use brk_vecs::{ AnyStoredVec, AnyVec, CollectableVec, CompressedVec, File, GenericStoredVec, Stamp, @@ -23,18 +23,16 @@ fn main() -> Result<(), Box> { }); let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(1)); - dbg!(iter.get(2)); - dbg!(iter.get(20)); - dbg!(iter.get(21)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(1) == Some(Cow::Borrowed(&1))); + assert!(iter.get(2) == Some(Cow::Borrowed(&2))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21).is_none()); drop(iter); - dbg!("flush"); vec.flush()?; - dbg!("flushed"); - dbg!(vec.header()); + assert!(vec.header().stamp() == Stamp::new(0)); } { @@ -42,26 +40,32 @@ fn main() -> Result<(), Box> { vec.mut_header().update_stamp(Stamp::new(100)); + assert!(vec.header().stamp() == Stamp::new(100)); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(1)); - dbg!(iter.get(2)); - dbg!(iter.get(3)); - dbg!(iter.get(4)); - dbg!(iter.get(5)); - dbg!(iter.get(20)); - dbg!(iter.get(20)); - dbg!(iter.get(0)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(1) == Some(Cow::Borrowed(&1))); + assert!(iter.get(2) == Some(Cow::Borrowed(&2))); + assert!(iter.get(3) == Some(Cow::Borrowed(&3))); + assert!(iter.get(4) == Some(Cow::Borrowed(&4))); + assert!(iter.get(5) == Some(Cow::Borrowed(&5))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); drop(iter); vec.push(21); vec.push(22); + assert!(vec.stored_len() == 21); + assert!(vec.pushed_len() == 2); + assert!(vec.len() == 23); + let mut iter = vec.into_iter(); - dbg!(iter.get(20)); - dbg!(iter.get(21)); - dbg!(iter.get(22)); - dbg!(iter.get(23)); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21) == Some(Cow::Borrowed(&21))); + assert!(iter.get(22) == Some(Cow::Borrowed(&22))); + assert!(iter.get(23).is_none()); drop(iter); vec.flush()?; @@ -70,27 +74,42 @@ fn main() -> Result<(), Box> { { let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; + assert!(vec.header().stamp() == Stamp::new(100)); + + assert!(vec.stored_len() == 23); + assert!(vec.pushed_len() == 0); + assert!(vec.len() == 23); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(20)); - dbg!(iter.get(21)); - dbg!(iter.get(22)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21) == Some(Cow::Borrowed(&21))); + assert!(iter.get(22) == Some(Cow::Borrowed(&22))); drop(iter); vec.truncate_if_needed(14)?; + assert!(vec.stored_len() == 14); + assert!(vec.pushed_len() == 0); + assert!(vec.len() == 14); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(5)); - dbg!(iter.get(20)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(5) == Some(Cow::Borrowed(&5))); + assert!(iter.get(20).is_none()); drop(iter); - dbg!(vec.collect_signed_range(Some(-5), None)?); + assert!(vec.collect_signed_range(Some(-5), None)? == vec![9, 10, 11, 12, 13]); vec.push(vec.len() as u32); - dbg!(VecIterator::last(vec.into_iter())); + assert!(VecIterator::last(vec.into_iter()) == Some((14, Cow::Borrowed(&14)))); - dbg!(vec.into_iter().collect::>()); + assert!( + vec.into_iter() + .map(|(_, v)| v.into_owned()) + .collect::>() + == vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); } { @@ -98,46 +117,54 @@ fn main() -> Result<(), Box> { vec.reset()?; - dbg!(vec.header(), vec.pushed_len(), vec.stored_len(), vec.len()); + // dbg!(vec.header()); + // assert len + + assert!(vec.pushed_len() == 0); + assert!(vec.stored_len() == 0); + assert!(vec.len() == 0); (0..21_u32).for_each(|v| { vec.push(v); }); + assert!(vec.pushed_len() == 21); + assert!(vec.stored_len() == 0); + assert!(vec.len() == 21); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(20)); - dbg!(iter.get(21)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21).is_none()); drop(iter); // let reader = vec.create_static_reader(); - // dbg!(vec.take(10, &reader)?); - // dbg!(vec.get_or_read(10, &reader)?); - // dbg!(vec.holes()); + // assert!(vec.take(10, &reader)? == Some(10)); + // assert!(vec.holes() == &BTreeSet::from([10])); + // assert!(vec.get_or_read(10, &reader)?.is_none()); // drop(reader); vec.flush()?; - dbg!(vec.holes()); + + // assert!(vec.holes() == &BTreeSet::from([10])); } { let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - dbg!(vec.holes()); + // assert!(vec.holes() == &BTreeSet::from([10])); - let reader = vec.create_static_reader(); - dbg!(vec.get_or_read(10, &reader)?); - drop(reader); + // let reader = vec.create_static_reader(); + // assert!(vec.get_or_read(10, &reader)?.is_none()); + // drop(reader); // vec.update(10, 10)?; // vec.update(0, 10)?; let reader = vec.create_static_reader(); - dbg!( - vec.holes(), - vec.get_or_read(0, &reader)?, - vec.get_or_read(10, &reader)? - ); + assert!(vec.holes() == &BTreeSet::new()); + assert!(vec.get_or_read(0, &reader)? == Some(Cow::Borrowed(&0))); + assert!(vec.get_or_read(10, &reader)? == Some(Cow::Borrowed(&10))); drop(reader); vec.flush()?; @@ -146,7 +173,12 @@ fn main() -> Result<(), Box> { { let vec: VEC = CompressedVec::forced_import(&file, "vec", version)?; - dbg!(vec.collect()?); + assert!( + vec.collect()? + == vec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); } Ok(()) diff --git a/crates/brk_vecs/examples/raw.rs b/crates/brk_vecs/examples/raw.rs index 6ca1069a6..83ea0aff6 100644 --- a/crates/brk_vecs/examples/raw.rs +++ b/crates/brk_vecs/examples/raw.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path, sync::Arc}; +use std::{borrow::Cow, collections::BTreeSet, fs, path::Path, sync::Arc}; use brk_vecs::{ AnyStoredVec, AnyVec, CollectableVec, File, GenericStoredVec, RawVec, Stamp, VecIterator, @@ -23,16 +23,16 @@ fn main() -> Result<(), Box> { }); let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(1)); - dbg!(iter.get(2)); - dbg!(iter.get(20)); - dbg!(iter.get(21)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(1) == Some(Cow::Borrowed(&1))); + assert!(iter.get(2) == Some(Cow::Borrowed(&2))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21).is_none()); drop(iter); vec.flush()?; - dbg!(vec.header()); + assert!(vec.header().stamp() == Stamp::new(0)); } { @@ -40,26 +40,32 @@ fn main() -> Result<(), Box> { vec.mut_header().update_stamp(Stamp::new(100)); + assert!(vec.header().stamp() == Stamp::new(100)); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(1)); - dbg!(iter.get(2)); - dbg!(iter.get(3)); - dbg!(iter.get(4)); - dbg!(iter.get(5)); - dbg!(iter.get(20)); - dbg!(iter.get(20)); - dbg!(iter.get(0)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(1) == Some(Cow::Borrowed(&1))); + assert!(iter.get(2) == Some(Cow::Borrowed(&2))); + assert!(iter.get(3) == Some(Cow::Borrowed(&3))); + assert!(iter.get(4) == Some(Cow::Borrowed(&4))); + assert!(iter.get(5) == Some(Cow::Borrowed(&5))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); drop(iter); vec.push(21); vec.push(22); + assert!(vec.stored_len() == 21); + assert!(vec.pushed_len() == 2); + assert!(vec.len() == 23); + let mut iter = vec.into_iter(); - dbg!(iter.get(20)); - dbg!(iter.get(21)); - dbg!(iter.get(22)); - dbg!(iter.get(23)); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21) == Some(Cow::Borrowed(&21))); + assert!(iter.get(22) == Some(Cow::Borrowed(&22))); + assert!(iter.get(23).is_none()); drop(iter); vec.flush()?; @@ -68,27 +74,42 @@ fn main() -> Result<(), Box> { { let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; + assert!(vec.header().stamp() == Stamp::new(100)); + + assert!(vec.stored_len() == 23); + assert!(vec.pushed_len() == 0); + assert!(vec.len() == 23); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(20)); - dbg!(iter.get(21)); - dbg!(iter.get(22)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21) == Some(Cow::Borrowed(&21))); + assert!(iter.get(22) == Some(Cow::Borrowed(&22))); drop(iter); vec.truncate_if_needed(14)?; + assert!(vec.stored_len() == 14); + assert!(vec.pushed_len() == 0); + assert!(vec.len() == 14); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(5)); - dbg!(iter.get(20)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(5) == Some(Cow::Borrowed(&5))); + assert!(iter.get(20).is_none()); drop(iter); - dbg!(vec.collect_signed_range(Some(-5), None)?); + assert!(vec.collect_signed_range(Some(-5), None)? == vec![9, 10, 11, 12, 13]); vec.push(vec.len() as u32); - dbg!(VecIterator::last(vec.into_iter())); + assert!(VecIterator::last(vec.into_iter()) == Some((14, Cow::Borrowed(&14)))); - dbg!(vec.into_iter().collect::>()); + assert!( + vec.into_iter() + .map(|(_, v)| v.into_owned()) + .collect::>() + == vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); } { @@ -96,46 +117,54 @@ fn main() -> Result<(), Box> { vec.reset()?; - dbg!(vec.header(), vec.pushed_len(), vec.stored_len(), vec.len()); + // dbg!(vec.header()); + // assert len + + assert!(vec.pushed_len() == 0); + assert!(vec.stored_len() == 0); + assert!(vec.len() == 0); (0..21_u32).for_each(|v| { vec.push(v); }); + assert!(vec.pushed_len() == 21); + assert!(vec.stored_len() == 0); + assert!(vec.len() == 21); + let mut iter = vec.into_iter(); - dbg!(iter.get(0)); - dbg!(iter.get(20)); - dbg!(iter.get(21)); + assert!(iter.get(0) == Some(Cow::Borrowed(&0))); + assert!(iter.get(20) == Some(Cow::Borrowed(&20))); + assert!(iter.get(21).is_none()); drop(iter); let reader = vec.create_static_reader(); - dbg!(vec.take(10, &reader)?); - dbg!(vec.get_or_read(10, &reader)?); - dbg!(vec.holes()); + assert!(vec.take(10, &reader)? == Some(10)); + assert!(vec.holes() == &BTreeSet::from([10])); + assert!(vec.get_or_read(10, &reader)?.is_none()); drop(reader); vec.flush()?; - dbg!(vec.holes()); + + assert!(vec.holes() == &BTreeSet::from([10])); } { let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?; - dbg!(vec.holes()); + assert!(vec.holes() == &BTreeSet::from([10])); let reader = vec.create_static_reader(); - dbg!(vec.get_or_read(10, &reader)?); + assert!(vec.get_or_read(10, &reader)?.is_none()); drop(reader); vec.update(10, 10)?; vec.update(0, 10)?; let reader = vec.create_static_reader(); - dbg!( - vec.holes(), - vec.get_or_read(0, &reader)?, - vec.get_or_read(10, &reader)? - ); + assert!(vec.holes() == &BTreeSet::new()); + assert!(vec.get_or_read(0, &reader)? == Some(Cow::Borrowed(&10))); + assert!(vec.get_or_read(10, &reader)? == Some(Cow::Borrowed(&10))); drop(reader); vec.flush()?; @@ -144,7 +173,12 @@ fn main() -> Result<(), Box> { { let vec: VEC = RawVec::forced_import(&file, "vec", version)?; - dbg!(vec.collect()?); + assert!( + vec.collect()? + == vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); } Ok(()) diff --git a/crates/brk_vecs/src/traits/generic.rs b/crates/brk_vecs/src/traits/generic.rs index 65f95ecd1..909bdd8e1 100644 --- a/crates/brk_vecs/src/traits/generic.rs +++ b/crates/brk_vecs/src/traits/generic.rs @@ -20,16 +20,20 @@ where { const SIZE_OF_T: usize = size_of::(); + /// /// Be careful with deadlocks /// /// You'll want to drop the reader before mutable ops + /// fn create_reader(&self) -> Reader { self.create_static_reader() } + /// /// Be careful with deadlocks /// /// You'll want to drop the reader before mutable ops + /// fn create_static_reader(&self) -> Reader<'static> { unsafe { std::mem::transmute( @@ -58,6 +62,11 @@ where fn get_or_read_(&self, index: usize, reader: &Reader) -> Result>> { let stored_len = self.stored_len(); + let holes = self.holes(); + if !holes.is_empty() && holes.contains(&index) { + return Ok(None); + } + if index >= stored_len { let pushed = self.pushed(); let j = index - stored_len; @@ -74,11 +83,6 @@ where return Ok(Some(Cow::Borrowed(updated))); } - let holes = self.holes(); - if !holes.is_empty() && holes.contains(&index) { - return Ok(None); - } - Ok(Some(Cow::Owned(self.read_(index, reader)?))) } @@ -112,7 +116,7 @@ where Ok(()) } Ordering::Less => { - dbg!(index, value, len, self.header()); + dbg!(index, value, len, self.header(), self.region_index()); Err(Error::IndexTooHigh) } }