Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 14:11:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-08-04 11:38:46 +02:00
parent a2f5704581
commit 5b855fd835
8 changed files with 191 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_indexer/examples/indexer.rs
+10 -4
View File
@@ -13,21 +13,27 @@ use brk_vecs::Exit;
fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = brk_structs::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let blocks_dir = bitcoin_dir.join("blocks");
let outputs_dir = Path::new("./_outputs");
fs::create_dir_all(outputs_dir)?;
// let outputs_dir = brk_structs::default_brk_path().join("outputs");
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();
exit.set_ctrlc_handler();
let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);
let parser = Parser::new(blocks_dir, outputs_dir.to_path_buf(), rpc);
fs::create_dir_all(outputs_dir)?;
crates/brk_indexer/src/lib.rs
+5 -3
View File
@@ -124,7 +124,7 @@ impl Indexer {
let mut p2traddressindex_to_p2trbytes_reader_opt = None;
let mut p2aaddressindex_to_p2abytes_reader_opt = None;
let reset_mmaps_options =
let reset_readers =
|vecs: &mut Vecs,
txindex_to_first_outputindex_reader_opt: &mut Option<Reader<'static>>,
p2pk65addressindex_to_p2pk65bytes_reader_opt: &mut Option<Reader<'static>>,
@@ -161,7 +161,7 @@ impl Indexer {
.replace(vecs.p2aaddressindex_to_p2abytes.create_static_reader());
};
reset_mmaps_options(
reset_readers(
vecs,
&mut txindex_to_first_outputindex_reader_opt,
&mut p2pk65addressindex_to_p2pk65bytes_reader_opt,
@@ -762,8 +762,10 @@ impl Indexer {
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
export(stores, vecs, height, exit)?;
reset_mmaps_options(
reset_readers(
vecs,
&mut txindex_to_first_outputindex_reader_opt,
&mut p2pk65addressindex_to_p2pk65bytes_reader_opt,
crates/brk_server/src/api/interface.rs
+2 -16
View File
@@ -89,21 +89,7 @@ fn req_to_response_res(
Response::new(Body::from(v))
} else {
match interface.format(vecs, &params.rest)? {
Output::CSV(s) => {
if let GuardResult::Guard(g) = guard_res {
g.insert(s.clone().into())
.map_err(|_| Error::QuickCacheError)?;
}
s.into_response()
}
Output::TSV(s) => {
if let GuardResult::Guard(g) = guard_res {
g.insert(s.clone().into())
.map_err(|_| Error::QuickCacheError)?;
}
s.into_response()
}
Output::MD(s) => {
Output::CSV(s) | Output::TSV(s) | Output::MD(s) => {
if let GuardResult::Guard(g) = guard_res {
g.insert(s.clone().into())
.map_err(|_| Error::QuickCacheError)?;
@@ -143,7 +129,7 @@ fn req_to_response_res(
Format::JSON => headers.insert_content_type_application_json(),
}
}
_ => headers.insert_content_type_application_json(),
None => headers.insert_content_type_application_json(),
};
Ok(response)
crates/brk_store/src/trait.rs → crates/brk_store/src/any.rs
View File
crates/brk_store/src/lib.rs
+2 -2
View File
@@ -19,12 +19,12 @@ use fjall::{
TransactionalPartitionHandle,
};
mod any;
mod meta;
mod r#trait;
pub use any::*;
use log::info;
use meta::*;
pub use r#trait::*;
pub struct Store<Key, Value> {
meta: StoreMeta,
crates/brk_vecs/examples/compressed.rs
+82 -50
View File
@@ -1,4 +1,4 @@
use std::{fs, path::Path, sync::Arc};
use std::{borrow::Cow, collections::BTreeSet, fs, path::Path, sync::Arc};
use brk_vecs::{
AnyStoredVec, AnyVec, CollectableVec, CompressedVec, File, GenericStoredVec, Stamp,
@@ -23,18 +23,16 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
});
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(1));
dbg!(iter.get(2));
dbg!(iter.get(20));
dbg!(iter.get(21));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(1) == Some(Cow::Borrowed(&1)));
assert!(iter.get(2) == Some(Cow::Borrowed(&2)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21).is_none());
drop(iter);
dbg!("flush");
vec.flush()?;
dbg!("flushed");
dbg!(vec.header());
assert!(vec.header().stamp() == Stamp::new(0));
}
{
@@ -42,26 +40,32 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.mut_header().update_stamp(Stamp::new(100));
assert!(vec.header().stamp() == Stamp::new(100));
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(1));
dbg!(iter.get(2));
dbg!(iter.get(3));
dbg!(iter.get(4));
dbg!(iter.get(5));
dbg!(iter.get(20));
dbg!(iter.get(20));
dbg!(iter.get(0));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(1) == Some(Cow::Borrowed(&1)));
assert!(iter.get(2) == Some(Cow::Borrowed(&2)));
assert!(iter.get(3) == Some(Cow::Borrowed(&3)));
assert!(iter.get(4) == Some(Cow::Borrowed(&4)));
assert!(iter.get(5) == Some(Cow::Borrowed(&5)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
drop(iter);
vec.push(21);
vec.push(22);
assert!(vec.stored_len() == 21);
assert!(vec.pushed_len() == 2);
assert!(vec.len() == 23);
let mut iter = vec.into_iter();
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
dbg!(iter.get(23));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21) == Some(Cow::Borrowed(&21)));
assert!(iter.get(22) == Some(Cow::Borrowed(&22)));
assert!(iter.get(23).is_none());
drop(iter);
vec.flush()?;
@@ -70,27 +74,42 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?;
assert!(vec.header().stamp() == Stamp::new(100));
assert!(vec.stored_len() == 23);
assert!(vec.pushed_len() == 0);
assert!(vec.len() == 23);
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21) == Some(Cow::Borrowed(&21)));
assert!(iter.get(22) == Some(Cow::Borrowed(&22)));
drop(iter);
vec.truncate_if_needed(14)?;
assert!(vec.stored_len() == 14);
assert!(vec.pushed_len() == 0);
assert!(vec.len() == 14);
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(5));
dbg!(iter.get(20));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(5) == Some(Cow::Borrowed(&5)));
assert!(iter.get(20).is_none());
drop(iter);
dbg!(vec.collect_signed_range(Some(-5), None)?);
assert!(vec.collect_signed_range(Some(-5), None)? == vec![9, 10, 11, 12, 13]);
vec.push(vec.len() as u32);
dbg!(VecIterator::last(vec.into_iter()));
assert!(VecIterator::last(vec.into_iter()) == Some((14, Cow::Borrowed(&14))));
dbg!(vec.into_iter().collect::<Vec<_>>());
assert!(
vec.into_iter()
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>()
== vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
);
}
{
@@ -98,46 +117,54 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.reset()?;
dbg!(vec.header(), vec.pushed_len(), vec.stored_len(), vec.len());
// dbg!(vec.header());
// assert len
assert!(vec.pushed_len() == 0);
assert!(vec.stored_len() == 0);
assert!(vec.len() == 0);
(0..21_u32).for_each(|v| {
vec.push(v);
});
assert!(vec.pushed_len() == 21);
assert!(vec.stored_len() == 0);
assert!(vec.len() == 21);
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21).is_none());
drop(iter);
// let reader = vec.create_static_reader();
// dbg!(vec.take(10, &reader)?);
// dbg!(vec.get_or_read(10, &reader)?);
// dbg!(vec.holes());
// assert!(vec.take(10, &reader)? == Some(10));
// assert!(vec.holes() == &BTreeSet::from([10]));
// assert!(vec.get_or_read(10, &reader)?.is_none());
// drop(reader);
vec.flush()?;
dbg!(vec.holes());
// assert!(vec.holes() == &BTreeSet::from([10]));
}
{
let mut vec: VEC = CompressedVec::forced_import(&file, "vec", version)?;
dbg!(vec.holes());
// assert!(vec.holes() == &BTreeSet::from([10]));
let reader = vec.create_static_reader();
dbg!(vec.get_or_read(10, &reader)?);
drop(reader);
// let reader = vec.create_static_reader();
// assert!(vec.get_or_read(10, &reader)?.is_none());
// drop(reader);
// vec.update(10, 10)?;
// vec.update(0, 10)?;
let reader = vec.create_static_reader();
dbg!(
vec.holes(),
vec.get_or_read(0, &reader)?,
vec.get_or_read(10, &reader)?
);
assert!(vec.holes() == &BTreeSet::new());
assert!(vec.get_or_read(0, &reader)? == Some(Cow::Borrowed(&0)));
assert!(vec.get_or_read(10, &reader)? == Some(Cow::Borrowed(&10)));
drop(reader);
vec.flush()?;
@@ -146,7 +173,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let vec: VEC = CompressedVec::forced_import(&file, "vec", version)?;
dbg!(vec.collect()?);
assert!(
vec.collect()?
== vec![
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
]
);
}
Ok(())
crates/brk_vecs/examples/raw.rs
+80 -46
View File
@@ -1,4 +1,4 @@
use std::{fs, path::Path, sync::Arc};
use std::{borrow::Cow, collections::BTreeSet, fs, path::Path, sync::Arc};
use brk_vecs::{
AnyStoredVec, AnyVec, CollectableVec, File, GenericStoredVec, RawVec, Stamp, VecIterator,
@@ -23,16 +23,16 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
});
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(1));
dbg!(iter.get(2));
dbg!(iter.get(20));
dbg!(iter.get(21));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(1) == Some(Cow::Borrowed(&1)));
assert!(iter.get(2) == Some(Cow::Borrowed(&2)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21).is_none());
drop(iter);
vec.flush()?;
dbg!(vec.header());
assert!(vec.header().stamp() == Stamp::new(0));
}
{
@@ -40,26 +40,32 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.mut_header().update_stamp(Stamp::new(100));
assert!(vec.header().stamp() == Stamp::new(100));
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(1));
dbg!(iter.get(2));
dbg!(iter.get(3));
dbg!(iter.get(4));
dbg!(iter.get(5));
dbg!(iter.get(20));
dbg!(iter.get(20));
dbg!(iter.get(0));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(1) == Some(Cow::Borrowed(&1)));
assert!(iter.get(2) == Some(Cow::Borrowed(&2)));
assert!(iter.get(3) == Some(Cow::Borrowed(&3)));
assert!(iter.get(4) == Some(Cow::Borrowed(&4)));
assert!(iter.get(5) == Some(Cow::Borrowed(&5)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
drop(iter);
vec.push(21);
vec.push(22);
assert!(vec.stored_len() == 21);
assert!(vec.pushed_len() == 2);
assert!(vec.len() == 23);
let mut iter = vec.into_iter();
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
dbg!(iter.get(23));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21) == Some(Cow::Borrowed(&21)));
assert!(iter.get(22) == Some(Cow::Borrowed(&22)));
assert!(iter.get(23).is_none());
drop(iter);
vec.flush()?;
@@ -68,27 +74,42 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
assert!(vec.header().stamp() == Stamp::new(100));
assert!(vec.stored_len() == 23);
assert!(vec.pushed_len() == 0);
assert!(vec.len() == 23);
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21) == Some(Cow::Borrowed(&21)));
assert!(iter.get(22) == Some(Cow::Borrowed(&22)));
drop(iter);
vec.truncate_if_needed(14)?;
assert!(vec.stored_len() == 14);
assert!(vec.pushed_len() == 0);
assert!(vec.len() == 14);
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(5));
dbg!(iter.get(20));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(5) == Some(Cow::Borrowed(&5)));
assert!(iter.get(20).is_none());
drop(iter);
dbg!(vec.collect_signed_range(Some(-5), None)?);
assert!(vec.collect_signed_range(Some(-5), None)? == vec![9, 10, 11, 12, 13]);
vec.push(vec.len() as u32);
dbg!(VecIterator::last(vec.into_iter()));
assert!(VecIterator::last(vec.into_iter()) == Some((14, Cow::Borrowed(&14))));
dbg!(vec.into_iter().collect::<Vec<_>>());
assert!(
vec.into_iter()
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>()
== vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
);
}
{
@@ -96,46 +117,54 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
vec.reset()?;
dbg!(vec.header(), vec.pushed_len(), vec.stored_len(), vec.len());
// dbg!(vec.header());
// assert len
assert!(vec.pushed_len() == 0);
assert!(vec.stored_len() == 0);
assert!(vec.len() == 0);
(0..21_u32).for_each(|v| {
vec.push(v);
});
assert!(vec.pushed_len() == 21);
assert!(vec.stored_len() == 0);
assert!(vec.len() == 21);
let mut iter = vec.into_iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
assert!(iter.get(0) == Some(Cow::Borrowed(&0)));
assert!(iter.get(20) == Some(Cow::Borrowed(&20)));
assert!(iter.get(21).is_none());
drop(iter);
let reader = vec.create_static_reader();
dbg!(vec.take(10, &reader)?);
dbg!(vec.get_or_read(10, &reader)?);
dbg!(vec.holes());
assert!(vec.take(10, &reader)? == Some(10));
assert!(vec.holes() == &BTreeSet::from([10]));
assert!(vec.get_or_read(10, &reader)?.is_none());
drop(reader);
vec.flush()?;
dbg!(vec.holes());
assert!(vec.holes() == &BTreeSet::from([10]));
}
{
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
dbg!(vec.holes());
assert!(vec.holes() == &BTreeSet::from([10]));
let reader = vec.create_static_reader();
dbg!(vec.get_or_read(10, &reader)?);
assert!(vec.get_or_read(10, &reader)?.is_none());
drop(reader);
vec.update(10, 10)?;
vec.update(0, 10)?;
let reader = vec.create_static_reader();
dbg!(
vec.holes(),
vec.get_or_read(0, &reader)?,
vec.get_or_read(10, &reader)?
);
assert!(vec.holes() == &BTreeSet::new());
assert!(vec.get_or_read(0, &reader)? == Some(Cow::Borrowed(&10)));
assert!(vec.get_or_read(10, &reader)? == Some(Cow::Borrowed(&10)));
drop(reader);
vec.flush()?;
@@ -144,7 +173,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let vec: VEC = RawVec::forced_import(&file, "vec", version)?;
dbg!(vec.collect()?);
assert!(
vec.collect()?
== vec![
10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
]
);
}
Ok(())
crates/brk_vecs/src/traits/generic.rs
+10 -6
View File
@@ -20,16 +20,20 @@ where
{
const SIZE_OF_T: usize = size_of::<T>();
///
/// Be careful with deadlocks
///
/// You'll want to drop the reader before mutable ops
///
fn create_reader(&self) -> Reader {
self.create_static_reader()
}
///
/// Be careful with deadlocks
///
/// You'll want to drop the reader before mutable ops
///
fn create_static_reader(&self) -> Reader<'static> {
unsafe {
std::mem::transmute(
@@ -58,6 +62,11 @@ where
fn get_or_read_(&self, index: usize, reader: &Reader) -> Result<Option<Cow<T>>> {
let stored_len = self.stored_len();
let holes = self.holes();
if !holes.is_empty() && holes.contains(&index) {
return Ok(None);
}
if index >= stored_len {
let pushed = self.pushed();
let j = index - stored_len;
@@ -74,11 +83,6 @@ where
return Ok(Some(Cow::Borrowed(updated)));
}
let holes = self.holes();
if !holes.is_empty() && holes.contains(&index) {
return Ok(None);
}
Ok(Some(Cow::Owned(self.read_(index, reader)?)))
}
@@ -112,7 +116,7 @@ where
Ok(())
}
Ordering::Less => {
dbg!(index, value, len, self.header());
dbg!(index, value, len, self.header(), self.region_index());
Err(Error::IndexTooHigh)
}
}