Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-12 13:18:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: fixed Fjalls rollback

Browse Source
This commit is contained in:
nym21
2025-02-24 18:15:13 +01:00
parent 53175c9ed7
commit 01ecae8979
8 changed files with 108 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@@ -327,6 +327,7 @@ name = "brk"
version = "0.0.0" version = "0.0.0"
dependencies = [ dependencies = [
"brk_computer", "brk_computer",
"brk_core",
"brk_fetcher", "brk_fetcher",
"brk_indexer", "brk_indexer",
"brk_logger", "brk_logger",
@@ -701,9 +702,9 @@ checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]] [[package]]
name = "either" name = "either"
version = "1.13.0" version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d"
[[package]] [[package]]
name = "enum_dispatch" name = "enum_dispatch"
@@ -797,9 +798,9 @@ dependencies = [
[[package]] [[package]]
name = "flate2" name = "flate2"
version = "1.0.35" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
dependencies = [ dependencies = [
"crc32fast", "crc32fast",
"miniz_oxide 0.8.5", "miniz_oxide 0.8.5",
@@ -1782,9 +1783,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
[[package]] [[package]]
name = "portable-atomic" name = "portable-atomic"
version = "1.10.0" version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
[[package]] [[package]]
name = "portable-atomic-util" name = "portable-atomic-util"

4
crates/brk/Cargo.toml
View File

@@ -6,7 +6,8 @@ edition.workspace = true
version.workspace = true version.workspace = true
[features] [features]
full = ["computer", "fetcher", "indexer", "logger", "parser", "server"] full = ["core", "computer", "fetcher", "indexer", "logger", "parser", "server"]
core = ["brk_core"]
computer = ["brk_computer"] computer = ["brk_computer"]
fetcher = ["brk_fetcher"] fetcher = ["brk_fetcher"]
indexer = ["brk_indexer"] indexer = ["brk_indexer"]
@@ -15,6 +16,7 @@ parser = ["brk_parser"]
server = ["brk_server"] server = ["brk_server"]
[dependencies] [dependencies]
brk_core = { workspace = true, optional = true }
brk_computer = { workspace = true, optional = true } brk_computer = { workspace = true, optional = true }
brk_fetcher = { workspace = true, optional = true } brk_fetcher = { workspace = true, optional = true }
brk_indexer = { workspace = true, optional = true } brk_indexer = { workspace = true, optional = true }

6
crates/brk/src/lib.rs
View File

@@ -1,3 +1,9 @@
#[cfg(feature = "core")]
pub mod core {
#[doc(inline)]
pub use brk_core::*;
}
#[cfg(feature = "computer")] #[cfg(feature = "computer")]
pub mod computer { pub mod computer {
#[doc(inline)] #[doc(inline)]

4
crates/brk_core/src/structs/height.rs
View File

@@ -32,6 +32,10 @@ impl Height {
pub const ZERO: Self = Height(0); pub const ZERO: Self = Height(0);
pub const MAX: Self = Height(u32::MAX); pub const MAX: Self = Height(u32::MAX);
pub fn new(height: u32) -> Self {
Self(height)
}
pub fn write(&self, path: &std::path::Path) -> Result<(), std::io::Error> { pub fn write(&self, path: &std::path::Path) -> Result<(), std::io::Error> {
std::fs::write(path, self.as_bytes()) std::fs::write(path, self.as_bytes())
} }

10
crates/brk_indexer/src/lib.rs
View File

@@ -60,6 +60,9 @@ impl Indexer<CACHED_GETS> {
indexes indexes
}); });
// dbg!(starting_indexes);
// panic!();
exit.block(); exit.block();
self.stores.rollback(&self.vecs, &starting_indexes)?; self.stores.rollback(&self.vecs, &starting_indexes)?;
self.vecs.rollback(&starting_indexes)?; self.vecs.rollback(&starting_indexes)?;
@@ -88,7 +91,7 @@ impl Indexer<CACHED_GETS> {
info!("Started indexing..."); info!("Started indexing...");
parser.parse(Some(idxs.height), Some(400_000_u32.into())) parser.parse(Some(idxs.height), None)
.iter() .iter()
.try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {height}..."); info!("Indexing block {height}...");
@@ -628,7 +631,10 @@ impl Indexer<CACHED_GETS> {
Ok(()) Ok(())
})?; })?;
export(stores, vecs, idxs.height)?; if idxs.height % SNAPSHOT_BLOCK_RANGE != 0 {
export(stores, vecs, idxs.height)?;
}
sleep(Duration::from_millis(100)); sleep(Duration::from_millis(100));
Ok(()) Ok(())

166
crates/brk_indexer/src/storage/fjalls/mod.rs
View File

@@ -48,103 +48,87 @@ impl Fjalls {
Ok(()) Ok(())
})?; })?;
vecs.height_to_first_p2pk65index if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes .p2pk65index_to_p2pk65addressbytes
.get(index.into_inner())? .get(index)?
.map(Value::into_inner) .map(Value::into_inner)
{ {
let bytes = Addressbytes::from(typedbytes); let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK65)); let hash = AddressHash::from((&bytes, Addresstype::P2PK65));
self.addresshash_to_addressindex.remove(hash); self.addresshash_to_addressindex.remove(hash);
} index.increment();
Ok(()) }
})?; }
vecs.height_to_first_p2pk33index if let Some(index) = vecs.height_to_first_p2pk33index.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes .p2pk33index_to_p2pk33addressbytes
.get(index.into_inner())? .get(index)?
.map(Value::into_inner) .map(Value::into_inner)
{ {
let bytes = Addressbytes::from(typedbytes); let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK33)); let hash = AddressHash::from((&bytes, Addresstype::P2PK33));
self.addresshash_to_addressindex.remove(hash); self.addresshash_to_addressindex.remove(hash);
} index.increment();
Ok(()) }
})?; }
vecs.height_to_first_p2pkhindex if let Some(index) = vecs.height_to_first_p2pkhindex.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs.p2pkhindex_to_p2pkhaddressbytes.get(index)?.map(Value::into_inner) {
.p2pkhindex_to_p2pkhaddressbytes let bytes = Addressbytes::from(typedbytes);
.get(index.into_inner())? let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
.map(Value::into_inner) self.addresshash_to_addressindex.remove(hash);
{ index.increment();
let bytes = Addressbytes::from(typedbytes); }
let hash = AddressHash::from((&bytes, Addresstype::P2PKH)); }
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
vecs.height_to_first_p2shindex if let Some(index) = vecs.height_to_first_p2shindex.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs.p2shindex_to_p2shaddressbytes.get(index)?.map(Value::into_inner) {
.p2shindex_to_p2shaddressbytes let bytes = Addressbytes::from(typedbytes);
.get(index.into_inner())? let hash = AddressHash::from((&bytes, Addresstype::P2SH));
.map(Value::into_inner) self.addresshash_to_addressindex.remove(hash);
{ index.increment();
let bytes = Addressbytes::from(typedbytes); }
let hash = AddressHash::from((&bytes, Addresstype::P2SH)); }
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
vecs.height_to_first_p2trindex if let Some(index) = vecs.height_to_first_p2trindex.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs.p2trindex_to_p2traddressbytes.get(index)?.map(Value::into_inner) {
.p2trindex_to_p2traddressbytes let bytes = Addressbytes::from(typedbytes);
.get(index.into_inner())? let hash = AddressHash::from((&bytes, Addresstype::P2TR));
.map(Value::into_inner) self.addresshash_to_addressindex.remove(hash);
{ index.increment();
let bytes = Addressbytes::from(typedbytes); }
let hash = AddressHash::from((&bytes, Addresstype::P2TR)); }
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
vecs.height_to_first_p2wpkhindex if let Some(index) = vecs.height_to_first_p2wpkhindex.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes .p2wpkhindex_to_p2wpkhaddressbytes
.get(index.into_inner())? .get(index)?
.map(Value::into_inner) .map(Value::into_inner)
{ {
let bytes = Addressbytes::from(typedbytes); let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WPKH)); let hash = AddressHash::from((&bytes, Addresstype::P2WPKH));
self.addresshash_to_addressindex.remove(hash); self.addresshash_to_addressindex.remove(hash);
} index.increment();
Ok(()) }
})?; }
vecs.height_to_first_p2wshindex if let Some(index) = vecs.height_to_first_p2wshindex.get(starting_indexes.height)? {
.iter_from(starting_indexes.height, |(_, index)| { let mut index = index.into_inner();
if let Some(typedbytes) = vecs while let Some(typedbytes) = vecs.p2wshindex_to_p2wshaddressbytes.get(index)?.map(Value::into_inner) {
.p2wshindex_to_p2wshaddressbytes let bytes = Addressbytes::from(typedbytes);
.get(index.into_inner())? let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
.map(Value::into_inner) self.addresshash_to_addressindex.remove(hash);
{ index.increment();
let bytes = Addressbytes::from(typedbytes); }
let hash = AddressHash::from((&bytes, Addresstype::P2WSH)); }
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
self.commit(starting_indexes.height.decremented())?; self.commit(starting_indexes.height.decremented())?;

2
crates/brk_indexer/src/structs/indexes.rs
View File

@@ -98,8 +98,6 @@ impl TryFrom<(&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)> for Indexes {
}) })
.unwrap_or(starting_height); .unwrap_or(starting_height);
// let height = 885000_u32.into();
Ok(Self { Ok(Self {
addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?, addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?, emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,

7
crates/brk_parser/src/lib.rs
View File

@@ -191,7 +191,6 @@ impl Parser {
} }
let height = Height::from(header.height); let height = Height::from(header.height);
// println!("{height}");
let len = blk_index_to_blk_recap.tree.len(); let len = blk_index_to_blk_recap.tree.len();
if blk_metadata.index == len as u16 || blk_metadata.index + 1 == len as u16 { if blk_metadata.index == len as u16 || blk_metadata.index + 1 == len as u16 {
@@ -235,9 +234,13 @@ impl Parser {
None None
} }
}) { }) {
if end.is_some_and(|end| end < current_height) {
return ControlFlow::Break(());
}
send_height_block_hash.send((current_height, block, hash)).unwrap(); send_height_block_hash.send((current_height, block, hash)).unwrap();
if end == Some(current_height) { if end.is_some_and(|end| end == current_height) {
return ControlFlow::Break(()); return ControlFlow::Break(());
} }