Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-16 04:44:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: fixed Fjalls rollback

Browse Source
This commit is contained in:
nym21
2025-02-24 18:15:13 +01:00
parent 53175c9ed7
commit 01ecae8979
8 changed files with 108 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@@ -327,6 +327,7 @@ name = "brk"
version = "0.0.0"
dependencies = [
"brk_computer",
"brk_core",
"brk_fetcher",
"brk_indexer",
"brk_logger",
@@ -701,9 +702,9 @@ checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]]
name = "either"
version = "1.13.0"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d"
[[package]]
name = "enum_dispatch"
@@ -797,9 +798,9 @@ dependencies = [
[[package]]
name = "flate2"
version = "1.0.35"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
dependencies = [
"crc32fast",
"miniz_oxide 0.8.5",
@@ -1782,9 +1783,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
[[package]]
name = "portable-atomic"
version = "1.10.0"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
[[package]]
name = "portable-atomic-util"

4
crates/brk/Cargo.toml
View File

@@ -6,7 +6,8 @@ edition.workspace = true
version.workspace = true
[features]
full = ["computer", "fetcher", "indexer", "logger", "parser", "server"]
full = ["core", "computer", "fetcher", "indexer", "logger", "parser", "server"]
core = ["brk_core"]
computer = ["brk_computer"]
fetcher = ["brk_fetcher"]
indexer = ["brk_indexer"]
@@ -15,6 +16,7 @@ parser = ["brk_parser"]
server = ["brk_server"]
[dependencies]
brk_core = { workspace = true, optional = true }
brk_computer = { workspace = true, optional = true }
brk_fetcher = { workspace = true, optional = true }
brk_indexer = { workspace = true, optional = true }

6
crates/brk/src/lib.rs
View File

@@ -1,3 +1,9 @@
#[cfg(feature = "core")]
pub mod core {
#[doc(inline)]
pub use brk_core::*;
}
#[cfg(feature = "computer")]
pub mod computer {
#[doc(inline)]

4
crates/brk_core/src/structs/height.rs
View File

@@ -32,6 +32,10 @@ impl Height {
pub const ZERO: Self = Height(0);
pub const MAX: Self = Height(u32::MAX);
pub fn new(height: u32) -> Self {
Self(height)
}
pub fn write(&self, path: &std::path::Path) -> Result<(), std::io::Error> {
std::fs::write(path, self.as_bytes())
}

10
crates/brk_indexer/src/lib.rs
View File

@@ -60,6 +60,9 @@ impl Indexer<CACHED_GETS> {
indexes
});
// dbg!(starting_indexes);
// panic!();
exit.block();
self.stores.rollback(&self.vecs, &starting_indexes)?;
self.vecs.rollback(&starting_indexes)?;
@@ -88,7 +91,7 @@ impl Indexer<CACHED_GETS> {
info!("Started indexing...");
parser.parse(Some(idxs.height), Some(400_000_u32.into()))
parser.parse(Some(idxs.height), None)
.iter()
.try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {height}...");
@@ -628,7 +631,10 @@ impl Indexer<CACHED_GETS> {
Ok(())
})?;
export(stores, vecs, idxs.height)?;
if idxs.height % SNAPSHOT_BLOCK_RANGE != 0 {
export(stores, vecs, idxs.height)?;
}
sleep(Duration::from_millis(100));
Ok(())

166
crates/brk_indexer/src/storage/fjalls/mod.rs
View File

@@ -48,103 +48,87 @@ impl Fjalls {
Ok(())
})?;
vecs.height_to_first_p2pk65index
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK65));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65addressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK65));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
vecs.height_to_first_p2pk33index
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK33));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2pk33index.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33addressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PK33));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
vecs.height_to_first_p2pkhindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2pkhindex.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2pkhindex_to_p2pkhaddressbytes.get(index)?.map(Value::into_inner) {
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2PKH));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
vecs.height_to_first_p2shindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2shindex_to_p2shaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2SH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2shindex.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2shindex_to_p2shaddressbytes.get(index)?.map(Value::into_inner) {
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2SH));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
vecs.height_to_first_p2trindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2trindex_to_p2traddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2TR));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2trindex.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2trindex_to_p2traddressbytes.get(index)?.map(Value::into_inner) {
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2TR));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
vecs.height_to_first_p2wpkhindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WPKH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2wpkhindex.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhaddressbytes
.get(index)?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WPKH));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
vecs.height_to_first_p2wshindex
.iter_from(starting_indexes.height, |(_, index)| {
if let Some(typedbytes) = vecs
.p2wshindex_to_p2wshaddressbytes
.get(index.into_inner())?
.map(Value::into_inner)
{
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
self.addresshash_to_addressindex.remove(hash);
}
Ok(())
})?;
if let Some(index) = vecs.height_to_first_p2wshindex.get(starting_indexes.height)? {
let mut index = index.into_inner();
while let Some(typedbytes) = vecs.p2wshindex_to_p2wshaddressbytes.get(index)?.map(Value::into_inner) {
let bytes = Addressbytes::from(typedbytes);
let hash = AddressHash::from((&bytes, Addresstype::P2WSH));
self.addresshash_to_addressindex.remove(hash);
index.increment();
}
}
self.commit(starting_indexes.height.decremented())?;

2
crates/brk_indexer/src/structs/indexes.rs
View File

@@ -98,8 +98,6 @@ impl TryFrom<(&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)> for Indexes {
})
.unwrap_or(starting_height);
// let height = 885000_u32.into();
Ok(Self {
addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,

7
crates/brk_parser/src/lib.rs
View File

@@ -191,7 +191,6 @@ impl Parser {
}
let height = Height::from(header.height);
// println!("{height}");
let len = blk_index_to_blk_recap.tree.len();
if blk_metadata.index == len as u16 || blk_metadata.index + 1 == len as u16 {
@@ -235,9 +234,13 @@ impl Parser {
None
}
}) {
if end.is_some_and(|end| end < current_height) {
return ControlFlow::Break(());
}
send_height_block_hash.send((current_height, block, hash)).unwrap();
if end == Some(current_height) {
if end.is_some_and(|end| end == current_height) {
return ControlFlow::Break(());
}