Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-29 17:19:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

bitbase: snapshot

Browse Source
This commit is contained in:
nym21
2025-01-03 21:34:10 +01:00
parent dea853d840
commit 1380b42c1d
40 changed files with 906 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
src/crates/barser/src/main.rs
View File

@@ -1,100 +0,0 @@
use std::path::Path;
use biter::bitcoincore_rpc::{Auth, Client};
mod databases;
mod structs;
use databases::Databases;
use structs::{Exit, Height, Txindex, Txoutindex};
// https://github.com/fjall-rs/fjall/discussions/72
// https://github.com/romanz/electrs/blob/master/doc/schema.md
const DAILY_BLOCK_TARGET: usize = 144;
const MONTHLY_BLOCK_TARGET: usize = DAILY_BLOCK_TARGET * 30;
fn main() -> color_eyre::Result<()> {
let i = std::time::Instant::now();
let data_dir = Path::new("../bitcoin");
let cookie = Path::new(data_dir).join(".cookie");
let rpc = Client::new("http://localhost:8332", Auth::CookieFile(cookie)).unwrap();
let exit = Exit::new();
let mut dbs = Databases::import()?;
let mut height = dbs.start_height(&rpc)?;
let mut txindex = dbs
.height_to_last_txindex
.get(height)?
.unwrap_or(Txindex::default());
let export = |dbs: &mut Databases, height: Height| -> color_eyre::Result<()> {
exit.block();
println!("Exporting...");
dbs.export(height)?;
println!("Export done");
exit.unblock();
Ok(())
};
biter::new(data_dir, Some(height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
height = Height::from(_height);
if dbs.has_different_blockhash(height, &blockhash)? {
dbs.erase_from(height)?;
}
dbs.blockhash_prefix_to_height.insert(&blockhash, height)?;
dbs.height_to_blockhash.insert(height, &blockhash);
let txlen = block.txdata.len();
block.txdata.into_iter().enumerate().try_for_each(
|(i, tx)| -> color_eyre::Result<()> {
if i == txlen - 1 {
dbs.height_to_last_txindex.insert(height, txindex);
}
if !dbs.txindex_to_txid.is_safe(height)
|| !dbs.txid_prefix_to_txindex.is_safe(height)
{
let txid = tx.compute_txid();
dbs.txindex_to_txid.insert(txindex, &txid, height);
dbs.txid_prefix_to_txindex.insert(&txid, txindex, height)?;
}
txindex.increment();
tx.output.into_iter().enumerate().for_each(|(vout, txout)| {
let vout = vout as u16;
let txoutindex = Txoutindex::from((txindex, vout));
let amount = txout.value.into();
dbs.txoutindex_to_amount.insert(txoutindex, amount, height);
});
Ok(())
},
)?;
let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active();
if should_snapshot {
export(&mut dbs, height)?;
}
Ok(())
})?;
export(&mut dbs, height)?;
dbg!(i.elapsed());
Ok(())
}

53
src/crates/barser/src/structs/txoutindex.rs
View File

@@ -1,53 +0,0 @@
use super::{SliceExtended, Txindex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
pub struct Txoutindex {
pub txindex: Txindex,
pub vout: u16,
}
const SHIFT: u64 = 16;
const AND: u64 = (1 << SHIFT) - 1;
impl From<Txindex> for Txoutindex {
fn from(value: Txindex) -> Self {
Self {
txindex: value,
vout: 0,
}
}
}
impl From<(Txindex, u16)> for Txoutindex {
fn from(value: (Txindex, u16)) -> Self {
Self {
txindex: value.0,
vout: value.1,
}
}
}
impl From<u64> for Txoutindex {
fn from(value: u64) -> Self {
Self {
txindex: (value >> SHIFT).into(),
vout: (value & AND) as u16,
}
}
}
impl From<Txoutindex> for u64 {
fn from(value: Txoutindex) -> Self {
(u64::from(value.txindex) << SHIFT) + value.vout as u64
}
}
impl From<Txoutindex> for fjall::Slice {
fn from(value: Txoutindex) -> Self {
u64::from(value).to_be_bytes().into()
}
}
impl From<fjall::Slice> for Txoutindex {
fn from(value: fjall::Slice) -> Self {
fjall::Slice::read_u64(&value).into()
}
}

1
src/crates/bitbase/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/database

79
src/crates/barser/Cargo.lock → src/crates/bitbase/Cargo.lock generated
View File

@@ -50,8 +50,8 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
dependencies = [
"bitcoin-internals",
"bitcoin_hashes",
"bitcoin-internals 0.3.0",
"bitcoin_hashes 0.14.0",
]
[[package]]
@@ -66,6 +66,19 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "bitbase"
version = "0.1.0"
dependencies = [
"bitcoin_hashes 0.16.0",
"biter",
"color-eyre",
"ctrlc",
"derive_deref",
"fjall",
"rayon",
]
[[package]]
name = "bitcoin"
version = "0.32.5"
@@ -74,11 +87,11 @@ checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026"
dependencies = [
"base58ck",
"bech32",
"bitcoin-internals",
"bitcoin-io",
"bitcoin-internals 0.3.0",
"bitcoin-io 0.1.3",
"bitcoin-units",
"bitcoin_hashes",
"hex-conservative",
"bitcoin_hashes 0.14.0",
"hex-conservative 0.2.1",
"hex_lit",
"secp256k1",
"serde",
@@ -93,19 +106,34 @@ dependencies = [
"serde",
]
[[package]]
name = "bitcoin-internals"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b854212e29b96c8f0fe04cab11d57586c8f3257de0d146c76cb3b42b3eb9118"
[[package]]
name = "bitcoin-io"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf"
[[package]]
name = "bitcoin-io"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26792cd2bf245069a1c5acb06aa7ad7abe1de69b507c90b490bca81e0665d0ee"
dependencies = [
"bitcoin-internals 0.4.0",
]
[[package]]
name = "bitcoin-units"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2"
dependencies = [
"bitcoin-internals",
"bitcoin-internals 0.3.0",
"serde",
]
@@ -115,11 +143,21 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
dependencies = [
"bitcoin-io",
"hex-conservative",
"bitcoin-io 0.1.3",
"hex-conservative 0.2.1",
"serde",
]
[[package]]
name = "bitcoin_hashes"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e5d09f16329cd545d7e6008b2c6b2af3a90bc678cf41ac3d2f6755943301b16"
dependencies = [
"bitcoin-io 0.2.0",
"hex-conservative 0.3.0",
]
[[package]]
name = "bitcoincore-rpc"
version = "0.19.0"
@@ -441,6 +479,15 @@ dependencies = [
"arrayvec",
]
[[package]]
name = "hex-conservative"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4afe881d0527571892c4034822e59bb10c6c991cce6abe8199b6f5cf10766f55"
dependencies = [
"arrayvec",
]
[[package]]
name = "hex_lit"
version = "0.1.1"
@@ -732,18 +779,6 @@ dependencies = [
"bitflags",
]
[[package]]
name = "rust-playground"
version = "0.1.0"
dependencies = [
"biter",
"color-eyre",
"ctrlc",
"derive_deref",
"fjall",
"rayon",
]
[[package]]
name = "rustc-demangle"
version = "0.1.24"
@@ -787,7 +822,7 @@ version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113"
dependencies = [
"bitcoin_hashes",
"bitcoin_hashes 0.14.0",
"rand",
"secp256k1-sys",
"serde",

3
src/crates/barser/Cargo.toml → src/crates/bitbase/Cargo.toml
View File

@@ -1,9 +1,10 @@
[package]
name = "barser"
name = "bitbase"
version = "0.1.0"
edition = "2021"
[dependencies]
bitcoin_hashes = "0.16.0"
biter = "0.2.2"
color-eyre = "0.6.3"
ctrlc = "3.4.5"

41
src/crates/bitbase/src/database/addressbytes_prefix_to_addressindex.rs Normal file
View File

@@ -0,0 +1,41 @@
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use crate::structs::{Addressbytes, Addressindex, Database, DatabaseTrait, Height, Version};
#[derive(Deref, DerefMut)]
pub struct AddressbytesPrefixToAddressindex(Database);
impl AddressbytesPrefixToAddressindex {
pub fn import() -> color_eyre::Result<Self> {
Ok(Self(Database::import(
"address_prefix_to_addressindex",
Self::version(),
)?))
}
pub fn insert(
&mut self,
addressbytes: &Addressbytes,
addressindex: Addressindex,
height: Height,
) -> color_eyre::Result<()> {
if let Some(_height) =
self.fetch_update(addressbytes.to_prefix_slice(), addressindex.into(), height)?
{
dbg!(addressbytes, addressindex);
return Err(eyre!("AddressPrefixToAddressindex: key collision"));
}
Ok(())
}
pub fn remove(&mut self, addressbytes: &Addressbytes) {
self.0.remove(addressbytes.to_prefix_slice())
}
}
impl DatabaseTrait for AddressbytesPrefixToAddressindex {
fn version() -> Version {
Version::from(1)
}
}

43
src/crates/bitbase/src/database/addressindex_to_addressbytes.rs Normal file
View File

@@ -0,0 +1,43 @@
use derive_deref::{Deref, DerefMut};
use crate::structs::{Addressbytes, Addressindex, Database, DatabaseTrait, Height, Version};
#[derive(Deref, DerefMut)]
pub struct AddressindexToAddressbytes(Database);
impl AddressindexToAddressbytes {
pub fn import() -> color_eyre::Result<Self> {
Ok(Self(Database::import(
"addressindex_to_addressbytes",
Self::version(),
)?))
}
pub fn get(&self, addressindex: Addressindex) -> color_eyre::Result<Option<Addressbytes>> {
if let Some(address) = self.0.get(addressindex.into())?.map(Addressbytes::try_from) {
Ok(Some(address?))
} else {
Ok(None)
}
}
pub fn insert(
&mut self,
addressindex: Addressindex,
addressbytes: &Addressbytes,
height: Height,
) {
self.0
.insert(addressindex.into(), addressbytes.into(), height)
}
pub fn remove(&mut self, addressindex: Addressindex) {
self.0.remove(addressindex.into())
}
}
impl DatabaseTrait for AddressindexToAddressbytes {
fn version() -> Version {
Version::from(1)
}
}

38
src/crates/bitbase/src/database/addressindex_to_addresstype.rs Normal file
View File

@@ -0,0 +1,38 @@
use derive_deref::{Deref, DerefMut};
use crate::structs::{Addressindex, Addresstype, Database, DatabaseTrait, Height, Version};
#[derive(Deref, DerefMut)]
pub struct AddressindexToAddresstype(Database);
impl AddressindexToAddresstype {
pub fn import() -> color_eyre::Result<Self> {
Ok(Self(Database::import(
"addressindex_to_addresstype",
Self::version(),
)?))
}
// pub fn get(&self, addressindex: Addressindex) -> color_eyre::Result<Option<Addresstype>> {
// if let Some(addresstype) = self.0.get(addressindex.into())?.map(Addresstype::try_from) {
// Ok(Some(addresstype?))
// } else {
// Ok(None)
// }
// }
pub fn insert(&mut self, addressindex: Addressindex, addresstype: Addresstype, height: Height) {
self.0
.insert(addressindex.into(), addresstype.into(), height)
}
pub fn remove(&mut self, addressindex: Addressindex) {
self.0.remove(addressindex.into())
}
}
impl DatabaseTrait for AddressindexToAddresstype {
fn version() -> Version {
Version::from(1)
}
}

42
src/crates/bitbase/src/database/addressindex_to_txoutindexes.rs Normal file
View File

@@ -0,0 +1,42 @@
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use crate::structs::{
Addressindex, Addresstxoutindex, Database, DatabaseTrait, Height, SliceExtended, Txoutindex,
Version,
};
#[derive(Deref, DerefMut)]
pub struct AddressindexToTxoutindexes(Database);
impl AddressindexToTxoutindexes {
pub fn import() -> color_eyre::Result<Self> {
Ok(Self(Database::import(
"addressindex_to_txoutindexes",
Self::version(),
)?))
}
pub fn insert(&mut self, addressindex: Addressindex, txoutindex: Txoutindex, height: Height) {
self.0.insert(
Addresstxoutindex::from((addressindex, txoutindex)).into(),
Slice::default(),
height,
)
}
pub fn remove(&mut self, addressindex: Addressindex, txoutindex: Txoutindex) {
self.0
.remove(Addresstxoutindex::from((addressindex, txoutindex)).into());
}
pub fn is_empty(&self, addressindex: Addressindex) -> bool {
self.prefix(Slice::from(addressindex)).next().is_none()
}
}
impl DatabaseTrait for AddressindexToTxoutindexes {
fn version() -> Version {
Version::from(1)
}
}

0
src/crates/barser/src/databases/blockhash_prefix_to_height.rs → src/crates/bitbase/src/database/blockhash_prefix_to_height.rs
View File

0
src/crates/barser/src/databases/height_to_blockhash.rs → src/crates/bitbase/src/database/height_to_blockhash.rs
View File

0
src/crates/barser/src/databases/height_to_txindex.rs → src/crates/bitbase/src/database/height_to_txindex.rs
View File

119
src/crates/barser/src/databases/mod.rs → src/crates/bitbase/src/database/mod.rs
View File

@@ -1,43 +1,65 @@
use std::{ops::Sub, thread};
use std::{collections::BTreeSet, ops::Sub, thread};
use biter::{
bitcoin::{hashes::Hash, BlockHash, Txid},
bitcoincore_rpc::Client,
};
pub use blockhash_prefix_to_height::*;
use biter::bitcoin::{hashes::Hash, BlockHash, Txid};
use color_eyre::eyre::ContextCompat;
use fjall::Slice;
mod addressbytes_prefix_to_addressindex;
mod addressindex_to_addressbytes;
mod addressindex_to_addresstype;
mod addressindex_to_txoutindexes;
mod blockhash_prefix_to_height;
mod height_to_blockhash;
mod height_to_txindex;
mod txid_prefix_to_txindex;
mod txindex_to_height;
mod txindex_to_txid;
mod txoutindex_to_addressindex;
mod txoutindex_to_amount;
pub use addressbytes_prefix_to_addressindex::*;
pub use addressindex_to_addressbytes::*;
pub use addressindex_to_addresstype::*;
pub use addressindex_to_txoutindexes::*;
pub use blockhash_prefix_to_height::*;
pub use height_to_blockhash::*;
pub use height_to_txindex::*;
pub use txid_prefix_to_txindex::*;
pub use txindex_to_height::*;
pub use txindex_to_txid::*;
pub use txoutindex_to_addressindex::*;
pub use txoutindex_to_amount::*;
use crate::structs::{Height, Txindex, Txoutindex};
use crate::structs::{Addressindex, Exit, Height, Txindex, Txoutindex};
pub struct Databases {
pub struct Database {
pub addressbytes_prefix_to_addressindex: AddressbytesPrefixToAddressindex,
pub addressindex_to_addressbytes: AddressindexToAddressbytes,
pub addressindex_to_addresstype: AddressindexToAddresstype,
pub addressindex_to_txoutindexes: AddressindexToTxoutindexes,
pub blockhash_prefix_to_height: BlockhashPrefixToHeight,
pub height_to_blockhash: HeightToBlockhash,
pub height_to_first_txindex: HeightToTxindex,
pub height_to_last_txindex: HeightToTxindex,
pub txid_prefix_to_txindex: TxidPrefixToTxindex,
pub txindex_to_txid: TxindexToTxid,
pub txindex_to_height: TxindexToHeight,
pub txoutindex_to_addressindex: TxoutindexToAddressindex,
pub txoutindex_to_amount: TxoutindexToAmount,
}
const UNSAFE_BLOCKS: usize = 100;
impl Databases {
impl Database {
pub fn import() -> color_eyre::Result<Self> {
thread::scope(|scope| {
let addressbytes_prefix_to_addressindex_handle =
scope.spawn(AddressbytesPrefixToAddressindex::import);
let addressindex_to_addressbytes_handle =
scope.spawn(AddressindexToAddressbytes::import);
let addressindex_to_addresstype_handle = scope.spawn(AddressindexToAddresstype::import);
let addressindex_to_txoutindexes_handle =
scope.spawn(AddressindexToTxoutindexes::import);
let blockhash_prefix_to_height_handle = scope.spawn(BlockhashPrefixToHeight::import);
let height_to_blockhash_handle = scope.spawn(HeightToBlockhash::import);
let height_to_first_txindex_handle =
@@ -45,16 +67,30 @@ impl Databases {
let height_to_last_txindex_handle =
scope.spawn(|| HeightToTxindex::import(HeightToTxindexPosition::Last));
let txid_prefix_to_txindex_handle = scope.spawn(TxidPrefixToTxindex::import);
let txindex_to_height_handle = scope.spawn(TxindexToHeight::import);
let txindex_to_txid_handle = scope.spawn(TxindexToTxid::import);
let txoutindex_to_addressindex_handle = scope.spawn(TxoutindexToAddressindex::import);
let txoutindex_to_amount_handle = scope.spawn(TxoutindexToAmount::import);
Ok(Self {
addressbytes_prefix_to_addressindex: addressbytes_prefix_to_addressindex_handle
.join()
.unwrap()?,
addressindex_to_addressbytes: addressindex_to_addressbytes_handle
.join()
.unwrap()?,
addressindex_to_addresstype: addressindex_to_addresstype_handle.join().unwrap()?,
addressindex_to_txoutindexes: addressindex_to_txoutindexes_handle
.join()
.unwrap()?,
blockhash_prefix_to_height: blockhash_prefix_to_height_handle.join().unwrap()?,
height_to_blockhash: height_to_blockhash_handle.join().unwrap()?,
height_to_first_txindex: height_to_first_txindex_handle.join().unwrap()?,
height_to_last_txindex: height_to_last_txindex_handle.join().unwrap()?,
txid_prefix_to_txindex: txid_prefix_to_txindex_handle.join().unwrap()?,
txindex_to_height: txindex_to_height_handle.join().unwrap()?,
txindex_to_txid: txindex_to_txid_handle.join().unwrap()?,
txoutindex_to_addressindex: txoutindex_to_addressindex_handle.join().unwrap()?,
txoutindex_to_amount: txoutindex_to_amount_handle.join().unwrap()?,
})
})
@@ -62,34 +98,47 @@ impl Databases {
pub fn export(&mut self, height: Height) -> color_eyre::Result<()> {
thread::scope(|scope| {
scope.spawn(|| {
self.addressbytes_prefix_to_addressindex
.export(height)
.unwrap()
});
scope.spawn(|| self.addressindex_to_addressbytes.export(height).unwrap());
scope.spawn(|| self.addressindex_to_addresstype.export(height).unwrap());
scope.spawn(|| self.addressindex_to_txoutindexes.export(height).unwrap());
scope.spawn(|| self.blockhash_prefix_to_height.export(height).unwrap());
scope.spawn(|| self.height_to_blockhash.export(height).unwrap());
scope.spawn(|| self.height_to_first_txindex.export(height).unwrap());
scope.spawn(|| self.height_to_last_txindex.export(height).unwrap());
scope.spawn(|| self.txid_prefix_to_txindex.export(height).unwrap());
scope.spawn(|| self.txindex_to_height.export(height).unwrap());
scope.spawn(|| self.txindex_to_txid.export(height).unwrap());
scope.spawn(|| self.txoutindex_to_addressindex.export(height).unwrap());
scope.spawn(|| self.txoutindex_to_amount.export(height).unwrap());
});
Ok(())
}
pub fn start_height(&self, rpc: &Client) -> color_eyre::Result<Height> {
let safe_height = Height::try_from(rpc)?.sub(UNSAFE_BLOCKS);
Ok(self
.min_height()
pub fn start_height(&self) -> Height {
self.min_height()
.map(|h| h.sub(UNSAFE_BLOCKS))
.unwrap_or_default()
.min(safe_height))
}
fn min_height(&self) -> Option<Height> {
[
self.addressbytes_prefix_to_addressindex.height(),
self.addressindex_to_addressbytes.height(),
self.addressindex_to_addresstype.height(),
self.addressindex_to_txoutindexes.height(),
self.blockhash_prefix_to_height.height(),
self.height_to_blockhash.height(),
self.height_to_first_txindex.height(),
self.height_to_last_txindex.height(),
self.txid_prefix_to_txindex.height(),
self.txindex_to_height.height(),
self.txindex_to_txid.height(),
self.txoutindex_to_addressindex.height(),
self.txoutindex_to_amount.height(),
]
.into_iter()
@@ -109,7 +158,11 @@ impl Databases {
.is_some_and(|saved_blockhash| blockhash != &saved_blockhash))
}
pub fn erase_from(&mut self, height: Height) -> color_eyre::Result<()> {
pub fn rollback_from(&mut self, height: Height, exit: &Exit) -> color_eyre::Result<()> {
exit.block();
self.export(height)?;
let mut txindex = None;
self.height_to_blockhash
@@ -144,6 +197,7 @@ impl Databases {
let txid = Txid::from_slice(&slice_txid)?;
self.txindex_to_txid.remove(txindex);
self.txindex_to_height.remove(txindex);
self.txid_prefix_to_txindex.remove(&txid);
Ok(())
@@ -151,6 +205,8 @@ impl Databases {
let txoutindex = Txoutindex::from(txindex);
let mut addressindexes = BTreeSet::new();
self.txoutindex_to_amount
.range(Slice::from(txoutindex)..)
.try_for_each(|slice| -> color_eyre::Result<()> {
@@ -159,9 +215,42 @@ impl Databases {
self.txoutindex_to_amount.remove(txoutindex);
if let Some(addressindex_slice) =
self.txoutindex_to_addressindex.get(txoutindex.into())?
{
self.txoutindex_to_addressindex.remove(txoutindex);
let addressindex = Addressindex::from(addressindex_slice);
addressindexes.insert(addressindex);
self.addressindex_to_txoutindexes
.remove(addressindex, txoutindex);
}
Ok(())
})?;
self.export(height)?;
addressindexes
.into_iter()
.filter(|addressindex| self.addressindex_to_txoutindexes.is_empty(*addressindex))
.try_for_each(|addressindex| -> color_eyre::Result<()> {
let addressbytes = self
.addressindex_to_addressbytes
.get(addressindex)?
.context("addressindex_to_address to have value")?;
self.addressbytes_prefix_to_addressindex
.remove(&addressbytes);
self.addressindex_to_addressbytes.remove(addressindex);
self.addressindex_to_addresstype.remove(addressindex);
Ok(())
})?;
self.export(height)?;
exit.unblock();
Ok(())
}
}

0
src/crates/barser/src/databases/txid_prefix_to_txindex.rs → src/crates/bitbase/src/database/txid_prefix_to_txindex.rs
View File

29
src/crates/bitbase/src/database/txindex_to_height.rs Normal file
View File

@@ -0,0 +1,29 @@
use derive_deref::{Deref, DerefMut};
use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version};
#[derive(Deref, DerefMut)]
pub struct TxindexToHeight(Database);
impl TxindexToHeight {
pub fn import() -> color_eyre::Result<Self> {
Ok(Self(Database::import(
"txindex_to_height",
Self::version(),
)?))
}
pub fn insert(&mut self, txindex: Txindex, height: Height) {
self.0.insert(txindex.into(), height.into(), height)
}
pub fn remove(&mut self, txindex: Txindex) {
self.0.remove(txindex.into())
}
}
impl DatabaseTrait for TxindexToHeight {
fn version() -> Version {
Version::from(1)
}
}

3
src/crates/barser/src/databases/txindex_to_txid.rs → src/crates/bitbase/src/database/txindex_to_txid.rs
View File

@@ -1,6 +1,5 @@
use biter::bitcoin::Txid;
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use crate::structs::{Database, DatabaseTrait, Height, Txindex, Version};
@@ -17,7 +16,7 @@ impl TxindexToTxid {
}
pub fn remove(&mut self, txindex: Txindex) {
self.0.remove(Slice::from(txindex))
self.0.remove(txindex.into())
}
}

30
src/crates/bitbase/src/database/txoutindex_to_addressindex.rs Normal file
View File

@@ -0,0 +1,30 @@
use derive_deref::{Deref, DerefMut};
use crate::structs::{Addressindex, Database, DatabaseTrait, Height, Txoutindex, Version};
#[derive(Deref, DerefMut)]
pub struct TxoutindexToAddressindex(Database);
impl TxoutindexToAddressindex {
pub fn import() -> color_eyre::Result<Self> {
Ok(Self(Database::import(
"txoutindex_to_addressindex",
Self::version(),
)?))
}
pub fn insert(&mut self, txoutindex: Txoutindex, addressindex: Addressindex, height: Height) {
self.0
.insert(txoutindex.into(), addressindex.into(), height)
}
pub fn remove(&mut self, txoutindex: Txoutindex) {
self.0.remove(txoutindex.into())
}
}
impl DatabaseTrait for TxoutindexToAddressindex {
fn version() -> Version {
Version::from(1)
}
}

3
src/crates/barser/src/databases/txoutindex_to_amount.rs → src/crates/bitbase/src/database/txoutindex_to_amount.rs
View File

@@ -1,5 +1,4 @@
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use crate::structs::{Amount, Database, DatabaseTrait, Height, Txoutindex, Version};
@@ -19,7 +18,7 @@ impl TxoutindexToAmount {
}
pub fn remove(&mut self, txoutindex: Txoutindex) {
self.0.remove(Slice::from(txoutindex))
self.0.remove(txoutindex.into())
}
}

167
src/crates/bitbase/src/main.rs Normal file
View File

@@ -0,0 +1,167 @@
use std::path::Path;
use biter::bitcoincore_rpc::{Auth, Client};
mod database;
mod structs;
use database::Database;
use fjall::Slice;
use structs::{Addressbytes, Addressindex, Addresstype, Exit, Height, Txindex, Txoutindex};
// https://github.com/fjall-rs/fjall/discussions/72
// https://github.com/romanz/electrs/blob/master/doc/schema.md
const DAILY_BLOCK_TARGET: usize = 144;
const MONTHLY_BLOCK_TARGET: usize = DAILY_BLOCK_TARGET * 30;
fn main() -> color_eyre::Result<()> {
let i = std::time::Instant::now();
let data_dir = Path::new("../../../../bitcoin");
let cookie = Path::new(data_dir).join(".cookie");
let rpc = Client::new("http://localhost:8332", Auth::CookieFile(cookie)).unwrap();
let exit = Exit::new();
let mut db = Database::import()?;
let mut height = db.start_height();
let mut txindex = db
.height_to_last_txindex
.get(height)?
.map(Txindex::incremented)
.unwrap_or(Txindex::default());
let mut addressindex = db
.txoutindex_to_addressindex
.prefix(Slice::from(txindex))
.last()
.map(|res| -> color_eyre::Result<Addressindex> {
Ok(Addressindex::from(res?.1).incremented())
})
.unwrap_or(Ok(Addressindex::default()))?;
let export = |db: &mut Database, height: Height| -> color_eyre::Result<()> {
exit.block();
println!("Exporting...");
db.export(height)?;
println!("Export done");
exit.unblock();
Ok(())
};
biter::new(data_dir, Some(height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
height = Height::from(_height);
if db.has_different_blockhash(height, &blockhash)? {
db.rollback_from(height, &exit)?;
}
db.blockhash_prefix_to_height.insert(&blockhash, height)?;
db.height_to_blockhash.insert(height, &blockhash);
let txlen = block.txdata.len();
let last_txindex = txlen - 1;
block.txdata.into_iter().enumerate().try_for_each(
|(txi, tx)| -> color_eyre::Result<()> {
if txi == 0 {
db.height_to_first_txindex.insert(height, txindex);
}
if txi == last_txindex {
db.height_to_last_txindex.insert(height, txindex);
}
if !db.txindex_to_txid.is_safe(height)
|| !db.txid_prefix_to_txindex.is_safe(height)
{
let txid = tx.compute_txid();
db.txindex_to_txid.insert(txindex, &txid, height);
db.txid_prefix_to_txindex.insert(&txid, txindex, height)?;
}
db.txindex_to_height.insert(txindex, height);
txindex.increment();
tx.output.into_iter().enumerate().try_for_each(
|(vout, txout)| -> color_eyre::Result<()> {
let vout = vout as u16;
let txoutindex = Txoutindex::from((txindex, vout));
let amount = txout.value.into();
db.txoutindex_to_amount.insert(txoutindex, amount, height);
let script = &txout.script_pubkey;
let addressbytes =
Addressbytes::try_from(script).inspect_err(|_| {
dbg!(&txout, height, txi);
})?;
let addresstype = Addresstype::try_from(script)?;
let mut addressindex_local = addressindex;
if let Some(addressindex_slice) = db
.addressbytes_prefix_to_addressindex
.get((&addressbytes).into())?
{
addressindex_local = addressindex_slice.into()
} else {
db.addressbytes_prefix_to_addressindex
.insert(&addressbytes, addressindex_local, height)
.inspect_err(|_| {
dbg!(addresstype);
})?;
db.addressindex_to_addressbytes.insert(
addressindex_local,
&addressbytes,
height,
);
db.addressindex_to_addresstype.insert(
addressindex_local,
addresstype,
height,
);
addressindex.increment();
}
db.txoutindex_to_addressindex.insert(
txoutindex,
addressindex_local,
height,
);
db.addressindex_to_txoutindexes.insert(
addressindex_local,
txoutindex,
height,
);
Ok(())
},
)?;
Ok(())
},
)?;
let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active();
if should_snapshot {
export(&mut db, height)?;
}
Ok(())
})?;
export(&mut db, height)?;
dbg!(i.elapsed());
Ok(())
}

53
src/crates/bitbase/src/structs/addressbytes.rs Normal file
View File

@@ -0,0 +1,53 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use fjall::Slice;
#[derive(Debug)]
pub struct Addressbytes(Slice);
impl Addressbytes {
pub fn to_prefix_slice(&self) -> Slice {
self.0[..8].into()
}
}
impl TryFrom<&ScriptBuf> for Addressbytes {
type Error = color_eyre::Report;
fn try_from(script: &ScriptBuf) -> Result<Self, Self::Error> {
if script.is_p2pk() {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
67 => &script.as_bytes()[1..66],
35 => &script.as_bytes()[1..34],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
if bytes[0] != 4 {
dbg!(bytes);
return Err(eyre!("Doesn't start with a 4"));
}
Ok(Self(bytes.into()))
} else if script.is_p2pkh() {
let bytes = &script.as_bytes()[3..23];
Ok(Self(bytes.into()))
} else {
Err(eyre!("Unsupported address type"))
}
}
}
impl From<Slice> for Addressbytes {
fn from(value: Slice) -> Self {
Self(value)
}
}
impl From<&Addressbytes> for Slice {
fn from(value: &Addressbytes) -> Self {
value.0.clone()
}
}

47
src/crates/bitbase/src/structs/addressindex.rs Normal file
View File

@@ -0,0 +1,47 @@
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::SliceExtended;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Addressindex(u32);
impl Addressindex {
pub const BYTES: usize = size_of::<Self>();
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
}
impl From<u32> for Addressindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Addressindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Addressindex> for u64 {
fn from(value: Addressindex) -> Self {
value.0 as u64
}
}
impl From<Slice> for Addressindex {
fn from(slice: Slice) -> Self {
Self(slice.read_u32())
}
}
impl From<Addressindex> for Slice {
fn from(value: Addressindex) -> Self {
value.to_be_bytes().into()
}
}

35
src/crates/bitbase/src/structs/addresstxoutindex.rs Normal file
View File

@@ -0,0 +1,35 @@
use fjall::Slice;
use super::{Addressindex, Txoutindex};
pub struct Addresstxoutindex {
addressindex: Addressindex,
txoutindex: Txoutindex,
}
impl From<(Addressindex, Txoutindex)> for Addresstxoutindex {
fn from(value: (Addressindex, Txoutindex)) -> Self {
Self {
addressindex: value.0,
txoutindex: value.1,
}
}
}
impl From<Addresstxoutindex> for Slice {
fn from(value: Addresstxoutindex) -> Self {
let txindex_slice = Self::from(value.addressindex);
let vout_slice = Self::from(value.txoutindex);
Self::from([txindex_slice, vout_slice].concat())
}
}
impl From<Slice> for Addresstxoutindex {
fn from(value: Slice) -> Self {
let addressindex = Addressindex::from(Slice::from(&value[..Addressindex::BYTES]));
let txoutindex = Txoutindex::from(Slice::from(&value[Addressindex::BYTES..]));
Self {
addressindex,
txoutindex,
}
}
}

40
src/crates/bitbase/src/structs/addresstype.rs Normal file
View File

@@ -0,0 +1,40 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use fjall::Slice;
use super::SliceExtended;
#[derive(Debug, Clone, Copy)]
pub enum Addresstype {
P2PK,
P2PKH,
}
impl TryFrom<&ScriptBuf> for Addresstype {
type Error = color_eyre::Report;
fn try_from(value: &ScriptBuf) -> Result<Self, Self::Error> {
if value.is_p2pk() {
Ok(Self::P2PK)
} else if value.is_p2pkh() {
Ok(Self::P2PKH)
} else {
Err(eyre!("Not compatible script"))
}
}
}
impl TryFrom<Slice> for Addresstype {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
match value.read_u8() {
x if x == Addresstype::P2PK as u8 => Ok(Addresstype::P2PK),
x if x == Addresstype::P2PKH as u8 => Ok(Addresstype::P2PKH),
_ => Err(eyre!("Unknown type")),
}
}
}
impl From<Addresstype> for Slice {
fn from(addresstype: Addresstype) -> Self {
(addresstype as u8).to_be_bytes().into()
}
}

3
src/crates/barser/src/structs/amount.rs → src/crates/bitbase/src/structs/amount.rs
View File

@@ -5,6 +5,7 @@ use std::{
use biter::bitcoin;
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::Height;
@@ -29,7 +30,7 @@ impl From<bitcoin::Amount> for Amount {
}
}
impl From<Amount> for fjall::Slice {
impl From<Amount> for Slice {
fn from(value: Amount) -> Self {
value.to_sat().to_be_bytes().into()
}

0
src/crates/barser/src/structs/exit.rs → src/crates/bitbase/src/structs/exit.rs
View File

15
src/crates/barser/src/structs/database.rs → src/crates/bitbase/src/structs/fjall.rs
View File

@@ -1,10 +1,8 @@
use std::{mem, ops::RangeBounds};
use fjall::{
Batch, Keyspace, KvPair, PartitionCreateOptions, PartitionHandle, PersistMode, Result, Slice,
};
pub use fjall::*;
use super::{Height, Version};
use crate::structs::{Height, Version};
pub struct Database {
keyspace: Keyspace,
@@ -19,7 +17,7 @@ const HEIGHT: &str = "height";
impl Database {
pub fn import(name: &str, version: Version) -> Result<Self> {
let keyspace = fjall::Config::new(format!("./databases/{name}")).open()?;
let keyspace = fjall::Config::new(format!("./database/{name}")).open()?;
let data = Self::open_data(&keyspace)?;
let meta = Self::open_meta(&keyspace)?;
@@ -69,6 +67,13 @@ impl Database {
self.data.range(range)
}
pub fn prefix<'a, K: AsRef<[u8]> + 'a>(
&'a self,
prefix: K,
) -> impl DoubleEndedIterator<Item = Result<KvPair>> + 'static {
self.data.prefix(prefix)
}
pub fn insert(&mut self, key: Slice, value: Slice, height: Height) {
if self.is_safe(height) {
return;

7
src/crates/barser/src/structs/height.rs → src/crates/bitbase/src/structs/height.rs
View File

@@ -5,18 +5,19 @@ use std::{
use biter::bitcoincore_rpc::{self, RpcApi};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::SliceExtended;
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct Height(u32);
impl From<fjall::Slice> for Height {
fn from(slice: fjall::Slice) -> Self {
impl From<Slice> for Height {
fn from(slice: Slice) -> Self {
Self(slice.read_u32())
}
}
impl From<Height> for fjall::Slice {
impl From<Height> for Slice {
fn from(value: Height) -> Self {
value.to_be_bytes().into()
}

12
src/crates/barser/src/structs/mod.rs → src/crates/bitbase/src/structs/mod.rs
View File

@@ -1,15 +1,23 @@
mod addressbytes;
mod addressindex;
mod addresstxoutindex;
mod addresstype;
mod amount;
mod database;
mod exit;
mod fjall;
mod height;
mod slice;
mod txindex;
mod txoutindex;
mod version;
pub use addressbytes::*;
pub use addressindex::*;
pub use addresstxoutindex::*;
pub use addresstype::*;
pub use amount::*;
pub use database::*;
pub use exit::*;
pub use fjall::*;
pub use height::*;
pub use slice::*;
pub use txindex::*;

17
src/crates/barser/src/structs/slice.rs → src/crates/bitbase/src/structs/slice.rs
View File

@@ -1,19 +1,34 @@
use std::io::Read;
use fjall::Slice;
#[allow(unused)]
pub trait SliceExtended {
fn default() -> Self;
fn read_u8(&self) -> u8;
fn read_u16(&self) -> u16;
fn read_u32(&self) -> u32;
fn read_u64(&self) -> u64;
fn read_exact(&self, buf: &mut [u8]);
}
impl SliceExtended for fjall::Slice {
impl SliceExtended for Slice {
fn default() -> Self {
Self::new(&[])
}
fn read_u8(&self) -> u8 {
let mut buf: [u8; 1] = [0; 1];
self.read_exact(&mut buf);
u8::from_be_bytes(buf)
}
fn read_u16(&self) -> u16 {
let mut buf: [u8; 2] = [0; 2];
self.read_exact(&mut buf);
u16::from_be_bytes(buf)
}
fn read_u32(&self) -> u32 {
let mut buf: [u8; 4] = [0; 4];
self.read_exact(&mut buf);

9
src/crates/barser/src/structs/txindex.rs → src/crates/bitbase/src/structs/txindex.rs
View File

@@ -1,4 +1,5 @@
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::SliceExtended;
@@ -6,6 +7,8 @@ use super::SliceExtended;
pub struct Txindex(u32);
impl Txindex {
pub const BYTES: usize = size_of::<Self>();
pub fn increment(&mut self) {
self.0 += 1;
}
@@ -32,12 +35,12 @@ impl From<Txindex> for u64 {
}
}
impl From<fjall::Slice> for Txindex {
fn from(slice: fjall::Slice) -> Self {
impl From<Slice> for Txindex {
fn from(slice: Slice) -> Self {
Self(slice.read_u32())
}
}
impl From<Txindex> for fjall::Slice {
impl From<Txindex> for Slice {
fn from(value: Txindex) -> Self {
value.to_be_bytes().into()
}

46
src/crates/bitbase/src/structs/txoutindex.rs Normal file
View File

@@ -0,0 +1,46 @@
use fjall::Slice;
use super::{SliceExtended, Txindex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
pub struct Txoutindex {
pub txindex: Txindex,
pub vout: u16,
}
impl Txoutindex {
pub const BYTES: usize = size_of::<Self>();
}
impl From<Txindex> for Txoutindex {
fn from(value: Txindex) -> Self {
Self {
txindex: value,
vout: 0,
}
}
}
impl From<(Txindex, u16)> for Txoutindex {
fn from(value: (Txindex, u16)) -> Self {
Self {
txindex: value.0,
vout: value.1,
}
}
}
impl From<Txoutindex> for Slice {
fn from(value: Txoutindex) -> Self {
let txindex_slice = Self::from(value.txindex);
let vout_slice = Self::from(value.vout.to_be_bytes());
Self::from([txindex_slice, vout_slice].concat())
}
}
impl From<Slice> for Txoutindex {
fn from(value: Slice) -> Self {
let txindex = Txindex::from(Slice::from(&value[..Txindex::BYTES]));
let vout = Slice::from(&value[Txindex::BYTES..]).read_u16();
Self { txindex, vout }
}
}

5
src/crates/barser/src/structs/version.rs → src/crates/bitbase/src/structs/version.rs
View File

@@ -1,4 +1,5 @@
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use super::SliceExtended;
@@ -11,8 +12,8 @@ impl From<u8> for Version {
}
}
impl From<fjall::Slice> for Version {
fn from(slice: fjall::Slice) -> Self {
impl From<Slice> for Version {
fn from(slice: Slice) -> Self {
Self(slice.read_u8())
}
}

24
src/crates/biter/Cargo.lock generated
View File

@@ -243,9 +243,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
[[package]]
name = "itoa"
version = "1.0.11"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "jsonrpc"
@@ -308,9 +308,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.37"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
dependencies = [
"proc-macro2",
]
@@ -394,18 +394,18 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.216"
version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.216"
version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
dependencies = [
"proc-macro2",
"quote",
@@ -414,9 +414,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.133"
version = "1.0.134"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d"
dependencies = [
"itoa",
"memchr",
@@ -432,9 +432,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "syn"
version = "2.0.90"
version = "2.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058"
dependencies = [
"proc-macro2",
"quote",

4
src/crates/biter/Cargo.toml
View File

@@ -12,8 +12,8 @@ edition = "2021"
bitcoin = { version = "0.32.5", features = ["serde"] }
rayon = "1.10.0"
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
serde = { version = "1.0.216", features = ["derive"] }
serde_json = "1.0.133"
serde = { version = "1.0.217", features = ["derive"] }
serde_json = "1.0.134"
derived-deref = "2.1.0"
bitcoincore-rpc = "0.19.0"
# tokio = { version = "1.39.2", features = ["rt-multi-thread"] }

46
src/crates/biter/src/blk_index_to_blk_path.rs Normal file
View File

@@ -0,0 +1,46 @@
use std::{
collections::BTreeMap,
fs,
path::{Path, PathBuf},
};
use derived_deref::{Deref, DerefMut};
const BLK: &str = "blk";
const DAT: &str = ".dat";
#[derive(Debug, Deref, DerefMut)]
pub struct BlkIndexToBlkPath(BTreeMap<usize, PathBuf>);
impl BlkIndexToBlkPath {
pub fn scan(data_dir: &Path) -> Self {
let blocks_dir = data_dir.join("blocks");
Self(
fs::read_dir(blocks_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| {
let is_file = path.is_file();
if is_file {
let file_name = path.file_name().unwrap().to_str().unwrap();
file_name.starts_with(BLK) && file_name.ends_with(DAT)
} else {
false
}
})
.map(|path| {
let file_name = path.file_name().unwrap().to_str().unwrap();
let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
.parse::<usize>()
.unwrap();
(blk_index, path)
})
.collect::<BTreeMap<_, _>>(),
)
}
}

6
src/crates/biter/src/blk_index_to_blk_recap.rs
View File

@@ -8,7 +8,7 @@ use std::{
use derived_deref::{Deref, DerefMut};
use crate::{blk_recap::BlkRecap, BlkMetadataAndBlock};
use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock};
const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30;
@@ -21,7 +21,7 @@ pub struct BlkIndexToBlkRecap {
}
impl BlkIndexToBlkRecap {
pub fn import(blocks_dir: &BTreeMap<usize, PathBuf>, data_dir: &Path) -> Self {
pub fn import(blocks_dir: &BlkIndexToBlkPath, data_dir: &Path) -> Self {
let path = data_dir.join("blk_index_to_blk_recap.json");
let tree = {
@@ -46,7 +46,7 @@ impl BlkIndexToBlkRecap {
this
}
pub fn clean_outdated(&mut self, blocks_dir: &BTreeMap<usize, PathBuf>) {
pub fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) {
let mut unprocessed_keys = self.keys().copied().collect::<BTreeSet<_>>();
blocks_dir.iter().for_each(|(blk_index, blk_path)| {

12
src/crates/biter/src/lib.rs
View File

@@ -13,12 +13,14 @@ use bitcoin::{
Block, BlockHash,
};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use crossbeam::channel::{bounded, Receiver};
use rayon::prelude::*;
pub use bitcoin;
pub use bitcoincore_rpc;
mod blk_index_to_blk_path;
mod blk_index_to_blk_recap;
mod blk_metadata;
mod blk_metadata_and_block;
@@ -88,7 +90,7 @@ pub fn new(
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
let blk_index_to_blk_path = scan_blocks_dir(data_dir);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(data_dir);
let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blk_index_to_blk_path, data_dir);
@@ -97,12 +99,12 @@ pub fn new(
thread::spawn(move || {
blk_index_to_blk_path
.into_iter()
.filter(|(blk_index, _)| blk_index >= &starting_blk_index)
.iter()
.filter(|(blk_index, _)| blk_index >= &&starting_blk_index)
.try_for_each(move |(blk_index, blk_path)| {
let blk_metadata = BlkMetadata::new(blk_index, &blk_path);
let blk_metadata = BlkMetadata::new(*blk_index, blk_path);
let blk_bytes = fs::read(&blk_path).unwrap();
let blk_bytes = fs::read(blk_path).unwrap();
let blk_bytes_len = blk_bytes.len() as u64;
let mut cursor = Cursor::new(blk_bytes.as_slice());

39
src/crates/biter/src/utils.rs
View File

@@ -1,41 +1,4 @@
use std::{
collections::BTreeMap,
fs,
path::{Path, PathBuf},
time::UNIX_EPOCH,
};
const BLK: &str = "blk";
const DAT: &str = ".dat";
pub fn scan_blocks_dir(data_dir: &Path) -> BTreeMap<usize, PathBuf> {
let blocks_dir = data_dir.join("blocks");
fs::read_dir(blocks_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| {
let is_file = path.is_file();
if is_file {
let file_name = path.file_name().unwrap().to_str().unwrap();
file_name.starts_with(BLK) && file_name.ends_with(DAT)
} else {
false
}
})
.map(|path| {
let file_name = path.file_name().unwrap().to_str().unwrap();
let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
.parse::<usize>()
.unwrap();
(blk_index, path)
})
.collect::<BTreeMap<_, _>>()
}
use std::{fs, path::PathBuf, time::UNIX_EPOCH};
pub fn path_to_modified_time(path: &PathBuf) -> u64 {
fs::metadata(path)

2
src/parser/actions/iter_blocks.rs
View File

@@ -21,7 +21,7 @@ use crate::{
pub fn iter_blocks(
config: &Config,
rpc: &biter::bitcoincore_rpc::Client,
rpc: &Client,
approx_block_count: usize,
exit: Exit,
databases: &mut Databases,