Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-28 19:28:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

bindex: snapshot

Browse Source
This commit is contained in:
nym21
2025-01-23 11:11:39 +01:00
parent 1296a2e9ec
commit d629ae8fbb
64 changed files with 334 additions and 3844 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/crates/bindex/.gitignore
-1
View File
@@ -1 +0,0 @@
/database
src/crates/bindex/Cargo.lock
Generated
-1065
View File
File diff suppressed because it is too large Load Diff
src/crates/bindex/Cargo.toml
-15
View File
@@ -1,15 +0,0 @@
[package]
name = "bindex"
version = "0.1.0"
edition = "2021"
[dependencies]
bitcoin_hashes = "0.16.0"
biter = "0.2.2"
color-eyre = "0.6.3"
ctrlc = "3.4.5"
derive_deref = "1.1.1"
jiff = "0.1.24"
rayon = "1.10.0"
snkrj = { path = "../snkrj" }
storable_vec = { path = "../storable_vec" }
src/crates/bindex/rustfmt.toml
-1
View File
@@ -1 +0,0 @@
max_width = 120
src/crates/bindex/src/main.rs
-566
View File
@@ -1,566 +0,0 @@
use std::{
collections::BTreeMap,
io::{Read, Write},
path::Path,
str::FromStr,
thread::{self},
};
use biter::{
bitcoin::{Transaction, TxIn, TxOut, Txid},
bitcoincore_rpc::{Auth, Client},
};
mod structs;
use color_eyre::eyre::{eyre, ContextCompat};
use rayon::prelude::*;
use structs::{
Addressbytes, AddressbytesPrefix, Addressindex, Addressindextxoutindex, Addresstype, Addresstypeindex, Amount,
BlockHashPrefix, Date, Exit, Height, Stores, Timestamp, TxidPrefix, Txindex, Txindexvout, Txoutindex, Vecs,
};
// https://github.com/romanz/electrs/blob/master/doc/schema.md
#[derive(Debug)]
enum TxInOrAddressindextoutindex<'a> {
TxIn(&'a TxIn),
Addressindextoutindex(Addressindextxoutindex),
}
const UNSAFE_BLOCKS: u32 = 100;
const SNAPSHOT_BLOCK_RANGE: usize = 4_200; // MUST 210_000 % THIS == 0
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let i = std::time::Instant::now();
let check_collisions = true;
let data_dir = Path::new("../../../../bitcoin");
let cookie = Path::new(data_dir).join(".cookie");
let rpc = Client::new("http://localhost:8332", Auth::CookieFile(cookie))?;
let exit = Exit::new();
let path_database = Path::new("./database");
let path_stores = path_database.join("stores");
let stores = Stores::open(&path_stores)?;
let mut vecs = Vecs::import(&path_database.join("vecs"))?;
let mut height = vecs
.min_height()
.unwrap_or_default()
.min(stores.min_height())
.and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
.map(Height::from)
.unwrap_or_default();
let mut txindex = vecs
.height_to_first_txindex
.get(height)?
.cloned()
.unwrap_or(Txindex::default());
let mut txoutindex = vecs
.height_to_first_txoutindex
.get(height)?
.cloned()
.unwrap_or(Txoutindex::default());
let mut addressindex = vecs
.height_to_first_addressindex
.get(height)?
.cloned()
.unwrap_or(Addressindex::default());
let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> {
exit.block();
println!("Exporting...");
// Memory: 2.87
// Real Memory: 16.23
// Private Memory: 10.8
if height > Height::from(400_000_u32) {
pause();
}
vecs.reset_cache();
println!("Resetted cache");
// Memory: 2.87
// Real Memory: 13.24
// Private Memory: 10.8
if height > Height::from(400_000_u32) {
pause();
}
vecs.flush(height)?;
println!("Vecs flushed");
// Memory: 3.36
// Real Memory: 13.55
// Private Memory: 10.66
// Gone up wtf
if height > Height::from(400_000_u32) {
pause();
}
stores.export(height);
println!("Export done");
if height > Height::from(400_000_u32) {
pause();
}
exit.unblock();
Ok(())
};
let mut stores_opt = Some(stores);
biter::new(data_dir, Some(height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
height = Height::from(_height);
let timestamp = Timestamp::try_from(block.header.time)?;
let date = Date::from(&timestamp);
let mut stores = stores_opt.take().context("option should have wtx")?;
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
if &blockhash != saved_blockhash {
todo!("Rollback not implemented");
// parts.rollback_from(&mut wtx, height, &exit)?;
}
}
if stores.blockhash_prefix_to_height.needs(height) {
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
if check_collisions {
if let Some(prev_height) =
stores.blockhash_prefix_to_height.get(&blockhash_prefix)
{
dbg!(blockhash, prev_height);
return Err(eyre!("Collision, expect prefix to need be set yet"));
}
}
stores.blockhash_prefix_to_height.insert(blockhash_prefix,height);
}
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
vecs.height_to_first_txindex.push_if_needed(height, txindex)?;
vecs.height_to_first_txoutindex.push_if_needed(height, txoutindex)?;
vecs.height_to_first_addressindex.push_if_needed(height, addressindex)?;
vecs.height_to_timestamp.push_if_needed(height, timestamp)?;
vecs.height_to_date.push_if_needed(height, date)?;
let outputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (Txindex::from(index), vout as u32, txout))
}).collect::<Vec<_>>();
let tx_len = block.txdata.len();
let outputs_len = outputs.len();
let (txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle, txin_or_addressindextxoutindex_vec_handle, txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle) = thread::scope(|scope| {
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle = scope.spawn(|| -> color_eyre::Result<_> {
block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| -> color_eyre::Result<_> {
let txid = tx.compute_txid();
let txid_prefix = TxidPrefix::try_from(&txid)?;
let prev_txindex_slice_opt = if check_collisions && stores.txid_prefix_to_txindex.needs(height) {
// Should only find collisions for two txids (duplicates), see below
stores.txid_prefix_to_txindex.get(&txid_prefix).cloned()
} else {
None
};
Ok((txid_prefix, (tx, txid, Txindex::from(index), prev_txindex_slice_opt)))
})
.try_fold(
BTreeMap::new,
|mut map, tuple| {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
},
)
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
})});
let txin_or_addressindextxoutindex_vec_handle = scope.spawn(|| -> color_eyre::Result<Vec<TxInOrAddressindextoutindex>> {
block
.txdata
.par_iter()
.filter(|tx| !tx.is_coinbase())
.flat_map(|tx| &tx.input)
.map(|txin| -> color_eyre::Result<_> {
let outpoint = txin.previous_output;
let txid = outpoint.txid;
let vout = outpoint.vout;
let txindex_local = if let Some(txindex_local) = stores.txid_prefix_to_txindex
.get(&TxidPrefix::try_from(&txid)?).and_then(|txindex_local| {
// Checking if not finding txindex from the future
(txindex_local < &txindex).then_some(txindex_local)
})
{
*txindex_local
} else {
return Ok(TxInOrAddressindextoutindex::TxIn(txin));
};
let txindexvout = Txindexvout::from((txindex_local, vout));
let txoutindex =
*stores.txindexvout_to_txoutindex.get(&txindexvout)
.context("Expect txoutindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, txindex_local, vout, txindexvout);
})?;
let addressindex = *vecs.txoutindex_to_addressindex.get(txoutindex)?
.context("Expect addressindex to not be none")
.inspect_err(|_| {
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
// .expect("Expect height for txindex");
dbg!(outpoint.txid, txindex_local, vout, txindexvout);
})?;
Ok(TxInOrAddressindextoutindex::Addressindextoutindex(Addressindextxoutindex::from((
addressindex,
txoutindex,
))))
})
.try_fold(
Vec::new,
|mut vec, addressindextxoutindex| {
vec.push(addressindextxoutindex?);
Ok(vec)
},
)
.try_reduce(Vec::new, |mut v, mut v2| {
if v.len() > v2.len() {
v.append(&mut v2);
Ok(v)
} else {
v2.append(&mut v);
Ok(v2)
}
})
});
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| {
outputs.into_par_iter().enumerate()
.map(
#[allow(clippy::type_complexity)]
|(block_txoutindex, (block_txindex, vout, txout))| -> color_eyre::Result<(Txoutindex,
(&TxOut, Txindexvout, Addresstype, color_eyre::Result<Addressbytes>, Option<Addressindex>))> {
let txindex_local = txindex + block_txindex;
let txindexvout = Txindexvout::from((txindex_local, vout));
let txoutindex_local = txoutindex + Txoutindex::from(block_txoutindex);
let script = &txout.script_pubkey;
let addresstype = Addresstype::from(script);
let addressbytes_res = Addressbytes::try_from((script, addresstype)).inspect_err(|_| {
// dbg!(&txout, height, txi, &tx.compute_txid());
});
let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
stores.addressbytes_prefix_to_addressindex.get(
&AddressbytesPrefix::from((addressbytes, addresstype)),
)
.cloned()
// Checking if not in the future
.and_then(|addressindex_local| (addressindex_local < addressindex)
.then_some(addressindex_local))
});
if let Some(Some(addressindex)) = check_collisions.then_some(addressindex_opt) {
let addressbytes = addressbytes_res.as_ref().unwrap();
let prev_addresstype = *vecs.addressindex_to_addresstype.get(
addressindex,
)?.context("Expect to have address type")?;
let addresstypeindex = *vecs.addressindex_to_addresstypeindex.get(
addressindex,
)?.context("Expect to have address type index")?;
let prev_addressbytes_opt= vecs.get_addressbytes(prev_addresstype, addresstypeindex)?;
let prev_addressbytes = prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
if (vecs.addressindex_to_addresstype.hasnt(addressindex) && addresstype != prev_addresstype) || (stores.addressbytes_prefix_to_addressindex.needs(height) && prev_addressbytes != addressbytes) {
dbg!(addresstype, prev_addresstype, prev_addressbytes, addressbytes, addressindex, addresstypeindex, txout, AddressbytesPrefix::from((addressbytes, addresstype)), AddressbytesPrefix::from((prev_addressbytes, prev_addresstype)));
panic!()
}
}
Ok((
txoutindex_local,
(txout, txindexvout, addresstype, addressbytes_res, addressindex_opt),
))
},
)
.try_fold(
BTreeMap::new,
|mut map, tuple| -> color_eyre::Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
},
)
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
})
});
(txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(), txin_or_addressindextxoutindex_vec_handle.join(), txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.join())
});
let txid_prefix_to_txid_and_block_txindex_and_prev_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle.ok().context("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join")??;
let txin_or_addressindextxoutindex_vec = txin_or_addressindextxoutindex_vec_handle.ok().context("Export txin_or_addressindextxoutindex_vec_handle to join")??;
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt = txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle.ok().context("Expect txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle to join")??;
let mut new_txindexvout_to_addressindextxoutindex: BTreeMap<Txindexvout, Addressindextxoutindex> = BTreeMap::new();
let mut already_added_addressbytes_prefix: BTreeMap<AddressbytesPrefix, Addressindex> = BTreeMap::new();
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt
.into_iter()
.try_for_each(|(txoutindex, (txout, txindexvout, addresstype, addressbytes_res, addressindex_opt))| -> color_eyre::Result<()> {
let amount = Amount::from(txout.value);
stores.txindexvout_to_txoutindex.insert_if_needed(
txindexvout,
txoutindex,
height,
);
vecs.txoutindex_to_amount.push_if_needed(
txoutindex,
amount,
)?;
let mut addressindex_local = addressindex;
let mut addressbytes_prefix= None;
if let Some(addressindex) = addressindex_opt.or_else(|| addressbytes_res.as_ref().ok().and_then(|addressbytes| {
// Check if address was first seen before in this iterator
// Example: https://mempool.space/address/046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0c
addressbytes_prefix.replace(AddressbytesPrefix::from((addressbytes, addresstype)));
already_added_addressbytes_prefix.get(
addressbytes_prefix.as_ref().unwrap(),
).cloned()
})) {
addressindex_local = addressindex;
} else {
addressindex.increment();
// TODO: Create counter of other addresstypes instead
let addresstypeindex = Addresstypeindex::from(vecs.addresstype_to_addressbytes(addresstype).map_or(0, |vec| vec.len()));
vecs.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?;
vecs.addressindex_to_addresstypeindex.push_if_needed(addressindex_local, addresstypeindex)?;
if let Ok(addressbytes) = addressbytes_res {
let addressbytes_prefix = addressbytes_prefix.unwrap();
already_added_addressbytes_prefix.insert(addressbytes_prefix.clone(), addressindex_local);
stores.addressbytes_prefix_to_addressindex.insert_if_needed(
addressbytes_prefix,
addressindex_local,
height
);
vecs.push_addressbytes_if_needed(addresstypeindex, addressbytes)?;
}
}
let addressindextxoutindex = Addressindextxoutindex::from((addressindex_local, txoutindex));
new_txindexvout_to_addressindextxoutindex.insert(txindexvout, addressindextxoutindex);
vecs.txoutindex_to_addressindex.push_if_needed(
txoutindex,
addressindex_local,
)?;
stores.addressindextxoutindex_in.insert_if_needed(
addressindextxoutindex,
(),
height,
);
Ok(())
})?;
drop(already_added_addressbytes_prefix);
if stores.addressindextxoutindex_out.needs(height) {
txin_or_addressindextxoutindex_vec
.into_iter()
.map(|txin_or_addressindextxoutindex| -> color_eyre::Result<Addressindextxoutindex> {
match txin_or_addressindextxoutindex {
TxInOrAddressindextoutindex::Addressindextoutindex(addressindextxoutindex) => Ok(addressindextxoutindex),
TxInOrAddressindextoutindex::TxIn(txin) => {
let outpoint = txin.previous_output;
let txid = outpoint.txid;
let vout = outpoint.vout;
let index = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.get(&TxidPrefix::try_from(&txid)?)
.context("txid should be in same block")?.2;
let txindex_local = txindex + index;
let txindexvout = Txindexvout::from((txindex_local, vout));
new_txindexvout_to_addressindextxoutindex
.remove(&txindexvout)
.context("should have found addressindex from same block").inspect_err(|_| {
dbg!(&new_txindexvout_to_addressindextxoutindex, txin, txindexvout, txid);
})
}
}
})
.try_for_each(|addressindextxoutindex| -> color_eyre::Result<()> {
stores.addressindextxoutindex_out.insert(
addressindextxoutindex?,
(),
);
Ok(())
})?;
}
drop(new_txindexvout_to_addressindextxoutindex);
let mut txindex_to_tx_and_txid: BTreeMap<Txindex, (&Transaction, Txid)> = BTreeMap::default();
txid_prefix_to_txid_and_block_txindex_and_prev_txindex.into_iter().try_for_each(
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
let txindex_local = txindex + index;
txindex_to_tx_and_txid.insert(txindex_local, (tx, txid));
match prev_txindex_opt {
None => {
stores.txid_prefix_to_txindex.insert_if_needed(txid_prefix, txindex_local, height);
},
Some(prev_txindex) => {
// In case if we start at an already parsed height
if txindex_local == prev_txindex {
return Ok(())
}
let len = vecs.txindex_to_txid.len();
// Ok if `get` is not par as should happen only twice
let prev_txid =
vecs.txindex_to_txid.get(prev_txindex)?
.context("To have txid for txindex").inspect_err(|_| {
dbg!(txindex_local, txid, len);
})?;
// If another Txid needs to be added to the list
// We need to check that it's also a coinbase tx otherwise par_iter inputs needs to be updated
let only_known_dup_txids = [
Txid::from_str("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599")?,
Txid::from_str("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468")?,
];
let is_dup = only_known_dup_txids.contains(prev_txid);
if !is_dup {
let prev_height = vecs.txindex_to_height.get(prev_txindex)?.expect("To have height");
dbg!(height, txid, txindex_local, prev_height, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
}
}
}
Ok(())
},
)?;
txindex_to_tx_and_txid.into_iter().try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version)?;
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_height.push_if_needed(txindex, height)?;
vecs.txindex_to_inputcount.push_if_needed(txindex, tx.input.len() as u32)?;
vecs.txindex_to_outputcount.push_if_needed(txindex, tx.output.len() as u32)?;
Ok(())
})?;
vecs.height_to_last_txindex.push_if_needed(height, txindex.decremented())?;
vecs.height_to_last_txoutindex.push_if_needed(height, txoutindex.decremented())?;
vecs.height_to_last_addressindex.push_if_needed(height, addressindex.decremented())?;
let should_snapshot = _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active();
if should_snapshot {
export(stores, &mut vecs, height)?;
stores_opt.replace(Stores::open(&path_stores)?);
} else {
stores_opt.replace(stores);
}
txindex += Txindex::from(tx_len);
txoutindex += Txoutindex::from(outputs_len);
Ok(())
})?;
dbg!(i.elapsed());
pause();
let stores = stores_opt.take().context("option should have wtx")?;
export(stores, &mut vecs, height)?;
pause();
dbg!(i.elapsed());
Ok(())
}
fn pause() {
let mut stdin = std::io::stdin();
let mut stdout = std::io::stdout();
// We want the cursor to stay at the end of the line, so we print without a newline and flush manually.
write!(stdout, "Press any key to continue...").unwrap();
stdout.flush().unwrap();
// Read a single byte and discard
let _ = stdin.read(&mut [0u8]).unwrap();
}
src/crates/bindex/src/structs/addressbytes.rs
-194
View File
@@ -1,194 +0,0 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use super::Addresstype;
#[derive(Debug, PartialEq, Eq)]
pub enum Addressbytes {
P2PK65(P2PK65AddressBytes),
P2PK33(P2PK33AddressBytes),
P2PKH(P2PKHAddressBytes),
P2SH(P2SHAddressBytes),
P2WPKH(P2WPKHAddressBytes),
P2WSH(P2WSHAddressBytes),
P2TR(P2TRAddressBytes),
}
impl Addressbytes {
pub fn as_slice(&self) -> &[u8] {
match self {
Addressbytes::P2PK65(bytes) => &bytes[..],
Addressbytes::P2PK33(bytes) => &bytes[..],
Addressbytes::P2PKH(bytes) => &bytes[..],
Addressbytes::P2SH(bytes) => &bytes[..],
Addressbytes::P2WPKH(bytes) => &bytes[..],
Addressbytes::P2WSH(bytes) => &bytes[..],
Addressbytes::P2TR(bytes) => &bytes[..],
}
}
}
impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
type Error = color_eyre::Report;
fn try_from(tuple: (&ScriptBuf, Addresstype)) -> Result<Self, Self::Error> {
let (script, addresstype) = tuple;
match addresstype {
Addresstype::P2PK65 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
67 => &bytes[1..66],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
Ok(Self::P2PK65(P2PK65AddressBytes(U8x65::from(bytes))))
}
Addresstype::P2PK33 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
35 => &bytes[1..34],
_ => {
dbg!(bytes);
return Err(eyre!("Wrong len"));
}
};
Ok(Self::P2PK33(P2PK33AddressBytes(U8x33::from(bytes))))
}
Addresstype::P2PKH => {
let bytes = &script.as_bytes()[3..23];
Ok(Self::P2PKH(P2PKHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2SH => {
let bytes = &script.as_bytes()[2..22];
Ok(Self::P2SH(P2SHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2WPKH => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2WPKH(P2WPKHAddressBytes(U8x20::from(bytes))))
}
Addresstype::P2WSH => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2WSH(P2WSHAddressBytes(U8x32::from(bytes))))
}
Addresstype::P2TR => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2TR(P2TRAddressBytes(U8x32::from(bytes))))
}
Addresstype::Multisig => Err(eyre!("multisig address type")),
Addresstype::PushOnly => Err(eyre!("push_only address type")),
Addresstype::Unknown => Err(eyre!("unknown address type")),
Addresstype::Empty => Err(eyre!("empty address type")),
Addresstype::OpReturn => Err(eyre!("op_return address type")),
}
}
}
impl From<P2PK65AddressBytes> for Addressbytes {
fn from(value: P2PK65AddressBytes) -> Self {
Self::P2PK65(value)
}
}
impl From<P2PK33AddressBytes> for Addressbytes {
fn from(value: P2PK33AddressBytes) -> Self {
Self::P2PK33(value)
}
}
impl From<P2PKHAddressBytes> for Addressbytes {
fn from(value: P2PKHAddressBytes) -> Self {
Self::P2PKH(value)
}
}
impl From<P2SHAddressBytes> for Addressbytes {
fn from(value: P2SHAddressBytes) -> Self {
Self::P2SH(value)
}
}
impl From<P2WPKHAddressBytes> for Addressbytes {
fn from(value: P2WPKHAddressBytes) -> Self {
Self::P2WPKH(value)
}
}
impl From<P2WSHAddressBytes> for Addressbytes {
fn from(value: P2WSHAddressBytes) -> Self {
Self::P2WSH(value)
}
}
impl From<P2TRAddressBytes> for Addressbytes {
fn from(value: P2TRAddressBytes) -> Self {
Self::P2TR(value)
}
}
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2PK65AddressBytes(U8x65);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2PK33AddressBytes(U8x33);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2PKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2SHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2WPKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2WSHAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
pub struct P2TRAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x20([u8; 20]);
impl From<&[u8]> for U8x20 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 20];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x32([u8; 32]);
impl From<&[u8]> for U8x32 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 32];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x33([u8; 33]);
impl From<&[u8]> for U8x33 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 33];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x64([u8; 64]);
impl From<&[u8]> for U8x64 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 64];
arr.copy_from_slice(slice);
Self(arr)
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
pub struct U8x65([u8; 65]);
impl From<&[u8]> for U8x65 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 65];
arr.copy_from_slice(slice);
Self(arr)
}
}
src/crates/bindex/src/structs/addressindex.rs
-50
View File
@@ -1,50 +0,0 @@
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Addressindex(u32);
direct_repr!(Addressindex);
impl Addressindex {
pub const BYTES: usize = size_of::<Self>();
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
}
impl From<u32> for Addressindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Addressindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Addressindex> for u64 {
fn from(value: Addressindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Addressindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Addressindex> for usize {
fn from(value: Addressindex) -> Self {
value.0 as usize
}
}
src/crates/bindex/src/structs/addressindextxoutindex.rs
-19
View File
@@ -1,19 +0,0 @@
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::{Addressindex, Txoutindex};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Addressindextxoutindex {
addressindex: Addressindex,
txoutindex: Txoutindex,
}
direct_repr!(Addressindextxoutindex);
impl From<(Addressindex, Txoutindex)> for Addressindextxoutindex {
fn from(value: (Addressindex, Txoutindex)) -> Self {
Self {
addressindex: value.0,
txoutindex: value.1,
}
}
}
src/crates/bindex/src/structs/addresstype.rs
-54
View File
@@ -1,54 +0,0 @@
use biter::bitcoin::ScriptBuf;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Addresstype {
P2PK65,
P2PK33,
P2PKH,
P2SH,
P2WPKH,
P2WSH,
P2TR,
Multisig = 251,
PushOnly = 252,
OpReturn = 253,
Empty = 254,
Unknown = 255,
}
impl From<&ScriptBuf> for Addresstype {
fn from(script: &ScriptBuf) -> Self {
if script.is_p2pk() {
let bytes = script.as_bytes();
match bytes.len() {
67 => Self::P2PK65,
35 => Self::P2PK33,
_ => {
dbg!(bytes);
unreachable!()
}
}
} else if script.is_p2pkh() {
Self::P2PKH
} else if script.is_p2sh() {
Self::P2SH
} else if script.is_p2wpkh() {
Self::P2WPKH
} else if script.is_p2wsh() {
Self::P2WSH
} else if script.is_p2tr() {
Self::P2TR
} else if script.is_empty() {
Self::Empty
} else if script.is_op_return() {
Self::OpReturn
} else if script.is_push_only() {
Self::PushOnly
} else if script.is_multisig() {
Self::Multisig
} else {
Self::Unknown
}
}
}
src/crates/bindex/src/structs/addresstypeindex.rs
-48
View File
@@ -1,48 +0,0 @@
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Addresstypeindex(u32);
direct_repr!(Addresstypeindex);
impl Addresstypeindex {
pub fn decremented(self) -> Self {
Self(*self - 1)
}
pub fn increment(&mut self) {
self.0 += 1;
}
pub fn incremented(self) -> Self {
Self(*self + 1)
}
}
impl From<u32> for Addresstypeindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Addresstypeindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Addresstypeindex> for u64 {
fn from(value: Addresstypeindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Addresstypeindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Addresstypeindex> for usize {
fn from(value: Addresstypeindex) -> Self {
value.0 as usize
}
}
src/crates/bindex/src/structs/amount.rs
-90
View File
@@ -1,90 +0,0 @@
use std::{
iter::Sum,
ops::{Add, AddAssign, Mul, Sub, SubAssign},
};
use biter::bitcoin;
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Height;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Amount(bitcoin::Amount);
direct_repr!(Amount);
impl Amount {
pub const ZERO: Self = Self(bitcoin::Amount::ZERO);
pub const ONE_BTC_F32: f32 = 100_000_000.0;
pub const ONE_BTC_F64: f64 = 100_000_000.0;
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
}
}
impl Add for Amount {
type Output = Amount;
fn add(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() + rhs.to_sat())
}
}
impl AddAssign for Amount {
fn add_assign(&mut self, rhs: Self) {
*self = Amount::from(self.to_sat() + rhs.to_sat());
}
}
impl Sub for Amount {
type Output = Amount;
fn sub(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() - rhs.to_sat())
}
}
impl SubAssign for Amount {
fn sub_assign(&mut self, rhs: Self) {
*self = Amount::from(self.to_sat() - rhs.to_sat());
}
}
impl Mul<Amount> for Amount {
type Output = Amount;
fn mul(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() * rhs.to_sat())
}
}
impl Mul<u64> for Amount {
type Output = Amount;
fn mul(self, rhs: u64) -> Self::Output {
Amount::from(self.to_sat() * rhs)
}
}
impl Mul<Height> for Amount {
type Output = Amount;
fn mul(self, rhs: Height) -> Self::Output {
Amount::from(self.to_sat() * *rhs as u64)
}
}
impl Sum for Amount {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
let sats: u64 = iter.map(|amt| amt.to_sat()).sum();
Amount::from(sats)
}
}
impl From<u64> for Amount {
fn from(value: u64) -> Self {
Self(bitcoin::Amount::from_sat(value))
}
}
impl From<bitcoin::Amount> for Amount {
fn from(value: bitcoin::Amount) -> Self {
Self(value)
}
}
src/crates/bindex/src/structs/date.rs
-12
View File
@@ -1,12 +0,0 @@
use jiff::tz::TimeZone;
use super::Timestamp;
#[derive(Debug)]
pub struct Date(jiff::civil::Date);
impl From<&Timestamp> for Date {
fn from(value: &Timestamp) -> Self {
Self(jiff::civil::Date::from(value.to_zoned(TimeZone::UTC)))
}
}
src/crates/bindex/src/structs/exit.rs
-60
View File
@@ -1,60 +0,0 @@
use std::{
process::exit,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
thread::sleep,
time::Duration,
};
#[derive(Default, Clone)]
pub struct Exit {
blocked: Arc<AtomicBool>,
active: Arc<AtomicBool>,
}
impl Exit {
pub fn new() -> Self {
let s = Self {
active: Arc::new(AtomicBool::new(false)),
blocked: Arc::new(AtomicBool::new(false)),
};
let active = s.active.clone();
let _blocked = s.blocked.clone();
let blocked = move || _blocked.load(Ordering::SeqCst);
ctrlc::set_handler(move || {
println!("Exitting...");
active.store(true, Ordering::SeqCst);
if blocked() {
println!("Waiting to exit safely");
while blocked() {
sleep(Duration::from_millis(50));
}
}
exit(0);
})
.expect("Error setting Ctrl-C handler");
s
}
pub fn block(&self) {
self.blocked.store(true, Ordering::SeqCst);
}
pub fn unblock(&self) {
self.blocked.store(false, Ordering::SeqCst);
}
pub fn active(&self) -> bool {
self.active.load(Ordering::SeqCst)
}
}
src/crates/bindex/src/structs/height.rs
-121
View File
@@ -1,121 +0,0 @@
use std::{
fmt, fs, io,
ops::{Add, AddAssign, Rem, Sub},
path::Path,
};
use biter::bitcoincore_rpc::{self, RpcApi};
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use storable_vec::UnsafeSizedSerDe;
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct Height(u32);
direct_repr!(Height);
impl Height {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.unsafe_as_slice())
}
}
impl PartialEq<u64> for Height {
fn eq(&self, other: &u64) -> bool {
**self == *other as u32
}
}
impl Add<u32> for Height {
type Output = Height;
fn add(self, rhs: u32) -> Self::Output {
Self::from(*self + rhs)
}
}
impl Add<usize> for Height {
type Output = Height;
fn add(self, rhs: usize) -> Self::Output {
Self::from(*self + rhs as u32)
}
}
impl Sub<Height> for Height {
type Output = Height;
fn sub(self, rhs: Height) -> Self::Output {
Self::from(*self - *rhs)
}
}
impl Sub<i32> for Height {
type Output = Height;
fn sub(self, rhs: i32) -> Self::Output {
Self::from(*self - rhs as u32)
}
}
impl Sub<u32> for Height {
type Output = Height;
fn sub(self, rhs: u32) -> Self::Output {
Self::from(*self - rhs)
}
}
impl Sub<usize> for Height {
type Output = Height;
fn sub(self, rhs: usize) -> Self::Output {
Self::from(*self - rhs as u32)
}
}
impl AddAssign<usize> for Height {
fn add_assign(&mut self, rhs: usize) {
*self = self.add(rhs);
}
}
impl Rem<usize> for Height {
type Output = Height;
fn rem(self, rhs: usize) -> Self::Output {
Self(self.abs_diff(Height::from(rhs).0))
}
}
impl fmt::Display for Height {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", **self)
}
}
impl From<u32> for Height {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<usize> for Height {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Height> for usize {
fn from(value: Height) -> Self {
value.0 as usize
}
}
impl TryFrom<&Path> for Height {
type Error = color_eyre::Report;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
}
}
impl TryFrom<&bitcoincore_rpc::Client> for Height {
type Error = bitcoincore_rpc::Error;
fn try_from(value: &bitcoincore_rpc::Client) -> Result<Self, Self::Error> {
Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
}
}
src/crates/bindex/src/structs/mod.rs
-41
View File
@@ -1,41 +0,0 @@
mod addressbytes;
mod addressindex;
mod addressindextxoutindex;
mod addresstype;
mod addresstypeindex;
mod amount;
mod date;
mod exit;
mod height;
mod prefix;
mod slice;
mod store;
mod stores;
mod timestamp;
mod txindex;
mod txindexvout;
mod txoutindex;
mod vec;
mod vecs;
mod version;
pub use addressbytes::*;
pub use addressindex::*;
pub use addressindextxoutindex::*;
pub use addresstype::*;
pub use addresstypeindex::*;
pub use amount::*;
pub use date::*;
pub use exit::*;
pub use height::*;
pub use prefix::*;
pub use slice::*;
pub use store::*;
pub use stores::*;
pub use timestamp::*;
pub use txindex::*;
pub use txindexvout::*;
pub use txoutindex::*;
pub use vec::*;
pub use vecs::*;
pub use version::*;
src/crates/bindex/src/structs/prefix.rs
-50
View File
@@ -1,50 +0,0 @@
use biter::bitcoin::{BlockHash, Txid};
use derive_deref::Deref;
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::{Addressbytes, Addresstype, SliceExtended};
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct AddressbytesPrefix([u8; 8]);
direct_repr!(AddressbytesPrefix);
impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix {
fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self {
let shorten = |slice: &[u8]| {
let len = slice.len();
let mut buf: [u8; 8] = [0; 8];
// Using both ends for collision reasons despite rehashing the addresses
(0..4_usize).for_each(|i| {
buf[i] = slice[i];
buf[4 + i] = slice[len - 4 + i];
});
buf[4] = addresstype as u8;
// Put in the middle and not at the start because either the first or the last byte can be used to split and if the type is used it wouldn't have the 0..256 range
// End result:
// [ i=0, i=1, i=2, i=3, type as u8, i=len-3, i=len-2, i=len-1 ]
buf
};
Self(shorten(
bitcoin_hashes::hash160::Hash::hash(addressbytes.as_slice()).as_byte_array(),
))
}
}
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct BlockHashPrefix([u8; 8]);
direct_repr!(BlockHashPrefix);
impl TryFrom<&BlockHash> for BlockHashPrefix {
type Error = color_eyre::Report;
fn try_from(value: &BlockHash) -> Result<Self, Self::Error> {
Ok(Self((&value[..]).read_8x_u8()?))
}
}
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct TxidPrefix([u8; 8]);
direct_repr!(TxidPrefix);
impl TryFrom<&Txid> for TxidPrefix {
type Error = color_eyre::Report;
fn try_from(value: &Txid) -> Result<Self, Self::Error> {
Ok(Self((&value[..]).read_8x_u8()?))
}
}
src/crates/bindex/src/structs/slice.rs
-55
View File
@@ -1,55 +0,0 @@
use color_eyre::eyre::eyre;
#[allow(unused)]
pub trait SliceExtended {
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]>;
fn read_be_u8(&self) -> color_eyre::Result<u8>;
fn read_be_u16(&self) -> color_eyre::Result<u16>;
fn read_be_u32(&self) -> color_eyre::Result<u32>;
fn read_be_u64(&self) -> color_eyre::Result<u64>;
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()>;
}
impl SliceExtended for &[u8] {
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
(&self[..8]).read_exact(&mut buf)?;
Ok(buf)
}
fn read_be_u8(&self) -> color_eyre::Result<u8> {
let mut buf: [u8; 1] = [0; 1];
self.read_exact(&mut buf)?;
Ok(u8::from_be_bytes(buf))
}
fn read_be_u16(&self) -> color_eyre::Result<u16> {
let mut buf: [u8; 2] = [0; 2];
self.read_exact(&mut buf)?;
Ok(u16::from_be_bytes(buf))
}
fn read_be_u32(&self) -> color_eyre::Result<u32> {
let mut buf: [u8; 4] = [0; 4];
self.read_exact(&mut buf)?;
Ok(u32::from_be_bytes(buf))
}
fn read_be_u64(&self) -> color_eyre::Result<u64> {
let mut buf: [u8; 8] = [0; 8];
self.read_exact(&mut buf)?;
Ok(u64::from_be_bytes(buf))
}
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()> {
let buf_len = buf.len();
if self.len() != buf_len {
dbg!(self.len(), buf_len);
return Err(eyre!("Not exact len"));
}
self.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(())
}
}
src/crates/bindex/src/structs/store.rs
-153
View File
@@ -1,153 +0,0 @@
use std::{
array, fs,
path::{Path, PathBuf},
sync::OnceLock,
};
use rayon::prelude::*;
use snkrj::{Database, DatabaseKey, DatabaseValue, UnitDatabase};
use storable_vec::UnsafeSizedSerDe;
use super::{Height, Version};
pub struct Store<K, V>
where
K: DatabaseKey,
V: DatabaseValue,
{
pathbuf: PathBuf,
version: Version,
height: Option<Height>,
len: usize,
pub parts: [OnceLock<Box<Database<K, V>>>; 256],
}
impl<K, V> Store<K, V>
where
K: DatabaseKey,
V: DatabaseValue,
{
pub fn open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
fs::create_dir_all(path)?;
let is_same_version =
Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
fs::remove_dir(path)?;
fs::create_dir_all(path)?;
}
let height = Height::try_from(Self::path_height_(path).as_path()).ok();
Ok(Self {
pathbuf: path.to_owned(),
version,
height,
len: UnitDatabase::read_length_(path),
parts: array::from_fn(|_| OnceLock::new()),
})
}
#[allow(unused)]
pub fn len(&self) -> usize {
self.len
}
fn key_to_byte(key: &K) -> u8 {
let slice = key.unsafe_as_slice();
*(if cfg!(target_endian = "big") {
slice.last()
} else {
slice.first()
})
.unwrap()
}
fn get_or_init_store(&self, key: &K) -> &Database<K, V> {
self.get_or_init_store_(Self::key_to_byte(key) as usize)
}
fn get_or_init_store_(&self, storeindex: usize) -> &Database<K, V> {
self.parts[storeindex]
.get_or_init(|| Box::new(Database::open(self.path_parts().join(storeindex.to_string())).unwrap()))
}
fn get_or_init_mut_store(&mut self, key: &K) -> &mut Database<K, V> {
self.get_or_init_store(key);
self.parts
.get_mut(Self::key_to_byte(key) as usize)
.unwrap()
.get_mut()
.unwrap()
}
#[allow(unused)]
pub fn open_all(&self) {
(0..=(u8::MAX) as usize).for_each(|storeindex| {
self.get_or_init_store_(storeindex);
});
}
pub fn get(&self, key: &K) -> Option<&V> {
self.get_or_init_store(key).get(key)
}
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
self.len += 1;
self.get_or_init_mut_store(&key).insert(key, value)
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
self.insert(key, value);
}
}
pub fn export(mut self, height: Height) -> Result<(), snkrj::Error> {
if self.height.is_some_and(|self_height| self_height >= height) {
return Ok(());
}
self.height = Some(height);
self.version.write(&self.path_version())?;
height.write(&self.path_height())?;
UnitDatabase::write_length_(&self.pathbuf, self.len)?;
self.parts.into_par_iter().try_for_each(|s| {
if let Some(db) = s.into_inner() {
db.export()
} else {
Ok(())
}
})
}
fn path_parts(&self) -> PathBuf {
Self::path_parts_(&self.pathbuf)
}
fn path_parts_(path: &Path) -> PathBuf {
path.join("parts")
}
fn path_version(&self) -> PathBuf {
Self::path_version_(&self.pathbuf)
}
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
pub fn height(&self) -> Option<&Height> {
self.height.as_ref()
}
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
}
fn path_height(&self) -> PathBuf {
Self::path_height_(&self.pathbuf)
}
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
}
src/crates/bindex/src/structs/stores.rs
-178
View File
@@ -1,178 +0,0 @@
use std::{path::Path, thread};
use crate::structs::Height;
use super::{
AddressbytesPrefix, Addressindex, Addressindextxoutindex, BlockHashPrefix, Store, TxidPrefix, Txindex, Txindexvout,
Txoutindex, Version,
};
pub struct Stores {
pub addressbytes_prefix_to_addressindex: Store<AddressbytesPrefix, Addressindex>,
pub addressindextxoutindex_in: Store<Addressindextxoutindex, ()>,
pub addressindextxoutindex_out: Store<Addressindextxoutindex, ()>,
pub blockhash_prefix_to_height: Store<BlockHashPrefix, Height>,
pub txid_prefix_to_txindex: Store<TxidPrefix, Txindex>,
pub txindexvout_to_txoutindex: Store<Txindexvout, Txoutindex>,
}
impl Stores {
pub fn open(path: &Path) -> color_eyre::Result<Self> {
Ok(Self {
addressbytes_prefix_to_addressindex: Store::open(
&path.join("addressbytes_prefix_to_addressindex"),
Version::from(1),
)?,
addressindextxoutindex_in: Store::open(&path.join("addresstxoutindexes_in"), Version::from(1))?,
addressindextxoutindex_out: Store::open(&path.join("addresstxoutindexes_out"), Version::from(1))?,
blockhash_prefix_to_height: Store::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
txid_prefix_to_txindex: Store::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
txindexvout_to_txoutindex: Store::open(&path.join("txindexvout_to_txoutindex"), Version::from(1))?,
})
}
// pub fn rollback_from(
// &mut self,
// _wtx: &mut WriteTransaction,
// _height: Height,
// _exit: &Exit,
// ) -> color_eyre::Result<()> {
// panic!();
// let mut txindex = None;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
// Ok(())
// })?;
// let txindex = txindex.context("txindex to not be none by now")?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
// Ok(())
// })?;
// let txoutindex = Txoutindex::from(txindex);
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// todo!("clear txindexvout_to_txoutindex")
// Ok(())
// }
pub fn min_height(&self) -> Option<Height> {
[
self.addressbytes_prefix_to_addressindex.height(),
self.addressindextxoutindex_in.height(),
self.addressindextxoutindex_out.height(),
self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(),
self.txindexvout_to_txoutindex.height(),
]
.into_iter()
.min()
.flatten()
.cloned()
}
pub fn export(self, height: Height) {
thread::scope(|scope| {
scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height));
scope.spawn(|| self.addressindextxoutindex_in.export(height));
scope.spawn(|| self.addressindextxoutindex_out.export(height));
scope.spawn(|| self.blockhash_prefix_to_height.export(height));
scope.spawn(|| self.txid_prefix_to_txindex.export(height));
scope.spawn(|| self.txindexvout_to_txoutindex.export(height));
});
}
}
src/crates/bindex/src/structs/timestamp.rs
-11
View File
@@ -1,11 +0,0 @@
use derive_deref::Deref;
#[derive(Debug, Deref, Clone)]
pub struct Timestamp(jiff::Timestamp);
impl TryFrom<u32> for Timestamp {
type Error = jiff::Error;
fn try_from(value: u32) -> Result<Self, Self::Error> {
Ok(Self(jiff::Timestamp::from_second(value as i64)?))
}
}
src/crates/bindex/src/structs/txindex.rs
-59
View File
@@ -1,59 +0,0 @@
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txindex(u32);
direct_repr!(Txindex);
impl Txindex {
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txindex> for Txindex {
type Output = Self;
fn add(self, rhs: Txindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign<Txindex> for Txindex {
fn add_assign(&mut self, rhs: Txindex) {
self.0 += rhs.0
}
}
impl From<u32> for Txindex {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<u64> for Txindex {
fn from(value: u64) -> Self {
Self(value as u32)
}
}
impl From<Txindex> for u64 {
fn from(value: Txindex) -> Self {
value.0 as u64
}
}
impl From<usize> for Txindex {
fn from(value: usize) -> Self {
Self(value as u32)
}
}
impl From<Txindex> for usize {
fn from(value: Txindex) -> Self {
value.0 as usize
}
}
src/crates/bindex/src/structs/txindexvout.rs
-28
View File
@@ -1,28 +0,0 @@
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Txindex;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
pub struct Txindexvout {
pub txindex: Txindex,
pub vout: u32,
}
direct_repr!(Txindexvout);
impl From<Txindex> for Txindexvout {
fn from(value: Txindex) -> Self {
Self {
txindex: value,
vout: 0,
}
}
}
impl From<(Txindex, u32)> for Txindexvout {
fn from(value: (Txindex, u32)) -> Self {
Self {
txindex: value.0,
vout: value.1,
}
}
}
src/crates/bindex/src/structs/txoutindex.rs
-53
View File
@@ -1,53 +0,0 @@
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txoutindex(u64);
direct_repr!(Txoutindex);
impl Txoutindex {
pub fn incremented(self) -> Self {
Self(*self + 1)
}
pub fn decremented(self) -> Self {
Self(*self - 1)
}
}
impl Add<Txoutindex> for Txoutindex {
type Output = Self;
fn add(self, rhs: Txoutindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl AddAssign<Txoutindex> for Txoutindex {
fn add_assign(&mut self, rhs: Txoutindex) {
self.0 += rhs.0
}
}
impl From<u64> for Txoutindex {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<Txoutindex> for u64 {
fn from(value: Txoutindex) -> Self {
value.0
}
}
impl From<usize> for Txoutindex {
fn from(value: usize) -> Self {
Self(value as u64)
}
}
impl From<Txoutindex> for usize {
fn from(value: Txoutindex) -> Self {
value.0 as usize
}
}
src/crates/bindex/src/structs/vec.rs
-116
View File
@@ -1,116 +0,0 @@
use std::{
fmt::Debug,
fs, io,
ops::{Deref, DerefMut},
path::{Path, PathBuf},
};
use super::{Height, Version};
pub struct StorableVec<I, T> {
pathbuf: PathBuf,
version: Version,
vec: storable_vec::StorableVec<I, T>,
}
impl<I, T> StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug,
{
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
fs::create_dir_all(path)?;
let pathbuf = path.to_owned();
let path_vec = Self::_path_vec(path);
let path_version = Self::_path_version(path);
let is_same_version =
Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
let _ = fs::remove_file(&path_vec);
let _ = fs::remove_file(&path_version);
let _ = fs::remove_file(Self::_path_height(path));
}
Ok(Self {
pathbuf,
version,
vec: storable_vec::StorableVec::import(&path_vec)?,
})
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
height.write(&self.path_height())?;
self.version.write(&self.path_version())?;
self.vec.flush()
}
// fn path_vec(&self) -> PathBuf {
// Self::_path_vec(&self.path)
// }
fn _path_vec(path: &Path) -> PathBuf {
path.join("vec")
}
fn path_version(&self) -> PathBuf {
Self::_path_version(&self.pathbuf)
}
fn _path_version(path: &Path) -> PathBuf {
path.join("version")
}
pub fn height(&self) -> color_eyre::Result<Height> {
Height::try_from(self.path_height().as_path())
}
fn path_height(&self) -> PathBuf {
Self::_path_height(&self.pathbuf)
}
fn _path_height(path: &Path) -> PathBuf {
path.join("height")
}
fn reset_cache(&mut self) {
self.vec.reset_cache();
}
// pub fn needs(&self, height: Height) -> bool {
// self.height() // store height in struct
// }
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = storable_vec::StorableVec<I, T>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
}
pub trait AnyBindexVec {
fn height(&self) -> color_eyre::Result<Height>;
fn reset_cache(&mut self);
fn flush(&mut self, height: Height) -> io::Result<()>;
}
impl<I, T> AnyBindexVec for StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug,
{
fn height(&self) -> color_eyre::Result<Height> {
self.height()
}
fn reset_cache(&mut self) {
self.reset_cache();
}
fn flush(&mut self, height: Height) -> io::Result<()> {
self.flush(height)
}
}
src/crates/bindex/src/structs/vecs.rs
-410
View File
@@ -1,410 +0,0 @@
use std::{fs, io, path::Path};
use biter::bitcoin::{transaction, BlockHash, Txid};
use color_eyre::eyre::eyre;
use rayon::prelude::*;
use storable_vec::AnyStorableVec;
use super::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, AnyBindexVec, Date, Exit, Height,
P2PK33AddressBytes, P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes,
P2WSHAddressBytes, StorableVec, Timestamp, Txindex, Txoutindex, Version,
};
pub struct Vecs {
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
pub height_to_blockhash: StorableVec<Height, BlockHash>,
pub height_to_date: StorableVec<Height, Date>,
pub height_to_totalfees: StorableVec<Height, Amount>,
pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
pub height_to_first_txindex: StorableVec<Height, Txindex>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
pub height_to_inputcount: StorableVec<Txindex, u32>,
pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
pub height_to_outputcount: StorableVec<Txindex, u32>,
pub height_to_timestamp: StorableVec<Height, Timestamp>,
pub height_to_txcount: StorableVec<Txindex, u32>,
// pub height_to_size: StorableVec<Txindex, u32>,
// pub height_to_weight: StorableVec<Txindex, u32>,
// pub height_to_subsidy: StorableVec<Txindex, u32>,
// pub height_to_minfeerate: StorableVec<Txindex, u32>,
// pub height_to_maxfeerate: StorableVec<Txindex, u32>,
// pub height_to_medianfeerate: StorableVec<Txindex, u32>,
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
pub txindex_to_fee: StorableVec<Txindex, Amount>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_inputcount: StorableVec<Txindex, u32>,
pub txindex_to_outputcount: StorableVec<Txindex, u32>,
pub txindex_to_txid: StorableVec<Txindex, Txid>,
pub txindex_to_txversion: StorableVec<Txindex, transaction::Version>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
}
// const UNSAFE_BLOCKS: usize = 100;
impl Vecs {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
addressindex_to_addresstype: StorableVec::import(
&path.join("addressindex_to_addresstype"),
Version::from(1),
)?,
addressindex_to_addresstypeindex: StorableVec::import(
&path.join("addressindex_to_addresstypeindex"),
Version::from(1),
)?,
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?,
height_to_date: StorableVec::import(&path.join("height_to_date"), Version::from(1))?,
height_to_first_addressindex: StorableVec::import(
&path.join("height_to_first_addressindex"),
Version::from(1),
)?,
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?,
height_to_first_txoutindex: StorableVec::import(
&path.join("height_to_first_txoutindex"),
Version::from(1),
)?,
height_to_inputcount: StorableVec::import(&path.join("height_to_inputcount"), Version::from(1))?,
height_to_last_addressindex: StorableVec::import(
&path.join("height_to_last_addressindex"),
Version::from(1),
)?,
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
height_to_outputcount: StorableVec::import(&path.join("height_to_outputcount"), Version::from(1))?,
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?,
height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?,
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
&path.join("p2pk33index_to_p2pk33addressbytes"),
Version::from(1),
)?,
p2pk65index_to_p2pk65addressbytes: StorableVec::import(
&path.join("p2pk65index_to_p2pk65addressbytes"),
Version::from(1),
)?,
p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(
&path.join("p2pkhindex_to_p2pkhaddressbytes"),
Version::from(1),
)?,
p2shindex_to_p2shaddressbytes: StorableVec::import(
&path.join("p2shindex_to_p2shaddressbytes"),
Version::from(1),
)?,
p2trindex_to_p2traddressbytes: StorableVec::import(
&path.join("p2trindex_to_p2traddressbytes"),
Version::from(1),
)?,
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(
&path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
Version::from(1),
)?,
p2wshindex_to_p2wshaddressbytes: StorableVec::import(
&path.join("p2wshindex_to_p2wshaddressbytes"),
Version::from(1),
)?,
txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?,
txindex_to_outputcount: StorableVec::import(&path.join("txindex_to_outputcount"), Version::from(1))?,
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?,
txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?,
txoutindex_to_addressindex: StorableVec::import(
&path.join("txoutindex_to_addressindex"),
Version::from(1),
)?,
txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"), Version::from(1))?,
})
}
pub fn addresstype_to_addressbytes(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyStorableVec> {
match addresstype {
Addresstype::P2PK65 => Ok(&*self.p2pk65index_to_p2pk65addressbytes),
Addresstype::P2PK33 => Ok(&*self.p2pk33index_to_p2pk33addressbytes),
Addresstype::P2PKH => Ok(&*self.p2pkhindex_to_p2pkhaddressbytes),
Addresstype::P2SH => Ok(&*self.p2shindex_to_p2shaddressbytes),
Addresstype::P2WPKH => Ok(&*self.p2wpkhindex_to_p2wpkhaddressbytes),
Addresstype::P2WSH => Ok(&*self.p2wshindex_to_p2wshaddressbytes),
Addresstype::P2TR => Ok(&*self.p2trindex_to_p2traddressbytes),
_ => Err(eyre!("wrong address type")),
}
}
pub fn push_addressbytes_if_needed(
&mut self,
index: Addresstypeindex,
addressbytes: Addressbytes,
) -> storable_vec::Result<()> {
match addressbytes {
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
}
}
pub fn get_addressbytes(
&self,
addresstype: Addresstype,
addresstypeindex: Addresstypeindex,
) -> storable_vec::Result<Option<Addressbytes>> {
Ok(match addresstype {
Addresstype::P2PK65 => self
.p2pk65index_to_p2pk65addressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
Addresstype::P2PK33 => self
.p2pk33index_to_p2pk33addressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
Addresstype::P2PKH => self
.p2pkhindex_to_p2pkhaddressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
Addresstype::P2SH => self
.p2shindex_to_p2shaddressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
Addresstype::P2WPKH => self
.p2wpkhindex_to_p2wpkhaddressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
Addresstype::P2WSH => self
.p2wshindex_to_p2wshaddressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
Addresstype::P2TR => self
.p2trindex_to_p2traddressbytes
.get(addresstypeindex)?
.cloned()
.map(Addressbytes::from),
_ => unreachable!(),
})
}
#[allow(unused)]
pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
panic!();
// let mut txindex = None;
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (height_slice, slice_blockhash) = slice?;
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
// wtx.remove(self.height_to_blockhash.data(), height_slice);
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
// if txindex.is_none() {
// txindex.replace(
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
// .context("for height to have first txindex")?,
// );
// }
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
// Ok(())
// })?;
// let txindex = txindex.context("txindex to not be none by now")?;
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (slice_txindex, slice_txid) = slice?;
// let txindex = Txindex::from(slice_txindex);
// let txid = Txid::from_slice(&slice_txid)?;
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
// Ok(())
// })?;
// let txoutindex = Txoutindex::from(txindex);
// let mut addressindexes = BTreeSet::new();
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
// .try_for_each(|slice| -> color_eyre::Result<()> {
// let (txoutindex_slice, _) = slice?;
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
// if let Some(addressindex_slice) =
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
// {
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
// let addressindex = Addressindex::from(addressindex_slice);
// addressindexes.insert(addressindex);
// let txoutindex = Txoutindex::from(txoutindex_slice);
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
// wtx.remove(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(addresstxoutindex),
// );
// }
// Ok(())
// })?;
// addressindexes
// .into_iter()
// .filter(|addressindex| {
// let is_empty = wtx
// .prefix(
// self.addressindex_to_txoutindexes.data(),
// Slice::from(*addressindex),
// )
// .next()
// .is_none();
// is_empty
// })
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
// let addressindex_slice = Slice::from(addressindex);
// let addressbytes = Addressbytes::from(
// wtx.get(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// )?
// .context("addressindex_to_address to have value")?,
// );
// wtx.remove(
// self.addressbytes_prefix_to_addressindex.data(),
// addressbytes.prefix(),
// );
// wtx.remove(
// self.addressindex_to_addressbytes.data(),
// &addressindex_slice,
// );
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
// Ok(())
// })?;
//
// todo!("clear addresstxoutindexes_out")
// todo!("clear addresstxoutindexes_in")
// todo!("clear zero_txoutindexes")
// Ok(())
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_slice()
.into_par_iter()
.try_for_each(|vec| vec.flush(height))
}
pub fn reset_cache(&mut self) {
self.as_mut_slice().par_iter_mut().for_each(|vec| {
vec.reset_cache();
})
}
pub fn min_height(&self) -> color_eyre::Result<Option<Height>> {
Ok(self
.as_slice()
.into_iter()
.map(|vec| vec.height().unwrap_or_default())
.min())
}
pub fn as_slice(&self) -> [&dyn AnyBindexVec; 30] {
[
&self.addressindex_to_addresstype as &dyn AnyBindexVec,
&self.addressindex_to_addresstypeindex,
&self.height_to_blockhash,
&self.height_to_date,
&self.height_to_totalfees,
&self.height_to_first_addressindex,
&self.height_to_first_txindex,
&self.height_to_first_txoutindex,
&self.height_to_inputcount,
&self.height_to_last_addressindex,
&self.height_to_last_txindex,
&self.height_to_last_txoutindex,
&self.height_to_outputcount,
&self.height_to_timestamp,
&self.height_to_txcount,
&self.p2pk33index_to_p2pk33addressbytes,
&self.p2pk65index_to_p2pk65addressbytes,
&self.p2pkhindex_to_p2pkhaddressbytes,
&self.p2shindex_to_p2shaddressbytes,
&self.p2trindex_to_p2traddressbytes,
&self.p2wpkhindex_to_p2wpkhaddressbytes,
&self.p2wshindex_to_p2wshaddressbytes,
&self.txindex_to_fee,
&self.txindex_to_height,
&self.txindex_to_inputcount,
&self.txindex_to_outputcount,
&self.txindex_to_txid,
&self.txindex_to_txversion,
&self.txoutindex_to_addressindex,
&self.txoutindex_to_amount,
]
}
pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyBindexVec + Send + Sync); 30] {
[
&mut self.addressindex_to_addresstype as &mut (dyn AnyBindexVec + Send + Sync),
&mut self.addressindex_to_addresstypeindex,
&mut self.height_to_blockhash,
&mut self.height_to_date,
&mut self.height_to_totalfees, // <-
&mut self.height_to_first_addressindex,
&mut self.height_to_first_txindex,
&mut self.height_to_first_txoutindex,
&mut self.height_to_inputcount, // <-
&mut self.height_to_last_addressindex,
&mut self.height_to_last_txindex,
&mut self.height_to_last_txoutindex,
&mut self.height_to_outputcount, // <-
&mut self.height_to_timestamp,
&mut self.height_to_txcount, // <-
&mut self.p2pk33index_to_p2pk33addressbytes,
&mut self.p2pk65index_to_p2pk65addressbytes,
&mut self.p2pkhindex_to_p2pkhaddressbytes,
&mut self.p2shindex_to_p2shaddressbytes,
&mut self.p2trindex_to_p2traddressbytes,
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
&mut self.p2wshindex_to_p2wshaddressbytes,
&mut self.txindex_to_fee, // <-
&mut self.txindex_to_height,
&mut self.txindex_to_inputcount, // <-
&mut self.txindex_to_outputcount, // <-
&mut self.txindex_to_txid,
&mut self.txindex_to_txversion,
&mut self.txoutindex_to_addressindex,
&mut self.txoutindex_to_amount,
]
}
}
src/crates/bindex/src/structs/version.rs
-25
View File
@@ -1,25 +0,0 @@
use std::{fs, io, path::Path};
use storable_vec::UnsafeSizedSerDe;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.unsafe_as_slice())
}
}
impl From<u32> for Version {
fn from(value: u32) -> Self {
Self(value)
}
}
impl TryFrom<&Path> for Version {
type Error = color_eyre::Report;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
}
}
src/crates/biter/CHANGELOG.md
-12
View File
@@ -1,12 +0,0 @@
# v0.2.1
- Clean `.json` if necessary
- Only save `.json` if needed
- Updated benchmarks
- Updated packages
# v0.2.0
- Removed the need for an output directory path
- Changed the location of the saved json file from the previously needed output directory path to the Bitcoin data directory
- Added a save of the json file every 144 * 30 blocks instead of only at the end
src/crates/biter/Cargo.lock
Generated
-475
View File
@@ -1,475 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "arrayvec"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "base58ck"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
dependencies = [
"bitcoin-internals",
"bitcoin_hashes",
]
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "bech32"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "bitcoin"
version = "0.32.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026"
dependencies = [
"base58ck",
"bech32",
"bitcoin-internals",
"bitcoin-io",
"bitcoin-units",
"bitcoin_hashes",
"hex-conservative",
"hex_lit",
"secp256k1",
"serde",
]
[[package]]
name = "bitcoin-internals"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2"
dependencies = [
"serde",
]
[[package]]
name = "bitcoin-io"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56"
[[package]]
name = "bitcoin-units"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2"
dependencies = [
"bitcoin-internals",
"serde",
]
[[package]]
name = "bitcoin_hashes"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
dependencies = [
"bitcoin-io",
"hex-conservative",
"serde",
]
[[package]]
name = "bitcoincore-rpc"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee"
dependencies = [
"bitcoincore-rpc-json",
"jsonrpc",
"log",
"serde",
"serde_json",
]
[[package]]
name = "bitcoincore-rpc-json"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a"
dependencies = [
"bitcoin",
"serde",
"serde_json",
]
[[package]]
name = "biter"
version = "0.2.2"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"crossbeam",
"derived-deref",
"rayon",
"serde",
"serde_json",
]
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "cc"
version = "1.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f"
dependencies = [
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "derived-deref"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "either"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "hex-conservative"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd"
dependencies = [
"arrayvec",
]
[[package]]
name = "hex_lit"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
[[package]]
name = "itoa"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "jsonrpc"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
dependencies = [
"base64",
"minreq",
"serde",
"serde_json",
]
[[package]]
name = "libc"
version = "0.2.161"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
[[package]]
name = "log"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "memchr"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "minreq"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0"
dependencies = [
"log",
"serde",
"serde_json",
]
[[package]]
name = "ppv-lite86"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "ryu"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "secp256k1"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113"
dependencies = [
"bitcoin_hashes",
"rand",
"secp256k1-sys",
"serde",
]
[[package]]
name = "secp256k1-sys"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9"
dependencies = [
"cc",
]
[[package]]
name = "serde"
version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.135"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "syn"
version = "2.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
src/crates/biter/Cargo.toml
-19
View File
@@ -1,19 +0,0 @@
[package]
name = "biter"
description = "A very fast Bitcoin block iterator"
version = "0.2.2"
license = "MIT"
repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter"
keywords = ["bitcoin", "block", "iterator"]
categories = ["cryptography::cryptocurrencies", "encoding"]
edition = "2021"
[dependencies]
bitcoin = { version = "0.32.5", features = ["serde"] }
rayon = "1.10.0"
crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
serde = { version = "1.0.217", features = ["derive"] }
serde_json = "1.0.135"
derived-deref = "2.1.0"
bitcoincore-rpc = "0.19.0"
# tokio = { version = "1.39.2", features = ["rt-multi-thread"] }
src/crates/biter/LICENSE.md
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2024 biter
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
src/crates/biter/README.md
-65
View File
@@ -1,65 +0,0 @@
# biter
Biter (Bitcoin Block Iterator) is a very fast and simple Rust library which reads raw block files (*blkXXXXX.dat*) from Bitcoin Core Node and creates an iterator over all the requested blocks in sequential order (0, 1, 2, ...).
The element returned by the iterator is a tuple which includes the:
- Height: `usize`
- Block: `Block` (from `bitcoin-rust`)
- Block's Hash: `BlockHash` (also from `bitcoin-rust`)
## Example
```rust
use std::path::Path;
use bitcoincore_rpc::{Auth, Client};
fn main() {
let i = std::time::Instant::now();
// Path to the Bitcoin data directory
let data_dir = "../../bitcoin";
// Inclusive starting height of the blocks received, `None` for 0
let start = Some(850_000);
// Inclusive ending height of the blocks received, `None` for the last one
let end = None;
// RPC client to filter out forks
let url = "http://localhost:8332";
let cookie = Path::new(data_dir).join(".cookie");
let auth = Auth::CookieFile(cookie);
let rpc = Client::new(url, auth).unwrap();
if cookie.is_file() {
Ok()
// Create channel receiver then iterate over the blocks
biter::new(data_dir, start, end, rpc)
.iter()
.for_each(|(height, _block, hash)| {
println!("{height}: {hash}");
});
dbg!(i.elapsed());
}
```
## Requirements
Even though it reads *blkXXXXX.dat* files, it **needs** `bitcoind` to run with the RPC server to filter out block forks.
Peak memory should be around 500MB.
## Comparaison
| | [biter](https://crates.io/crates/biter) | [bitcoin-explorer (deprecated)](https://crates.io/crates/bitcoin-explorer) | [blocks_iterator](https://crates.io/crates/blocks_iterator) |
| --- | --- | --- | --- |
| Runs **with** `bitcoind` | Yes ✅ | No ❌ | Yes ✅ |
| Runs **without** `bitcoind` | No ❌ | Yes ✅ | Yes ✅ |
| `0..=855_000` | 4mn 10s | 4mn 45s | > 2h |
| `800_000..=855_000` | 0mn 52s (4mn 10s if first run) | 0mn 55s | > 2h |
*Benchmarked on a Macbook Pro M3 Pro*
src/crates/biter/src/blk_index_to_blk_path.rs
-46
View File
@@ -1,46 +0,0 @@
use std::{
collections::BTreeMap,
fs,
path::{Path, PathBuf},
};
use derived_deref::{Deref, DerefMut};
const BLK: &str = "blk";
const DAT: &str = ".dat";
#[derive(Debug, Deref, DerefMut)]
pub struct BlkIndexToBlkPath(BTreeMap<usize, PathBuf>);
impl BlkIndexToBlkPath {
pub fn scan(data_dir: &Path) -> Self {
let blocks_dir = data_dir.join("blocks");
Self(
fs::read_dir(blocks_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| {
let is_file = path.is_file();
if is_file {
let file_name = path.file_name().unwrap().to_str().unwrap();
file_name.starts_with(BLK) && file_name.ends_with(DAT)
} else {
false
}
})
.map(|path| {
let file_name = path.file_name().unwrap().to_str().unwrap();
let blk_index = file_name[BLK.len()..(file_name.len() - DAT.len())]
.parse::<usize>()
.unwrap();
(blk_index, path)
})
.collect::<BTreeMap<_, _>>(),
)
}
}
src/crates/biter/src/blk_index_to_blk_recap.rs
-127
View File
@@ -1,127 +0,0 @@
use std::{
cmp::Ordering,
collections::{BTreeMap, BTreeSet},
fs::{self, File},
io::{BufReader, BufWriter},
path::{Path, PathBuf},
};
use derived_deref::{Deref, DerefMut};
use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock};
const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30;
#[derive(Deref, DerefMut, Debug)]
pub struct BlkIndexToBlkRecap {
path: PathBuf,
#[target]
tree: BTreeMap<usize, BlkRecap>,
last_safe_height: Option<usize>,
}
impl BlkIndexToBlkRecap {
pub fn import(blocks_dir: &BlkIndexToBlkPath, data_dir: &Path) -> Self {
let path = data_dir.join("blk_index_to_blk_recap.json");
let tree = {
fs::create_dir_all(data_dir).unwrap();
if let Ok(file) = File::open(&path) {
let reader = BufReader::new(file);
serde_json::from_reader(reader).unwrap_or_default()
} else {
BTreeMap::default()
}
};
let mut this = Self {
path,
tree,
last_safe_height: None,
};
this.clean_outdated(blocks_dir);
this
}
pub fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) {
let mut unprocessed_keys = self.keys().copied().collect::<BTreeSet<_>>();
blocks_dir.iter().for_each(|(blk_index, blk_path)| {
unprocessed_keys.remove(blk_index);
if let Some(blk_recap) = self.get(blk_index) {
if blk_recap.has_different_modified_time(blk_path) {
self.remove(blk_index);
}
}
});
unprocessed_keys.into_iter().for_each(|blk_index| {
self.remove(&blk_index);
});
self.last_safe_height = self.iter().map(|(_, recap)| recap.height()).max();
}
pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
if let Some(start) = start {
let (last_key, last_value) = self.last_key_value()?;
if last_value.height() < start {
return Some((*last_key, *last_value));
} else if let Some((blk_index, _)) = self
.iter()
.find(|(_, blk_recap)| blk_recap.is_younger_than(start))
{
if *blk_index != 0 {
let blk_index = *blk_index - 1;
return Some((blk_index, *self.get(&blk_index).unwrap()));
}
}
}
None
}
pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) {
let blk_index = blk_metadata_and_block.blk_metadata.index;
if let Some(last_entry) = self.last_entry() {
match last_entry.key().cmp(&blk_index) {
Ordering::Greater => {
last_entry.remove_entry();
}
Ordering::Less => {
self.insert(blk_index, BlkRecap::from(height, blk_metadata_and_block));
}
Ordering::Equal => {}
};
} else {
if blk_index != 0 || height != 0 {
// dbg!(blk_index, height);
unreachable!();
}
self.insert(blk_index, BlkRecap::first(blk_metadata_and_block));
}
if self
.last_safe_height
.map_or(true, |safe_height| height >= safe_height)
&& (height % TARGET_BLOCKS_PER_MONTH) == 0
{
self.export();
}
}
pub fn export(&self) {
let file = File::create(&self.path).unwrap_or_else(|_| {
dbg!(&self.path);
panic!("No such file or directory")
});
serde_json::to_writer_pretty(&mut BufWriter::new(file), &self.tree).unwrap();
}
}
src/crates/biter/src/blk_metadata.rs
-18
View File
@@ -1,18 +0,0 @@
use std::path::PathBuf;
use crate::path_to_modified_time;
#[derive(Clone, Copy)]
pub struct BlkMetadata {
pub index: usize,
pub modified_time: u64,
}
impl BlkMetadata {
pub fn new(index: usize, path: &PathBuf) -> Self {
Self {
index,
modified_time: path_to_modified_time(path),
}
}
}
src/crates/biter/src/blk_metadata_and_block.rs
-17
View File
@@ -1,17 +0,0 @@
use bitcoin::Block;
use crate::BlkMetadata;
pub struct BlkMetadataAndBlock {
pub blk_metadata: BlkMetadata,
pub block: Block,
}
impl BlkMetadataAndBlock {
pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self {
Self {
blk_metadata,
block,
}
}
}
src/crates/biter/src/blk_recap.rs
-47
View File
@@ -1,47 +0,0 @@
use std::path::PathBuf;
use bitcoin::{hashes::Hash, BlockHash};
use serde::{Deserialize, Serialize};
use crate::{path_to_modified_time, BlkMetadataAndBlock};
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct BlkRecap {
min_continuous_height: usize,
min_continuous_prev_hash: BlockHash,
modified_time: u64,
}
impl BlkRecap {
pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
Self {
min_continuous_height: 0,
min_continuous_prev_hash: BlockHash::all_zeros(),
modified_time: blk_metadata_and_block.blk_metadata.modified_time,
}
}
pub fn from(height: usize, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
Self {
min_continuous_height: height,
min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash,
modified_time: blk_metadata_and_block.blk_metadata.modified_time,
}
}
pub fn has_different_modified_time(&self, blk_path: &PathBuf) -> bool {
self.modified_time != path_to_modified_time(blk_path)
}
pub fn is_younger_than(&self, height: usize) -> bool {
self.min_continuous_height > height
}
pub fn height(&self) -> usize {
self.min_continuous_height
}
pub fn prev_hash(&self) -> &BlockHash {
&self.min_continuous_prev_hash
}
}
src/crates/biter/src/lib.rs
-387
View File
@@ -1,387 +0,0 @@
use std::{
collections::{BTreeMap, BTreeSet, VecDeque},
fs::{self},
ops::ControlFlow,
path::Path,
thread,
};
use bitcoin::{
consensus::{Decodable, ReadExt},
hashes::Hash,
io::{Cursor, Read},
Block, BlockHash,
};
use bitcoincore_rpc::RpcApi;
use blk_index_to_blk_path::*;
use crossbeam::channel::{bounded, Receiver};
use rayon::prelude::*;
pub use bitcoin;
pub use bitcoincore_rpc;
mod blk_index_to_blk_path;
mod blk_index_to_blk_recap;
mod blk_metadata;
mod blk_metadata_and_block;
mod blk_recap;
mod utils;
use blk_index_to_blk_recap::*;
use blk_metadata::*;
use blk_metadata_and_block::*;
use utils::*;
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 210;
enum BlockState {
Raw(Vec<u8>),
Decoded(Block),
}
///
/// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order.
///
/// # Arguments
///
/// * `data_dir` - Path to the Bitcoin data directory
/// * `start` - Inclusive starting height of the blocks received, `None` for 0
/// * `end` - Inclusive ending height of the blocks received, `None` for the last one
/// * `rpc` - RPC client to filter out forks
///
/// # Example
///
/// ```rust
/// use std::path::Path;
///
/// use bitcoincore_rpc::{Auth, Client};
///
/// fn main() {
/// let i = std::time::Instant::now();
///
/// let data_dir = Path::new("../../bitcoin");
/// let url = "http://localhost:8332";
/// let cookie = Path::new(data_dir).join(".cookie");
/// let auth = Auth::CookieFile(cookie);
/// let rpc = Client::new(url, auth).unwrap();
///
/// let start = Some(850_000);
/// let end = None;
///
/// biter::new(data_dir, start, end, rpc)
/// .iter()
/// .for_each(|(height, _block, hash)| {
/// println!("{height}: {hash}");
/// });
///
/// dbg!(i.elapsed());
/// }
/// ```
///
pub fn new(
data_dir: &Path,
start: Option<usize>,
end: Option<usize>,
rpc: bitcoincore_rpc::Client,
) -> Receiver<(usize, Block, BlockHash)> {
let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
let (send_block, recv_block) = bounded(BOUND_CAP);
let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
let blk_index_to_blk_path = BlkIndexToBlkPath::scan(data_dir);
let mut blk_index_to_blk_recap = BlkIndexToBlkRecap::import(&blk_index_to_blk_path, data_dir);
let start_recap = blk_index_to_blk_recap.get_start_recap(start);
let starting_blk_index = start_recap.as_ref().map_or(0, |(index, _)| *index);
thread::spawn(move || {
blk_index_to_blk_path
.iter()
.filter(|(blk_index, _)| blk_index >= &&starting_blk_index)
.try_for_each(move |(blk_index, blk_path)| {
let blk_metadata = BlkMetadata::new(*blk_index, blk_path);
let blk_bytes = fs::read(blk_path).unwrap();
let blk_bytes_len = blk_bytes.len() as u64;
let mut cursor = Cursor::new(blk_bytes.as_slice());
let mut current_4bytes = [0; 4];
'parent: loop {
if cursor.position() == blk_bytes_len {
break;
}
// Read until we find a valid suite of MAGIC_BYTES
loop {
current_4bytes.rotate_left(1);
if let Ok(byte) = cursor.read_u8() {
current_4bytes[3] = byte;
} else {
break 'parent;
}
if current_4bytes == MAGIC_BYTES {
break;
}
}
let block_size = cursor.read_u32().unwrap();
let mut raw_block = vec![0u8; block_size as usize];
cursor.read_exact(&mut raw_block).unwrap();
if send_block_reader
.send((blk_metadata, BlockState::Raw(raw_block)))
.is_err()
{
return ControlFlow::Break(());
}
}
ControlFlow::Continue(())
})
});
// thread::spawn(move || {
// recv_block_reader.iter().par_bridge().try_for_each(
// move |(blk_metadata, mut block_state)| {
// let raw_block = match block_state {
// BlockState::Raw(vec) => vec,
// _ => unreachable!(),
// };
// let mut cursor = Cursor::new(raw_block);
// block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
// if send_block
// .send(BlkMetadataAndBlock::new(
// blk_metadata,
// match block_state {
// BlockState::Decoded(block) => block,
// _ => unreachable!(),
// },
// ))
// .is_err()
// {
// return ControlFlow::Break(());
// }
// ControlFlow::Continue(())
// },
// );
// });
// Can't use the previous code because .send() blocks all the threads if full
// And other .par_iter() are also stuck because of that
thread::spawn(move || {
let mut bulk = vec![];
let drain_and_send = |bulk: &mut Vec<_>| {
// Using a vec and sending after to not end up with stuck threads in par iter
bulk.par_iter_mut().for_each(|(_, block_state)| {
let raw_block = match block_state {
BlockState::Raw(vec) => vec,
_ => unreachable!(),
};
let mut cursor = Cursor::new(raw_block);
*block_state = BlockState::Decoded(Block::consensus_decode(&mut cursor).unwrap());
});
bulk.drain(..).try_for_each(|(blk_metadata, block_state)| {
let block = match block_state {
BlockState::Decoded(block) => block,
_ => unreachable!(),
};
if send_block
.send(BlkMetadataAndBlock::new(blk_metadata, block))
.is_err()
{
return ControlFlow::Break(());
}
ControlFlow::Continue(())
})
};
recv_block_reader.iter().try_for_each(|tuple| {
bulk.push(tuple);
if bulk.len() < BOUND_CAP / 2 {
return ControlFlow::Continue(());
}
drain_and_send(&mut bulk)
});
drain_and_send(&mut bulk)
});
// Tokio version: 1022s
// Slighlty slower than rayon version
// thread::spawn(move || {
// let rt = tokio::runtime::Runtime::new().unwrap();
// let _guard = rt.enter();
// let mut tasks = VecDeque::with_capacity(BOUND);
// recv_block_reader
// .iter()
// .try_for_each(move |(blk_metadata, block_state)| {
// let raw_block = match block_state {
// BlockState::Raw(vec) => vec,
// _ => unreachable!(),
// };
// tasks.push_back(tokio::task::spawn(async move {
// let block = Block::consensus_decode(&mut Cursor::new(raw_block)).unwrap();
// (blk_metadata, block)
// }));
// while tasks.len() > BOUND {
// let (blk_metadata, block) = rt.block_on(tasks.pop_front().unwrap()).unwrap();
// if send_block
// .send(BlkMetadataAndBlock::new(blk_metadata, block))
// .is_err()
// {
// return ControlFlow::Break(());
// }
// }
// ControlFlow::Continue(())
// });
//
// todo!("Send the rest")
// });
thread::spawn(move || {
let mut height = start_recap.map_or(0, |(_, recap)| recap.height());
let mut future_blocks = BTreeMap::default();
let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default();
let mut recent_hashes: BTreeSet<BlockHash> = BTreeSet::default();
let mut prev_hash =
start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash());
let mut prepare_and_send = |(hash, tuple): (BlockHash, BlkMetadataAndBlock)| {
blk_index_to_blk_recap.update(&tuple, height);
if start.map_or(true, |start| start <= height) {
send_height_block_hash
.send((height, tuple.block, hash))
.unwrap();
}
if end.map_or(false, |end| height == end) {
return ControlFlow::Break(());
}
height += 1;
ControlFlow::Continue(())
};
let mut update_tip = |prev_hash: &mut BlockHash,
recent_hashes: &mut BTreeSet<BlockHash>,
recent_chain: &mut VecDeque<(BlockHash, BlkMetadataAndBlock)>,
future_blocks: &mut BTreeMap<BlockHash, BlkMetadataAndBlock>,
tuple: BlkMetadataAndBlock| {
let mut tuple = Some(tuple);
while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) {
let hash = tuple.block.block_hash();
*prev_hash = hash;
recent_hashes.insert(hash);
recent_chain.push_back((hash, tuple));
}
while recent_chain.len() > NUMBER_OF_UNSAFE_BLOCKS {
let (hash, tuple) = recent_chain.pop_front().unwrap();
recent_hashes.remove(&hash);
if prepare_and_send((hash, tuple)).is_break() {
return ControlFlow::Break(());
}
}
ControlFlow::Continue(())
};
let flow = recv_block.iter().try_for_each(|tuple| {
// block isn't next after current tip
if prev_hash != tuple.block.header.prev_blockhash {
let is_block_active =
|hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0;
// block prev has already been processed
if recent_hashes.contains(&tuple.block.header.prev_blockhash) {
let hash = tuple.block.block_hash();
if is_block_active(&hash) {
let prev_index = recent_chain
.iter()
.position(|(hash, ..)| hash == &tuple.block.header.prev_blockhash)
.unwrap();
let bad_index_start = prev_index + 1;
recent_chain.drain(bad_index_start..).for_each(|(hash, _)| {
recent_hashes.remove(&hash);
});
return update_tip(
&mut prev_hash,
&mut recent_hashes,
&mut recent_chain,
&mut future_blocks,
tuple,
);
}
// Check if there was already a future block with the same prev hash
} else if let Some(prev_tuple) =
future_blocks.insert(tuple.block.header.prev_blockhash, tuple)
{
// If the previous was the active one
if is_block_active(&prev_tuple.block.block_hash()) {
// Rollback the insert
future_blocks.insert(prev_tuple.block.header.prev_blockhash, prev_tuple);
}
}
} else {
return update_tip(
&mut prev_hash,
&mut recent_hashes,
&mut recent_chain,
&mut future_blocks,
tuple,
);
}
ControlFlow::Continue(())
});
if flow.is_continue() {
// Send the last (up to 100) blocks
recent_chain.into_iter().try_for_each(prepare_and_send);
}
blk_index_to_blk_recap.export();
});
recv_height_block_hash
}
src/crates/biter/src/main.rs
-24
View File
@@ -1,24 +0,0 @@
use std::path::Path;
use bitcoincore_rpc::{Auth, Client};
fn main() {
let i = std::time::Instant::now();
let data_dir = Path::new("../../../bitcoin");
let url = "http://localhost:8332";
let cookie = Path::new(data_dir).join(".cookie");
let auth = Auth::CookieFile(cookie);
let rpc = Client::new(url, auth).unwrap();
let start = Some(810078);
let end = None;
biter::new(data_dir, start, end, rpc)
.iter()
.for_each(|(height, _block, hash)| {
println!("{height}: {hash}");
});
dbg!(i.elapsed());
}
src/crates/biter/src/utils.rs
-11
View File
@@ -1,11 +0,0 @@
use std::{fs, path::PathBuf, time::UNIX_EPOCH};
pub fn path_to_modified_time(path: &PathBuf) -> u64 {
fs::metadata(path)
.unwrap()
.modified()
.unwrap()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs()
}
src/crates/iterable/Cargo.lock
Generated
-60
View File
@@ -1,60 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "proc-macro2"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
[[package]]
name = "struct_iterable"
version = "0.1.2"
dependencies = [
"struct_iterable_derive",
"struct_iterable_internal",
]
[[package]]
name = "struct_iterable_derive"
version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
"struct_iterable_internal",
"syn",
]
[[package]]
name = "struct_iterable_internal"
version = "0.1.1"
[[package]]
name = "syn"
version = "2.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
src/crates/iterable/Cargo.toml
-26
View File
@@ -1,26 +0,0 @@
[package]
name = "struct_iterable"
version = "0.1.2"
authors = ["André de Moraes <deco.moraes@icloud.com>"]
edition = "2021"
description = "A Rust library providing a proc macro to make a struct iterable."
license = "MIT"
repository = "https://github.com/decomoraes/rust_struct_iterable"
readme = "README.md"
keywords = ["proc-macro", "struct", "iterable"]
categories = ["development-tools::cargo-plugins"]
homepage = "https://github.com/decomoraes/rust_struct_iterable"
documentation = "https://docs.rs/struct_iterable"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
struct_iterable_derive = { path = "./struct_iterable_derive" }
struct_iterable_internal = { path = "./struct_iterable_internal" }
[lib]
name = "struct_iterable"
path = "src/lib.rs"
[package.metadata.docs.rs]
all-features = true
src/crates/iterable/README.md
-105
View File
@@ -1,105 +0,0 @@
# Struct Iterable
`Struct Iterable` is a Rust library that provides a proc macro to make a struct iterable. This allows you to iterate over the fields of your struct in a generic way, with each iteration returning a tuple containing the name of the field as a static string and a reference to the field's value as a `dyn Any`.
## How to Use
First, add `Struct Iterable` to your `Cargo.toml`:
```toml
[dependencies]
struct_iterable = "0.1.1"
```
Next, include the library at the top of your Rust file:
```rust
use struct_iterable::Iterable;
```
Finally, add the `#[derive(Iterable)]` attribute to your struct:
```rust
#[derive(Iterable)]
struct MyStruct {
field1: u32,
field2: String,
// etc.
}
```
Now, you can iterate over the fields of an instance of your struct:
```rust
let my_instance = MyStruct {
field1: 42,
field2: "Hello, world!".to_string(),
};
for (field_name, field_value) in my_instance.iter() {
println!("{}: {:?}", field_name, field_value);
}
```
## Limitations
- Only structs with named fields are supported.
- Only structs are supported, not enums or unions.
## Implementation
Here is the implementation of the proc macro:
```rust
extern crate proc_macro;
use proc_macro::TokenStream;
use quote::quote;
use syn::{parse_macro_input, Data, DeriveInput, Fields};
use iterable_structs::Iterable;
#[proc_macro_derive(Iterable)]
pub fn derive_iterable(input: TokenStream) -> TokenStream {
let input = parse_macro_input!(input as DeriveInput);
let struct_name = input.ident;
let fields = match input.data {
Data::Struct(data_struct) => match data_struct.fields {
Fields::Named(fields_named) => fields_named.named,
_ => panic!("Only structs with named fields are supported"),
},
_ => panic!("Only structs are supported"),
};
let fields_iter = fields.iter().map(|field| {
let field_ident = &field.ident;
let field_name = field_ident.as_ref().unwrap().to_string();
quote! {
(#field_name, &(self.#field_ident) as &dyn std::any::Any)
}
});
let expanded = quote! {
impl Iterable for #struct_name {
fn iter<'a>(&'a self) -> std::vec::IntoIter<(&'static str, &'a dyn std::any::Any)> {
vec![
#(#fields_iter),*
].into_iter()
}
}
};
TokenStream::from(expanded)
}
```
The macro takes in the TokenStream of a struct and expands it into an implementation of the Iterable trait for that struct. This trait provides an iter method that returns an iterator over tuples of field names and values.
## Contributing and License
`Struct Iterable` is an open-source project, and contributions are warmly welcomed. Whether you're fixing bugs, improving the documentation, or proposing new features, your efforts are highly appreciated!
If you're interested in contributing, please feel free to submit a pull request. For major changes, please open an issue first to discuss what you would like to change.
Please note that this project is released with a Contributor Code of Conduct. By participating in this project, you agree to abide by its terms.
`Struct Iterable` is distributed under the terms of the MIT license. As such, you're free to use, modify, distribute, and privately use it in any way you see fit, in accordance with the terms of the license.
src/crates/iterable/src/lib.rs
-91
View File
@@ -1,91 +0,0 @@
/// The `Iterable` proc macro.
///
/// This macro provides a convenient way to make a struct iterable.
/// The struct fields' names are returned as static strings and their values as `dyn Any`.
/// This allows to iterate over the struct fields in a generic way.
///
/// Note that only structs with named fields are supported.
///
/// # Example
///
/// ```
/// use struct_iterable::Iterable;
///
/// #[derive(Iterable)]
/// struct MyStruct {
/// field1: i32,
/// field2: String,
/// // etc.
/// }
///
/// let my_instance = MyStruct {
/// field1: 42,
/// field2: "Hello, world!".to_string(),
/// };
///
/// for (field_name, field_value) in my_instance.iter() {
/// println!("{}: {:?}", field_name, field_value);
/// }
/// ```
pub use struct_iterable_derive::Iterable;
/// The `Iterable` trait.
///
/// This trait is implemented by the struct once the `Iterable` proc macro is derived.
/// It provides an `iter` method that returns an iterator over tuples of field names and values.
///
/// # Example
///
/// ```
/// use struct_iterable::Iterable;
///
/// #[derive(Iterable)]
/// struct MyStruct {
/// field1: i32,
/// field2: String,
/// // etc.
/// }
///
/// let my_instance = MyStruct {
/// field1: 42,
/// field2: "Hello, world!".to_string(),
/// };
///
/// for (field_name, field_value) in my_instance.iter() {
/// println!("{}: {:?}", field_name, field_value);
/// }
/// ```
pub use struct_iterable_internal::Iterable;
#[cfg(test)]
mod tests {
use super::*;
#[derive(Iterable)]
struct MyStruct {
field1: i32,
field2: String,
// etc.
}
#[test]
fn it_works() {
let mut my_instance = MyStruct {
field1: 42,
field2: "Hello, world!".to_string(),
};
for (field_name, field_value) in my_instance.iter() {
dbg!("{}: {:?}", field_name, field_value);
}
for (field_name, field_value) in my_instance.iter_mut() {
dbg!("{}: {:?}", field_name, &field_value);
if let Some(i32) = field_value.downcast_mut::<i32>() {
*i32 += 1;
dbg!(i32);
}
dbg!("{}: {:?}", field_name, &field_value);
}
}
}
src/crates/iterable/struct_iterable_derive/Cargo.toml
-17
View File
@@ -1,17 +0,0 @@
[package]
name = "struct_iterable_derive"
version = "0.1.0"
authors = ["André de Moraes <deco.moraes@icloud.com>"]
edition = "2021"
description = "An internal crate for struct_iterable"
license = "MIT"
[lib]
proc-macro = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
syn = "2.0.85"
quote = "1.0.37"
proc-macro2 = "1.0.89"
struct_iterable_internal = { path = "../struct_iterable_internal" }
src/crates/iterable/struct_iterable_derive/README.md
-7
View File
@@ -1,7 +0,0 @@
# Struct Iterable Derive
This crate is a supporting library for the `struct_iterable` crate. It provides the proc macro `Iterable` which is used in conjunction with the `struct_iterable_internal` crate to provide an easy way to make a struct iterable in Rust.
**Please note:** This crate is not intended to be used directly. If you want to make your structs iterable, please use the `struct_iterable` crate instead.
Please visit the [`struct_iterable` crate on crates.io](https://crates.io/crates/struct_iterable) for more information and usage examples.
src/crates/iterable/struct_iterable_derive/src/lib.rs
-88
View File
@@ -1,88 +0,0 @@
extern crate proc_macro;
use proc_macro::TokenStream;
use quote::quote;
use syn::{parse_macro_input, Data, DeriveInput, Fields};
/// The `Iterable` proc macro.
///
/// Deriving this macro for your struct will make it "iterable". An iterable struct allows you to iterate over its fields, returning a tuple containing the field name as a static string and a reference to the field's value as `dyn Any`.
///
/// # Limitations
///
/// - Only structs are supported, not enums or unions.
/// - Only structs with named fields are supported.
///
/// # Usage
///
/// Add the derive attribute (`#[derive(Iterable)]`) above your struct definition.
///
/// ```
/// use struct_iterable::Iterable;
///
/// #[derive(Iterable)]
/// struct MyStruct {
/// field1: i32,
/// field2: String,
/// }
/// ```
///
/// You can now call the `iter` method on instances of your struct to get an iterator over its fields:
///
/// ```
/// let my_instance = MyStruct {
/// field1: 42,
/// field2: "Hello, world!".to_string(),
/// };
///
/// for (field_name, field_value) in my_instance.iter() {
/// println!("{}: {:?}", field_name, field_value);
/// }
/// ```
#[proc_macro_derive(Iterable)]
pub fn derive_iterable(input: TokenStream) -> TokenStream {
let input = parse_macro_input!(input as DeriveInput);
let struct_name = input.ident;
let fields = match input.data {
Data::Struct(data_struct) => match data_struct.fields {
Fields::Named(fields_named) => fields_named.named,
_ => panic!("Only structs with named fields are supported"),
},
_ => panic!("Only structs are supported"),
};
let fields_iter = fields.iter().map(|field| {
let field_ident = &field.ident;
let field_name = field_ident.as_ref().unwrap().to_string();
quote! {
(#field_name, &(self.#field_ident) as &dyn std::any::Any)
}
});
let fields_iter_mut = fields.iter().map(|field| {
let field_ident = &field.ident;
let field_name = field_ident.as_ref().unwrap().to_string();
quote! {
(#field_name, &mut (self.#field_ident) as &mut dyn std::any::Any)
}
});
let expanded = quote! {
impl Iterable for #struct_name {
fn iter<'a>(&'a self) -> std::vec::IntoIter<(&'static str, &'a dyn std::any::Any)> {
vec![
#(#fields_iter),*
].into_iter()
}
fn iter_mut<'a>(&'a mut self) -> std::vec::IntoIter<(&'static str, &'a mut dyn std::any::Any)> {
vec![
#(#fields_iter_mut),*
].into_iter()
}
}
};
TokenStream::from(expanded)
}
src/crates/iterable/struct_iterable_internal/Cargo.toml
-11
View File
@@ -1,11 +0,0 @@
[package]
name = "struct_iterable_internal"
version = "0.1.1"
authors = ["André de Moraes <deco.moraes@icloud.com>"]
edition = "2021"
description = "An internal crate for struct_iterable"
license = "MIT"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
src/crates/iterable/struct_iterable_internal/README.md
-7
View File
@@ -1,7 +0,0 @@
# Struct Iterable Internal
This crate is a supporting library for the `struct_iterable` crate. It provides the `Iterable` trait which is used in conjunction with the `struct_iterable_derive` crate to provide an easy way to make a struct iterable in Rust.
**Please note:** This crate is not intended to be used directly. If you want to make your structs iterable, please use the `struct_iterable` crate instead.
Please visit the [`struct_iterable` crate on crates.io](https://crates.io/crates/struct_iterable) for more information and usage examples.
src/crates/iterable/struct_iterable_internal/src/lib.rs
-58
View File
@@ -1,58 +0,0 @@
/// The `Iterable` trait.
///
/// This trait is implemented for structs that derive the `Iterable` proc macro.
/// It provides the `iter` method which returns an iterator over the struct's fields as tuples, containing the field name as a static string and a reference to the field's value as `dyn Any`.
///
/// You usually don't need to implement this trait manually, as it is automatically derived when using the `#[derive(Iterable)]` proc macro.
///
/// # Example
///
/// ```
/// use struct_iterable::Iterable;
///
/// #[derive(Iterable)]
/// struct MyStruct {
/// field1: i32,
/// field2: String,
/// }
///
/// let my_instance = MyStruct {
/// field1: 42,
/// field2: "Hello, world!".to_string(),
/// };
///
/// // Iterate over the fields of `my_instance`:
/// for (field_name, field_value) in my_instance.iter() {
/// println!("{}: {:?}", field_name, field_value);
/// }
/// ```
pub trait Iterable {
/// Returns an iterator over the struct's fields as tuples.
///
/// Each tuple contains a field's name as a static string and a reference to the field's value as `dyn Any`.
///
/// # Example
///
/// ```
/// use struct_iterable::Iterable;
///
/// #[derive(Iterable)]
/// struct MyStruct {
/// field1: i32,
/// field2: String,
/// }
///
/// let my_instance = MyStruct {
/// field1: 42,
/// field2: "Hello, world!".to_string(),
/// };
///
/// // Iterate over the fields of `my_instance`:
/// for (field_name, field_value) in my_instance.iter() {
/// println!("{}: {:?}", field_name, field_value);
/// }
/// ```
fn iter(&self) -> std::vec::IntoIter<(&'static str, &'_ dyn std::any::Any)>;
fn iter_mut(&mut self) -> std::vec::IntoIter<(&'static str, &'_ mut dyn std::any::Any)>;
}
src/crates/snkrj/Cargo.lock
Generated
-242
View File
@@ -1,242 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "autocfg"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "libc"
version = "0.2.168"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
[[package]]
name = "lock_api"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "memmap2"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
dependencies = [
"libc",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]]
name = "proc-macro2"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]]
name = "sanakirja"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11"
dependencies = [
"fs2",
"log",
"memmap2",
"parking_lot",
"sanakirja-core",
"serde",
"thiserror",
]
[[package]]
name = "sanakirja-core"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "snkrj"
version = "0.1.1"
dependencies = [
"sanakirja",
]
[[package]]
name = "syn"
version = "2.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "unicode-ident"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
src/crates/snkrj/Cargo.toml
-12
View File
@@ -1,12 +0,0 @@
[package]
name = "snkrj"
description = "A very simple wrapper around Sanakirja"
version = "0.1.1"
license = "MIT"
repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/snkrj"
keywords = ["database", "sanakirja", "btreemap"]
categories = ["database"]
edition = "2021"
[dependencies]
sanakirja = "1.4.3"
src/crates/snkrj/LICENSE.md
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2024 snkrj
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
src/crates/snkrj/README.md
-37
View File
@@ -1,37 +0,0 @@
# snkrj
A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap.
## Example
```rust
use snkrj::{AnyDatabase, Database};
fn main() {
let path = std::env::temp_dir().join("./db");
let database: Database<i32, i32> = Database::open(path.clone()).unwrap();
let _ = database.destroy();
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
database.insert(64, 128);
database.export(false).unwrap();
let mut database: Database<i32, i32> = Database::open(path).unwrap();
database.insert(1, 2);
database.insert(128, 256);
println!("iter_ram:");
database.iter_ram().for_each(|pair| {
println!("{:?}", pair);
});
println!("iter_disk:");
database.iter_disk().for_each(|pair| {
println!("{:?}", pair.unwrap());
});
println!("iter_ram_then_disk:");
database.iter_ram_then_disk().for_each(|pair| {
println!("{:?}", pair);
});
database.export(false).unwrap();
}
```
src/crates/snkrj/src/lib.rs
-346
View File
@@ -1,346 +0,0 @@
// https://docs.rs/sanakirja/latest/sanakirja/index.html
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
use core::panic;
use std::{
collections::{BTreeMap, BTreeSet},
fmt::Debug,
fs::{self, File},
io, mem,
path::{Path, PathBuf},
result::Result,
};
use sanakirja::btree::{page, Db_};
pub use sanakirja::*;
///
/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap.
///
/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program.
///
pub struct Database<Key, Value>
where
Key: DatabaseKey,
Value: DatabaseValue,
{
pathbuf: PathBuf,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
len: usize,
db: Db_<Key, Value, page::Page<Key, Value>>,
txn: MutTxn<Env, ()>,
}
const ROOT_DB: usize = 0;
const PAGE_SIZE: u64 = 4096;
pub type UnitDatabase = Database<(), ()>;
const DEFRAGMENT_RATIO_THRESHOLD: f64 = 0.5;
impl<Key, Value> Database<Key, Value>
where
Key: DatabaseKey,
Value: DatabaseValue,
{
const KEY_SIZE: usize = size_of::<Key>();
const VALUE_SIZE: usize = size_of::<Value>();
const KEY_AND_VALUE_SIZE: usize = Self::KEY_SIZE + Self::VALUE_SIZE;
/// Open a database without a lock file where only one instance is safe to open.
pub fn open(pathbuf: PathBuf) -> Result<Self, Error> {
fs::create_dir_all(&pathbuf)?;
let env = unsafe { Env::new_nolock(Self::path_sanakirja_(&pathbuf), PAGE_SIZE, 1)? };
let mut txn = Env::mut_txn_begin(env)?;
let db = txn
.root_db(ROOT_DB)
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
Ok(Self {
len: Self::read_length_(&pathbuf),
pathbuf,
puts: BTreeMap::default(),
dels: BTreeSet::default(),
db,
txn,
})
}
pub fn path_sanakirja(&self) -> PathBuf {
Self::path_sanakirja_(&self.pathbuf)
}
fn path_sanakirja_(path: &Path) -> PathBuf {
path.join("sanakirja")
}
pub fn read_length(&self) -> usize {
Self::read_length_(&self.pathbuf)
}
pub fn read_length_(path: &Path) -> usize {
fs::read(Self::path_length(path))
.map(|v| {
let mut buf = [0_u8; 8];
v.iter().enumerate().take(8).for_each(|(i, b)| {
buf[i] = *b;
});
usize::from_le_bytes(buf)
})
.unwrap_or_default()
}
pub fn write_length(&self) -> Result<(), io::Error> {
Self::write_length_(&self.pathbuf, self.len)
}
pub fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> {
fs::write(Self::path_length(path), len.to_le_bytes())
}
fn path_length(path: &Path) -> PathBuf {
path.join("length")
}
#[inline]
pub fn get(&self, key: &Key) -> Option<&Value> {
if let Some(cached_put) = self.get_from_ram(key) {
return Some(cached_put);
}
self.get_from_disk(key)
}
/// Get only from the uncommited tree (ram) without checking the database (disk)
#[inline]
pub fn get_from_ram(&self, key: &Key) -> Option<&Value> {
self.puts.get(key)
}
/// Get mut only from the uncommited tree (ram) without checking the database (disk)
#[inline]
pub fn get_mut_from_ram(&mut self, key: &Key) -> Option<&mut Value> {
self.puts.get_mut(key)
}
/// Get only from the database (disk) without checking the uncommited tree (ram)
#[inline]
pub fn get_from_disk(&self, key: &Key) -> Option<&Value> {
let option = btree::get(&self.txn, &self.db, key, None).unwrap();
if let Some((key_found, v)) = option {
if key == key_found {
return Some(v);
}
}
None
}
#[inline]
pub fn insert(&mut self, key: Key, value: Value) -> Option<Value> {
self.dels.remove(&key);
self.insert_to_ram(key, value)
}
/// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set
#[inline]
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
self.len += 1;
self.puts.insert(key, value)
}
#[inline]
pub fn update(&mut self, key: Key, value: Value) -> Option<Value> {
self.dels.insert(key.clone());
self.puts.insert(key, value)
}
#[inline]
pub fn remove(&mut self, key: &Key) -> Option<Value> {
self.len -= 1;
self.puts.remove(key).or_else(|| {
self.dels.insert(key.clone());
None
})
}
/// Get only from the uncommited tree (ram) without checking the database (disk)
#[inline]
pub fn remove_from_ram(&mut self, key: &Key) -> Option<Value> {
self.len -= 1;
self.puts.remove(key)
}
/// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key
#[inline]
pub fn remove_later_from_disk(&mut self, key: &Key) {
self.len -= 1;
self.dels.insert(key.clone());
}
/// Iterate over key/value pairs from the uncommited tree (ram)
#[inline]
pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Value> {
self.puts.iter()
}
/// Iterate over key/value pairs from the database (disk)
#[inline]
pub fn iter_disk(
&self,
) -> btree::Iter<'_, MutTxn<Env, ()>, Key, Value, page::Page<Key, Value>> {
btree::iter(&self.txn, &self.db, None).unwrap()
}
/// Iterate over key/value pairs
#[inline]
pub fn iter_ram_then_disk(&self) -> impl Iterator<Item = (&Key, &Value)> {
self.iter_ram().chain(self.iter_disk().map(|r| r.unwrap()))
}
/// Collect a **clone** of all uncommited key/value pairs (ram)
pub fn collect_ram(&self) -> BTreeMap<Key, Value> {
self.puts.clone()
}
/// Collect a **clone** of all key/value pairs from the database (disk)
pub fn collect_disk(&self) -> BTreeMap<Key, Value> {
self.iter_disk()
.map(|r| r.unwrap())
.map(|(key, value)| (key.clone(), value.clone()))
.collect::<_>()
}
#[inline]
pub fn len(&self) -> usize {
self.len
}
#[inline]
pub fn is_empty(&self) -> bool {
self.len == 0
}
// pub fn export(self) -> Result<(), Error> {
// self.boxed().boxed_export()
// }
// pub fn boxed(self) -> Box<Self> {
// Box::new(self)
// }
pub fn get_file_size_to_data_ratio(&self) -> Result<f64, Error> {
let data_bytes = (self.len() * Self::KEY_AND_VALUE_SIZE) as f64;
let file_bytes = File::open(&self.pathbuf)?.metadata()?.len() as f64;
Ok(file_bytes / data_bytes)
}
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
///
/// /!\ Do not kill the program while this function is runnning /!\
pub fn export(mut self) -> Result<(), Error> {
let defragment = self.get_file_size_to_data_ratio()? >= DEFRAGMENT_RATIO_THRESHOLD;
if defragment {
let mut btree = self.collect_disk();
let disk_len = btree.len();
let dels_len = self.dels.len();
let puts_len = self.puts.len();
let path = self.pathbuf.to_owned();
self.dels.iter().for_each(|key| {
btree.remove(key);
});
btree.append(&mut self.puts);
let len = btree.len();
if len != self.len {
dbg!(len, self.len, path, disk_len, dels_len, puts_len);
panic!("Len should be the same");
}
self.destroy()?;
self = Self::open(path).unwrap();
if !self.is_empty() {
panic!()
}
self.len = len;
self.puts = btree;
}
self.write_length()?;
if self.dels.is_empty() && self.puts.is_empty() {
return Ok(());
}
mem::take(&mut self.dels)
.into_iter()
.try_for_each(|key| -> Result<(), Error> {
btree::del(&mut self.txn, &mut self.db, &key, None)?;
Ok(())
})?;
mem::take(&mut self.puts).into_iter().try_for_each(
|(key, value)| -> Result<(), Error> {
btree::put(&mut self.txn, &mut self.db, &key, &value)?;
Ok(())
},
)?;
self.txn.set_root(ROOT_DB, self.db.db.into());
self.txn.commit()
}
pub fn destroy(self) -> io::Result<()> {
let path = self.pathbuf.to_owned();
drop(self);
fs::remove_dir_all(&path)
}
}
pub trait AnyDatabase {
fn export(self) -> Result<(), Error>;
// fn boxed_export(self: Box<Self>) -> Result<(), Error>;
fn destroy(self) -> io::Result<()>;
}
impl<Key, Value> AnyDatabase for Database<Key, Value>
where
Key: DatabaseKey,
Value: DatabaseValue,
{
fn export(self) -> Result<(), Error> {
self.export()
}
// fn boxed_export(self: Box<Self>) -> Result<(), Error> {
// self.boxed_export()
// }
fn destroy(self) -> io::Result<()> {
self.destroy()
}
}
pub trait DatabaseKey
where
Self: Ord + Clone + Debug + Storable + Send + Sync,
{
}
impl<T> DatabaseKey for T where T: Ord + Clone + Debug + Storable + Send + Sync {}
pub trait DatabaseValue
where
Self: Clone + Storable + PartialEq + Send + Sync,
{
}
impl<T> DatabaseValue for T where T: Clone + Storable + PartialEq + Send + Sync {}
src/crates/snkrj/src/main.rs
-29
View File
@@ -1,29 +0,0 @@
use snkrj::Database;
fn main() {
let path = std::env::temp_dir().join("./db");
let database: Database<i32, i32> = Database::open(path.clone()).unwrap();
let _ = database.destroy();
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
database.insert(64, 128);
database.export().unwrap();
let mut database: Database<i32, i32> = Database::open(path).unwrap();
database.insert(1, 2);
database.insert(128, 256);
println!("iter_ram:");
database.iter_ram().for_each(|pair| {
println!("{:?}", pair);
});
println!("iter_disk:");
database.iter_disk().for_each(|pair| {
println!("{:?}", pair.unwrap());
});
println!("iter_ram_then_disk:");
database.iter_ram_then_disk().for_each(|pair| {
println!("{:?}", pair);
});
database.export().unwrap();
}
src/crates/storable_vec/.gitignore
-1
View File
@@ -1 +0,0 @@
/v
src/crates/storable_vec/Cargo.lock
Generated
-25
View File
@@ -1,25 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "libc"
version = "0.2.169"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
[[package]]
name = "memmap2"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
dependencies = [
"libc",
]
[[package]]
name = "storable_vec"
version = "0.1.2"
dependencies = [
"memmap2",
]
src/crates/storable_vec/Cargo.toml
-11
View File
@@ -1,11 +0,0 @@
[package]
name = "storable_vec"
description = "A very small, fast, efficient and simple storable Vec"
version = "0.1.2"
license = "MIT"
keywords = ["vec", "disk", "data"]
categories = ["database"]
edition = "2021"
[dependencies]
memmap2 = "0.9.5"
src/crates/storable_vec/LICENSE.md
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2024 storable_vec
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
src/crates/storable_vec/README.md
-53
View File
@@ -1,53 +0,0 @@
# storable_vec
A very small, fast, efficient and simple storable `vec` which uses `mmap2` for speed.
## Features
- [x] Get (Rayon compatible)
- [x] Push
- [ ] Update
- [ ] Insert
- [ ] Remove
## Example
```rust
use std::path::Path;
use storable_vec::{AnyStorableVec, StorableVec};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
{
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
vec.push(21);
dbg!(vec.get(0)?); // 21
vec.flush()?;
}
{
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
dbg!(vec.get(0)?); // 21
}
Ok(())
}
```
## Disclaimer
Portability will depend on the type of values.
Non bytes/slices types (`u8`, `u16`, ...) will be read as slice in an unsafe manner (using `std::slice::from_raw_parts`) and thus have the endianness of the system. On the other hand, `&[u8]` should be inserted as is.
If portability is important to you, just create a wrapper struct which has custom `get`, `push`, ... methods and does something like:
```rust
impl StorableVecU64 {
pub fn push(&mut self, value: u64) {
self.push(&value.to_be_bytes())
}
}
```
src/crates/storable_vec/src/lib.rs
-352
View File
@@ -1,352 +0,0 @@
use std::{
cmp::Ordering,
fmt::{self, Debug},
fs::{File, OpenOptions},
io::{self, Write},
marker::PhantomData,
mem,
ops::Range,
path::Path,
sync::OnceLock,
};
use memmap2::{Mmap, MmapOptions};
///
/// A very small, fast, efficient and simple storable Vec
///
/// Reads (imports of Mmap) are lazy
///
/// Stores only raw data without any overhead, and doesn't even have a header (TODO: which it should, at least to Err if wrong endian)
///
/// The file isn't portable for speed reasons (TODO: but could be ?)
///
/// If you don't call `.flush()` it just acts as a normal Vec
///
#[derive(Debug)]
pub struct StorableVec<I, T> {
file: File,
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed to reduce the size of the lock (24 > 16)
disk_len: usize,
pushed: Vec<T>,
// updated: BTreeMap<usize, T>,
// inserted: BTreeMap<usize, T>,
// removed: BTreeSet<usize>,
phantom: PhantomData<I>,
}
/// In bytes
const MAX_PAGE_SIZE: usize = 4096;
const ONE_MB: usize = 1024 * 1024;
const MAX_CACHE_SIZE: usize = 50 * ONE_MB;
impl<I, T> StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug,
{
pub const SIZE: usize = size_of::<T>();
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE;
/// In bytes
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE;
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
pub fn import(path: &Path) -> Result<Self, io::Error> {
let file = Self::open_file(path)?;
let mut this = Self {
disk_len: Self::byte_index_to_index(file.metadata()?.len() as usize),
file,
cache: vec![],
pushed: vec![],
// updated: BTreeMap::new(),
// inserted: BTreeMap::new(),
// removed: BTreeSet::new(),
phantom: PhantomData,
};
this.reset_cache();
Ok(this)
}
pub fn reset_cache(&mut self) {
let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
self.cache.clear();
self.cache.resize_with(len, Default::default);
}
fn open_file(path: &Path) -> Result<File, io::Error> {
OpenOptions::new()
.read(true)
.create(true)
.truncate(false)
.append(true)
.open(path)
}
#[inline]
fn index_to_mmap_index(index: usize) -> usize {
Self::index_to_byte_index(index) / Self::PAGE_SIZE
}
#[inline]
fn index_to_range(index: usize) -> Range<usize> {
let index = Self::index_to_byte_index(index) % Self::PAGE_SIZE;
index..(index + Self::SIZE)
}
#[inline]
fn index_to_byte_index(index: usize) -> usize {
index * Self::SIZE
}
#[inline]
fn byte_index_to_index(byte_index: usize) -> usize {
byte_index / Self::SIZE
}
fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
if index >= self.disk_len {
let index = index - self.disk_len;
if index >= self.pushed.len() {
Err(Error::IndexTooHigh)
} else {
Ok(Some(index))
}
} else {
Ok(None)
}
}
#[allow(unused)]
#[inline]
pub fn get(&self, index: I) -> Result<Option<&T>> {
self.get_(index.into())
}
pub fn get_(&self, index: usize) -> Result<Option<&T>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.pushed.get(index));
}
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(error) => return Err(error),
}
// if !self.updated.is_empty() {
// if let Some(v) = self.updated.get(&index) {
// return Ok(Some(v));
// }
// }
let mmap_index = Self::index_to_mmap_index(index);
let mmap = &**self
.cache
.get(mmap_index)
.ok_or(Error::MmapsVecIsTooSmall)?
.get_or_init(|| {
Box::new(unsafe {
MmapOptions::new()
.len(MAX_PAGE_SIZE)
.offset((mmap_index * Self::PAGE_SIZE) as u64)
.map(&self.file)
.unwrap()
})
});
let range = Self::index_to_range(index);
let src = &mmap[range];
Ok(Some(T::unsafe_try_from_slice(src)?))
}
#[allow(unused)]
pub fn first(&self) -> Result<Option<&T>> {
self.get_(0)
}
#[allow(unused)]
pub fn last(&self) -> Result<Option<&T>> {
let len = self.len();
if len == 0 {
return Ok(None);
}
self.get_(len - 1)
}
pub fn push(&mut self, value: T) {
self.pushed.push(value)
}
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
self.push_if_needed_(index.into(), value)
}
pub fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> {
let len = self.len();
match len.cmp(&index) {
Ordering::Greater => Ok(()),
Ordering::Equal => {
self.push(value);
Ok(())
}
Ordering::Less => Err(Error::IndexTooHigh),
}
}
// pub fn update(&mut self, index: I, value: T) -> Result<()> {
// self._update(index.into(), value)
// }
// pub fn update_(&mut self, index: usize, value: T) -> Result<()> {
// if let Some(index) = self.index_to_pushed_index(index) {
// self.pushed[index] = value;
// } else {
// self.updated.insert(index, value);
// }
// Ok(())
// }
// pub fn fetch_update(&mut self, index: I, value: T) -> Result<T>
// where
// T: Clone,
// {
// self._fetch_update(index.into(), value)
// }
// pub fn fetch_update_(&mut self, index: usize, value: T) -> Result<T>
// where
// T: Clone,
// {
// let prev_opt = self.updated.insert(index, value);
// if let Some(prev) = prev_opt {
// Ok(prev)
// } else {
// Ok(self
// ._get(index)?
// .ok_or(Error::ExpectFileToHaveIndex)?
// .clone())
// }
// }
// pub fn remove(&mut self, index: I) {
// self._remove(index.into())
// }
// pub fn remove_(&mut self, index: usize) {
// self.removed.insert(index);
// }
pub fn len(&self) -> usize {
self.disk_len + self.pushed.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn has(&self, index: I) -> bool {
self.has_(index.into())
}
pub fn has_(&self, index: usize) -> bool {
index < self.len()
}
pub fn hasnt(&self, index: I) -> bool {
self.hasnt_(index.into())
}
pub fn hasnt_(&self, index: usize) -> bool {
!self.has_(index)
}
pub fn flush(&mut self) -> io::Result<()> {
self.disk_len += self.pushed.len();
self.reset_cache();
let mut bytes: Vec<u8> = vec![];
mem::take(&mut self.pushed)
.into_iter()
.for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice()));
self.file.write_all(&bytes)
}
}
pub trait AnyStorableVec {
fn len(&self) -> usize;
fn is_empty(&self) -> bool;
fn reset_cache(&mut self);
fn flush(&mut self) -> io::Result<()>;
}
impl<I, T> AnyStorableVec for StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug,
{
fn len(&self) -> usize {
self.len()
}
fn is_empty(&self) -> bool {
self.is_empty()
}
fn reset_cache(&mut self) {
self.reset_cache();
}
fn flush(&mut self) -> io::Result<()> {
self.flush()
}
}
pub trait UnsafeSizedSerDe
where
Self: Sized,
{
const SIZE: usize = size_of::<Self>();
fn unsafe_try_from_slice(slice: &[u8]) -> Result<&Self> {
let (prefix, shorts, suffix) = unsafe { slice.align_to::<Self>() };
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
// dbg!(&slice, &prefix, &shorts, &suffix);
return Err(Error::FailedToAlignToSelf);
}
Ok(&shorts[0])
}
fn unsafe_as_slice(&self) -> &[u8] {
let data: *const Self = self;
let data: *const u8 = data as *const u8;
unsafe { std::slice::from_raw_parts(data, Self::SIZE) }
}
}
impl<T> UnsafeSizedSerDe for T {}
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
MmapsVecIsTooSmall,
FailedToAlignToSelf,
IndexTooHigh,
ExpectFileToHaveIndex,
ExpectVecToHaveIndex,
}
impl fmt::Display for Error {
// This trait requires `fmt` with this exact signature.
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::FailedToAlignToSelf => write!(f, "Failed to align_to for T"),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"),
}
}
}
impl std::error::Error for Error {}
src/crates/storable_vec/src/main.rs
-25
View File
@@ -1,25 +0,0 @@
use std::path::Path;
use storable_vec::StorableVec;
fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
vec.push(0);
vec.push(1);
vec.push(2);
dbg!(vec.get(0)?); // Some(0)
dbg!(vec.get(21)?); // None
vec.flush()?;
}
{
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
dbg!(vec.get(0)?); // 0
}
Ok(())
}