Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-19 11:19:44 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-11 19:01:30 +01:00
parent cb7ea40e7c
commit cf26696d12
34 changed files with 1379 additions and 1871 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+577 -1689
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+8 -4
View File
@@ -4,6 +4,7 @@ members = [
"exit",
"indexer",
"iterator",
"logger",
"server",
"storable_vec",
"struct_iterable",
@@ -16,12 +17,15 @@ color-eyre = "0.6.3"
computer = { path = "computer", package = "bomputer" }
derive_deref = "1.1.1"
exit = { path = "exit" }
fjall = "2.5.0"
fjall = "2.6.2"
indexer = { path = "indexer", package = "bindex" }
iterator = { path = "iterator", package = "biter" }
jiff = "0.1.29"
jiff = "0.2.0"
logger = { path = "logger" }
rayon = "1.10.0"
server = { path = "server" }
rlimit = { version = "0.10.2" }
serde = { version = "1.0.217", features = ["derive"] }
server = { path = "server", package = "berver" }
storable_vec = { path = "storable_vec" }
struct_iterable = { path = "struct_iterable" }
zerocopy = { version = "0.8.16", features = ["derive"] }
zerocopy = { version = "0.8.17", features = ["derive"] }
computer/src/lib.rs
+11 -20
View File
@@ -2,48 +2,35 @@ use std::path::{Path, PathBuf};
use exit::Exit;
use indexer::Indexer;
use iterator::rpc;
pub use iterator::rpc;
mod storage;
mod structs;
use storable_vec::{CACHED_GETS, SINGLE_THREAD};
use storable_vec::SINGLE_THREAD;
use storage::{Fjalls, StorableVecs};
use structs::*;
pub use structs::*;
pub struct Computer<const MODE: u8> {
outputs_dir: PathBuf,
path: PathBuf,
pub vecs: StorableVecs<MODE>,
pub trees: Fjalls,
}
impl<const MODE: u8> Computer<MODE> {
pub fn import(outputs_dir: &Path) -> color_eyre::Result<Self> {
let outputs_dir = outputs_dir.to_owned();
let computed_dir = outputs_dir.join("computed");
pub fn import(computed_dir: &Path) -> color_eyre::Result<Self> {
let vecs = StorableVecs::import(&computed_dir.join("vecs"))?;
let trees = Fjalls::import(&computed_dir.join("fjall"))?;
Ok(Self {
outputs_dir,
path: computed_dir.to_owned(),
vecs,
trees,
})
}
fn open_indexer<const MODE_IDX: u8>(&self) -> color_eyre::Result<Indexer<MODE_IDX>> {
Indexer::import(&self.outputs_dir.join("indexes"))
}
}
impl Computer<SINGLE_THREAD> {
pub fn compute(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
if false {
let mut indexer: Indexer<CACHED_GETS> = self.open_indexer()?;
indexer.index(bitcoin_dir, rpc, exit)?;
}
let mut indexer: Indexer<SINGLE_THREAD> = self.open_indexer()?;
pub fn compute(&mut self, mut indexer: Indexer<SINGLE_THREAD>, exit: &Exit) -> color_eyre::Result<()> {
let height_count = indexer.vecs.height_to_size.len();
let txindexes_count = indexer.vecs.txindex_to_txid.len();
let txinindexes_count = indexer.vecs.txinindex_to_txoutindex.len();
@@ -117,4 +104,8 @@ impl Computer<SINGLE_THREAD> {
Ok(())
}
pub fn path(&self) -> &Path {
&self.path
}
}
computer/src/main.rs
+3 -9
View File
@@ -2,26 +2,20 @@ use std::path::Path;
use bomputer::Computer;
use exit::Exit;
use iterator::rpc;
use storable_vec::SINGLE_THREAD;
use indexer::Indexer;
mod structs;
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let data_dir = Path::new("../../bitcoin");
let rpc = rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
)?;
let exit = Exit::new();
let i = std::time::Instant::now();
let mut computer: Computer<SINGLE_THREAD> = Computer::import(Path::new("../_outputs"))?;
let outputs_dir = Path::new("../_outputs");
computer.compute(data_dir, rpc, &exit)?;
Computer::import(&outputs_dir.join("computed"))?.compute(Indexer::import(&outputs_dir.join("indexes"))?, &exit)?;
dbg!(i.elapsed());
computer/src/storage/storable_vecs.rs
+8 -7
View File
@@ -11,20 +11,21 @@ use crate::structs::{Date, Feerate};
pub struct StorableVecs<const MODE: u8> {
pub date_to_first_height: StorableVec<Date, Height, MODE>,
// pub date_to_last_height: StorableVec<Date, Height, MODE>,
// pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
pub height_to_date: StorableVec<Height, Date, MODE>,
// pub height_to_fee: StorableVec<Txindex, Amount, MODE>,
// pub height_to_inputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_inputcount: StorableVec<Height, u32, MODE>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex, MODE>,
pub height_to_last_txindex: StorableVec<Height, Txindex, MODE>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex, MODE>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_minfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_outputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_subsidy: StorableVec<Txindex, u32, MODE>,
// pub height_to_maxfeerate: StorableVec<Height, Feerate, MODE>,
// pub height_to_medianfeerate: StorableVec<Height, Feerate, MODE>,
// pub height_to_minfeerate: StorableVec<Height, Feerate, MODE>,
// pub height_to_outputcount: StorableVec<Height, u32, MODE>,
// pub height_to_subsidy: StorableVec<Height, u32, MODE>,
// pub height_to_totalfees: StorableVec<Height, Amount, MODE>,
// pub height_to_txcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_txcount: StorableVec<Height, u32, MODE>,
pub txindex_to_fee: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool, MODE>,
exit/Cargo.toml
+1
View File
@@ -5,3 +5,4 @@ edition = "2021"
[dependencies]
ctrlc = "3.4.5"
logger = { workspace = true }
exit/src/lib.rs
+4 -2
View File
@@ -8,6 +8,8 @@ use std::{
time::Duration,
};
use logger::info;
#[derive(Default, Clone)]
pub struct Exit {
blocked: Arc<AtomicBool>,
@@ -27,12 +29,12 @@ impl Exit {
let blocked = move || _blocked.load(Ordering::SeqCst);
ctrlc::set_handler(move || {
println!("Exitting...");
info!("Exitting...");
active.store(true, Ordering::SeqCst);
if blocked() {
println!("Waiting to exit safely");
info!("Waiting to exit safely");
while blocked() {
sleep(Duration::from_millis(50));
indexer/Cargo.toml
+4
View File
@@ -11,7 +11,11 @@ exit = { workspace = true }
fjall = { workspace = true }
iterator = { workspace = true }
jiff = { workspace = true }
logger = { workspace = true }
rapidhash = "1.3.0"
rayon = { workspace = true }
rlimit = { workspace = true }
serde = { workspace = true }
serde_bytes = "0.11.15"
storable_vec = { workspace = true }
zerocopy = { workspace = true }
indexer/src/lib.rs
+20 -35
View File
@@ -1,6 +1,5 @@
use std::{
collections::BTreeMap,
io::{Read, Write},
path::Path,
str::FromStr,
thread::{self, sleep},
@@ -12,6 +11,7 @@ pub use iterator::*;
use bitcoin::{Transaction, TxIn, TxOut};
use color_eyre::eyre::{eyre, ContextCompat};
use exit::Exit;
use logger::info;
use rayon::prelude::*;
use storable_vec::CACHED_GETS;
@@ -19,15 +19,10 @@ mod storage;
mod structs;
pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta};
use structs::{BlockHash, Txid};
use storage::{Fjalls, StorableVecs};
pub use structs::{
AddressHash, Addressbytes, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp, TxidPrefix,
Txindex, Txinindex, Txoutindex, Vin, Vout,
};
pub use structs::*;
const UNSAFE_BLOCKS: u32 = 100;
const UNSAFE_BLOCKS: u32 = 1000;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
pub struct Indexer<const MODE: u8> {
@@ -39,6 +34,7 @@ impl<const MODE: u8> Indexer<MODE> {
pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?;
let trees = Fjalls::import(&indexes_dir.join("fjall"))?;
Ok(Self { vecs, trees })
}
}
@@ -77,7 +73,7 @@ impl Indexer<CACHED_GETS> {
let export =
|trees: &mut Fjalls, vecs: &mut StorableVecs<CACHED_GETS>, height: Height| -> color_eyre::Result<()> {
println!("Exporting...");
info!("Exporting...");
exit.block();
@@ -93,10 +89,10 @@ impl Indexer<CACHED_GETS> {
Ok(())
};
iterator::new(bitcoin_dir, Some(height.into()), None, rpc)
iterator::new(bitcoin_dir, Some(height.into()), Some(400_000), rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
info!("Processing block {_height}...");
let blockhash = BlockHash::from(blockhash);
height = Height::from(_height);
@@ -327,7 +323,7 @@ impl Indexer<CACHED_GETS> {
.and_then(|addressindex_local| {
(addressindex_local < addressindex_global).then_some(addressindex_local)
})
}); // OK
});
if let Some(Some(addressindex)) = check_collisions.then_some(addressindex_opt) {
let addressbytes = addressbytes_res.as_ref().unwrap();
@@ -341,8 +337,6 @@ impl Indexer<CACHED_GETS> {
.addressindex_to_addresstypeindex
.get(addressindex)?
.context("Expect to have address type index")?;
// Good first time
// Wrong after rerun
let prev_addressbytes_opt =
vecs.get_addressbytes(prev_addresstype, addresstypeindex)?;
@@ -473,18 +467,18 @@ impl Indexer<CACHED_GETS> {
addressindex_global.increment();
let addresstypeindex = match addresstype {
Addresstype::Empty => emptyindex_global.clone_then_increment(),
Addresstype::Multisig => multisigindex_global.clone_then_increment(),
Addresstype::OpReturn => opreturnindex_global.clone_then_increment(),
Addresstype::PushOnly => pushonlyindex_global.clone_then_increment(),
Addresstype::Unknown => unknownindex_global.clone_then_increment(),
Addresstype::P2PK65 => p2pk65index_global.clone_then_increment(),
Addresstype::P2PK33 => p2pk33index_global.clone_then_increment(),
Addresstype::P2PKH => p2pkhindex_global.clone_then_increment(),
Addresstype::P2SH => p2shindex_global.clone_then_increment(),
Addresstype::P2WPKH => p2wpkhindex_global.clone_then_increment(),
Addresstype::P2WSH => p2wshindex_global.clone_then_increment(),
Addresstype::P2TR => p2trindex_global.clone_then_increment(),
Addresstype::Empty => emptyindex_global.copy_then_increment(),
Addresstype::Multisig => multisigindex_global.copy_then_increment(),
Addresstype::OpReturn => opreturnindex_global.copy_then_increment(),
Addresstype::PushOnly => pushonlyindex_global.copy_then_increment(),
Addresstype::Unknown => unknownindex_global.copy_then_increment(),
Addresstype::P2PK65 => p2pk65index_global.copy_then_increment(),
Addresstype::P2PK33 => p2pk33index_global.copy_then_increment(),
Addresstype::P2PKH => p2pkhindex_global.copy_then_increment(),
Addresstype::P2SH => p2shindex_global.copy_then_increment(),
Addresstype::P2WPKH => p2wpkhindex_global.copy_then_increment(),
Addresstype::P2WSH => p2wshindex_global.copy_then_increment(),
Addresstype::P2TR => p2trindex_global.copy_then_increment(),
};
vecs.addressindex_to_addresstype
@@ -674,12 +668,3 @@ enum InputSource<'a> {
PreviousBlock((Vin, Txindex, Txoutindex)),
SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)),
}
#[allow(unused)]
fn pause() {
let mut stdin = std::io::stdin();
let mut stdout = std::io::stdout();
write!(stdout, "Press any key to continue...").unwrap();
stdout.flush().unwrap();
let _ = stdin.read(&mut [0u8]).unwrap();
}
indexer/src/main.rs
+8
View File
@@ -8,6 +8,14 @@ use storable_vec::CACHED_GETS;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
logger::init_log(None);
rlimit::setrlimit(
rlimit::Resource::NOFILE,
21_000,
rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
)?;
let data_dir = Path::new("../../bitcoin");
let rpc = rpc::Client::new(
"http://localhost:8332",
indexer/src/storage/storable_vecs/mod.rs
+45 -37
View File
@@ -5,9 +5,11 @@ use rayon::prelude::*;
use storable_vec::{Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Height, LockTime, P2PK33AddressBytes,
P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes,
Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Weight,
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Emptyindex, Height, LockTime,
Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes,
P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex,
P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex,
Unknownindex, Weight,
};
mod base;
@@ -21,31 +23,31 @@ pub struct StorableVecs<const MODE: u8> {
pub height_to_blockhash: StorableVec<Height, BlockHash, MODE>,
pub height_to_difficulty: StorableVec<Height, f64, MODE>,
pub height_to_first_addressindex: StorableVec<Height, Addressindex, MODE>,
pub height_to_first_emptyindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_multisigindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_opreturnindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_pushonlyindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_emptyindex: StorableVec<Height, Emptyindex, MODE>,
pub height_to_first_multisigindex: StorableVec<Height, Multisigindex, MODE>,
pub height_to_first_opreturnindex: StorableVec<Height, Opreturnindex, MODE>,
pub height_to_first_pushonlyindex: StorableVec<Height, Pushonlyindex, MODE>,
pub height_to_first_txindex: StorableVec<Height, Txindex, MODE>,
pub height_to_first_txinindex: StorableVec<Height, Txinindex, MODE>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex, MODE>,
pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_unknownindex: StorableVec<Height, Unknownindex, MODE>,
pub height_to_first_p2pk33index: StorableVec<Height, P2PK33index, MODE>,
pub height_to_first_p2pk65index: StorableVec<Height, P2PK65index, MODE>,
pub height_to_first_p2pkhindex: StorableVec<Height, P2PKHindex, MODE>,
pub height_to_first_p2shindex: StorableVec<Height, P2SHindex, MODE>,
pub height_to_first_p2trindex: StorableVec<Height, P2TRindex, MODE>,
pub height_to_first_p2wpkhindex: StorableVec<Height, P2WPKHindex, MODE>,
pub height_to_first_p2wshindex: StorableVec<Height, P2WSHindex, MODE>,
pub height_to_size: StorableVec<Height, usize, MODE>,
pub height_to_timestamp: StorableVec<Height, Timestamp, MODE>,
pub height_to_weight: StorableVec<Height, Weight, MODE>,
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes, MODE>,
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes, MODE>,
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes, MODE>,
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes, MODE>,
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes, MODE>,
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes, MODE>,
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes, MODE>,
pub p2pk33index_to_p2pk33addressbytes: StorableVec<P2PK33index, P2PK33AddressBytes, MODE>,
pub p2pk65index_to_p2pk65addressbytes: StorableVec<P2PK65index, P2PK65AddressBytes, MODE>,
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<P2PKHindex, P2PKHAddressBytes, MODE>,
pub p2shindex_to_p2shaddressbytes: StorableVec<P2SHindex, P2SHAddressBytes, MODE>,
pub p2trindex_to_p2traddressbytes: StorableVec<P2TRindex, P2TRAddressBytes, MODE>,
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<P2WPKHindex, P2WPKHAddressBytes, MODE>,
pub p2wshindex_to_p2wshaddressbytes: StorableVec<P2WSHindex, P2WSHAddressBytes, MODE>,
pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
@@ -57,7 +59,7 @@ pub struct StorableVecs<const MODE: u8> {
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount, MODE>,
}
// const UNSAFE_BLOCKS: usize = 100;
// const UNSAFE_BLOCKS: usize = 1000;
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
@@ -407,37 +409,37 @@ impl StorableVecs<CACHED_GETS> {
Ok(match addresstype {
Addresstype::P2PK65 => self
.p2pk65index_to_p2pk65addressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PK33 => self
.p2pk33index_to_p2pk33addressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PKH => self
.p2pkhindex_to_p2pkhaddressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2SH => self
.p2shindex_to_p2shaddressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WPKH => self
.p2wpkhindex_to_p2wpkhaddressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WSH => self
.p2wshindex_to_p2wshaddressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2TR => self
.p2trindex_to_p2traddressbytes
.get(addresstypeindex)?
.get(addresstypeindex.into())?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
_ => unreachable!(),
@@ -450,13 +452,19 @@ impl StorableVecs<CACHED_GETS> {
addressbytes: Addressbytes,
) -> storable_vec::Result<()> {
match addressbytes {
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK65(bytes) => self
.p2pk65index_to_p2pk65addressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2PK33(bytes) => self
.p2pk33index_to_p2pk33addressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2WPKH(bytes) => self
.p2wpkhindex_to_p2wpkhaddressbytes
.push_if_needed(index.into(), bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index.into(), bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index.into(), bytes),
}
}
}
indexer/src/structs/addressbytes.rs
+16 -15
View File
@@ -1,6 +1,7 @@
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use iterator::bitcoin::ScriptBuf;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Addresstype;
@@ -123,28 +124,28 @@ impl From<P2TRAddressBytes> for Addressbytes {
}
}
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2PK65AddressBytes(U8x65);
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2PK33AddressBytes(U8x33);
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2PKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2SHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2WPKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2WSHAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct P2TRAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct U8x20([u8; 20]);
impl From<&[u8]> for U8x20 {
fn from(slice: &[u8]) -> Self {
@@ -154,7 +155,7 @@ impl From<&[u8]> for U8x20 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct U8x32([u8; 32]);
impl From<&[u8]> for U8x32 {
fn from(slice: &[u8]) -> Self {
@@ -164,8 +165,8 @@ impl From<&[u8]> for U8x32 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x33([u8; 33]);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct U8x33(#[serde(with = "serde_bytes")] [u8; 33]);
impl From<&[u8]> for U8x33 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 33];
@@ -174,8 +175,8 @@ impl From<&[u8]> for U8x33 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x64([u8; 64]);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct U8x64(#[serde(with = "serde_bytes")] [u8; 64]);
impl From<&[u8]> for U8x64 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 64];
@@ -184,8 +185,8 @@ impl From<&[u8]> for U8x64 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x65([u8; 65]);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct U8x65(#[serde(with = "serde_bytes")] [u8; 65]);
impl From<&[u8]> for U8x65 {
fn from(slice: &[u8]) -> Self {
let mut arr = [0; 65];
indexer/src/structs/addressindex.rs
+2
View File
@@ -2,6 +2,7 @@ use std::ops::Add;
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
@@ -19,6 +20,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Addressindex(u32);
indexer/src/structs/addresstype.rs
+4 -1
View File
@@ -1,7 +1,10 @@
use iterator::bitcoin::ScriptBuf;
use serde::Serialize;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromBytes, Immutable, IntoBytes, KnownLayout)]
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
)]
#[repr(u8)]
pub enum Addresstype {
P2PK65,
indexer/src/structs/addresstypeindex.rs
+483 -1
View File
@@ -1,6 +1,7 @@
use std::ops::Add;
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
@@ -18,6 +19,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Addresstypeindex(u32);
@@ -34,7 +36,7 @@ impl Addresstypeindex {
Self(*self + 1)
}
pub fn clone_then_increment(&mut self) -> Self {
pub fn copy_then_increment(&mut self) -> Self {
let i = *self;
self.increment();
i
@@ -82,3 +84,483 @@ impl Add<Addresstypeindex> for Addresstypeindex {
Self(self.0 + rhs.0)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Emptyindex(Addresstypeindex);
impl From<Addresstypeindex> for Emptyindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<Emptyindex> for usize {
fn from(value: Emptyindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for Emptyindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for Emptyindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Multisigindex(Addresstypeindex);
impl From<Addresstypeindex> for Multisigindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<Multisigindex> for usize {
fn from(value: Multisigindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for Multisigindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for Multisigindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Opreturnindex(Addresstypeindex);
impl From<Addresstypeindex> for Opreturnindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<Opreturnindex> for usize {
fn from(value: Opreturnindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for Opreturnindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for Opreturnindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Pushonlyindex(Addresstypeindex);
impl From<Addresstypeindex> for Pushonlyindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<Pushonlyindex> for usize {
fn from(value: Pushonlyindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for Pushonlyindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for Pushonlyindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Unknownindex(Addresstypeindex);
impl From<Addresstypeindex> for Unknownindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<Unknownindex> for usize {
fn from(value: Unknownindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for Unknownindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for Unknownindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2PK33index(Addresstypeindex);
impl From<Addresstypeindex> for P2PK33index {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2PK33index> for usize {
fn from(value: P2PK33index) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2PK33index {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2PK33index {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2PK65index(Addresstypeindex);
impl From<Addresstypeindex> for P2PK65index {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2PK65index> for usize {
fn from(value: P2PK65index) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2PK65index {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2PK65index {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2PKHindex(Addresstypeindex);
impl From<Addresstypeindex> for P2PKHindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2PKHindex> for usize {
fn from(value: P2PKHindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2PKHindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2PKHindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2SHindex(Addresstypeindex);
impl From<Addresstypeindex> for P2SHindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2SHindex> for usize {
fn from(value: P2SHindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2SHindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2SHindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2TRindex(Addresstypeindex);
impl From<Addresstypeindex> for P2TRindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2TRindex> for usize {
fn from(value: P2TRindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2TRindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2TRindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2WPKHindex(Addresstypeindex);
impl From<Addresstypeindex> for P2WPKHindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2WPKHindex> for usize {
fn from(value: P2WPKHindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2WPKHindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2WPKHindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct P2WSHindex(Addresstypeindex);
impl From<Addresstypeindex> for P2WSHindex {
fn from(value: Addresstypeindex) -> Self {
Self(value)
}
}
impl From<P2WSHindex> for usize {
fn from(value: P2WSHindex) -> Self {
Self::from(*value)
}
}
impl From<usize> for P2WSHindex {
fn from(value: usize) -> Self {
Self(Addresstypeindex::from(value))
}
}
impl Add<usize> for P2WSHindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(*self + rhs)
}
}
indexer/src/structs/amount.rs
+2
View File
@@ -5,6 +5,7 @@ use std::{
use derive_deref::{Deref, DerefMut};
use iterator::bitcoin;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
@@ -24,6 +25,7 @@ use super::Height;
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Amount(u64);
indexer/src/structs/blockhash.rs
+2 -1
View File
@@ -1,9 +1,10 @@
use std::mem;
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct BlockHash([u8; 32]);
impl From<bitcoin::BlockHash> for BlockHash {
indexer/src/structs/height.rs
+2
View File
@@ -7,6 +7,7 @@ use std::{
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use iterator::rpc::{self, RpcApi};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
@@ -24,6 +25,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Height(u32);
indexer/src/structs/locktime.rs
+2 -1
View File
@@ -1,8 +1,9 @@
use serde::Serialize;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
use super::{Height, Timestamp};
#[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes)]
#[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)]
#[repr(C)]
pub enum LockTime {
Height(Height),
indexer/src/structs/timestamp.rs
+4 -1
View File
@@ -1,7 +1,10 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
#[derive(
Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
)]
pub struct Timestamp(u32);
impl From<u32> for Timestamp {
indexer/src/structs/txid.rs
+2 -1
View File
@@ -1,9 +1,10 @@
use std::mem;
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct Txid([u8; 32]);
impl From<bitcoin::Txid> for Txid {
indexer/src/structs/txindex.rs
+2
View File
@@ -2,6 +2,7 @@ use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
@@ -19,6 +20,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Txindex(u32);
indexer/src/structs/txinindex.rs
+2
View File
@@ -1,6 +1,7 @@
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Vin;
@@ -20,6 +21,7 @@ use super::Vin;
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Txinindex(u64);
indexer/src/structs/txoutindex.rs
+2
View File
@@ -1,6 +1,7 @@
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Vout;
@@ -20,6 +21,7 @@ use super::Vout;
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Txoutindex(u64);
indexer/src/structs/txversion.rs
+2 -1
View File
@@ -1,7 +1,8 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct TxVersion(i32);
impl From<bitcoin::transaction::Version> for TxVersion {
indexer/src/structs/weight.rs
+2 -1
View File
@@ -1,7 +1,8 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes)]
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
pub struct Weight(u64);
impl From<bitcoin::Weight> for Weight {
iterator/src/lib.rs
+6 -15
View File
@@ -32,7 +32,7 @@ use blk_metadata::*;
use blk_metadata_and_block::*;
use utils::*;
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 1000;
const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
const BOUND_CAP: usize = 210;
@@ -171,10 +171,7 @@ pub fn new(
_ => unreachable!(),
};
if send_block
.send(BlkMetadataAndBlock::new(blk_metadata, block))
.is_err()
{
if send_block.send(BlkMetadataAndBlock::new(blk_metadata, block)).is_err() {
return ControlFlow::Break(());
}
@@ -242,16 +239,13 @@ pub fn new(
let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default();
let mut recent_hashes: BTreeSet<BlockHash> = BTreeSet::default();
let mut prev_hash =
start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash());
let mut prev_hash = start_recap.map_or_else(BlockHash::all_zeros, |(_, recap)| *recap.prev_hash());
let mut prepare_and_send = |(hash, tuple): (BlockHash, BlkMetadataAndBlock)| {
blk_index_to_blk_recap.update(&tuple, height);
if start.map_or(true, |start| start <= height) {
send_height_block_hash
.send((height, tuple.block, hash))
.unwrap();
send_height_block_hash.send((height, tuple.block, hash)).unwrap();
}
if end == Some(height) {
@@ -294,8 +288,7 @@ pub fn new(
let flow = recv_block.iter().try_for_each(|tuple| {
// block isn't next after current tip
if prev_hash != tuple.block.header.prev_blockhash {
let is_block_active =
|hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0;
let is_block_active = |hash| rpc.get_block_header_info(hash).unwrap().confirmations > 0;
// block prev has already been processed
if recent_hashes.contains(&tuple.block.header.prev_blockhash) {
@@ -322,9 +315,7 @@ pub fn new(
);
}
// Check if there was already a future block with the same prev hash
} else if let Some(prev_tuple) =
future_blocks.insert(tuple.block.header.prev_blockhash, tuple)
{
} else if let Some(prev_tuple) = future_blocks.insert(tuple.block.header.prev_blockhash, tuple) {
// If the previous was the active one
if is_block_active(&prev_tuple.block.block_hash()) {
// Rollback the insert
logger/Cargo.toml
+10
View File
@@ -0,0 +1,10 @@
[package]
name = "logger"
version = "0.1.0"
edition = "2021"
[dependencies]
color-eyre = { workspace = true }
env_logger = "0.11.6"
jiff = { workspace = true }
log = { version = "0.4.25" }
logger/src/lib.rs
+62
View File
@@ -0,0 +1,62 @@
use std::{
fmt::Display,
fs::{self, OpenOptions},
io::Write,
path::Path,
};
use color_eyre::owo_colors::OwoColorize;
use env_logger::{Builder, Env};
use jiff::{tz, Timestamp};
pub use log::{debug, error, info, trace, warn};
#[inline(always)]
pub fn init_log(path: Option<&Path>) {
let file = path.map(|path| {
let _ = fs::remove_file(path);
OpenOptions::new().create(true).append(true).open(path).unwrap()
});
Builder::from_env(Env::default().default_filter_or("info,fjall=off,lsm_tree=off"))
.format(move |buf, record| {
let date_time = Timestamp::now()
.to_zoned(tz::TimeZone::system())
.strftime("%Y-%m-%d %H:%M:%S")
.to_string();
let level = record.level().as_str().to_lowercase();
let level = format!("{:5}", level);
let target = record.target();
let dash = "-";
let args = record.args();
if let Some(file) = file.as_ref() {
let _ = write(file.try_clone().unwrap(), &date_time, target, &level, dash, args);
}
let colored_date_time = date_time.bright_black();
let colored_level = match level.chars().next().unwrap() {
'e' => level.red().to_string(),
'w' => level.yellow().to_string(),
'i' => level.green().to_string(),
'd' => level.blue().to_string(),
't' => level.cyan().to_string(),
_ => panic!(),
};
let colored_dash = dash.bright_black();
write(buf, colored_date_time, target, colored_level, colored_dash, args)
})
.init();
}
fn write(
mut buf: impl Write,
date_time: impl Display,
_target: impl Display,
level: impl Display,
dash: impl Display,
args: impl Display,
) -> Result<(), std::io::Error> {
writeln!(buf, "{} {} {} {}", date_time, dash, level, args)
// writeln!(buf, "{} {} {} {} {}", date_time, _target, level, dash, args)
}
logger/src/main.rs
+7
View File
@@ -0,0 +1,7 @@
use log::info;
use logger::init_log;
fn main() {
init_log(None);
info!("test");
}
server/src/api/mod.rs
+3 -4
View File
@@ -71,6 +71,7 @@ fn req_to_response_res(
AppState { vecs, .. }: AppState,
) -> color_eyre::Result<Response> {
let format = Format::try_from(format).ok();
let indexes = i
.to_lowercase()
.split(",")
@@ -78,13 +79,11 @@ fn req_to_response_res(
.collect::<Vec<_>>();
if indexes.len() > 1 {
return Err(eyre!("Multiple indexes aren't supported yet !"));
return Err(eyre!("Multiple indexes aren't supported"));
} else if indexes.is_empty() {
return Err(eyre!("Bad index(es)"));
return Err(eyre!("Unknown index"));
}
dbg!(format, &indexes, &v, from, to);
let values = v
.to_lowercase()
.split(",")
src/structs/epoch.rs
+1 -1
View File
@@ -9,7 +9,7 @@ impl Epoch {
impl From<Height> for Epoch {
fn from(height: Height) -> Self {
Self(((height.to_usize() / Self::BLOCKS_PER_EPOCH) + 1) as u16)
Self::from(&height)
}
}
storable_vec/src/enums/error.rs
+2
View File
@@ -21,6 +21,7 @@ pub enum Error {
ExpectVecToHaveIndex,
FailedKeyTryIntoUsize,
UnsupportedUnflushedState,
RangeFromAfterTo,
}
impl From<io::Error> for Error {
@@ -60,6 +61,7 @@ impl fmt::Display for Error {
write!(f, "Unsupported unflush state, please flush before using this function")
}
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
Error::RangeFromAfterTo => write!(f, "Range, from is after to"),
}
}
}
storable_vec/src/lib.rs
+70 -25
View File
@@ -37,7 +37,7 @@ pub const SINGLE_THREAD: u8 = 1;
/// Will spin up a new `File` for every read
///
/// Used in `/server`
pub const ASYNC_READ_ONLY: u8 = 2;
pub const STATELESS: u8 = 2;
///
/// A very small, fast, efficient and simple storable Vec
@@ -73,7 +73,7 @@ pub struct StorableVec<I, T, const MODE: u8> {
/// In bytes
const MAX_PAGE_SIZE: usize = 4 * 4096;
const ONE_MB: usize = 1000 * 1024;
const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
const MAX_CACHE_SIZE: usize = usize::MAX;
// const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
@@ -157,6 +157,13 @@ where
.open(path)
}
fn open_file_at_then_read(&self, index: usize) -> Result<T> {
let mut file = self.open_file()?;
Self::seek(&mut file, Self::index_to_byte_index(index))?;
let mut buf = Self::create_buffer();
Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned())
}
fn read_disk_len(&self) -> io::Result<usize> {
Self::read_disk_len_(&self.file)
}
@@ -165,7 +172,6 @@ where
}
fn reset_disk_related_state(&mut self) -> io::Result<()> {
self.file = self.open_file()?;
self.file_len = self.read_disk_len()?;
self.file_position = 0;
self.reset_cache()
@@ -193,13 +199,6 @@ where
}
}
fn open_file_at_then_read(&self, index: usize) -> Result<T> {
let mut file = self.open_file()?;
let mut buf = Self::create_buffer();
Self::seek(&mut file, Self::index_to_byte_index(index))?;
Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned())
}
#[inline]
fn seek(file: &mut File, byte_index: u64) -> io::Result<u64> {
file.seek(SeekFrom::Start(byte_index))
@@ -218,7 +217,7 @@ where
#[inline]
fn push_if_needed_(&mut self, index: I, value: T) -> Result<()> {
match self.pushed_len().cmp(&Self::i_to_usize(index)?) {
match self.len().cmp(&Self::i_to_usize(index)?) {
Ordering::Greater => {
// dbg!(len, index, &self.pathbuf);
// panic!();
@@ -265,22 +264,19 @@ where
}
pub fn flush(&mut self) -> io::Result<()> {
self.reset_disk_related_state()?;
if self.pushed.is_empty() {
return Ok(());
}
self.file_len += self.pushed.len();
let mut bytes: Vec<u8> = vec![];
let mut bytes: Vec<u8> = Vec::with_capacity(self.pushed_len() * Self::SIZE_OF_T);
mem::take(&mut self.pushed)
.into_iter()
.for_each(|v| bytes.extend_from_slice(v.as_bytes()));
self.file.write_all(&bytes)?;
self.reset_disk_related_state()?;
Ok(())
}
@@ -318,6 +314,10 @@ where
}
}
pub fn file_name(&self) -> String {
self.path().file_name().unwrap().to_str().unwrap().to_owned()
}
#[inline]
pub fn path(&self) -> &Path {
&self.pathbuf
@@ -368,7 +368,7 @@ where
let page_index = index / Self::PER_PAGE;
let last_index = self.file_len - 1;
let max_page_index = last_index / Self::PER_PAGE;
let min_page_index = (max_page_index + 1).checked_sub(self.cache.len()).unwrap_or_default();
let min_page_index = (max_page_index + 1) - self.cache.len();
// let min_open_page = self.min.load(AtomicOrdering::SeqCst);
@@ -429,7 +429,6 @@ where
fn get_(&mut self, index: usize) -> Result<&T> {
let byte_index = Self::index_to_byte_index(index);
if self.file_position != byte_index {
println!("seek");
self.file_position = Self::seek(&mut self.file, byte_index)?;
}
let res = Self::read_exact(&mut self.file, &mut self.buf);
@@ -595,7 +594,6 @@ where
&mut self,
first_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
last_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
callback: F,
) -> Result<()>
where
T: From<T2>,
@@ -612,20 +610,67 @@ where
}
}
impl<I, T> StorableVec<I, T, ASYNC_READ_ONLY>
impl<I, T> StorableVec<I, T, STATELESS>
where
I: StorableVecIndex,
T: StorableVecType,
{
#[inline]
pub fn get(&self, index: I) -> Result<Option<T>> {
self.get_(Self::i_to_usize(index)?)
Ok(Some(self.open_file_at_then_read(Self::i_to_usize(index)?)?))
}
#[inline]
fn get_(&self, index: usize) -> Result<Option<T>> {
Ok(Some(self.open_file_at_then_read(index)?.to_owned()))
pub fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
if !self.pushed.is_empty() {
return Err(Error::UnsupportedUnflushedState);
}
let mut file = self.open_file()?;
let len = Self::read_disk_len_(&file)?;
let from = from.map_or(0, |from| {
if from >= 0 {
from as usize
} else {
(len as i64 + from) as usize
}
});
let to = to.map_or(len, |to| {
if to >= 0 {
to as usize
} else {
(len as i64 + to) as usize
}
});
if from >= to {
return Err(Error::RangeFromAfterTo);
}
Self::seek(&mut file, Self::index_to_byte_index(from))?;
let mut buf = Self::create_buffer();
Ok((from..to)
.map(|_| Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned()).unwrap())
.collect::<Vec<_>>())
}
// Add iter iter_from iter_range collect..
// + add memory cap
}
impl<I, T> Clone for StorableVec<I, T, STATELESS>
where
I: StorableVecIndex,
T: StorableVecType,
{
fn clone(&self) -> Self {
let path = &self.pathbuf;
let path_version = Self::path_version_(path);
let version = Version::try_from(path_version.as_path()).unwrap();
Self::import(path, version).unwrap()
}
}