Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-12 16:03:31 -07:00
Code Issues Packages Projects Releases Wiki Activity

workspace: use folder name for packages

Browse Source
This commit is contained in:
nym21
2025-02-05 23:42:48 +01:00
parent d86d614520
commit 5aaa55197e
33 changed files with 3776 additions and 1655 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+3370 -37
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+6 -4
View File
@@ -4,22 +4,24 @@ members = [
"exit",
"indexer",
"iterator",
"server",
"storable_vec",
"struct_iterable",
]
resolver = "2"
[workspace.dependencies]
bindex = { path = "indexer" }
bitcoin = { version = "0.32.5", features = ["serde"] }
biter = { path = "iterator" }
bomputer = { path = "computer" }
color-eyre = "0.6.3"
computer = { path = "computer", package = "bomputer" }
derive_deref = "1.1.1"
exit = { path = "exit" }
fjall = "2.5.0"
indexer = { path = "indexer", package = "bindex" }
iterator = { path = "iterator", package = "biter" }
jiff = "0.1.29"
rayon = "1.10.0"
server = { path = "server" }
storable_vec = { path = "storable_vec" }
struct_iterable = { path = "struct_iterable" }
zerocopy = { version = "0.8.15", features = ["derive"] }
zerocopy = { version = "0.8.16", features = ["derive"] }
computer/Cargo.toml
+2 -2
View File
@@ -4,12 +4,12 @@ version = "0.1.0"
edition = "2021"
[dependencies]
bindex = { workspace = true }
biter = { workspace = true }
iterator = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
exit = { workspace = true }
fjall = { workspace = true }
indexer = { workspace = true }
jiff = { workspace = true }
storable_vec = { workspace = true }
zerocopy = { workspace = true }
computer/src/lib.rs
+6 -6
View File
@@ -1,8 +1,8 @@
use std::path::{Path, PathBuf};
use bindex::Indexer;
use biter::rpc;
use exit::Exit;
use indexer::Indexer;
use iterator::rpc;
mod storage;
mod structs;
@@ -89,10 +89,10 @@ impl Computer<SINGLE_THREAD> {
&mut indexer.vecs.height_to_first_txindex,
)?;
self.vecs.txindex_to_fee.compute_transform(
&mut self.vecs.txindex_to_height,
&mut indexer.vecs.height_to_first_txindex,
)?;
// self.vecs.txindex_to_fee.compute_transform(
// &mut self.vecs.txindex_to_height,
// &mut indexer.vecs.height_to_first_txindex,
// )?;
let date_count = self.vecs.height_to_date.len();
computer/src/main.rs
+1 -1
View File
@@ -1,8 +1,8 @@
use std::path::Path;
use biter::rpc;
use bomputer::Computer;
use exit::Exit;
use iterator::rpc;
use storable_vec::SINGLE_THREAD;
mod structs;
computer/src/storage/fjalls.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::path::Path;
use bindex::Store;
use indexer::Store;
use storable_vec::Version;
use crate::structs::{AddressindexTxoutindex, Unit};
computer/src/storage/storable_vecs.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::{fs, path::Path};
use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use indexer::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use storable_vec::{StorableVec, Version};
use crate::structs::{Date, Feerate};
computer/src/structs/addressindextxoutindex.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use bindex::{Addressindex, Txoutindex};
use fjall::Slice;
use indexer::{Addressindex, Txoutindex};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)]
computer/src/structs/date.rs
+1 -1
View File
@@ -1,7 +1,7 @@
use std::ops::Add;
use bindex::Timestamp;
use color_eyre::eyre::eyre;
use indexer::Timestamp;
use jiff::{civil::Date as Date_, tz::TimeZone, Span};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
indexer/Cargo.lock
Generated
-1065
View File
File diff suppressed because it is too large Load Diff
indexer/Cargo.toml
+1 -1
View File
@@ -5,11 +5,11 @@ edition = "2021"
[dependencies]
bitcoin = { workspace = true }
biter = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
exit = { workspace = true }
fjall = { workspace = true }
iterator = { workspace = true }
jiff = { workspace = true }
rapidhash = "1.3.0"
rayon = { workspace = true }
indexer/src/lib.rs
+2 -2
View File
@@ -7,7 +7,7 @@ use std::{
time::Duration,
};
pub use biter::*;
pub use iterator::*;
use bitcoin::{Transaction, TxIn, TxOut};
use color_eyre::eyre::{eyre, ContextCompat};
@@ -93,7 +93,7 @@ impl Indexer<CACHED_GETS> {
Ok(())
};
biter::new(bitcoin_dir, Some(height.into()), None, rpc)
iterator::new(bitcoin_dir, Some(height.into()), None, rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
indexer/src/main.rs
+1 -1
View File
@@ -1,8 +1,8 @@
use std::path::Path;
use bindex::Indexer;
use biter::rpc;
use exit::Exit;
use iterator::rpc;
use storable_vec::CACHED_GETS;
fn main() -> color_eyre::Result<()> {
indexer/src/structs/addressbytes.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use iterator::bitcoin::ScriptBuf;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Addresstype;
indexer/src/structs/addresstype.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use biter::bitcoin::ScriptBuf;
use iterator::bitcoin::ScriptBuf;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromBytes, Immutable, IntoBytes, KnownLayout)]
indexer/src/structs/amount.rs
+1 -1
View File
@@ -3,8 +3,8 @@ use std::{
ops::{Add, AddAssign, Mul, Sub, SubAssign},
};
use biter::bitcoin;
use derive_deref::{Deref, DerefMut};
use iterator::bitcoin;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
indexer/src/structs/height.rs
+1 -1
View File
@@ -4,9 +4,9 @@ use std::{
path::Path,
};
use biter::rpc::{self, RpcApi};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use iterator::rpc::{self, RpcApi};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(
server/Cargo.toml
+9 -9
View File
@@ -5,7 +5,7 @@ edition = "2021"
[dependencies]
# allocative = "0.3.4"
axum = "0.7.9"
axum = "0.8.1"
# bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
# "serde",
# ] }
@@ -13,7 +13,7 @@ axum = "0.7.9"
# biter = { path = "./src/crates/biter" }
# chrono = { version = "0.4.39", features = ["serde"] }
# clap = { version = "4.5.26", features = ["derive"] }
# color-eyre = "0.6.3"
color-eyre = { workspace = true }
# ctrlc = { version = "3.4.5", features = ["termination"] }
# derive_deref = "1.1.1"
# env_logger = "0.11.6"
@@ -22,16 +22,16 @@ axum = "0.7.9"
# log = { version = "0.4.25", features = ["std", "serde"] }
# ordered-float = "4.6.0"
# rayon = "1.10.0"
# regex = "1.11.1"
# reqwest = { version = "0.12.12", features = ["blocking", "json"] }
regex = "1.11.1"
reqwest = { version = "0.12.12", features = ["blocking", "json"] }
# rlimit = "0.10.2"
# snkrj = { path = "./src/crates/snkrj" }
# serde = { version = "1.0.217", features = ["derive"] }
# serde_json = "1.0.135"
# struct_iterable = { path = "./src/crates/iterable" }
# swc = "9.0.2"
# swc_common = "5.0.0"
# tokio = { version = "1.43.0", features = ["full"] }
swc = "13.0.1"
swc_common = "6.0.0"
tokio = { version = "1.43.0", features = ["full"] }
# toml = "0.8.19"
# tower-http = { version = "0.6.2", features = ["compression-full"] }
# zstd = "0.13.2"
tower-http = { version = "0.6.2", features = ["compression-full"] }
zstd = "0.13.2"
server/src/website/handlers/minify.rs
+8 -15
View File
@@ -6,24 +6,17 @@ use swc::{config::JsMinifyOptions, try_with_handler, JsMinifyExtras};
use swc_common::{SourceMap, GLOBALS};
pub fn minify_js(path: &Path) -> String {
let cm = Arc::<SourceMap>::default();
let source_map = Arc::<SourceMap>::default();
let compiler = swc::Compiler::new(source_map.clone());
let c = swc::Compiler::new(cm.clone());
let output = GLOBALS
GLOBALS
.set(&Default::default(), || {
try_with_handler(cm.clone(), Default::default(), |handler| {
let fm = cm.load_file(path).expect("failed to load file");
try_with_handler(source_map.clone(), Default::default(), |handler| {
let fm = source_map.load_file(path).expect("failed to load file");
c.minify(
fm,
handler,
&JsMinifyOptions::default(),
JsMinifyExtras::default(),
)
compiler.minify(fm, handler, &JsMinifyOptions::default(), JsMinifyExtras::default())
})
})
.unwrap();
output.code
.unwrap()
.code
}
src/main.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use biter::bitcoincore_rpc::Client;
use iterator::bitcoincore_rpc::Client;
use log::info;
use rlimit::{getrlimit, setrlimit, Resource};
src/parser/actions/iter_blocks.rs
+14 -25
View File
@@ -1,8 +1,8 @@
use std::{collections::BTreeSet, time::Instant};
use biter::bitcoincore_rpc::Client;
use chrono::Datelike;
use export::ExportedData;
use iterator::bitcoincore_rpc::Client;
use itertools::Itertools;
use log::info;
@@ -31,8 +31,7 @@ pub fn iter_blocks(
info!("Imported states");
let first_unsafe_heights =
find_first_inserted_unsafe_height(&mut states, databases, datasets, config);
let first_unsafe_heights = find_first_inserted_unsafe_height(&mut states, databases, datasets, config);
let mut height = first_unsafe_heights.min();
@@ -69,9 +68,7 @@ pub fn iter_blocks(
next_block_opt = block_iter.next();
if let Some((_current_block_height, current_block, _current_block_hash)) =
current_block_opt
{
if let Some((_current_block_height, current_block, _current_block_hash)) = current_block_opt {
let timestamp = Timestamp::from(current_block.header.time);
let current_block_date = timestamp.to_date();
@@ -82,9 +79,9 @@ pub fn iter_blocks(
panic!()
}
next_date_opt = next_block_opt.as_ref().map(|(_, next_block, _)| {
Timestamp::from(next_block.header.time).to_date()
});
next_date_opt = next_block_opt
.as_ref()
.map(|(_, next_block, _)| Timestamp::from(next_block.header.time).to_date());
// Always run for the first block of the loop
if blocks_loop_date.is_none() {
@@ -96,9 +93,7 @@ pub fn iter_blocks(
.map(|date_data| *date_data.date < *current_block_date)
.unwrap_or(true)
{
states
.date_data_vec
.push(DateData::new(current_block_date, vec![]));
states.date_data_vec.push(DateData::new(current_block_date, vec![]));
}
processed_dates.insert(current_block_date);
@@ -117,10 +112,8 @@ pub fn iter_blocks(
processed_heights.insert(current_block_height);
if first_unsafe_heights.inserted <= current_block_height {
let compute_addresses = databases.check_if_needs_to_compute_addresses(
current_block_height,
blocks_loop_date,
);
let compute_addresses =
databases.check_if_needs_to_compute_addresses(current_block_height, blocks_loop_date);
if states.address_cohorts_durable_states.is_none()
&& (compute_addresses
@@ -128,10 +121,9 @@ pub fn iter_blocks(
.address
.needs_durable_states(current_block_height, current_block_date))
{
states.address_cohorts_durable_states =
Some(AddressCohortsDurableStates::init(
&mut databases.address_index_to_address_data,
));
states.address_cohorts_durable_states = Some(AddressCohortsDurableStates::init(
&mut databases.address_index_to_address_data,
));
}
if states.utxo_cohorts_durable_states.is_none()
@@ -169,9 +161,7 @@ pub fn iter_blocks(
height += blocks_loop_i;
let is_check_point = next_date_opt
.as_ref()
.map_or(true, |date| date.is_first_of_month());
let is_check_point = next_date_opt.as_ref().map_or(true, |date| date.is_first_of_month());
if (is_check_point && instant.elapsed().as_secs() >= 1)
|| height.is_close_to_end(approx_block_count)
@@ -209,8 +199,7 @@ pub fn iter_blocks(
let defragment = is_safe
&& next_date_opt.is_some_and(|date| {
(date.year() >= 2020 && date.is_january()
|| date.year() >= 2022 && date.is_july())
(date.year() >= 2020 && date.is_january() || date.year() >= 2022 && date.is_july())
&& date.is_first_of_month()
});
src/parser/actions/parse.rs
+315 -395
View File
@@ -1,6 +1,6 @@
use std::{collections::BTreeMap, ops::ControlFlow, thread};
use biter::{
use iterator::{
bitcoin::{Block, Txid},
bitcoincore_rpc::RpcApi,
};
@@ -11,19 +11,18 @@ use rayon::prelude::*;
use crate::{
parser::{
databases::{
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex,
Databases, TxidToTxData, TxoutIndexToAddressIndex, TxoutIndexToAmount,
AddressIndexToAddressData, AddressIndexToEmptyAddressData, AddressToAddressIndex, Databases, TxidToTxData,
TxoutIndexToAddressIndex, TxoutIndexToAmount,
},
datasets::{Datasets, InsertData},
states::{
AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates,
States, UTXOCohortsOneShotStates, UTXOCohortsSentStates,
AddressCohortsInputStates, AddressCohortsOutputStates, AddressCohortsRealizedStates, States,
UTXOCohortsOneShotStates, UTXOCohortsSentStates,
},
},
structs::{
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Config, Counter,
Date, EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData,
TxoutIndex,
Address, AddressData, AddressRealizedData, Amount, BlockData, BlockPath, Config, Counter, Date,
EmptyAddressData, Height, PartialTxoutData, Price, SentData, Timestamp, TxData, TxoutIndex,
},
};
@@ -111,8 +110,7 @@ pub fn parse(
let mut block_path_to_sent_data: BTreeMap<BlockPath, SentData> = BTreeMap::default();
// let mut received_data: ReceivedData = ReceivedData::default();
let mut address_index_to_address_realized_data: BTreeMap<u32, AddressRealizedData> =
BTreeMap::default();
let mut address_index_to_address_realized_data: BTreeMap<u32, AddressRealizedData> = BTreeMap::default();
let mut coinbase = Amount::ZERO;
let mut satblocks_destroyed = Amount::ZERO;
@@ -171,261 +169,228 @@ pub fn parse(
)
});
block
.txdata
.iter()
.enumerate()
.try_for_each(|(block_tx_index, tx)| {
let txid = tx.compute_txid();
let tx_index = databases.txid_to_tx_data.metadata.serial as u32;
block.txdata.iter().enumerate().try_for_each(|(block_tx_index, tx)| {
let txid = tx.compute_txid();
let tx_index = databases.txid_to_tx_data.metadata.serial as u32;
transaction_count += 1;
transaction_count += 1;
// --
// outputs
// ---
// --
// outputs
// ---
let mut utxos = BTreeMap::new();
let mut spendable_amount = Amount::ZERO;
let mut utxos = BTreeMap::new();
let mut spendable_amount = Amount::ZERO;
let is_coinbase = tx.is_coinbase();
let is_coinbase = tx.is_coinbase();
if is_coinbase != (block_tx_index == 0) {
unreachable!();
}
if is_coinbase != (block_tx_index == 0) {
unreachable!();
}
let mut inputs_sum = Amount::ZERO;
let mut outputs_sum = Amount::ZERO;
let mut inputs_sum = Amount::ZERO;
let mut outputs_sum = Amount::ZERO;
let last_block = states.date_data_vec.last_mut_block().unwrap();
let last_block = states.date_data_vec.last_mut_block().unwrap();
// Before `input` to cover outputs being used in the same block as inputs
tx.output
.iter()
.enumerate()
.filter_map(|(vout, tx_out)| {
if vout > (u16::MAX as usize) {
panic!("vout can indeed be bigger than u16::MAX !");
}
// Before `input` to cover outputs being used in the same block as inputs
tx.output
.iter()
.enumerate()
.filter_map(|(vout, tx_out)| {
if vout > (u16::MAX as usize) {
panic!("vout can indeed be bigger than u16::MAX !");
}
let amount = Amount::wrap(tx_out.value);
let amount = Amount::wrap(tx_out.value);
if is_coinbase {
coinbase += amount;
} else {
outputs_sum += amount;
}
if is_coinbase {
coinbase += amount;
} else {
outputs_sum += amount;
}
partial_txout_data_vec
.pop()
.unwrap()
// None if not worth parsing (empty/op_return/...)
.map(|partial_txout_data| (vout, partial_txout_data))
})
.for_each(|(vout, partial_txout_data)| {
let vout = vout as u16;
partial_txout_data_vec
.pop()
.unwrap()
// None if not worth parsing (empty/op_return/...)
.map(|partial_txout_data| (vout, partial_txout_data))
})
.for_each(|(vout, partial_txout_data)| {
let vout = vout as u16;
let txout_index = TxoutIndex::new(tx_index, vout);
let txout_index = TxoutIndex::new(tx_index, vout);
let PartialTxoutData {
address,
address_index_opt,
amount,
} = partial_txout_data;
let PartialTxoutData {
address,
address_index_opt,
amount,
} = partial_txout_data;
spendable_amount += amount;
spendable_amount += amount;
last_block.receive(amount);
last_block.receive(amount);
utxos.insert(vout, amount);
utxos.insert(vout, amount);
databases.txout_index_to_amount.insert_to_ram(txout_index, amount);
if compute_addresses {
let address = address.unwrap();
let address_index_to_address_data = address_index_to_address_data.as_mut().unwrap();
let (address_data, address_index) = {
if let Some(address_index) = address_index_opt
.or_else(|| databases.address_to_address_index.get_from_ram(&address).cloned())
{
let address_data = address_index_to_address_data.get_mut(&address_index).unwrap();
(address_data, address_index)
} else {
let address_index = databases.address_to_address_index.metadata.serial as u32;
let address_type = address.to_type();
if let Some(previous) = databases.address_to_address_index.insert(address, address_index) {
dbg!(previous);
panic!("address #{address_index} shouldn't be present during put");
}
// Checked new
let address_data = address_index_to_address_data
.entry(address_index)
.and_modify(|_| {
panic!("Shouldn't exist");
})
// Will always insert, it's to avoid insert + get
.or_insert(AddressData::new(address_type));
(address_data, address_index)
}
};
// MUST be before received !
let address_realized_data = address_index_to_address_realized_data
.entry(address_index)
.or_insert_with(|| AddressRealizedData::default(address_data));
address_data.receive(amount, block_price);
address_realized_data.receive(amount);
databases
.txout_index_to_amount
.insert_to_ram(txout_index, amount);
.txout_index_to_address_index
.insert_to_ram(txout_index, address_index);
}
});
if compute_addresses {
let address = address.unwrap();
if !utxos.is_empty() {
databases.txid_to_tx_data.insert(
&txid,
TxData::new(
tx_index,
BlockPath::new(date_index as u16, block_index as u16),
utxos.len() as u16,
),
);
}
let address_index_to_address_data =
address_index_to_address_data.as_mut().unwrap();
// ---
// inputs
// ---
let (address_data, address_index) = {
if let Some(address_index) = address_index_opt.or_else(|| {
databases
.address_to_address_index
.get_from_ram(&address)
.cloned()
}) {
let address_data = address_index_to_address_data
.get_mut(&address_index)
.unwrap();
if !is_coinbase {
tx.input.iter().try_for_each(|txin| {
let outpoint = txin.previous_output;
let input_txid = outpoint.txid;
let input_vout = outpoint.vout;
(address_data, address_index)
} else {
let address_index =
databases.address_to_address_index.metadata.serial as u32;
let remove_tx_data_from_cached_puts = {
let mut is_tx_data_from_cached_puts = false;
let address_type = address.to_type();
let input_tx_data = txid_to_tx_data.get_mut(&input_txid).unwrap().as_mut().or_else(|| {
is_tx_data_from_cached_puts = true;
if let Some(previous) = databases
.address_to_address_index
.insert(address, address_index)
{
dbg!(previous);
panic!(
"address #{address_index} shouldn't be present during put"
);
}
databases.txid_to_tx_data.get_mut_from_ram(&input_txid)
});
// Checked new
let address_data = address_index_to_address_data
.entry(address_index)
.and_modify(|_| {
panic!("Shouldn't exist");
})
// Will always insert, it's to avoid insert + get
.or_insert(AddressData::new(address_type));
// Can be none because 0 sats inputs happen
// https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
(address_data, address_index)
}
};
if input_tx_data.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| rpc
.get_raw_transaction(&input_txid, None)
.unwrap()
.output
.get(input_vout as usize)
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
// MUST be before received !
let address_realized_data = address_index_to_address_realized_data
.entry(address_index)
.or_insert_with(|| AddressRealizedData::default(address_data));
address_data.receive(amount, block_price);
address_realized_data.receive(amount);
databases
.txout_index_to_address_index
.insert_to_ram(txout_index, address_index);
dbg!((input_txid, txid, tx_index, input_vout));
panic!("Txid to be in txid_to_tx_data");
}
});
if !utxos.is_empty() {
databases.txid_to_tx_data.insert(
&txid,
TxData::new(
tx_index,
BlockPath::new(date_index as u16, block_index as u16),
utxos.len() as u16,
),
);
}
let input_tx_data = input_tx_data.unwrap();
let input_tx_index = input_tx_data.index;
let input_vout = input_vout as u16;
let input_txout_index = TxoutIndex::new(input_tx_index, input_vout);
// ---
// inputs
// ---
// if input_tx_index == 2516 || input_tx_index == 2490 {
// dbg!(input_tx_index, &input_tx_data.utxos);
// }
if !is_coinbase {
tx.input.iter().try_for_each(|txin| {
let outpoint = txin.previous_output;
let input_txid = outpoint.txid;
let input_vout = outpoint.vout;
// let input_amount = input_tx_data.utxos.remove(&input_vout);
let remove_tx_data_from_cached_puts = {
let mut is_tx_data_from_cached_puts = false;
let input_amount_and_address_index = databases
.txout_index_to_amount
.remove(&input_txout_index)
.map(|amount| {
(
amount,
databases.txout_index_to_address_index.remove(&input_txout_index),
)
}) // Remove from cached puts
.or_else(|| txout_index_to_amount_and_address_index.remove(&input_txout_index));
let input_tx_data = txid_to_tx_data
.get_mut(&input_txid)
.unwrap()
.as_mut()
.or_else(|| {
is_tx_data_from_cached_puts = true;
databases.txid_to_tx_data.get_mut_from_ram(&input_txid)
});
// Can be none because 0 sats inputs happen
// https://mempool.space/tx/f329e55c2de9b821356e6f2c4bba923ea7030cad61120f5ced5d4429f5c86fda#vin=27
if input_tx_data.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| rpc
.get_raw_transaction(&input_txid, None)
.unwrap()
.output
.get(input_vout as usize)
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
dbg!((input_txid, txid, tx_index, input_vout));
panic!("Txid to be in txid_to_tx_data");
if input_amount_and_address_index.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| rpc
.get_raw_transaction(&input_txid, None)
.unwrap()
.output
.get(input_vout as usize)
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
let input_tx_data = input_tx_data.unwrap();
let input_tx_index = input_tx_data.index;
let input_vout = input_vout as u16;
let input_txout_index = TxoutIndex::new(input_tx_index, input_vout);
dbg!((input_txid, tx_index, input_tx_index, input_vout, input_tx_data, txid,));
panic!("Txout index to be in txout_index_to_txout_value");
}
// if input_tx_index == 2516 || input_tx_index == 2490 {
// dbg!(input_tx_index, &input_tx_data.utxos);
// }
input_tx_data.utxos -= 1;
// let input_amount = input_tx_data.utxos.remove(&input_vout);
let (input_amount, input_address_index) = input_amount_and_address_index.unwrap();
let input_amount_and_address_index = databases
.txout_index_to_amount
.remove(&input_txout_index)
.map(|amount| {
(
amount,
databases
.txout_index_to_address_index
.remove(&input_txout_index),
)
}) // Remove from cached puts
.or_else(|| {
txout_index_to_amount_and_address_index.remove(&input_txout_index)
});
let input_block_path = input_tx_data.block_path;
if input_amount_and_address_index.is_none() {
if !enable_check_if_txout_value_is_zero_in_db
|| rpc
.get_raw_transaction(&input_txid, None)
.unwrap()
.output
.get(input_vout as usize)
.unwrap()
.value
.to_sat()
== 0
{
return ControlFlow::Continue::<()>(());
}
let BlockPath {
date_index: input_date_index,
block_index: input_block_index,
} = input_block_path;
dbg!((
input_txid,
tx_index,
input_tx_index,
input_vout,
input_tx_data,
txid,
));
panic!("Txout index to be in txout_index_to_txout_value");
}
input_tx_data.utxos -= 1;
let (input_amount, input_address_index) =
input_amount_and_address_index.unwrap();
let input_block_path = input_tx_data.block_path;
let BlockPath {
date_index: input_date_index,
block_index: input_block_index,
} = input_block_path;
let input_date_data = states
let input_date_data =
states
.date_data_vec
.get_mut(input_date_index as usize)
.unwrap_or_else(|| {
@@ -433,121 +398,112 @@ pub fn parse(
panic!()
});
let input_block_data = input_date_data
.blocks
.get_mut(input_block_index as usize)
let input_block_data = input_date_data
.blocks
.get_mut(input_block_index as usize)
.unwrap_or_else(|| {
dbg!(
height,
&input_txid,
input_block_path,
input_date_index,
input_block_index,
);
panic!()
});
input_block_data.send(input_amount);
inputs_sum += input_amount;
block_path_to_sent_data
.entry(input_block_path)
.or_default()
.send(input_amount);
satblocks_destroyed += input_amount * (height - input_block_data.height);
satdays_destroyed +=
input_amount * date.signed_duration_since(*input_date_data.date).num_days() as u64;
if compute_addresses {
let input_address_index = input_address_index.unwrap_or_else(|| {
dbg!(
height,
input_amount,
&input_tx_data,
input_address_index,
input_txout_index,
txid,
input_txid,
input_vout
);
panic!()
});
let address_index_to_address_data = address_index_to_address_data.as_mut().unwrap();
let input_address_data = address_index_to_address_data
.get_mut(&input_address_index)
.unwrap_or_else(|| {
dbg!(
height,
&input_txid,
input_block_path,
input_date_index,
input_block_index,
);
panic!()
dbg!(input_address_index, input_txout_index, input_txid, input_vout);
panic!();
});
input_block_data.send(input_amount);
let input_address_realized_data = address_index_to_address_realized_data
.entry(input_address_index)
.or_insert_with(|| AddressRealizedData::default(input_address_data));
inputs_sum += input_amount;
let previous_price = input_block_data.price;
block_path_to_sent_data
.entry(input_block_path)
.or_default()
.send(input_amount);
satblocks_destroyed += input_amount * (height - input_block_data.height);
satdays_destroyed += input_amount
* date.signed_duration_since(*input_date_data.date).num_days() as u64;
if compute_addresses {
let input_address_index = input_address_index.unwrap_or_else(|| {
// MUST be after `or_insert_with`
input_address_data
.send(input_amount, previous_price)
.unwrap_or_else(|_| {
dbg!(
height,
input_amount,
&input_tx_data,
input_address_index,
input_txout_index,
txid,
input_txid,
input_vout
input_amount,
tx_index,
input_tx_index,
input_vout,
&input_address_data
);
panic!()
});
let address_index_to_address_data =
address_index_to_address_data.as_mut().unwrap();
let input_address_data = address_index_to_address_data
.get_mut(&input_address_index)
.unwrap_or_else(|| {
dbg!(
input_address_index,
input_txout_index,
input_txid,
input_vout
);
panic!();
});
let input_address_realized_data =
address_index_to_address_realized_data
.entry(input_address_index)
.or_insert_with(|| {
AddressRealizedData::default(input_address_data)
});
let previous_price = input_block_data.price;
// MUST be after `or_insert_with`
input_address_data
.send(input_amount, previous_price)
.unwrap_or_else(|_| {
dbg!(
input_address_index,
txid,
input_txid,
input_amount,
tx_index,
input_tx_index,
input_vout,
&input_address_data
);
panic!()
});
input_address_realized_data.send(
input_amount,
block_price,
previous_price,
timestamp,
input_block_data.timestamp,
);
};
is_tx_data_from_cached_puts && input_tx_data.is_empty()
input_address_realized_data.send(
input_amount,
block_price,
previous_price,
timestamp,
input_block_data.timestamp,
);
};
if remove_tx_data_from_cached_puts {
// Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
databases.txid_to_tx_data.remove_from_ram(&input_txid)
}
is_tx_data_from_cached_puts && input_tx_data.is_empty()
};
ControlFlow::Continue(())
})?;
}
if remove_tx_data_from_cached_puts {
// Pre remove tx_datas that are empty and weren't yet added to the database to avoid having it was in there or not (and thus avoid useless operations)
databases.txid_to_tx_data.remove_from_ram(&input_txid)
}
amount_sent += inputs_sum;
ControlFlow::Continue(())
})?;
}
let fee = inputs_sum - outputs_sum;
amount_sent += inputs_sum;
fees_total += fee;
fees.push(fee);
let fee = inputs_sum - outputs_sum;
ControlFlow::Continue(())
});
fees_total += fee;
fees.push(fee);
ControlFlow::Continue(())
});
if !partial_txout_data_vec.is_empty() {
panic!("partial_txout_data_vec should've been fully consumed");
@@ -580,24 +536,17 @@ pub fn parse(
if datasets.utxo.needs_durable_states(height, date) {
if let Some(previous_last_block_data) = previous_last_block_data {
block_path_to_sent_data
.iter()
.for_each(|(block_path, sent_data)| {
let block_data =
states.date_data_vec.get_block_data(block_path).unwrap();
block_path_to_sent_data.iter().for_each(|(block_path, sent_data)| {
let block_data = states.date_data_vec.get_block_data(block_path).unwrap();
if block_data.height != height {
states
.utxo_cohorts_durable_states
.as_mut()
.unwrap()
.subtract_moved(
block_data,
sent_data,
previous_last_block_data,
);
}
});
if block_data.height != height {
states.utxo_cohorts_durable_states.as_mut().unwrap().subtract_moved(
block_data,
sent_data,
previous_last_block_data,
);
}
});
}
let last_block_data = states.date_data_vec.last_block().unwrap();
@@ -615,11 +564,7 @@ pub fn parse(
.utxo_cohorts_durable_states
.as_mut()
.unwrap()
.udpate_age_if_needed(
block_data,
last_block_data,
previous_last_block_data,
);
.udpate_age_if_needed(block_data, last_block_data, previous_last_block_data);
});
}
@@ -628,14 +573,7 @@ pub fn parse(
.utxo_cohorts_durable_states
.as_ref()
.unwrap()
.compute_one_shot_states(
block_price,
if is_date_last_block {
Some(date_price)
} else {
None
},
);
.compute_one_shot_states(block_price, if is_date_last_block { Some(date_price) } else { None });
}
});
@@ -668,10 +606,10 @@ pub fn parse(
// TODO: Only compute if needed
address_cohorts_output_states.replace(AddressCohortsOutputStates::default());
address_index_to_address_realized_data.iter().for_each(
|(address_index, address_realized_data)| {
let current_address_data =
address_index_to_address_data.get(address_index).unwrap();
address_index_to_address_realized_data
.iter()
.for_each(|(address_index, address_realized_data)| {
let current_address_data = address_index_to_address_data.get(address_index).unwrap();
states
.address_cohorts_durable_states
@@ -711,42 +649,34 @@ pub fn parse(
&current_address_data.compute_liquidity_classification(),
)
.unwrap();
},
);
});
address_cohorts_one_shot_states.replace(
states
.address_cohorts_durable_states
.as_ref()
.unwrap()
.compute_one_shot_states(
block_price,
if is_date_last_block {
Some(date_price)
} else {
None
},
),
.compute_one_shot_states(block_price, if is_date_last_block { Some(date_price) } else { None }),
);
});
}
});
if compute_addresses {
address_index_to_address_data.unwrap().into_iter().for_each(
|(address_index, address_data)| {
address_index_to_address_data
.unwrap()
.into_iter()
.for_each(|(address_index, address_data)| {
if address_data.is_empty() {
databases.address_index_to_empty_address_data.insert_to_ram(
address_index,
EmptyAddressData::from_non_empty(&address_data),
);
databases
.address_index_to_empty_address_data
.insert_to_ram(address_index, EmptyAddressData::from_non_empty(&address_data));
} else {
databases
.address_index_to_address_data
.insert_to_ram(address_index, address_data);
}
},
)
})
}
datasets.insert(InsertData {
@@ -851,17 +781,15 @@ fn prepare_outputs(
.collect_vec();
if compute_addresses {
partial_txout_data_vec
.par_iter_mut()
.for_each(|partial_tx_out_data| {
if let Some(partial_tx_out_data) = partial_tx_out_data {
let address_index_opt = address_to_address_index
.unsafe_get(partial_tx_out_data.address.as_ref().unwrap())
.cloned();
partial_txout_data_vec.par_iter_mut().for_each(|partial_tx_out_data| {
if let Some(partial_tx_out_data) = partial_tx_out_data {
let address_index_opt = address_to_address_index
.unsafe_get(partial_tx_out_data.address.as_ref().unwrap())
.cloned();
partial_tx_out_data.address_index_opt = address_index_opt;
}
});
partial_tx_out_data.address_index_opt = address_index_opt;
}
});
}
TxoutsParsingResults {
@@ -937,11 +865,8 @@ fn prepare_inputs<'a>(
// https://mempool.space/tx/9d8a0d851c9fb2cdf1c6d9406ce97e19e6911ae3503ab2dd5f38640bacdac996
// which is used later as input
.map(|amount| {
let address_index = compute_addresses.then(|| {
*txout_index_to_address_index_db
.unsafe_get(&txout_index)
.unwrap()
});
let address_index =
compute_addresses.then(|| *txout_index_to_address_index_db.unsafe_get(&txout_index).unwrap());
(txout_index, (*amount, address_index))
})
@@ -989,17 +914,12 @@ fn compute_address_index_to_address_data(
address_index_to_address_data
.par_iter_mut()
.for_each(|(address_index, address_data)| {
if let Some(_address_data) =
address_index_to_address_data_db.get_from_ram(address_index)
{
if let Some(_address_data) = address_index_to_address_data_db.get_from_ram(address_index) {
_address_data.clone_into(address_data);
} else if let Some(empty_address_data) =
address_index_to_empty_address_data_db.get_from_ram(address_index)
} else if let Some(empty_address_data) = address_index_to_empty_address_data_db.get_from_ram(address_index)
{
*address_data = AddressData::from_empty(empty_address_data);
} else if let Some(_address_data) =
address_index_to_address_data_db.get_from_disk(address_index)
{
} else if let Some(_address_data) = address_index_to_address_data_db.get_from_disk(address_index) {
_address_data.clone_into(address_data);
} else {
let empty_address_data = address_index_to_empty_address_data_db
src/parser/databases/txid_to_tx_data.rs
+2 -5
View File
@@ -5,7 +5,7 @@ use std::{
};
use allocative::Allocative;
use biter::bitcoin::Txid;
use iterator::bitcoin::Txid;
use itertools::Itertools;
use snkrj::{AnyDatabase, Database as _Database};
@@ -49,10 +49,7 @@ impl TxidToTxData {
let db_index = Self::db_index(txid);
self.map
.get_mut(&db_index)
.unwrap()
.get_mut_from_ram(&txid_key)
self.map.get_mut(&db_index).unwrap().get_mut_from_ram(&txid_key)
}
pub fn remove_later_from_disk(&mut self, txid: &Txid) {
src/parser/mod.rs
+2 -9
View File
@@ -1,6 +1,6 @@
use std::{thread::sleep, time::Duration};
use biter::bitcoincore_rpc::{Client, RpcApi};
use iterator::bitcoincore_rpc::{Client, RpcApi};
mod actions;
mod databases;
@@ -25,14 +25,7 @@ pub fn main(config: &Config, rpc: &Client, exit: &Exit) -> color_eyre::Result<()
let mut databases = Databases::import(config);
let mut datasets = Datasets::import(config)?;
iter_blocks(
config,
rpc,
block_count,
exit.clone(),
&mut databases,
&mut datasets,
)?;
iter_blocks(config, rpc, block_count, exit.clone(), &mut databases, &mut datasets)?;
if let Some(delay) = config.delay() {
sleep(Duration::from_secs(delay))
src/structs/address.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use bitcoin_hashes::hash160;
use biter::bitcoin::TxOut;
use iterator::bitcoin::TxOut;
use super::{AddressType, Counter, U8x19, U8x31};
src/structs/amount.rs
+2 -15
View File
@@ -10,27 +10,14 @@ use bincode::{
error::{DecodeError, EncodeError},
BorrowDecode, Decode, Encode,
};
use biter::bitcoin::Amount as BitcoinAmount;
use derive_deref::{Deref, DerefMut};
use iterator::bitcoin::Amount as BitcoinAmount;
use serde::{Deserialize, Serialize};
use snkrj::{direct_repr, Storable, UnsizedStorable};
use super::Height;
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
Serialize,
Deserialize,
)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default, Serialize, Deserialize)]
pub struct Amount(BitcoinAmount);
direct_repr!(Amount);
src/structs/config.rs
+2 -4
View File
@@ -4,9 +4,9 @@ use std::{
path::{Path, PathBuf},
};
use biter::bitcoincore_rpc::Auth;
use clap::Parser;
use color_eyre::eyre::eyre;
use iterator::bitcoincore_rpc::Auth;
use log::info;
use serde::{Deserialize, Serialize};
@@ -270,9 +270,7 @@ impl Config {
fn fix_user_path(path: &str) -> PathBuf {
let fix = move |pattern: &str| {
if path.starts_with(pattern) {
let path = &path
.replace(&format!("{pattern}/"), "")
.replace(pattern, "");
let path = &path.replace(&format!("{pattern}/"), "").replace(pattern, "");
let home = std::env::var("HOME").unwrap();
src/structs/height.rs
+1 -1
View File
@@ -5,8 +5,8 @@ use std::{
use allocative::Allocative;
use bincode::{Decode, Encode};
use biter::NUMBER_OF_UNSAFE_BLOCKS;
use derive_deref::{Deref, DerefMut};
use iterator::NUMBER_OF_UNSAFE_BLOCKS;
use serde::{Deserialize, Serialize};
use super::{HeightMapChunkId, MapKey, HEIGHT_MAP_CHUNK_SIZE};
src/structs/rpc.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use biter::bitcoincore_rpc::Client;
use iterator::bitcoincore_rpc::Client;
use crate::structs::Config;
storable_vec/src/lib.rs
+20
View File
@@ -590,6 +590,26 @@ where
})?;
Ok(self.flush()?)
}
pub fn compute_sum_from_indexes<T2, F>(
&mut self,
first_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
last_indexes: &mut StorableVec<I, T2, SINGLE_THREAD>,
callback: F,
) -> Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
F: Fn(&T2) -> T,
{
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
let last_index = last_indexes.get(i)?;
let count = *last_index + 1_usize - *first_index;
self.push_if_needed(i, count.into())
})?;
Ok(self.flush()?)
}
}
impl<I, T> StorableVec<I, T, ASYNC_READ_ONLY>
storable_vec/src/main.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::path::Path;
use storable_vec::{StorableVec, Version, ASYNC_READ_ONLY, CACHED_GETS, SINGLE_THREAD};
use storable_vec::{StorableVec, Version, CACHED_GETS, SINGLE_THREAD};
fn main() -> Result<(), Box<dyn std::error::Error>> {
{
unsafe_slice_serde/Cargo.toml
-4
View File
@@ -1,4 +0,0 @@
[package]
name = "unsafe_slice_serde"
version = "0.1.0"
edition = "2021"
unsafe_slice_serde/src/lib.rs
-42
View File
@@ -1,42 +0,0 @@
use std::{fmt, slice};
pub trait UnsafeSliceSerde
where
Self: Sized,
{
const SIZE: usize = size_of::<Self>();
fn unsafe_try_from_slice(slice: &[u8]) -> Result<&Self> {
let (prefix, shorts, suffix) = unsafe { slice.align_to::<Self>() };
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
// dbg!(&slice, &prefix, &shorts, &suffix);
return Err(Error::FailedToAlignToSelf);
}
Ok(&shorts[0])
}
fn unsafe_as_slice(&self) -> &[u8] {
let data: *const Self = self;
let data: *const u8 = data as *const u8;
unsafe { slice::from_raw_parts(data, Self::SIZE) }
}
}
impl<T> UnsafeSliceSerde for T {}
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
FailedToAlignToSelf,
}
impl fmt::Display for Error {
// This trait requires `fmt` with this exact signature.
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::FailedToAlignToSelf => write!(f, "Failed to align_to for T"),
}
}
}
impl std::error::Error for Error {}