Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: fixes

Browse Source
This commit is contained in:
nym21
2025-07-12 11:18:51 +02:00
parent 2c867103ca
commit 0167a2ae59
4 changed files with 383 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

480
crates/brk_computer/src/vecs/stateful/mod.rs
View File

@@ -57,7 +57,7 @@ use range_map::*;
use r#trait::CohortVecs; use r#trait::CohortVecs;
pub use withaddressdatasource::WithAddressDataSource; pub use withaddressdatasource::WithAddressDataSource;
const VERSION: Version = Version::new(11); const VERSION: Version = Version::new(12);
#[derive(Clone)] #[derive(Clone)]
pub struct Vecs { pub struct Vecs {
@@ -666,11 +666,6 @@ impl Vecs {
let height_to_timestamp_fixed_vec = height_to_timestamp_fixed.collect().unwrap(); let height_to_timestamp_fixed_vec = height_to_timestamp_fixed.collect().unwrap();
let outputindex_range_to_height = RangeMap::from(height_to_first_outputindex); let outputindex_range_to_height = RangeMap::from(height_to_first_outputindex);
let mut outputindex_to_value_type_typeindex: BTreeMap<
OutputIndex,
(Sats, OutputType, Option<TypeIndex>),
> = BTreeMap::default();
let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() { let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_unspendable_supply self.height_to_unspendable_supply
.into_iter() .into_iter()
@@ -731,37 +726,36 @@ impl Vecs {
let output_count = height_to_output_count_iter.unwrap_get_inner(height); let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height); let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let first_addressindexes: ByAddressType<TypeIndex> = let first_addressindexes: ByAddressType<TypeIndex> = ByAddressType {
ByAddressType { p2a: height_to_first_p2aaddressindex_iter
p2a: height_to_first_p2aaddressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2pk33: height_to_first_p2pk33addressindex_iter
p2pk33: height_to_first_p2pk33addressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2pk65: height_to_first_p2pk65addressindex_iter
p2pk65: height_to_first_p2pk65addressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2pkh: height_to_first_p2pkhaddressindex_iter
p2pkh: height_to_first_p2pkhaddressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2sh: height_to_first_p2shaddressindex_iter
p2sh: height_to_first_p2shaddressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2tr: height_to_first_p2traddressindex_iter
p2tr: height_to_first_p2traddressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2wpkh: height_to_first_p2wpkhaddressindex_iter
p2wpkh: height_to_first_p2wpkhaddressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), p2wsh: height_to_first_p2wshaddressindex_iter
p2wsh: height_to_first_p2wshaddressindex_iter .unwrap_get_inner(height)
.unwrap_get_inner(height) .into(),
.into(), };
};
let ( let (
(sent_outputindexes, mut height_to_sent, addresstype_to_typedindex_to_sent_data), (mut height_to_sent, addresstype_to_typedindex_to_sent_data),
(mut received_outputindexes, mut transacted, addresstype_to_typedindex_to_received_data), (mut transacted, addresstype_to_typedindex_to_received_data),
) = thread::scope(|s| { ) = thread::scope(|s| {
if chain_state_starting_height <= height { if chain_state_starting_height <= height {
s.spawn(|| { s.spawn(|| {
@@ -772,158 +766,157 @@ impl Vecs {
let sent_handle = s.spawn(|| { let sent_handle = s.spawn(|| {
// Skip coinbase // Skip coinbase
(first_inputindex + 1..first_inputindex + *input_count) (first_inputindex + 1..first_inputindex + *input_count)
.into_par_iter() .into_par_iter()
.map(InputIndex::from) .map(InputIndex::from)
.map(|inputindex| { .map(|inputindex| {
let outputindex = inputindex_to_outputindex let outputindex = inputindex_to_outputindex
.get_or_read(inputindex, &inputindex_to_outputindex_mmap) .get_or_read(
.unwrap() inputindex,
.unwrap() &inputindex_to_outputindex_mmap,
.into_owned(); )
.unwrap()
.unwrap()
.into_owned();
let cached = outputindex_to_value_type_typeindex.get(&outputindex).cloned(); let value = outputindex_to_value
let (value, input_type) = cached.map_or_else(|| (
outputindex_to_value
.get_or_read(outputindex, &outputindex_to_value_mmap) .get_or_read(outputindex, &outputindex_to_value_mmap)
.unwrap() .unwrap()
.unwrap() .unwrap()
.into_owned(), .into_owned();
outputindex_to_outputtype
.get_or_read(outputindex, &outputindex_to_outputtype_mmap) let input_type = outputindex_to_outputtype
.get_or_read(
outputindex,
&outputindex_to_outputtype_mmap,
)
.unwrap() .unwrap()
.unwrap() .unwrap()
.into_owned() .into_owned();
), |(value,_type,..)| (value, _type));
let prev_height = *outputindex_range_to_height.get(outputindex).unwrap(); let prev_height =
*outputindex_range_to_height.get(outputindex).unwrap();
if input_type.is_unspendable() { if input_type.is_unspendable() {
unreachable!() unreachable!()
} else if input_type.is_not_address() { } else if input_type.is_not_address() {
return ( return (
outputindex, prev_height,
value,
input_type,
None,
);
}
let typeindex = outputindex_to_typeindex
.get_or_read(
outputindex,
&outputindex_to_typeindex_mmap,
)
.unwrap()
.unwrap()
.into_owned();
let addressdata_opt = if input_type.is_address()
&& *first_addressindexes.get(input_type).unwrap()
> typeindex
&& !addresstype_to_typeindex_to_addressdata
.get(input_type)
.unwrap()
.contains_key(&typeindex)
&& let Some(address_data) = stores
.get_addressdata(input_type, typeindex)
.unwrap()
// Otherwise it was empty and got funds in the same block before sending them
{
Some(WithAddressDataSource::FromAddressDataStore(
address_data,
))
} else {
None
};
(
prev_height, prev_height,
value, value,
input_type, input_type,
None Some((typeindex, addressdata_opt)),
) )
} })
.fold(
let typeindex = cached.map_or_else(|| outputindex_to_typeindex || {
.get_or_read(outputindex, &outputindex_to_typeindex_mmap) (
.unwrap() BTreeMap::<Height, Transacted>::default(),
.unwrap() HeightToAddressTypeToVec::<(
.into_owned(), |(_, _, typeindex)| typeindex.unwrap()); TypeIndex,
Sats,
let addressdata_opt = if input_type.is_address() Option<WithAddressDataSource<AddressData>>,
&& *first_addressindexes.get(input_type).unwrap() )>::default(
> typeindex ),
&& !addresstype_to_typeindex_to_addressdata )
.get(input_type) },
.unwrap() |(mut tree, mut tree2),
.contains_key(&typeindex) (
&& let Some(address_data) = height,
stores.get_addressdata(input_type, typeindex).unwrap() value,
// Otherwise it was empty and got funds in the same block before sending them input_type,
{ address_data_opt,
Some(WithAddressDataSource::FromAddressDataStore( )| {
address_data, tree.entry(height)
)) .or_default()
} else { .iterate(value, input_type);
None if let Some((typeindex, addressdata_opt)) =
}; address_data_opt
{
( tree2
outputindex, .entry(height)
prev_height, .or_default()
value, .get_mut(input_type)
input_type, .unwrap()
Some((typeindex, .push((typeindex, value, addressdata_opt));
addressdata_opt }
)) (tree, tree2)
},
) )
}) .reduce(
.fold( || {
|| { (
( BTreeMap::<Height, Transacted>::default(),
Vec::<OutputIndex>::default(), HeightToAddressTypeToVec::<(
BTreeMap::<Height, Transacted>::default(), TypeIndex,
HeightToAddressTypeToVec::<( Sats,
TypeIndex, Option<WithAddressDataSource<AddressData>>,
Sats, )>::default(
Option<WithAddressDataSource<AddressData>>, ),
)>::default(), )
) },
}, |(first_tree, first_tree2), (second_tree, second_tree2)| {
|(mut vec, mut tree, mut tree2), let (mut tree_source, tree_to_consume) =
( if first_tree.len() > second_tree.len() {
outputindex, (first_tree, second_tree)
height, } else {
value, (second_tree, first_tree)
input_type, };
address_data_opt tree_to_consume.into_iter().for_each(|(k, v)| {
)| { *tree_source.entry(k).or_default() += v;
vec.push(outputindex); });
tree.entry(height).or_default().iterate(value, input_type);
if let Some((typeindex,
addressdata_opt)) = address_data_opt {
tree2.entry(height).or_default().get_mut(input_type).unwrap().push((
typeindex,
value,
addressdata_opt,
));
}
(vec, tree, tree2)
},
)
.reduce(
|| {
(
Vec::<OutputIndex>::default(),
BTreeMap::<Height, Transacted>::default(),
HeightToAddressTypeToVec::<(
TypeIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
)>::default(),
)
},
|(mut first_vec, first_tree, first_tree2), (mut second_vec, second_tree, second_tree2)| {
let vec = if first_vec.len() > second_vec.len() {
first_vec.append(&mut second_vec);
first_vec
} else {
second_vec.append(&mut first_vec);
second_vec
};
let (mut tree_source, tree_to_consume) = let (mut tree_source2, tree_to_consume2) =
if first_tree.len() > second_tree.len() { if first_tree2.len() > second_tree2.len() {
(first_tree, second_tree) (first_tree2, second_tree2)
} else { } else {
(second_tree, first_tree) (second_tree2, first_tree2)
}; };
tree_to_consume.into_iter().for_each(|(k, v)| { tree_to_consume2.0.into_iter().for_each(|(k, v)| {
*tree_source.entry(k).or_default() += v; tree_source2.entry(k).or_default().merge(v);
}); });
let (mut tree_source2, tree_to_consume2) = (tree_source, tree_source2)
if first_tree2.len() > second_tree2.len() { },
(first_tree2, second_tree2) )
} else {
(second_tree2, first_tree2)
};
tree_to_consume2.0.into_iter().for_each(|(k, v)| {
tree_source2.entry(k).or_default().merge(v);
});
(vec, tree_source, tree_source2)
},
)
}); });
let received_output = (first_outputindex..first_outputindex + *output_count) let received_output = (first_outputindex
..first_outputindex + *output_count)
.into_par_iter() .into_par_iter()
.map(OutputIndex::from) .map(OutputIndex::from)
.map(|outputindex| { .map(|outputindex| {
@@ -940,7 +933,7 @@ impl Vecs {
.into_owned(); .into_owned();
if output_type.is_not_address() { if output_type.is_not_address() {
return (outputindex, value, output_type, None); return (value, output_type, None);
} }
let typeindex = outputindex_to_typeindex let typeindex = outputindex_to_typeindex
@@ -949,25 +942,26 @@ impl Vecs {
.unwrap() .unwrap()
.into_owned(); .into_owned();
let addressdata_opt = if *first_addressindexes.get(output_type).unwrap() let addressdata_opt = if *first_addressindexes
<= typeindex { .get(output_type)
.unwrap()
<= typeindex
{
Some(WithAddressDataSource::New(AddressData::default())) Some(WithAddressDataSource::New(AddressData::default()))
} else if !addresstype_to_typeindex_to_addressdata } else if !addresstype_to_typeindex_to_addressdata
.get(output_type) .get(output_type)
.unwrap() .unwrap()
.contains_key(&typeindex) .contains_key(&typeindex)
&& !addresstype_to_typeindex_to_emptyaddressdata && !addresstype_to_typeindex_to_emptyaddressdata
.get(output_type) .get(output_type)
.unwrap() .unwrap()
.contains_key(&typeindex) { .contains_key(&typeindex)
{
Some( Some(
if let Some(addressdata) = stores if let Some(addressdata) =
.get_addressdata(output_type, typeindex) stores.get_addressdata(output_type, typeindex).unwrap()
.unwrap()
{ {
WithAddressDataSource::FromAddressDataStore( WithAddressDataSource::FromAddressDataStore(addressdata)
addressdata,
)
} else if let Some(emptyaddressdata) = stores } else if let Some(emptyaddressdata) = stores
.get_emptyaddressdata(output_type, typeindex) .get_emptyaddressdata(output_type, typeindex)
.unwrap() .unwrap()
@@ -983,56 +977,15 @@ impl Vecs {
None None
}; };
(outputindex, value, output_type, Some((typeindex, addressdata_opt))) (
value,
output_type,
Some((typeindex, addressdata_opt)),
)
}) })
.fold( .fold(
|| { || {
( (
BTreeMap::<
OutputIndex,
(Sats, OutputType, Option<TypeIndex>),
>::default(),
Transacted::default(),
AddressTypeToVec::<(
TypeIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
)>::default(
),
)
},
|(mut received_outputindexes, mut transacted, mut vecs),
(
outputindex,
value,
output_type,
typeindex_with_addressdata_opt,
)| {
if output_type.is_address() {
received_outputindexes.insert(outputindex, (value, output_type, typeindex_with_addressdata_opt.as_ref().map(|(v, ..)| *v)));
}
transacted.iterate(value, output_type);
if let Some(vec) = vecs.get_mut(output_type) {
let (typeindex,
addressdata_opt) = typeindex_with_addressdata_opt.unwrap();
vec.push((
typeindex,
value,
addressdata_opt,
));
}
(received_outputindexes, transacted, vecs)
},
)
.reduce(
|| {
(
BTreeMap::<
OutputIndex,
(Sats, OutputType, Option<TypeIndex>),
>::default(),
Transacted::default(), Transacted::default(),
AddressTypeToVec::<( AddressTypeToVec::<(
TypeIndex, TypeIndex,
@@ -1041,18 +994,37 @@ impl Vecs {
)>::default(), )>::default(),
) )
}, },
|(received_outputindexes, transacted, mut vecs), (other_received_outputindexes, other_transacted, other_vecs)| { |(mut transacted, mut vecs),
let (mut tree_source, mut tree_to_consume) = (
if received_outputindexes.len() > other_received_outputindexes.len() { value,
(received_outputindexes, other_received_outputindexes) output_type,
} else { typeindex_with_addressdata_opt,
(other_received_outputindexes, received_outputindexes) )| {
}; transacted.iterate(value, output_type);
tree_source.append(&mut tree_to_consume); if let Some(vec) = vecs.get_mut(output_type) {
let (typeindex, addressdata_opt) =
typeindex_with_addressdata_opt.unwrap();
vec.push((typeindex, value, addressdata_opt));
}
(transacted, vecs)
},
)
.reduce(
|| {
(
Transacted::default(),
AddressTypeToVec::<(
TypeIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
)>::default(),
)
},
|(transacted, mut vecs), (other_transacted, other_vecs)| {
vecs.merge(other_vecs); vecs.merge(other_vecs);
(tree_source, transacted + other_transacted, vecs) (transacted + other_transacted, vecs)
}, },
); );
@@ -1088,26 +1060,7 @@ impl Vecs {
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65); .iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
}; };
thread::scope(|scope| -> Result<()> { thread::scope(|scope| -> Result<()> {
scope.spawn(|| {
let min_outputindex = if let Some(min_height) = height.checked_sub(Height::new(5000)) {
height_to_first_outputindex.into_iter().unwrap_get_inner(min_height)
} else {
OutputIndex::ZERO
};
outputindex_to_value_type_typeindex.retain(|k, _| *k >= min_outputindex);
sent_outputindexes
.into_iter()
.filter(|k| *k >= min_outputindex)
.for_each(|outputindex| {
outputindex_to_value_type_typeindex.remove(&outputindex);
});
outputindex_to_value_type_typeindex.append(&mut received_outputindexes);
});
scope.spawn(|| { scope.spawn(|| {
// Push current block state before processing sends and receives // Push current block state before processing sends and receives
chain_state.push(BlockState { chain_state.push(BlockState {
@@ -1147,7 +1100,7 @@ impl Vecs {
height_to_close_vec.as_ref(), height_to_close_vec.as_ref(),
&height_to_timestamp_fixed_vec, &height_to_timestamp_fixed_vec,
height, height,
timestamp timestamp,
)?; )?;
Ok(()) Ok(())
@@ -1199,18 +1152,15 @@ impl Vecs {
let dateindex = is_date_last_height.then_some(dateindex); let dateindex = is_date_last_height.then_some(dateindex);
separate_utxo_vecs separate_utxo_vecs
.into_par_iter() .into_par_iter()
.map(|(_, v)| v as &mut dyn DynCohortVecs).chain( .map(|(_, v)| v as &mut dyn DynCohortVecs)
.chain(
separate_address_vecs separate_address_vecs
.into_par_iter() .into_par_iter()
.map(|(_, v)| v as &mut dyn DynCohortVecs) .map(|(_, v)| v as &mut dyn DynCohortVecs),
) )
.try_for_each(|v| { .try_for_each(|v| {
v.compute_then_force_push_unrealized_states( v.compute_then_force_push_unrealized_states(
height, height, price, dateindex, date_price, exit,
price,
dateindex,
date_price,
exit,
) )
})?; })?;

340
crates/brk_indexer/src/lib.rs
View File

@@ -136,34 +136,6 @@ impl Indexer {
vecs.height_to_total_size.push_if_needed(height, block.total_size().into())?; vecs.height_to_total_size.push_if_needed(height, block.total_size().into())?;
vecs.height_to_weight.push_if_needed(height, block.weight().into())?; vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
let inputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>();
let outputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
})
.collect::<Vec<_>>();
let tx_len = block.txdata.len();
let outputs_len = outputs.len();
let inputs_len = inputs.len();
let ( let (
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle, txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle,
input_source_vec_handle, input_source_vec_handle,
@@ -173,9 +145,9 @@ impl Indexer {
scope.spawn(|| -> color_eyre::Result<_> { scope.spawn(|| -> color_eyre::Result<_> {
block block
.txdata .txdata
.par_iter() .iter()
.enumerate() .enumerate()
.map(|(index, tx)| -> color_eyre::Result<_> { .map(|(index, tx)| {
let txid = Txid::from(tx.compute_txid()); let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::from(&txid); let txid_prefix = TxidPrefix::from(&txid);
@@ -190,27 +162,25 @@ impl Indexer {
Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt))) Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt)))
}) })
.try_fold(BTreeMap::new, |mut map, tuple| { .collect::<color_eyre::Result<BTreeMap<_, _>>>()
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
})
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
})
}); });
let input_source_vec_handle = scope.spawn(|| { let input_source_vec_handle = scope.spawn(|| {
let txindex_to_first_outputindex_mmap = vecs let txindex_to_first_outputindex_mmap = vecs
.txindex_to_first_outputindex.mmap().load(); .txindex_to_first_outputindex.mmap().load();
let inputs = block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>();
inputs inputs
.into_par_iter() .into_par_iter()
.enumerate() .enumerate()
@@ -270,157 +240,165 @@ impl Indexer {
}) })
}); });
let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| { let p2pk65addressindex_to_p2pk65bytes_mmap = vecs
let p2pk65addressindex_to_p2pk65bytes_mmap = vecs .p2pk65addressindex_to_p2pk65bytes.mmap().load();
.p2pk65addressindex_to_p2pk65bytes.mmap().load(); let p2pk33addressindex_to_p2pk33bytes_mmap = vecs.p2pk33addressindex_to_p2pk33bytes.mmap().load();
let p2pk33addressindex_to_p2pk33bytes_mmap = vecs.p2pk33addressindex_to_p2pk33bytes.mmap().load(); let p2pkhaddressindex_to_p2pkhbytes_mmap = vecs.p2pkhaddressindex_to_p2pkhbytes.mmap().load();
let p2pkhaddressindex_to_p2pkhbytes_mmap = vecs.p2pkhaddressindex_to_p2pkhbytes.mmap().load(); let p2shaddressindex_to_p2shbytes_mmap = vecs.p2shaddressindex_to_p2shbytes.mmap().load();
let p2shaddressindex_to_p2shbytes_mmap = vecs.p2shaddressindex_to_p2shbytes.mmap().load(); let p2wpkhaddressindex_to_p2wpkhbytes_mmap = vecs.p2wpkhaddressindex_to_p2wpkhbytes.mmap().load();
let p2wpkhaddressindex_to_p2wpkhbytes_mmap = vecs.p2wpkhaddressindex_to_p2wpkhbytes.mmap().load(); let p2wshaddressindex_to_p2wshbytes_mmap = vecs.p2wshaddressindex_to_p2wshbytes.mmap().load();
let p2wshaddressindex_to_p2wshbytes_mmap = vecs.p2wshaddressindex_to_p2wshbytes.mmap().load(); let p2traddressindex_to_p2trbytes_mmap = vecs.p2traddressindex_to_p2trbytes.mmap().load();
let p2traddressindex_to_p2trbytes_mmap = vecs.p2traddressindex_to_p2trbytes.mmap().load(); let p2aaddressindex_to_p2abytes_mmap = vecs.p2aaddressindex_to_p2abytes.mmap().load();
let p2aaddressindex_to_p2abytes_mmap = vecs.p2aaddressindex_to_p2abytes.mmap().load();
outputs let outputs = block
.into_par_iter() .txdata
.enumerate() .iter()
.map( .enumerate()
#[allow(clippy::type_complexity)] .flat_map(|(index, tx)| {
|(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<( tx.output
OutputIndex, .iter()
( .enumerate()
&TxOut, .map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
TxIndex, }).collect::<Vec<_>>();
Vout,
OutputType,
brk_core::Result<AddressBytes>,
Option<TypeIndex>,
&Transaction,
),
)> {
let txindex = idxs.txindex + block_txindex;
let outputindex = idxs.outputindex + OutputIndex::from(block_outputindex);
let script = &txout.script_pubkey; let outputindex_to_txout_outputtype_addressbytes_res_addressindex = outputs.into_par_iter()
.enumerate()
.map(
#[allow(clippy::type_complexity)]
|(block_outputindex, (block_txindex, vout, txout, tx))| -> color_eyre::Result<(
OutputIndex,
(
&TxOut,
TxIndex,
Vout,
OutputType,
brk_core::Result<AddressBytes>,
Option<TypeIndex>,
&Transaction,
),
)> {
let txindex = idxs.txindex + block_txindex;
let outputindex = idxs.outputindex + OutputIndex::from(block_outputindex);
let outputtype = OutputType::from(script); let script = &txout.script_pubkey;
let address_bytes_res = let outputtype = OutputType::from(script);
AddressBytes::try_from((script, outputtype)).inspect_err(|_| {
// dbg!(&txout, height, txi, &tx.compute_txid());
});
let typeindex_opt = address_bytes_res.as_ref().ok().and_then(|addressbytes| { let address_bytes_res =
stores AddressBytes::try_from((script, outputtype)).inspect_err(|_| {
.addressbyteshash_to_typeindex // dbg!(&txout, height, txi, &tx.compute_txid());
.get(&AddressBytesHash::from((addressbytes, outputtype)))
.unwrap()
.map(|v| *v)
// Checking if not in the future
.and_then(|typeindex_local| {
(typeindex_local < idxs.typeindex(outputtype)).then_some(typeindex_local)
})
}); });
if let Some(Some(typeindex)) = check_collisions.then_some(typeindex_opt) { let typeindex_opt = address_bytes_res.as_ref().ok().and_then(|addressbytes| {
let addressbytes = address_bytes_res.as_ref().unwrap(); stores
.addressbyteshash_to_typeindex
.get(&AddressBytesHash::from((addressbytes, outputtype)))
.unwrap()
.map(|v| *v)
// Checking if not in the future
.and_then(|typeindex_local| {
(typeindex_local < idxs.typeindex(outputtype)).then_some(typeindex_local)
})
});
let prev_addressbytes_opt = match outputtype { if let Some(Some(typeindex)) = check_collisions.then_some(typeindex_opt) {
OutputType::P2PK65 => vecs let addressbytes = address_bytes_res.as_ref().unwrap();
.p2pk65addressindex_to_p2pk65bytes
.get_or_read(typeindex.into(), &p2pk65addressindex_to_p2pk65bytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2PK33 => vecs
.p2pk33addressindex_to_p2pk33bytes
.get_or_read(typeindex.into(), &p2pk33addressindex_to_p2pk33bytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2PKH => vecs
.p2pkhaddressindex_to_p2pkhbytes
.get_or_read(typeindex.into(), &p2pkhaddressindex_to_p2pkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2SH => vecs
.p2shaddressindex_to_p2shbytes
.get_or_read(typeindex.into(), &p2shaddressindex_to_p2shbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2WPKH => vecs
.p2wpkhaddressindex_to_p2wpkhbytes
.get_or_read(typeindex.into(), &p2wpkhaddressindex_to_p2wpkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2WSH => vecs
.p2wshaddressindex_to_p2wshbytes
.get_or_read(typeindex.into(), &p2wshaddressindex_to_p2wshbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2TR => vecs
.p2traddressindex_to_p2trbytes
.get_or_read(typeindex.into(), &p2traddressindex_to_p2trbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2A => vecs
.p2aaddressindex_to_p2abytes
.get_or_read(typeindex.into(), &p2aaddressindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
};
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
if stores.addressbyteshash_to_typeindex.needs(height) let prev_addressbytes_opt = match outputtype {
&& prev_addressbytes != addressbytes OutputType::P2PK65 => vecs
{ .p2pk65addressindex_to_p2pk65bytes
let txid = tx.compute_txid(); .get_or_read(typeindex.into(), &p2pk65addressindex_to_p2pk65bytes_mmap)?
dbg!( .map(|v| AddressBytes::from(v.into_owned())),
height, OutputType::P2PK33 => vecs
txid, .p2pk33addressindex_to_p2pk33bytes
vout, .get_or_read(typeindex.into(), &p2pk33addressindex_to_p2pk33bytes_mmap)?
block_txindex, .map(|v| AddressBytes::from(v.into_owned())),
outputtype, OutputType::P2PKH => vecs
prev_addressbytes, .p2pkhaddressindex_to_p2pkhbytes
addressbytes, .get_or_read(typeindex.into(), &p2pkhaddressindex_to_p2pkhbytes_mmap)?
&idxs, .map(|v| AddressBytes::from(v.into_owned())),
typeindex, OutputType::P2SH => vecs
typeindex, .p2shaddressindex_to_p2shbytes
txout, .get_or_read(typeindex.into(), &p2shaddressindex_to_p2shbytes_mmap)?
AddressBytesHash::from((addressbytes, outputtype)), .map(|v| AddressBytes::from(v.into_owned())),
); OutputType::P2WPKH => vecs
panic!() .p2wpkhaddressindex_to_p2wpkhbytes
.get_or_read(typeindex.into(), &p2wpkhaddressindex_to_p2wpkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2WSH => vecs
.p2wshaddressindex_to_p2wshbytes
.get_or_read(typeindex.into(), &p2wshaddressindex_to_p2wshbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2TR => vecs
.p2traddressindex_to_p2trbytes
.get_or_read(typeindex.into(), &p2traddressindex_to_p2trbytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::P2A => vecs
.p2aaddressindex_to_p2abytes
.get_or_read(typeindex.into(), &p2aaddressindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_owned())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
} }
} };
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
Ok(( if stores.addressbyteshash_to_typeindex.needs(height)
outputindex, && prev_addressbytes != addressbytes
( {
txout, let txid = tx.compute_txid();
txindex, dbg!(
height,
txid,
vout, vout,
block_txindex,
outputtype, outputtype,
address_bytes_res, prev_addressbytes,
typeindex_opt, addressbytes,
tx, &idxs,
), typeindex,
)) typeindex,
}, txout,
) AddressBytesHash::from((addressbytes, outputtype)),
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { );
let (key, value) = tuple?; panic!()
map.insert(key, value); }
Ok(map)
})
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
} }
})
}); Ok((
outputindex,
(
txout,
txindex,
vout,
outputtype,
address_bytes_res,
typeindex_opt,
tx,
),
))
},
)
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
let (key, value) = tuple?;
map.insert(key, value);
Ok(map)
})
.try_reduce(BTreeMap::new, |mut map, mut map2| {
if map.len() > map2.len() {
map.append(&mut map2);
Ok(map)
} else {
map2.append(&mut map);
Ok(map2)
}
});
( (
txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(), txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(),
input_source_vec_handle.join(), input_source_vec_handle.join(),
outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle.join(), outputindex_to_txout_outputtype_addressbytes_res_addressindex,
) )
}); });
@@ -440,7 +418,11 @@ impl Indexer {
.ok() .ok()
.context( .context(
"Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join", "Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join",
)??; )?;
let outputs_len = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt.len();
let inputs_len = input_source_vec.len();
let tx_len = block.txdata.len();
let mut new_txindexvout_to_outputindex: BTreeMap< let mut new_txindexvout_to_outputindex: BTreeMap<
(TxIndex, Vout), (TxIndex, Vout),

12
crates/brk_server/src/traits/header_map.rs
View File

@@ -8,7 +8,6 @@ use axum::http::{
header::{self, IF_MODIFIED_SINCE, IF_NONE_MATCH}, header::{self, IF_MODIFIED_SINCE, IF_NONE_MATCH},
}; };
use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone}; use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone};
use log::info;
const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT"; const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT";
@@ -140,7 +139,11 @@ impl HeaderMapExtended for HeaderMap {
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
fn insert_content_type(&mut self, path: &Path) { fn insert_content_type(&mut self, path: &Path) {
match path.extension().unwrap().to_str().unwrap() { match path
.extension()
.map(|s| s.to_str().unwrap_or_default())
.unwrap_or_default()
{
"js" => self.insert_content_type_application_javascript(), "js" => self.insert_content_type_application_javascript(),
"json" | "map" => self.insert_content_type_application_json(), "json" | "map" => self.insert_content_type_application_json(),
"html" => self.insert_content_type_text_html(), "html" => self.insert_content_type_text_html(),
@@ -152,10 +155,7 @@ impl HeaderMapExtended for HeaderMap {
"jpg" | "jpeg" => self.insert_content_type_image_jpeg(), "jpg" | "jpeg" => self.insert_content_type_image_jpeg(),
"png" => self.insert_content_type_image_png(), "png" => self.insert_content_type_image_png(),
"webmanifest" => self.insert_content_type_application_manifest_json(), "webmanifest" => self.insert_content_type_application_manifest_json(),
extension => { _ => {}
info!("Extension unsupported: {extension}");
panic!()
}
} }
} }

3
crates/brk_vec/src/traits/generic.rs
View File

@@ -122,8 +122,7 @@ where
fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> { fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> {
file.write_all(buf)?; file.write_all(buf)?;
// file.flush()?; file.flush()?;
file.sync_data()?;
self.update_mmap(file) self.update_mmap(file)
} }