Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snap

Browse Source
This commit is contained in:
nym21
2026-04-15 12:51:30 +02:00
parent 39da441d14
commit 08ba4ad996
24 changed files with 1076 additions and 620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
crates/brk_client/src/lib.rs
View File

@@ -1277,6 +1277,38 @@ impl<T: DeserializeOwned> AverageBaseCumulativeMaxMedianMinPct10Pct25Pct75Pct90S
}
}
/// Pattern struct for repeated tree structure.
pub struct IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern {
pub index: SeriesPattern1<StoredI8>,
pub pct0_5: CentsSatsUsdPattern,
pub pct1: CentsSatsUsdPattern,
pub pct2: CentsSatsUsdPattern,
pub pct5: CentsSatsUsdPattern,
pub pct95: CentsSatsUsdPattern,
pub pct98: CentsSatsUsdPattern,
pub pct99: CentsSatsUsdPattern,
pub pct99_5: CentsSatsUsdPattern,
pub score: SeriesPattern1<StoredI8>,
}
impl IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern {
/// Create a new pattern node with accumulated series name.
pub fn new(client: Arc<BrkClientBase>, acc: String) -> Self {
Self {
index: SeriesPattern1::new(client.clone(), _m(&acc, "index")),
pct0_5: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct0_5")),
pct1: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct01")),
pct2: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct02")),
pct5: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct05")),
pct95: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct95")),
pct98: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct98")),
pct99: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct99")),
pct99_5: CentsSatsUsdPattern::new(client.clone(), _m(&acc, "pct99_5")),
score: SeriesPattern1::new(client.clone(), _m(&acc, "score")),
}
}
}
/// Pattern struct for repeated tree structure.
pub struct AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5 {
pub all: AverageBlockCumulativeSumPattern<StoredU64>,
@@ -5619,7 +5651,7 @@ pub struct SeriesTree_Indicators {
pub dormancy: SeriesTree_Indicators_Dormancy,
pub stock_to_flow: SeriesPattern1<StoredF32>,
pub seller_exhaustion: SeriesPattern1<StoredF32>,
pub realized_envelope: SeriesTree_Indicators_RealizedEnvelope,
pub rarity_meter: SeriesTree_Indicators_RarityMeter,
}
impl SeriesTree_Indicators {
@@ -5635,7 +5667,7 @@ impl SeriesTree_Indicators {
dormancy: SeriesTree_Indicators_Dormancy::new(client.clone(), format!("{base_path}_dormancy")),
stock_to_flow: SeriesPattern1::new(client.clone(), "stock_to_flow".to_string()),
seller_exhaustion: SeriesPattern1::new(client.clone(), "seller_exhaustion".to_string()),
realized_envelope: SeriesTree_Indicators_RealizedEnvelope::new(client.clone(), format!("{base_path}_realized_envelope")),
rarity_meter: SeriesTree_Indicators_RarityMeter::new(client.clone(), format!("{base_path}_rarity_meter")),
}
}
}
@@ -5656,32 +5688,18 @@ impl SeriesTree_Indicators_Dormancy {
}
/// Series tree node.
pub struct SeriesTree_Indicators_RealizedEnvelope {
pub pct0_5: CentsSatsUsdPattern,
pub pct1: CentsSatsUsdPattern,
pub pct2: CentsSatsUsdPattern,
pub pct5: CentsSatsUsdPattern,
pub pct95: CentsSatsUsdPattern,
pub pct98: CentsSatsUsdPattern,
pub pct99: CentsSatsUsdPattern,
pub pct99_5: CentsSatsUsdPattern,
pub index: SeriesPattern1<StoredI8>,
pub score: SeriesPattern1<StoredI8>,
pub struct SeriesTree_Indicators_RarityMeter {
pub full: IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern,
pub local: IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern,
pub cycle: IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern,
}
impl SeriesTree_Indicators_RealizedEnvelope {
impl SeriesTree_Indicators_RarityMeter {
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
Self {
pct0_5: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct0_5".to_string()),
pct1: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct01".to_string()),
pct2: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct02".to_string()),
pct5: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct05".to_string()),
pct95: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct95".to_string()),
pct98: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct98".to_string()),
pct99: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct99".to_string()),
pct99_5: CentsSatsUsdPattern::new(client.clone(), "realized_envelope_pct99_5".to_string()),
index: SeriesPattern1::new(client.clone(), "realized_envelope_index".to_string()),
score: SeriesPattern1::new(client.clone(), "realized_envelope_score".to_string()),
full: IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern::new(client.clone(), "rarity_meter".to_string()),
local: IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern::new(client.clone(), "local_rarity_meter".to_string()),
cycle: IndexPct0Pct1Pct2Pct5Pct95Pct98Pct99ScorePattern::new(client.clone(), "cycle_rarity_meter".to_string()),
}
}
}

6
crates/brk_computer/src/indicators/import.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use brk_error::Result;
use brk_types::Version;
use super::{Vecs, realized_envelope::RealizedEnvelope};
use super::{Vecs, rarity_meter::RarityMeter};
use crate::{
indexes,
internal::{
@@ -40,7 +40,7 @@ impl Vecs {
let stock_to_flow = PerBlock::forced_import(&db, "stock_to_flow", v, indexes)?;
let seller_exhaustion = PerBlock::forced_import(&db, "seller_exhaustion", v, indexes)?;
let realized_envelope = RealizedEnvelope::forced_import(&db, v, indexes)?;
let rarity_meter = RarityMeter::forced_import(&db, v, indexes)?;
let this = Self {
db,
@@ -54,7 +54,7 @@ impl Vecs {
dormancy,
stock_to_flow,
seller_exhaustion,
realized_envelope,
rarity_meter,
};
finalize_db(&this.db, &this)?;
Ok(this)

2
crates/brk_computer/src/indicators/mod.rs
View File

@@ -1,7 +1,7 @@
mod compute;
mod gini;
mod import;
pub mod realized_envelope;
pub mod rarity_meter;
mod vecs;
pub use vecs::Vecs;

91
crates/brk_computer/src/indicators/realized_envelope.rs → crates/brk_computer/src/indicators/rarity_meter/inner.rs
View File

@@ -4,13 +4,12 @@ use brk_types::{Cents, Height, Indexes, StoredI8, Version};
use vecdb::{AnyVec, Database, Exit, ReadableVec, Rw, StorageMode, WritableVec};
use crate::{
cointime, distribution, indexes,
indexes,
internal::{PerBlock, Price, RatioPerBlockPercentiles},
prices,
};
#[derive(Traversable)]
pub struct RealizedEnvelope<M: StorageMode = Rw> {
pub struct RarityMeterInner<M: StorageMode = Rw> {
pub pct0_5: Price<PerBlock<Cents, M>>,
pub pct1: Price<PerBlock<Cents, M>>,
pub pct2: Price<PerBlock<Cents, M>>,
@@ -23,113 +22,85 @@ pub struct RealizedEnvelope<M: StorageMode = Rw> {
pub score: PerBlock<StoredI8, M>,
}
const VERSION: Version = Version::new(3);
impl RealizedEnvelope {
impl RarityMeterInner {
pub(crate) fn forced_import(
db: &Database,
prefix: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let v = version + VERSION;
Ok(Self {
pct0_5: Price::forced_import(db, "realized_envelope_pct0_5", v, indexes)?,
pct1: Price::forced_import(db, "realized_envelope_pct01", v, indexes)?,
pct2: Price::forced_import(db, "realized_envelope_pct02", v, indexes)?,
pct5: Price::forced_import(db, "realized_envelope_pct05", v, indexes)?,
pct95: Price::forced_import(db, "realized_envelope_pct95", v, indexes)?,
pct98: Price::forced_import(db, "realized_envelope_pct98", v, indexes)?,
pct99: Price::forced_import(db, "realized_envelope_pct99", v, indexes)?,
pct99_5: Price::forced_import(db, "realized_envelope_pct99_5", v, indexes)?,
index: PerBlock::forced_import(db, "realized_envelope_index", v, indexes)?,
score: PerBlock::forced_import(db, "realized_envelope_score", v, indexes)?,
pct0_5: Price::forced_import(db, &format!("{prefix}_pct0_5"), version, indexes)?,
pct1: Price::forced_import(db, &format!("{prefix}_pct01"), version, indexes)?,
pct2: Price::forced_import(db, &format!("{prefix}_pct02"), version, indexes)?,
pct5: Price::forced_import(db, &format!("{prefix}_pct05"), version, indexes)?,
pct95: Price::forced_import(db, &format!("{prefix}_pct95"), version, indexes)?,
pct98: Price::forced_import(db, &format!("{prefix}_pct98"), version, indexes)?,
pct99: Price::forced_import(db, &format!("{prefix}_pct99"), version, indexes)?,
pct99_5: Price::forced_import(db, &format!("{prefix}_pct99_5"), version, indexes)?,
index: PerBlock::forced_import(db, &format!("{prefix}_index"), version, indexes)?,
score: PerBlock::forced_import(db, &format!("{prefix}_score"), version, indexes)?,
})
}
pub(crate) fn compute(
pub(super) fn compute(
&mut self,
distribution: &distribution::Vecs,
cointime: &cointime::Vecs,
prices: &prices::Vecs,
models: &[&RatioPerBlockPercentiles],
spot: &impl ReadableVec<Height, Cents>,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let realized = &distribution.utxo_cohorts.all.metrics.realized;
let ct = &cointime.prices;
let sth_realized = &distribution.utxo_cohorts.sth.metrics.realized;
let lth_realized = &distribution.utxo_cohorts.lth.metrics.realized;
let models: [&RatioPerBlockPercentiles; 10] = [
&realized.price_ratio_percentiles,
&realized.investor.price.percentiles,
&sth_realized.price_ratio_percentiles,
&sth_realized.investor.price.percentiles,
&lth_realized.price_ratio_percentiles,
&lth_realized.investor.price.percentiles,
&ct.vaulted.percentiles,
&ct.active.percentiles,
&ct.true_market_mean.percentiles,
&ct.cointime.percentiles,
];
macro_rules! sources {
($pct:ident) => {
models.each_ref().map(|m| &m.$pct.price.cents.height)
};
}
let gather = |f: fn(&RatioPerBlockPercentiles) -> &_| -> Vec<_> {
models.iter().map(|m| f(m)).collect()
};
// Lower percentiles: max across all models (tightest lower bound)
self.pct0_5.cents.height.compute_max_of_others(
starting_indexes.height,
&sources!(pct0_5),
&gather(|m| &m.pct0_5.price.cents.height),
exit,
)?;
self.pct1.cents.height.compute_max_of_others(
starting_indexes.height,
&sources!(pct1),
&gather(|m| &m.pct1.price.cents.height),
exit,
)?;
self.pct2.cents.height.compute_max_of_others(
starting_indexes.height,
&sources!(pct2),
&gather(|m| &m.pct2.price.cents.height),
exit,
)?;
self.pct5.cents.height.compute_max_of_others(
starting_indexes.height,
&sources!(pct5),
&gather(|m| &m.pct5.price.cents.height),
exit,
)?;
// Upper percentiles: min across all models (tightest upper bound)
self.pct95.cents.height.compute_min_of_others(
starting_indexes.height,
&sources!(pct95),
&gather(|m| &m.pct95.price.cents.height),
exit,
)?;
self.pct98.cents.height.compute_min_of_others(
starting_indexes.height,
&sources!(pct98),
&gather(|m| &m.pct98.price.cents.height),
exit,
)?;
self.pct99.cents.height.compute_min_of_others(
starting_indexes.height,
&sources!(pct99),
&gather(|m| &m.pct99.price.cents.height),
exit,
)?;
self.pct99_5.cents.height.compute_min_of_others(
starting_indexes.height,
&sources!(pct99_5),
&gather(|m| &m.pct99_5.price.cents.height),
exit,
)?;
let spot = &prices.spot.cents.height;
// Zone: spot vs own envelope bands (-4 to +4)
self.compute_index(spot, starting_indexes, exit)?;
// Temperature: per-model band crossings (-40 to +40)
self.compute_score(&models, spot, starting_indexes, exit)?;
self.compute_score(models, spot, starting_indexes, exit)?;
Ok(())
}
@@ -140,7 +111,7 @@ impl RealizedEnvelope {
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let bands: [&_; 8] = [
let bands = [
&self.pct0_5.cents.height,
&self.pct1.cents.height,
&self.pct2.cents.height,
@@ -213,7 +184,7 @@ impl RealizedEnvelope {
fn compute_score(
&mut self,
models: &[&RatioPerBlockPercentiles; 10],
models: &[&RatioPerBlockPercentiles],
spot: &impl ReadableVec<Height, Cents>,
starting_indexes: &Indexes,
exit: &Exit,

88
crates/brk_computer/src/indicators/rarity_meter/mod.rs Normal file
View File

@@ -0,0 +1,88 @@
mod inner;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Indexes, Version};
use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{distribution, indexes, prices};
pub use inner::RarityMeterInner;
#[derive(Traversable)]
pub struct RarityMeter<M: StorageMode = Rw> {
pub full: RarityMeterInner<M>,
pub local: RarityMeterInner<M>,
pub cycle: RarityMeterInner<M>,
}
const VERSION: Version = Version::new(4);
impl RarityMeter {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let v = version + VERSION;
Ok(Self {
full: RarityMeterInner::forced_import(db, "rarity_meter", v, indexes)?,
local: RarityMeterInner::forced_import(db, "local_rarity_meter", v, indexes)?,
cycle: RarityMeterInner::forced_import(db, "cycle_rarity_meter", v, indexes)?,
})
}
pub(crate) fn compute(
&mut self,
distribution: &distribution::Vecs,
prices: &prices::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let realized = &distribution.utxo_cohorts.all.metrics.realized;
let sth_realized = &distribution.utxo_cohorts.sth.metrics.realized;
let lth_realized = &distribution.utxo_cohorts.lth.metrics.realized;
let spot = &prices.spot.cents.height;
// Full: all + sth + lth (rp + ip), 6 models
self.full.compute(
&[
&realized.price_ratio_percentiles,
&realized.investor.price.percentiles,
&sth_realized.price_ratio_percentiles,
&sth_realized.investor.price.percentiles,
&lth_realized.price_ratio_percentiles,
&lth_realized.investor.price.percentiles,
],
spot,
starting_indexes,
exit,
)?;
// Local: sth only, 2 models
self.local.compute(
&[
&sth_realized.price_ratio_percentiles,
&sth_realized.investor.price.percentiles,
],
spot,
starting_indexes,
exit,
)?;
// Cycle: all + lth, 4 models
self.cycle.compute(
&[
&realized.price_ratio_percentiles,
&realized.investor.price.percentiles,
&lth_realized.price_ratio_percentiles,
&lth_realized.investor.price.percentiles,
],
spot,
starting_indexes,
exit,
)?;
Ok(())
}
}

4
crates/brk_computer/src/indicators/vecs.rs
View File

@@ -2,7 +2,7 @@ use brk_traversable::Traversable;
use brk_types::{BasisPoints16, BasisPoints32, StoredF32};
use vecdb::{Database, Rw, StorageMode};
use super::realized_envelope::RealizedEnvelope;
use super::rarity_meter::RarityMeter;
use crate::internal::{PerBlock, PercentPerBlock, RatioPerBlock};
#[derive(Traversable)]
@@ -25,5 +25,5 @@ pub struct Vecs<M: StorageMode = Rw> {
pub dormancy: DormancyVecs<M>,
pub stock_to_flow: PerBlock<StoredF32, M>,
pub seller_exhaustion: PerBlock<StoredF32, M>,
pub realized_envelope: RealizedEnvelope<M>,
pub rarity_meter: RarityMeter<M>,
}

3
crates/brk_computer/src/lib.rs
View File

@@ -467,9 +467,8 @@ impl Computer {
Ok(())
})?;
self.indicators.realized_envelope.compute(
self.indicators.rarity_meter.compute(
&self.distribution,
&self.cointime,
&self.prices,
&starting_indexes,
exit,

52
crates/brk_reader/examples/last_n_bench.rs Normal file
View File

@@ -0,0 +1,52 @@
//! Times `Reader::after` for a handful of tail-clustered catchup
//! sizes. `N ≤ ~1024` lands in the tail strategy (chunked reverse
//! reader); `N = 10_000` falls through to the forward strategy since
//! it's past the 8-newest-files window.
//!
//! Run with:
//! cargo run --release -p brk_reader --example last_n_bench
//!
//! Requires a running bitcoind with a cookie file at the default path.
use std::time::Instant;
use brk_error::Result;
use brk_reader::Reader;
use brk_rpc::{Auth, Client};
use brk_types::Height;
const SCENARIOS: &[u32] = &[1, 10, 100, 1_000, 10_000];
fn main() -> Result<()> {
let bitcoin_dir = Client::default_bitcoin_path();
let client = Client::new(
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;
let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
let tip = client.get_last_height()?;
println!("Tip: {tip}");
println!();
println!("{:>6} {:>14} {:>10}", "blocks", "elapsed", "blk/s");
println!("{}", "-".repeat(36));
for &n in SCENARIOS {
let anchor_height = Height::from(tip.saturating_sub(n));
let anchor_hash = client.get_block_hash(*anchor_height as u64)?;
let anchor = Some(anchor_hash);
let start = Instant::now();
let mut count = 0usize;
for block in reader.after(anchor)? {
let _ = block?;
count += 1;
}
let elapsed = start.elapsed();
let blk_per_s = count as f64 / elapsed.as_secs_f64().max(f64::EPSILON);
println!("{n:>6} {elapsed:>14?} {blk_per_s:>10.0}");
}
Ok(())
}

2
crates/brk_reader/src/bisect.rs
View File

@@ -53,7 +53,7 @@ pub(crate) fn first_block_height(
}
xor_i.bytes(&mut buf[magic_end..header_end], xor_bytes);
let header = Header::consensus_decode(&mut &buf[magic_end + 4..header_end])?;
let header = Header::consensus_decode_from_finite_reader(&mut &buf[magic_end + 4..header_end])?;
let height = client.get_block_info(&header.block_hash())?.height as u32;
Ok(Height::new(height))

12
crates/brk_reader/src/parse.rs
View File

@@ -27,7 +27,7 @@ pub(crate) fn peek_canonical(
let mut header_buf = [0u8; HEADER_LEN];
header_buf.copy_from_slice(&bytes[..HEADER_LEN]);
xor_state.bytes(&mut header_buf, xor_bytes);
let header = Header::consensus_decode(&mut &header_buf[..]).ok()?;
let header = Header::consensus_decode_from_finite_reader(&mut &header_buf[..]).ok()?;
let offset = canonical.offset_of(&BlockHash::from(header.block_hash()))?;
Some((offset, header))
}
@@ -52,14 +52,20 @@ pub(crate) fn parse_canonical_body(
let mut cursor = Cursor::new(bytes);
cursor.set_position(HEADER_LEN as u64);
let tx_count = VarInt::consensus_decode(&mut cursor)?.0 as usize;
// `consensus_decode_from_finite_reader` skips the `Take<R>` wrap
// that `consensus_decode` applies to every nested field for
// memory-safety — our cursor is already a bounded `Vec<u8>`, so
// the extra wrapping is pure overhead. Per the crate docs it's
// "marginally faster", but for a ~2000-tx block the per-field
// compounding adds up.
let tx_count = VarInt::consensus_decode_from_finite_reader(&mut cursor)?.0 as usize;
let mut txdata = Vec::with_capacity(tx_count);
let mut tx_metadata = Vec::with_capacity(tx_count);
let mut tx_offsets = Vec::with_capacity(tx_count);
for _ in 0..tx_count {
let tx_start = cursor.position() as u32;
tx_offsets.push(tx_start);
let tx = Transaction::consensus_decode(&mut cursor)?;
let tx = Transaction::consensus_decode_from_finite_reader(&mut cursor)?;
let tx_len = cursor.position() as u32 - tx_start;
txdata.push(tx);
tx_metadata.push(BlkMetadata::new(metadata.position() + tx_start, tx_len));

1
crates/brk_reader/src/pipeline/forward.rs
View File

@@ -135,6 +135,7 @@ fn read_and_dispatch(
scan_bytes(
&mut bytes,
blk_index,
0,
xor_bytes,
|metadata, block_bytes, xor_state| {
if stop.get().is_some() {

136
crates/brk_reader/src/pipeline/tail.rs
View File

@@ -1,10 +1,10 @@
//! Tail pipeline: single-threaded reverse scan of the newest blk
//! files until every canonical hash is matched, then forward-emit
//! with an inline chain check. Avoids the forward pipeline's
//! bisection + out-of-order backoff (~2.7 GB of reads) for any
//! tip-clustered catchup.
//! files, reading each file in `TAIL_CHUNK`-sized slices from tail
//! to head so we only touch bytes covering the canonical window.
//! Matches fill offset slots and are emitted forward with an inline
//! chain check.
use std::{fs, ops::ControlFlow};
use std::{fs::File, ops::ControlFlow, os::unix::fs::FileExt};
use brk_error::{Error, Result};
use brk_rpc::Client;
@@ -18,6 +18,8 @@ use crate::{
scan::scan_bytes,
};
const TAIL_CHUNK: usize = 8 * 1024 * 1024;
pub(super) fn pipeline_tail(
client: &Client,
paths: &BlkIndexToBlkPath,
@@ -34,7 +36,7 @@ pub(super) fn pipeline_tail(
// miss doesn't scan the entire chain in reverse.
let mut below_floor_streak: usize = 0;
for (&blk_index, path) in paths.iter().rev() {
'files: for (&blk_index, path) in paths.iter().rev() {
// If this file's first block is below the lowest still-missing
// canonical height, we've walked past the window.
if let Some(missing_idx) = slots.iter().position(Option::is_none)
@@ -53,51 +55,85 @@ pub(super) fn pipeline_tail(
}
}
let mut bytes = fs::read(path)?;
scan_bytes(
&mut bytes,
blk_index,
xor_bytes,
|metadata, block_bytes, xor_state| {
let Some((offset, header)) =
peek_canonical(block_bytes, xor_state, xor_bytes, canonical)
else {
return ControlFlow::Continue(());
};
if slots[offset as usize].is_some() {
return ControlFlow::Continue(());
}
let height = Height::from(*canonical.start + offset);
match parse_canonical_body(
block_bytes.to_vec(),
metadata,
xor_state,
xor_bytes,
height,
header,
) {
Ok(block) => {
slots[offset as usize] = Some(block);
remaining -= 1;
}
Err(e) => {
parse_failure = Some(e);
return ControlFlow::Break(());
}
}
if remaining == 0 {
ControlFlow::Break(())
} else {
ControlFlow::Continue(())
}
},
);
if let Some(e) = parse_failure {
return Err(e);
let file = File::open(path)?;
let file_len = file.metadata()?.len() as usize;
if file_len == 0 {
continue;
}
if remaining == 0 {
break;
// Chunked reverse read. `end` is the file position we've
// already covered (exclusive). Each iteration reads
// [end - TAIL_CHUNK..end] and prepends it to any `spillover`
// carried from the previous iteration — the pre-first-magic
// bytes of that chunk, which must belong to a block that
// started in this earlier region.
let mut end = file_len;
let mut spillover: Vec<u8> = Vec::new();
while end > 0 && remaining > 0 {
let start = end.saturating_sub(TAIL_CHUNK);
let chunk_len = end - start;
let mut buf = vec![0u8; chunk_len + spillover.len()];
file.read_exact_at(&mut buf[..chunk_len], start as u64)?;
buf[chunk_len..].copy_from_slice(&spillover);
spillover.clear();
// `buf` now represents file bytes [start..start + buf.len()].
let result = scan_bytes(
&mut buf,
blk_index,
start,
xor_bytes,
|metadata, block_bytes, xor_state| {
let Some((offset, header)) =
peek_canonical(block_bytes, xor_state, xor_bytes, canonical)
else {
return ControlFlow::Continue(());
};
if slots[offset as usize].is_some() {
return ControlFlow::Continue(());
}
let height = Height::from(*canonical.start + offset);
match parse_canonical_body(
block_bytes.to_vec(),
metadata,
xor_state,
xor_bytes,
height,
header,
) {
Ok(block) => {
slots[offset as usize] = Some(block);
remaining -= 1;
}
Err(e) => {
parse_failure = Some(e);
return ControlFlow::Break(());
}
}
if remaining == 0 {
ControlFlow::Break(())
} else {
ControlFlow::Continue(())
}
},
);
if let Some(e) = parse_failure {
return Err(e);
}
if remaining == 0 {
break 'files;
}
// Carry pre-first-magic bytes into the next (earlier)
// chunk so a block that straddled this chunk's start is
// stitched back together.
end = start;
if end > 0 {
let prefix_len = result.first_magic.unwrap_or(buf.len());
spillover.extend_from_slice(&buf[..prefix_len]);
}
}
}

37
crates/brk_reader/src/scan.rs
View File

@@ -8,7 +8,7 @@ const MAGIC_BYTES: [u8; 4] = [0xF9, 0xBE, 0xB4, 0xD9];
/// Returns the position **immediately after** the matched magic, or
/// `None` if no match. Advances `xor_i` by the bytes consumed either
/// way.
/// way. First-byte fast-fail keeps the inner loop tight.
pub(crate) fn find_magic(bytes: &[u8], xor_i: &mut XORIndex, xor_bytes: XORBytes) -> Option<usize> {
let len = bytes.len();
if len < MAGIC_BYTES.len() {
@@ -42,36 +42,51 @@ pub(crate) fn find_magic(bytes: &[u8], xor_i: &mut XORIndex, xor_bytes: XORBytes
None
}
/// Scans `buf` (the full contents of one blk file) for blocks,
/// calling `on_block` for each. The block bytes are passed as a
/// mutable borrow so the callback can clone (to ship to a parser
/// thread) or process in place (to peek the header).
/// Position (relative to `buf`) of the first matched magic byte.
/// Used by the chunked tail pipeline to carry pre-first-magic bytes
/// into the next (earlier) chunk.
pub(crate) struct ScanResult {
pub first_magic: Option<usize>,
}
/// Scans `buf` for blocks and calls `on_block` for each. `file_offset`
/// is the absolute file position of `buf[0]` — used to seed the XOR
/// phase and to report absolute `BlkPosition`s so the chunked tail
/// pipeline can read mid-file slices.
pub(crate) fn scan_bytes(
buf: &mut [u8],
blk_index: u16,
file_offset: usize,
xor_bytes: XORBytes,
mut on_block: impl FnMut(BlkMetadata, &mut [u8], XORIndex) -> ControlFlow<()>,
) {
let mut xor_i = XORIndex::default();
) -> ScanResult {
let mut xor_i = XORIndex::at_offset(file_offset);
let mut first_magic: Option<usize> = None;
let mut i = 0;
while let Some(off) = find_magic(&buf[i..], &mut xor_i, xor_bytes) {
first_magic.get_or_insert(i + off - MAGIC_BYTES.len());
i += off;
if i + 4 > buf.len() {
return;
break;
}
let mut size_bytes = [buf[i], buf[i + 1], buf[i + 2], buf[i + 3]];
xor_i.bytes(&mut size_bytes, xor_bytes);
let len = u32::from_le_bytes(size_bytes) as usize;
i += 4;
if i + len > buf.len() {
return;
break;
}
let metadata = BlkMetadata::new(BlkPosition::new(blk_index, i as u32), len as u32);
let metadata = BlkMetadata::new(
BlkPosition::new(blk_index, (file_offset + i) as u32),
len as u32,
);
if on_block(metadata, &mut buf[i..i + len], xor_i).is_break() {
return;
break;
}
i += len;
xor_i.add_assign(len);
}
ScanResult { first_magic }
}