global: snapshot

This commit is contained in:
nym21
2026-03-04 23:21:56 +01:00
parent 9e23de4ba1
commit ef0b77baa8
51 changed files with 2109 additions and 5730 deletions
@@ -6,7 +6,9 @@ use vecdb::{
VecIndex, VecValue, Version,
};
use crate::internal::{ComputedVecValue, DistributionStats};
use crate::internal::{
ComputedVecValue, DistributionStats, compute_aggregations, compute_aggregations_nblock_window,
};
#[derive(Traversable)]
pub struct Distribution<I: VecIndex, T: ComputedVecValue + JsonSchema, M: StorageMode = Rw> {
@@ -50,7 +52,7 @@ impl<I: VecIndex, T: ComputedVecValue + JsonSchema> Distribution<I, T> {
where
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
{
crate::internal::compute_aggregations(
compute_aggregations(
max_from,
source,
first_indexes,
@@ -87,7 +89,7 @@ impl<I: VecIndex, T: ComputedVecValue + JsonSchema> Distribution<I, T> {
where
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
{
crate::internal::compute_aggregations_nblock_window(
compute_aggregations_nblock_window(
max_from,
source,
first_indexes,
@@ -6,7 +6,7 @@ use vecdb::{
VecIndex, VecValue, Version,
};
use crate::internal::ComputedVecValue;
use crate::internal::{ComputedVecValue, compute_aggregations};
use super::Distribution;
@@ -43,7 +43,7 @@ impl<I: VecIndex, T: ComputedVecValue + JsonSchema> Full<I, T> {
where
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
{
crate::internal::compute_aggregations(
compute_aggregations(
max_from,
source,
first_indexes,
@@ -68,11 +68,8 @@ where
} else {
0
};
let partial_values: Vec<f64> = values
.collect_range_at(range_start, end)
.into_iter()
.map(|a| f64::from(a))
.collect();
let mut partial_values: Vec<f64> = Vec::with_capacity(end - range_start);
values.for_each_range_at(range_start, end, |a: A| partial_values.push(f64::from(a)));
let capacity = if skip > 0 && skip < end {
let first_start = window_starts.collect_one_at(skip).unwrap().to_usize();
@@ -45,11 +45,8 @@ where
} else {
0
};
let partial_values: Vec<f64> = values
.collect_range_at(range_start, end)
.into_iter()
.map(|a| f64::from(a))
.collect();
let mut partial_values: Vec<f64> = Vec::with_capacity(end - range_start);
values.for_each_range_at(range_start, end, |a: A| partial_values.push(f64::from(a)));
let capacity = if skip > 0 && skip < end {
let first_start = window_starts.collect_one_at(skip).unwrap().to_usize();
@@ -39,9 +39,8 @@ impl SortedBlocks {
// Find the block where value belongs: first block whose max >= value
let block_idx = self
.blocks
.iter()
.position(|b| *b.last().unwrap() >= value)
.unwrap_or(self.blocks.len() - 1);
.partition_point(|b| *b.last().unwrap() < value)
.min(self.blocks.len() - 1);
let block = &mut self.blocks[block_idx];
let pos = block.partition_point(|a| *a < value);
@@ -126,25 +126,26 @@ impl TDigest {
return;
}
let total: f64 = self.centroids.iter().map(|c| c.weight).sum();
let mut merged: Vec<Centroid> = Vec::with_capacity(self.centroids.len());
let total = self.count as f64;
let mut cum = 0.0;
let mut write_idx = 0;
for c in &self.centroids {
if let Some(last) = merged.last_mut() {
let q = (cum + last.weight / 2.0) / total;
let limit = (4.0 * self.compression * q * (1.0 - q)).floor().max(1.0);
if last.weight + c.weight <= limit {
let new_weight = last.weight + c.weight;
last.mean = (last.mean * last.weight + c.mean * c.weight) / new_weight;
last.weight = new_weight;
continue;
}
for read_idx in 1..self.centroids.len() {
let c = self.centroids[read_idx];
let last = &mut self.centroids[write_idx];
let q = (cum + last.weight / 2.0) / total;
let limit = (4.0 * self.compression * q * (1.0 - q)).floor().max(1.0);
if last.weight + c.weight <= limit {
let new_weight = last.weight + c.weight;
last.mean = (last.mean * last.weight + c.mean * c.weight) / new_weight;
last.weight = new_weight;
} else {
cum += last.weight;
write_idx += 1;
self.centroids[write_idx] = c;
}
merged.push(*c);
}
self.centroids = merged;
self.centroids.truncate(write_idx + 1);
}
/// Batch quantile query in a single pass. `qs` must be sorted ascending.
@@ -167,7 +168,7 @@ impl TDigest {
return;
}
let total: f64 = self.centroids.iter().map(|c| c.weight).sum();
let total = self.count as f64;
let mut cum = 0.0;
let mut ci = 0;
@@ -33,20 +33,16 @@ impl<B: BpsType> PercentFromHeight<B> {
indexes: &indexes::Vecs,
) -> Result<Self> {
let bps = ComputedFromHeight::forced_import(db, &format!("{name}_bps"), version, indexes)?;
let bps_clone = bps.height.read_only_boxed_clone();
let ratio = LazyFromHeight::from_computed::<B::ToRatio>(
&format!("{name}_ratio"),
version,
bps.height.read_only_boxed_clone(),
bps_clone.clone(),
&bps,
);
let percent = LazyFromHeight::from_computed::<B::ToPercent>(
name,
version,
bps.height.read_only_boxed_clone(),
&bps,
);
let percent = LazyFromHeight::from_computed::<B::ToPercent>(name, version, bps_clone, &bps);
Ok(Self {
bps,
@@ -8,7 +8,7 @@ use vecdb::{
use crate::{
blocks, indexes,
internal::{ComputedFromHeightStdDevExtended, Price, TDigest},
internal::{ComputedFromHeightStdDevExtended, Price, PriceTimesRatioBp32Cents, TDigest},
};
use super::{super::ComputedFromHeight, ComputedFromHeightRatio};
@@ -200,8 +200,6 @@ impl ComputedFromHeightRatioExtension {
metric_price: &impl ReadableVec<Height, Cents>,
exit: &Exit,
) -> Result<()> {
use crate::internal::PriceTimesRatioBp32Cents;
macro_rules! compute_band {
($usd_field:ident, $band_source:expr) => {
self.$usd_field
@@ -6,9 +6,10 @@ use vecdb::{
WritableVec,
};
use crate::{blocks, indexes};
use crate::internal::{ComputedFromHeight, Price};
use crate::{
blocks, indexes,
internal::{ComputedFromHeight, Price, PriceTimesRatioCents},
};
use super::ComputedFromHeightStdDev;
@@ -172,8 +173,7 @@ impl ComputedFromHeightStdDevExtended {
const MULTIPLIERS: [f32; 12] = [
0.5, 1.0, 1.5, 2.0, 2.5, 3.0, -0.5, -1.0, -1.5, -2.0, -2.5, -3.0,
];
let band_vecs: Vec<_> = self.mut_band_height_vecs().collect();
for (vec, mult) in band_vecs.into_iter().zip(MULTIPLIERS) {
for (vec, mult) in self.mut_band_height_vecs().zip(MULTIPLIERS) {
for (offset, _) in source_data.iter().enumerate() {
let index = start + offset;
let average = sma_data[offset];
@@ -214,8 +214,6 @@ impl ComputedFromHeightStdDevExtended {
metric_price: &impl ReadableVec<Height, Cents>,
exit: &Exit,
) -> Result<()> {
use crate::internal::PriceTimesRatioCents;
macro_rules! compute_band {
($usd_field:ident, $band_source:expr) => {
self.$usd_field
@@ -52,6 +52,19 @@ impl ComputedFromHeightStdDev {
exit: &Exit,
source: &impl ReadableVec<Height, StoredF32>,
) -> Result<()> {
if self.days == usize::MAX {
self.sma
.height
.compute_sma_(starting_indexes.height, source, usize::MAX, exit, None)?;
self.sd.height.compute_expanding_sd(
starting_indexes.height,
source,
&self.sma.height,
exit,
)?;
return Ok(());
}
let window_starts = blocks.count.start_vec(self.days);
self.sma.height.compute_rolling_average(