clients: add .len()

This commit is contained in:
nym21
2026-04-29 12:06:22 +02:00
parent f1749472e7
commit a7e41df1c6
17 changed files with 265 additions and 149 deletions

View File

@@ -4,7 +4,7 @@ use std::fmt::Write;
use crate::{
Endpoint, Parameter,
generators::{normalize_return_type, write_description},
generators::{javascript::types::jsdoc_normalize, normalize_return_type, write_description},
to_camel_case,
};
@@ -16,8 +16,9 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
}
let method_name = endpoint_to_method_name(endpoint);
let base_return_type =
normalize_return_type(endpoint.response_type.as_deref().unwrap_or("*"));
let base_return_type = jsdoc_normalize(&normalize_return_type(
endpoint.response_type.as_deref().unwrap_or("*"),
));
let return_type = if endpoint.supports_csv {
format!("{} | string", base_return_type)
} else {
@@ -51,20 +52,17 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
for param in &endpoint.path_params {
let desc = format_param_desc(param.description.as_deref());
writeln!(
output,
" * @param {{{}}} {}{}",
param.param_type, param.name, desc
)
.unwrap();
let ty = jsdoc_normalize(&param.param_type);
writeln!(output, " * @param {{{}}} {}{}", ty, param.name, desc).unwrap();
}
for param in &endpoint.query_params {
let optional = if param.required { "" } else { "=" };
let desc = format_param_desc(param.description.as_deref());
let ty = jsdoc_normalize(&param.param_type);
writeln!(
output,
" * @param {{{}{}}} [{}]{}",
param.param_type, optional, param.name, desc
ty, optional, param.name, desc
)
.unwrap();
}

View File

@@ -198,7 +198,6 @@ function _wrapSeriesData(raw) {{
* @property {{number}} version - Version of the series data
* @property {{Index}} index - The index type used for this query
* @property {{string}} type - Value type (e.g. "f32", "u64", "Sats")
* @property {{number}} total - Total number of data points
* @property {{number}} start - Start index (inclusive)
* @property {{number}} end - End index (exclusive)
* @property {{string}} stamp - ISO 8601 timestamp of when the response was generated
@@ -236,6 +235,8 @@ function _wrapSeriesData(raw) {{
* @property {{(n: number) => SkippedBuilder<T>}} skip - Skip first n items, chain with take()
* @property {{(onUpdate?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch all data
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{() => Promise<number>}} len - Get total number of data points
* @property {{() => Promise<Version>}} version - Get the current version of the series
* @property {{Thenable<T>}} then - Thenable (await endpoint)
* @property {{string}} path - The endpoint path
*/
@@ -250,6 +251,8 @@ function _wrapSeriesData(raw) {{
* @property {{(n: number) => DateSkippedBuilder<T>}} skip - Skip first n items, chain with take()
* @property {{(onUpdate?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{() => Promise<number>}} len - Get total number of data points
* @property {{() => Promise<Version>}} version - Get the current version of the series
* @property {{DateThenable<T>}} then - Thenable (await endpoint)
* @property {{string}} path - The endpoint path
*/
@@ -308,7 +311,7 @@ function _wrapSeriesData(raw) {{
/**
* Create a series endpoint builder with typestate pattern.
* @template T
* @param {{BrkClientBase}} client
* @param {{BrkClient}} client
* @param {{string}} name - The series vec name
* @param {{Index}} index - The index name
* @returns {{DateSeriesEndpoint<T>}}
@@ -376,6 +379,8 @@ function _endpoint(client, name, index) {{
skip(n) {{ return skippedBuilder(n); }},
fetch(onUpdate) {{ return client._fetchSeriesData(buildPath(), onUpdate); }},
fetchCsv() {{ return client.getText(buildPath(undefined, undefined, 'csv')); }},
len() {{ return client.getSeriesLen(name, index); }},
version() {{ return client.getSeriesVersion(name, index); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
get path() {{ return p; }},
}};
@@ -626,7 +631,7 @@ pub fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern
r#"/**
* Generic series pattern factory.
* @template T
* @param {{BrkClientBase}} client
* @param {{BrkClient}} client
* @param {{string}} name - The series vec name
* @param {{readonly Index[]}} indexes - The supported indexes
*/
@@ -679,7 +684,7 @@ function _mp(client, name, indexes) {{
// Generate thin wrapper that calls the generic factory
writeln!(
output,
"/** @template T @param {{BrkClientBase}} client @param {{string}} name @returns {{{}<T>}} */",
"/** @template T @param {{BrkClient}} client @param {{string}} name @returns {{{}<T>}} */",
pattern.name
)
.unwrap();
@@ -741,7 +746,7 @@ pub fn generate_structural_patterns(
if pattern.is_generic {
writeln!(output, " * @template T").unwrap();
}
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
writeln!(output, " * @param {{BrkClient}} client").unwrap();
writeln!(output, " * @param {{string}} acc - Accumulated series name").unwrap();
if pattern.is_templated() {
writeln!(output, " * @param {{string}} disc - Discriminator suffix").unwrap();

View File

@@ -111,6 +111,25 @@ fn json_type_to_js(ty: &str, schema: &Value, current_type: Option<&str>) -> Stri
}
}
/// JSDoc has no `integer` keyword, only `number`. Map `integer` (and `integer[]`,
/// `Foo<integer>`, etc.) to `number` before emitting type strings to JS.
pub fn jsdoc_normalize(ty: &str) -> String {
let mut out = ty.to_string();
let mut prev = String::new();
while prev != out {
prev = out.clone();
out = out.replace("integer[]", "number[]");
out = out.replace("<integer>", "<number>");
out = out.replace("(integer)", "(number)");
out = out.replace("integer | ", "number | ");
out = out.replace(" | integer", " | number");
}
if out == "integer" {
return "number".to_string();
}
out
}
/// Convert a JSON schema to a JavaScript type string.
pub fn schema_to_js_type(schema: &Value, current_type: Option<&str>) -> String {
if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array()) {

View File

@@ -221,7 +221,6 @@ class SeriesData(Generic[T]):
version: int
index: Index
type: str
total: int
start: int
end: int
stamp: str

View File

@@ -364,7 +364,7 @@ fn single_type_to_name(t: &SchemaType, schema: &ObjectSchema) -> Option<String>
match t {
SchemaType::String => Some("string".to_string()),
SchemaType::Number => Some("number".to_string()),
SchemaType::Integer => Some("number".to_string()),
SchemaType::Integer => Some("integer".to_string()),
SchemaType::Boolean => Some("boolean".to_string()),
SchemaType::Array => {
let inner = match &schema.items {

View File

@@ -9350,7 +9350,7 @@ impl BrkClient {
/// Returns the total number of data points for a series at the given index.
///
/// Endpoint: `GET /api/series/{series}/{index}/len`
pub fn get_series_len(&self, series: SeriesName, index: Index) -> Result<f64> {
pub fn get_series_len(&self, series: SeriesName, index: Index) -> Result<i64> {
self.base.get_json(&format!("/api/series/{series}/{}/len", index.name()))
}
@@ -9845,7 +9845,7 @@ impl BrkClient {
/// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-transaction-times)*
///
/// Endpoint: `GET /api/v1/transaction-times`
pub fn get_transaction_times(&self) -> Result<Vec<f64>> {
pub fn get_transaction_times(&self) -> Result<Vec<i64>> {
self.base.get_json(&format!("/api/v1/transaction-times"))
}

View File

@@ -31,10 +31,6 @@ impl Applier {
}
}
/// Move one tx from the live mempool into the graveyard. Removes
/// from every store + tracker, then hands the body to
/// `graveyard.bury`. Silently bails if the entry or tx body is
/// already gone (idempotent under repeated removals).
fn bury_one(s: &mut LockedState, prefix: &TxidPrefix, reason: TxRemoval) {
let Some(entry) = s.entries.remove(prefix) else {
return;
@@ -58,9 +54,6 @@ impl Applier {
s.txs.extend(to_store);
}
/// Materialize a `TxAddition` into the (tx, entry) pair the Applier
/// will publish. Fresh additions are already-decoded; Revived ones
/// pull the cached body out of the graveyard and skip if it's gone.
fn resolve_addition(
s: &mut LockedState,
addition: TxAddition,
@@ -74,9 +67,6 @@ impl Applier {
}
}
/// Publish one tx into the live mempool: fold its fee into info,
/// register addr deltas, store the entry. Returns `(txid, tx)` for
/// the caller to batch into `txs.extend` once at the end.
fn publish_one(s: &mut LockedState, tx: Transaction, entry: TxEntry) -> (Txid, Transaction) {
s.info.add(&tx, entry.fee);
s.addrs.add_tx(&tx, &entry.txid);

View File

@@ -3,10 +3,10 @@ use std::{collections::BTreeMap, sync::LazyLock};
use brk_error::{Error, Result};
use brk_traversable::TreeNode;
use brk_types::{
BlockHashPrefix, Date, DetailedSeriesCount, Epoch, Format, Halving, Height, Index, IndexInfo,
LegacyValue, Limit, Output, OutputLegacy, PaginatedSeries, Pagination, PaginationIndex,
RangeIndex, RangeMap, SearchQuery, SeriesData, SeriesInfo, SeriesName, SeriesOutput,
SeriesOutputLegacy, SeriesSelection, Timestamp, Version,
BlockHashPrefix, CacheClass, Date, DetailedSeriesCount, Epoch, Format, Halving, Height, Index,
IndexInfo, LegacyValue, Limit, Output, OutputLegacy, PaginatedSeries, Pagination,
PaginationIndex, RangeIndex, RangeMap, SearchQuery, SeriesData, SeriesInfo, SeriesName,
SeriesOutput, SeriesOutputLegacy, SeriesSelection, Timestamp, Version,
};
use parking_lot::RwLock;
use vecdb::{AnyExportableVec, ReadableVec};
@@ -196,6 +196,13 @@ impl Query {
});
}
// Snapshot tip-derived state together so the historical-branch ETag stays
// self-consistent: stable_count is computed from tip_height, hash_prefix
// is the live tip.
let tip_height = self.indexed_height();
let hash_prefix = self.tip_hash_prefix();
let stable_count = self.stable_count(params.index, total, tip_height);
Ok(ResolvedQuery {
vecs,
format: params.format(),
@@ -204,10 +211,58 @@ impl Query {
total,
start,
end,
hash_prefix: self.tip_hash_prefix(),
hash_prefix,
stable_count,
})
}
/// Count of leading entries provably immutable across a 6-block reorg, used
/// to gate the historical-branch series ETag.
///
/// - Bucketed indexes: `total - margin`.
/// - Entity indexes: `first_X_index[tip_height - 6]`, falling back to 0 if
/// the tip is shallower than 6 blocks. Clamped to `total` so a query
/// whose vecs are shorter than the entity-type's own count never marks
/// its live tail as stable.
/// - Mutable (Funded/Empty addr): `None`. No immutable region exists, so
/// the caller must use the tip-bound ETag for every range.
pub fn stable_count(
&self,
index: Index,
total: usize,
tip_height: Height,
) -> Option<usize> {
match index.cache_class() {
CacheClass::Bucket { margin } => Some(total.saturating_sub(margin)),
CacheClass::Entity => {
let h = Height::from((*tip_height).saturating_sub(6));
let v = &self.indexer().vecs;
let n = match index {
Index::TxIndex => v.transactions.first_tx_index.collect_one(h).map(usize::from),
Index::TxInIndex => v.inputs.first_txin_index.collect_one(h).map(usize::from),
Index::TxOutIndex => v.outputs.first_txout_index.collect_one(h).map(usize::from),
Index::EmptyOutputIndex => v.scripts.empty.first_index.collect_one(h).map(usize::from),
Index::OpReturnIndex => v.scripts.op_return.first_index.collect_one(h).map(usize::from),
Index::P2MSOutputIndex => v.scripts.p2ms.first_index.collect_one(h).map(usize::from),
Index::UnknownOutputIndex => v.scripts.unknown.first_index.collect_one(h).map(usize::from),
Index::P2AAddrIndex => v.addrs.p2a.first_index.collect_one(h).map(usize::from),
Index::P2PK33AddrIndex => v.addrs.p2pk33.first_index.collect_one(h).map(usize::from),
Index::P2PK65AddrIndex => v.addrs.p2pk65.first_index.collect_one(h).map(usize::from),
Index::P2PKHAddrIndex => v.addrs.p2pkh.first_index.collect_one(h).map(usize::from),
Index::P2SHAddrIndex => v.addrs.p2sh.first_index.collect_one(h).map(usize::from),
Index::P2TRAddrIndex => v.addrs.p2tr.first_index.collect_one(h).map(usize::from),
Index::P2WPKHAddrIndex => v.addrs.p2wpkh.first_index.collect_one(h).map(usize::from),
Index::P2WSHAddrIndex => v.addrs.p2wsh.first_index.collect_one(h).map(usize::from),
_ => unreachable!("non-entity index in CacheClass::Entity arm"),
}
.unwrap_or(0)
.min(total);
Some(n)
}
CacheClass::Mutable => None,
}
}
/// Format a resolved query (expensive).
/// Call after ETag/cache checks to avoid unnecessary work.
pub fn format(&self, resolved: ResolvedQuery) -> Result<SeriesOutput> {
@@ -449,8 +504,9 @@ impl Query {
}
/// A resolved series query ready for formatting.
/// Carries the vecs plus the metadata (version, total, end, hash_prefix) callers
/// need to derive an etag or cache policy.
/// Carries the vecs plus the metadata callers need to derive an etag or cache
/// policy. `stable_count` is `None` for indexes whose entries can mutate
/// retroactively (Funded/Empty addr).
pub struct ResolvedQuery {
pub vecs: Vec<&'static dyn AnyExportableVec>,
pub format: Format,
@@ -460,6 +516,7 @@ pub struct ResolvedQuery {
pub start: usize,
pub end: usize,
pub hash_prefix: BlockHashPrefix,
pub stable_count: Option<usize>,
}
impl ResolvedQuery {

View File

@@ -44,8 +44,9 @@ pub(super) async fn serve(
let csv_filename = resolved.csv_filename();
let cache_params = CacheParams::series(
resolved.version,
resolved.total,
resolved.start,
resolved.end,
resolved.stable_count,
resolved.hash_prefix,
);

View File

@@ -67,23 +67,39 @@ impl CacheParams {
}
}
/// Series query: tail-bound (`end >= total`) gets LIVE, historical gets CACHED.
/// Etag distinguishes the two: tail uses tip hash (per-block + reorgs),
/// historical uses total length (only changes when new data is appended).
pub fn series(version: Version, total: usize, end: usize, hash: BlockHashPrefix) -> Self {
/// Series query: tail-bound gets LIVE, historical gets CACHED.
///
/// `stable_count` is the count of leading entries provably immutable across
/// a 6-block reorg (per `Index::cache_class()` + `Query::stable_count`).
/// `None` (Funded/Empty addr indexes) forces the tail branch for every range.
///
/// Etag shapes:
/// - historical (`end <= stable_count`): `s{v}-h{start}-{end}`. Pure
/// range, stable across appends and reorgs of the volatile tail.
/// - tail (`end > stable_count` or `stable_count.is_none()`):
/// `s{v}-t{tip_hash:x}`. Invalidates per-block, reorg-safe.
///
/// The `h`/`t` discriminator after `s{v}-` prevents collision with old
/// `s{v}-{number}` ETags from before the migration.
pub fn series(
version: Version,
start: usize,
end: usize,
stable_count: Option<usize>,
hash: BlockHashPrefix,
) -> Self {
let v = u32::from(version);
if end >= total {
Self {
etag: format!("s{v}-{:x}", *hash).into(),
cache_control: CC,
cdn_cache_control: CDN_LIVE,
}
} else {
Self {
etag: format!("s{v}-{total}").into(),
match stable_count {
Some(s) if end <= s => Self {
etag: format!("s{v}-h{start}-{end}").into(),
cache_control: CC,
cdn_cache_control: cdn_cached(),
}
},
_ => Self {
etag: format!("s{v}-t{:x}", *hash).into(),
cache_control: CC,
cdn_cache_control: CDN_LIVE,
},
}
}
@@ -139,28 +155,55 @@ mod tests {
}
#[test]
fn series_tail_uses_tip_hash() {
let p = CacheParams::series(v(3), 100, 100, h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-abcd");
fn series_tail_when_end_exceeds_stable_count() {
let p = CacheParams::series(v(3), 0, 60, Some(50), h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-tabcd");
}
#[test]
fn series_historical_uses_total() {
let p = CacheParams::series(v(3), 100, 50, h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-100");
fn series_historical_when_end_at_or_below_stable_count() {
let p = CacheParams::series(v(3), 10, 50, Some(50), h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-h10-50");
}
#[test]
fn series_historical_ignores_tip_hash() {
let a = CacheParams::series(v(3), 100, 50, h(0xabcd));
let b = CacheParams::series(v(3), 100, 50, h(0xdead));
let a = CacheParams::series(v(3), 0, 50, Some(100), h(0xabcd));
let b = CacheParams::series(v(3), 0, 50, Some(100), h(0xdead));
assert_eq!(a.etag.as_str(), b.etag.as_str());
}
#[test]
fn series_tail_changes_with_tip_hash() {
let a = CacheParams::series(v(3), 100, 100, h(0xabcd));
let b = CacheParams::series(v(3), 100, 100, h(0xdead));
let a = CacheParams::series(v(3), 0, 100, Some(50), h(0xabcd));
let b = CacheParams::series(v(3), 0, 100, Some(50), h(0xdead));
assert_ne!(a.etag.as_str(), b.etag.as_str());
}
#[test]
fn series_mutable_class_always_tail() {
let small = CacheParams::series(v(3), 0, 5, None, h(0xabcd));
let large = CacheParams::series(v(3), 0, 1_000_000, None, h(0xabcd));
assert_eq!(small.etag.as_str(), "s3-tabcd");
assert_eq!(large.etag.as_str(), "s3-tabcd");
}
#[test]
fn series_at_stable_boundary_is_historical() {
let p = CacheParams::series(v(3), 0, 50, Some(50), h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-h0-50");
}
#[test]
fn series_just_past_stable_boundary_is_tail() {
let p = CacheParams::series(v(3), 0, 51, Some(50), h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-tabcd");
}
#[test]
fn series_different_ranges_get_different_etags() {
let a = CacheParams::series(v(3), 0, 50, Some(100), h(0xabcd));
let b = CacheParams::series(v(3), 10, 50, Some(100), h(0xabcd));
assert_ne!(a.etag.as_str(), b.etag.as_str());
}
}

View File

@@ -73,6 +73,23 @@ pub enum Index {
EmptyAddrIndex,
}
/// How the trailing edge of an [`Index`] mutates over time. Drives the series
/// cache: bucketed indexes have a small fixed volatile tail, entity indexes
/// derive their stable count from a height→first-index mapping, and mutable
/// addr indexes have no immutable region (entries change retroactively).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CacheClass {
/// Append-only with `margin` trailing entries volatile per ≥6-block reorg.
Bucket { margin: usize },
/// Append-only entity index (one number per tx / input / output / typed
/// addr). Stable count must be looked up via the indexer's
/// `first_X_index[tip - 6]` mapping.
Entity,
/// Retroactively mutable: no immutable region (`FundedAddrIndex`,
/// `EmptyAddrIndex`).
Mutable,
}
impl Index {
pub const fn all() -> [Self; 33] {
[
@@ -195,22 +212,19 @@ impl Index {
}
}
/// Number of trailing entries that may still mutate due to a 6-block reorg.
/// Used to size the cache invalidation tail: ranges ending within this margin
/// of `total` use a tip-bound ETag, others may use the cheaper total-only ETag.
///
/// Panics for cohort indexes (per-tx, per-output, per-addr): series queries
/// shouldn't reach this codepath under those indexes. If they do, the cache
/// strategy needs rethinking.
pub const fn safety_margin(&self) -> usize {
/// Classifies how the trailing edge of an index mutates, so the series cache
/// can pick the right stable-count strategy.
pub const fn cache_class(&self) -> CacheClass {
match self {
Self::Minute10 => 6,
Self::Minute30 => 2,
Self::Hour1 | Self::Hour4 | Self::Hour12 => 1,
Self::Day1 | Self::Day3 | Self::Week1 => 1,
Self::Month1 | Self::Month3 | Self::Month6 => 1,
Self::Year1 | Self::Year10 | Self::Halving | Self::Epoch => 1,
Self::Height => 6,
Self::Minute10 => CacheClass::Bucket { margin: 8 },
Self::Minute30 => CacheClass::Bucket { margin: 3 },
Self::Hour1 | Self::Hour4 | Self::Hour12 => CacheClass::Bucket { margin: 2 },
Self::Day1 | Self::Day3 | Self::Week1 => CacheClass::Bucket { margin: 2 },
Self::Month1 | Self::Month3 | Self::Month6 => CacheClass::Bucket { margin: 1 },
Self::Year1 | Self::Year10 | Self::Halving | Self::Epoch => {
CacheClass::Bucket { margin: 1 }
}
Self::Height => CacheClass::Bucket { margin: 6 },
Self::TxIndex
| Self::TxInIndex
| Self::TxOutIndex
@@ -225,11 +239,8 @@ impl Index {
| Self::P2TRAddrIndex
| Self::P2WPKHAddrIndex
| Self::P2WSHAddrIndex
| Self::UnknownOutputIndex
| Self::FundedAddrIndex
| Self::EmptyAddrIndex => {
panic!("cohort index has no series cache safety margin")
}
| Self::UnknownOutputIndex => CacheClass::Entity,
Self::FundedAddrIndex | Self::EmptyAddrIndex => CacheClass::Mutable,
}
}

View File

@@ -20,8 +20,6 @@ pub struct SeriesData<T = Value> {
/// Value type (e.g. "f32", "u64", "Sats")
#[serde(rename = "type", default)]
pub value_type: String,
/// Total number of data points in the series
pub total: usize,
/// Start index (inclusive) of the returned range
pub start: usize,
/// End index (exclusive) of the returned range
@@ -33,7 +31,8 @@ pub struct SeriesData<T = Value> {
}
impl SeriesData {
/// Write series data as JSON to buffer: `{"version":N,"index":"...","total":N,"start":N,"end":N,"stamp":"...","data":[...]}`
/// Write series data as JSON to buffer: `{"version":N,"index":"...","start":N,"end":N,"stamp":"...","data":[...]}`.
/// `total` is omitted so historical-range bodies stay cacheable across appends. Clients call `/len` for the live count.
pub fn serialize(
vec: &dyn AnySerializableVec,
index: Index,
@@ -53,9 +52,7 @@ impl SeriesData {
buf.extend_from_slice(index.name().as_bytes());
buf.extend_from_slice(b"\",\"type\":\"");
buf.extend_from_slice(vec.value_type_to_string().as_bytes());
buf.extend_from_slice(b"\",\"total\":");
buf.extend_from_slice(itoa_buf.format(total).as_bytes());
buf.extend_from_slice(b",\"start\":");
buf.extend_from_slice(b"\",\"start\":");
buf.extend_from_slice(itoa_buf.format(start).as_bytes());
buf.extend_from_slice(b",\"end\":");
buf.extend_from_slice(itoa_buf.format(end).as_bytes());